mirror of https://github.com/scrapy/scrapy.git
Automated merge with ssh://hg.scrapy.org:2222/scrapy-0.12
This commit is contained in:
Pablo Hoffman
commit
5da6ffb57b
2
AUTHORS
2
AUTHORS
|
|
@ -27,3 +27,5 @@ Here is the list of the primary authors & contributors:
|
|||
* Shuaib Khan
|
||||
* Didier Deshommes
|
||||
* Vikas Dhiman
|
||||
* Jochen Maes
|
||||
* Darian Moody
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ include AUTHORS
|
|||
include INSTALL
|
||||
include LICENSE
|
||||
include MANIFEST.in
|
||||
include scrapy/core/downloader/responsetypes/mime.types
|
||||
include scrapy/mime.types
|
||||
include scrapyd/default_scrapyd.conf
|
||||
recursive-include scrapy/templates *
|
||||
recursive-include scrapy/tests/sample_data *
|
||||
|
|
|
|||
2
README
2
README
|
|
@ -1,4 +1,4 @@
|
|||
This is Scrapy, an opensource screen scraping framework written in Python.
|
||||
|
||||
For more visit the project home page at http://scrapy.org
|
||||
For more info visit the project home page at http://scrapy.org
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
#!/bin/sh
|
||||
|
||||
repotac=$(cd $(dirname $0)/../extras; pwd)/scrapyd.tac
|
||||
|
||||
if [ -f "$repotac" ]; then
|
||||
tacfile="$repotac"
|
||||
elif [ -f "/usr/share/scrapyd/scrapyd.tac" ]; then
|
||||
tacfile="/usr/share/scrapyd/scrapyd.tac"
|
||||
else
|
||||
echo "Unable to find scrapy.tac file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
twistd -ny "$tacfile"
|
||||
|
|
@ -2,13 +2,14 @@ Source: scrapy-SUFFIX
|
|||
Section: python
|
||||
Priority: optional
|
||||
Maintainer: Insophia Team <info@insophia.com>
|
||||
Build-Depends: debhelper (>= 7.0.50), python (>=2.5), python-twisted
|
||||
Build-Depends: debhelper (>= 7.0.50), python (>=2.6), python-twisted, python-w3lib
|
||||
Standards-Version: 3.8.4
|
||||
Homepage: http://scrapy.org/
|
||||
|
||||
Package: scrapy-SUFFIX
|
||||
Architecture: all
|
||||
Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl
|
||||
Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl, python-w3lib
|
||||
Recommends: python-setuptools
|
||||
Conflicts: python-scrapy, scrapy, scrapy-0.11
|
||||
Provides: python-scrapy, scrapy
|
||||
Description: Python web crawling and scraping framework
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
examples/*
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
usr/lib/python*/*-packages/scrapy
|
||||
usr/lib/python*/*-packages/scrapy*
|
||||
usr/bin
|
||||
extras/scrapy_bash_completion etc/bash_completion.d/
|
||||
|
|
|
|||
|
|
@ -1,3 +1,2 @@
|
|||
usr/lib/python*/*-packages/scrapyd
|
||||
debian/scrapyd-files/000-default etc/scrapyd/conf.d
|
||||
extras/scrapyd.tac usr/share/scrapyd
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
from docutils.parsers.rst.roles import set_classes
|
||||
from docutils import nodes
|
||||
|
||||
def setup(app):
|
||||
app.add_crossref_type(
|
||||
directivename = "setting",
|
||||
|
|
@ -19,3 +22,10 @@ def setup(app):
|
|||
rolename = "reqmeta",
|
||||
indextemplate = "pair: %s; reqmeta",
|
||||
)
|
||||
app.add_role('source', source_role)
|
||||
|
||||
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
url = 'http://dev.scrapy.org/browser/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
.. _api-stability:
|
||||
|
||||
============================
|
||||
Versioning and API Stability
|
||||
============================
|
||||
|
||||
API stability is one of Scrapy major goals.
|
||||
|
||||
Versioning
|
||||
==========
|
||||
|
||||
When Scrapy reaches 1.0, each release will consist of three version numbers:
|
||||
|
||||
* major - big, backwards-incompatible changes
|
||||
* minor - new features and backwards-compatible changes
|
||||
* micro - bug fixes only
|
||||
|
||||
Until Scrapy reaches 1.0, minor releases (0.7, 0.8, etc) will follow the same
|
||||
policy as major releases.
|
||||
|
||||
Sometimes the micro version can be omitted, for brevity, when it's not
|
||||
relevant.
|
||||
|
||||
API Stability
|
||||
=============
|
||||
|
||||
Methods or functions that start with a single dash (``_``) are private and
|
||||
should never be relied as stable. Besides those, the plan is to stabilize and
|
||||
document the entire API, as we approach the 1.0 release.
|
||||
|
||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||
new methods or functionality but the existing methods should keep working the
|
||||
same way.
|
||||
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
.. _topics-crawlspider-v2:
|
||||
|
||||
==============
|
||||
CrawlSpider v2
|
||||
==============
|
||||
|
||||
Introduction
|
||||
============
|
||||
|
||||
TODO: introduction
|
||||
|
||||
Rules Matching
|
||||
==============
|
||||
|
||||
TODO: describe purpose of rules
|
||||
|
||||
Request Extractors & Processors
|
||||
===============================
|
||||
|
||||
TODO: describe purpose of extractors & processors
|
||||
|
||||
Examples
|
||||
========
|
||||
|
||||
TODO: plenty of examples
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.spider
|
||||
:synopsis: CrawlSpider
|
||||
|
||||
|
||||
Reference
|
||||
=========
|
||||
|
||||
CrawlSpider
|
||||
-----------
|
||||
|
||||
TODO: describe crawlspider
|
||||
|
||||
.. class:: CrawlSpider
|
||||
|
||||
TODO: describe class
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.rules
|
||||
:synopsis: Rules
|
||||
|
||||
Rules
|
||||
-----
|
||||
|
||||
TODO: describe spider rules
|
||||
|
||||
.. class:: Rule
|
||||
|
||||
TODO: describe Rules class
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.reqext
|
||||
:synopsis: Request Extractors
|
||||
|
||||
Request Extractors
|
||||
------------------
|
||||
|
||||
TODO: describe extractors purpose
|
||||
|
||||
.. class:: BaseSgmlRequestExtractor
|
||||
|
||||
TODO: describe base extractor
|
||||
|
||||
.. class:: SgmlRequestExtractor
|
||||
|
||||
TODO: describe sgml extractor
|
||||
|
||||
.. class:: XPathRequestExtractor
|
||||
|
||||
TODO: describe xpath request extractor
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.reqproc
|
||||
:synopsis: Request Processors
|
||||
|
||||
Request Processors
|
||||
------------------
|
||||
|
||||
TODO: describe request processors
|
||||
|
||||
.. class:: Canonicalize
|
||||
|
||||
TODO: describe proc
|
||||
|
||||
.. class:: Unique
|
||||
|
||||
TODO: describe unique
|
||||
|
||||
.. class:: FilterDomain
|
||||
|
||||
TODO: describe filter domain
|
||||
|
||||
.. class:: FilterUrl
|
||||
|
||||
TODO: describe filter url
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.matchers
|
||||
:synopsis: Matchers
|
||||
|
||||
Request/Response Matchers
|
||||
-------------------------
|
||||
|
||||
TODO: describe matchers
|
||||
|
||||
.. class:: BaseMatcher
|
||||
|
||||
TODO: describe base matcher
|
||||
|
||||
.. class:: UrlMatcher
|
||||
|
||||
TODO: describe url matcher
|
||||
|
||||
.. class:: UrlRegexMatcher
|
||||
|
||||
TODO: describe UrlListMatcher
|
||||
|
||||
.. class:: UrlListMatcher
|
||||
|
||||
TODO: describe url list matcher
|
||||
|
||||
|
||||
|
|
@ -20,4 +20,3 @@ it's properly merged) . Use at your own risk.
|
|||
:maxdepth: 1
|
||||
|
||||
djangoitems
|
||||
crawlspider-v2
|
||||
|
|
|
|||
74
docs/faq.rst
74
docs/faq.rst
|
|
@ -3,7 +3,7 @@
|
|||
Frequently Asked Questions
|
||||
==========================
|
||||
|
||||
How does Scrapy compare to BeautifulSoul or lxml?
|
||||
How does Scrapy compare to BeautifulSoup or lxml?
|
||||
-------------------------------------------------
|
||||
|
||||
`BeautifulSoup`_ and `lxml`_ are libraries for parsing HTML and XML. Scrapy is
|
||||
|
|
@ -84,10 +84,11 @@ How can I simulate a user login in my spider?
|
|||
|
||||
See :ref:`topics-request-response-ref-request-userlogin`.
|
||||
|
||||
Can I crawl in breadth-first order instead of depth-first order?
|
||||
----------------------------------------------------------------
|
||||
Does Scrapy crawl in breath-first or depth-first order?
|
||||
-------------------------------------------------------
|
||||
|
||||
Yes, there's a setting for that: :setting:`SCHEDULER_ORDER`.
|
||||
It crawls on breath-first order by default, but you can change it to
|
||||
depth-first order by setting the :setting:`DEPTH_PRIORITY` setting to ``-1``.
|
||||
|
||||
My Scrapy crawler has memory leaks. What can I do?
|
||||
--------------------------------------------------
|
||||
|
|
@ -115,24 +116,10 @@ Try changing the default `Accept-Language`_ request header by overriding the
|
|||
|
||||
.. _Accept-Language: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
|
||||
|
||||
Where can I find some example code using Scrapy?
|
||||
------------------------------------------------
|
||||
Where can I find some example Scrapy projects?
|
||||
----------------------------------------------
|
||||
|
||||
Scrapy comes with a built-in, fully functional project to scrape the `Google
|
||||
Directory`_. You can find it in the `examples/googledir`_ directory of the
|
||||
Scrapy distribution.
|
||||
|
||||
Also, there's a site for sharing code snippets (spiders, middlewares,
|
||||
extensions) called `Scrapy snippets`_.
|
||||
|
||||
Finally, you can find some example code for performing not-so-trivial tasks in
|
||||
the `Scrapy Recipes`_ wiki page.
|
||||
|
||||
.. _Google Directory: http://www.google.com/dirhp
|
||||
.. _examples/googledir: http://dev.scrapy.org/browser/examples/googledir
|
||||
.. _Community Spiders: http://dev.scrapy.org/wiki/CommunitySpiders
|
||||
.. _Scrapy Recipes: http://dev.scrapy.org/wiki/ScrapyRecipes
|
||||
.. _Scrapy snippets: http://snippets.scrapy.org/
|
||||
See :ref:`intro-examples`.
|
||||
|
||||
Can I run a spider without creating a project?
|
||||
----------------------------------------------
|
||||
|
|
@ -240,3 +227,48 @@ In order to avoid parsing all the entire feed at once in memory, you can use
|
|||
the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators``
|
||||
module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use
|
||||
under the cover.
|
||||
|
||||
Does Scrapy manage cookies automatically?
|
||||
-----------------------------------------
|
||||
|
||||
Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them
|
||||
back on subsequent requests, like any regular web browser does.
|
||||
|
||||
For more info see :ref:`topics-request-response` and :ref:`cookies-mw`.
|
||||
|
||||
How can I see the cookies being sent and received from Scrapy?
|
||||
--------------------------------------------------------------
|
||||
|
||||
Enable the :setting:`COOKIES_DEBUG` setting.
|
||||
|
||||
How can I instruct a spider to stop itself?
|
||||
-------------------------------------------
|
||||
|
||||
Raise the :exc:`~scrapy.exceptions.CloseSpider` exception from a callback. For
|
||||
more info see: :exc:`~scrapy.exceptions.CloseSpider`.
|
||||
|
||||
How can I prevent my Scrapy bot from getting banned?
|
||||
----------------------------------------------------
|
||||
|
||||
Some websites implement certain measures to prevent bots from crawling them,
|
||||
with varying degrees of sophistication. Getting around those measures can be
|
||||
difficult and tricky, and may sometimes require special infrastructure.
|
||||
|
||||
Here are some tips to keep in mind when dealing with these kind of sites:
|
||||
|
||||
* rotate your user agent from a pool of well-known ones from browsers (google
|
||||
around to get a list of them)
|
||||
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
|
||||
cookies to spot bot behaviour
|
||||
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
|
||||
* is possible, use `Google cache`_ to fetch pages, instead of hitting the sites
|
||||
directly
|
||||
* use a pool of rotating IPs. For example, the free `Tor project`_.
|
||||
|
||||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
||||
.. _user agents: http://en.wikipedia.org/wiki/User_agent
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: http://scrapy.org/support/
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ First steps
|
|||
intro/overview
|
||||
intro/install
|
||||
intro/tutorial
|
||||
intro/examples
|
||||
|
||||
:doc:`intro/overview`
|
||||
Understand what Scrapy is and how it can help you.
|
||||
|
|
@ -43,9 +44,13 @@ First steps
|
|||
:doc:`intro/tutorial`
|
||||
Write your first Scrapy project.
|
||||
|
||||
:doc:`intro/examples`
|
||||
Learn more by playing with a pre-made Scrapy project.
|
||||
|
||||
Scraping basics
|
||||
===============
|
||||
.. _section-basics:
|
||||
|
||||
Basic concepts
|
||||
==============
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
|
@ -59,6 +64,7 @@ Scraping basics
|
|||
topics/shell
|
||||
topics/item-pipeline
|
||||
topics/feed-exports
|
||||
topics/link-extractors
|
||||
|
||||
:doc:`topics/commands`
|
||||
Learn about the command-line tool used to manage your Scrapy project.
|
||||
|
|
@ -84,6 +90,8 @@ Scraping basics
|
|||
:doc:`topics/feed-exports`
|
||||
Output your scraped data using different formats and storages.
|
||||
|
||||
:doc:`topics/link-extractors`
|
||||
Convenient classes to extract links to follow from pages.
|
||||
|
||||
Built-in services
|
||||
=================
|
||||
|
|
@ -213,13 +221,13 @@ All the rest
|
|||
:hidden:
|
||||
|
||||
contributing
|
||||
api-stability
|
||||
versioning
|
||||
experimental/index
|
||||
|
||||
:doc:`contributing`
|
||||
Learn how to contribute to the Scrapy project.
|
||||
|
||||
:doc:`api-stability`
|
||||
:doc:`versioning`
|
||||
Understand Scrapy versioning and API stability.
|
||||
|
||||
:doc:`experimental/index`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
.. _intro-examples:
|
||||
|
||||
========
|
||||
Examples
|
||||
========
|
||||
|
||||
The best way to learn is with examples. For this reason, there is an example
|
||||
Scrapy project named dirbot_, that you can use to play and learn more about
|
||||
Scrapy. It contains the dmoz spider described in the tutorial.
|
||||
|
||||
This dirbot_ project is available at: https://github.com/scrapy/dirbot
|
||||
|
||||
It contains a README file with a detailed description of the project contents.
|
||||
|
||||
If you're familiar with git, you can checkout the code. Otherwise you can
|
||||
download a tarball or zip file of the project by clicking on `Downloads`_.
|
||||
|
||||
Other sources for examples
|
||||
==========================
|
||||
|
||||
1. There is a site for sharing code snippets (spiders, middlewares, extensions)
|
||||
called `Scrapy snippets`_.
|
||||
|
||||
2. There is also a `Scrapy Recipes`_ page on the wiki that contains code
|
||||
snippets for performing not-so-trivial tasks. This contains code posted before
|
||||
the `Scrapy snippets`_ site was lunched. New code should be posted there.
|
||||
|
||||
.. _dirbot: https://github.com/scrapy/dirbot
|
||||
.. _Downloads: https://github.com/scrapy/dirbot/archives/master
|
||||
.. _Scrapy Recipes: http://dev.scrapy.org/wiki/ScrapyRecipes
|
||||
.. _Scrapy snippets: http://snippets.scrapy.org/
|
||||
|
|
@ -16,6 +16,8 @@ Requirements
|
|||
* `Twisted`_ 2.5.0, 8.0 or above (Windows users: you'll need to install
|
||||
`Zope.Interface`_ and maybe `pywin32`_ because of `this Twisted bug`_)
|
||||
|
||||
* `w3lib`_
|
||||
|
||||
* `lxml`_ or `libxml2`_ (if using `libxml2`_, version 2.6.28 or above is highly recommended)
|
||||
|
||||
* `simplejson`_ (not required if using Python 2.6 or above)
|
||||
|
|
@ -111,6 +113,10 @@ If you're running Debian Linux, run the following command as root::
|
|||
|
||||
apt-get install python-twisted python-libxml2 python-pyopenssl python-simplejson
|
||||
|
||||
Then::
|
||||
|
||||
easy_install -U w3lib
|
||||
|
||||
And then follow the instructions in :ref:`intro-install-scrapy`.
|
||||
|
||||
Arch Linux
|
||||
|
|
@ -120,13 +126,17 @@ If you are running Arch Linux, run the following command as root::
|
|||
|
||||
pacman -S twisted libxml2 pyopenssl python-simplejson
|
||||
|
||||
Then::
|
||||
|
||||
easy_install -U w3lib
|
||||
|
||||
And then follow the instructions in :ref:`intro-install-scrapy`.
|
||||
|
||||
Other Linux distros
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The easiest way to install Scrapy in other Linux distros is through
|
||||
``easy_install``, which will automatically install Twisted and lxml as
|
||||
``easy_install``, which will automatically install Twisted, w3lib and lxml as
|
||||
dependencies. See :ref:`intro-install-easy`.
|
||||
|
||||
Another way would be to install dependencies, if you know the packages in your
|
||||
|
|
@ -135,8 +145,8 @@ distros that meets them. See :ref:`intro-install-requirements`.
|
|||
Mac OS X
|
||||
--------
|
||||
|
||||
The easiest way to install Scrapy on Mac is through ``easy_install``, which
|
||||
will automatically install Twisted and lxml as dependencies.
|
||||
The easiest way to install Scrapy on Mac is through ``easy_install`` or
|
||||
``pip``, which will automatically install Twisted, w3lib and lxml dependencies.
|
||||
|
||||
See :ref:`intro-install-easy`.
|
||||
|
||||
|
|
@ -164,6 +174,7 @@ There are two ways to install Scrapy in Windows:
|
|||
|
||||
.. _Python: http://www.python.org
|
||||
.. _Twisted: http://twistedmatrix.com
|
||||
.. _w3lib: http://pypi.python.org/pypi/w3lib
|
||||
.. _lxml: http://codespeak.net/lxml/
|
||||
.. _libxml2: http://xmlsoft.org
|
||||
.. _pywin32: http://sourceforge.net/projects/pywin32/
|
||||
|
|
|
|||
|
|
@ -246,6 +246,10 @@ scraping easy and efficient, such as:
|
|||
* :ref:`Logging <topics-logging>` facility that you can hook on to for catching
|
||||
errors during the scraping process.
|
||||
|
||||
* Support for crawling based on URLs discovered through `Sitemaps`_
|
||||
|
||||
* A caching DNS resolver
|
||||
|
||||
What's next?
|
||||
============
|
||||
|
||||
|
|
@ -262,3 +266,4 @@ interest!
|
|||
.. _XPath: http://www.w3.org/TR/xpath
|
||||
.. _XPath reference: http://www.w3.org/TR/xpath
|
||||
.. _Amazon S3: http://aws.amazon.com/s3/
|
||||
.. _Sitemaps: http://www.sitemaps.org
|
||||
|
|
|
|||
|
|
@ -36,13 +36,13 @@ Creating a project
|
|||
Before you start scraping, you will have set up a new Scrapy project. Enter a
|
||||
directory where you'd like to store your code and then run::
|
||||
|
||||
scrapy startproject dmoz
|
||||
scrapy startproject tutorial
|
||||
|
||||
This will create a ``dmoz`` directory with the following contents::
|
||||
This will create a ``tutorial`` directory with the following contents::
|
||||
|
||||
dmoz/
|
||||
tutorial/
|
||||
scrapy.cfg
|
||||
dmoz/
|
||||
tutorial/
|
||||
__init__.py
|
||||
items.py
|
||||
pipelines.py
|
||||
|
|
@ -54,19 +54,19 @@ This will create a ``dmoz`` directory with the following contents::
|
|||
These are basically:
|
||||
|
||||
* ``scrapy.cfg``: the project configuration file
|
||||
* ``dmoz/``: the project's python module, you'll later import your code from
|
||||
* ``tutorial/``: the project's python module, you'll later import your code from
|
||||
here.
|
||||
* ``dmoz/items.py``: the project's items file.
|
||||
* ``dmoz/pipelines.py``: the project's pipelines file.
|
||||
* ``dmoz/settings.py``: the project's settings file.
|
||||
* ``dmoz/spiders/``: a directory where you'll later put your spiders.
|
||||
* ``tutorial/items.py``: the project's items file.
|
||||
* ``tutorial/pipelines.py``: the project's pipelines file.
|
||||
* ``tutorial/settings.py``: the project's settings file.
|
||||
* ``tutorial/spiders/``: a directory where you'll later put your spiders.
|
||||
|
||||
Defining our Item
|
||||
=================
|
||||
|
||||
`Items` are containers that will be loaded with the scraped data; they work
|
||||
like simple python dicts but they offer some additional features like providing
|
||||
default values.
|
||||
like simple python dicts but provide additional protecting against populating
|
||||
undeclared fields, to prevent typos.
|
||||
|
||||
They are declared by creating an :class:`scrapy.item.Item` class an defining
|
||||
its attributes as :class:`scrapy.item.Field` objects, like you will in an ORM
|
||||
|
|
@ -78,8 +78,6 @@ from dmoz.org, as we want to capture the name, url and description of the
|
|||
sites, we define fields for each of these three attributes. To do that, we edit
|
||||
items.py, found in the dmoz directory. Our Item class looks like this::
|
||||
|
||||
# Define here the models for your scraped items
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
class DmozItem(Item):
|
||||
|
|
@ -128,7 +126,7 @@ This is the code for our first Spider; save it in a file named
|
|||
from scrapy.spider import BaseSpider
|
||||
|
||||
class DmozSpider(BaseSpider):
|
||||
name = "dmoz.org"
|
||||
name = "dmoz"
|
||||
allowed_domains = ["dmoz.org"]
|
||||
start_urls = [
|
||||
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
|
||||
|
|
@ -144,27 +142,25 @@ Crawling
|
|||
|
||||
To put our spider to work, go to the project's top level directory and run::
|
||||
|
||||
scrapy crawl dmoz.org
|
||||
scrapy crawl dmoz
|
||||
|
||||
The ``crawl dmoz.org`` command runs the spider for the ``dmoz.org`` domain. You
|
||||
The ``crawl dmoz`` command runs the spider for the ``dmoz.org`` domain. You
|
||||
will get an output similar to this::
|
||||
|
||||
2008-08-20 03:51:13-0300 [scrapy] INFO: Started project: dmoz
|
||||
2008-08-20 03:51:13-0300 [dmoz] INFO: Enabled extensions: ...
|
||||
2008-08-20 03:51:13-0300 [dmoz] INFO: Enabled scheduler middlewares: ...
|
||||
2008-08-20 03:51:13-0300 [dmoz] INFO: Enabled downloader middlewares: ...
|
||||
2008-08-20 03:51:13-0300 [dmoz] INFO: Enabled spider middlewares: ...
|
||||
2008-08-20 03:51:13-0300 [dmoz] INFO: Enabled item pipelines: ...
|
||||
2008-08-20 03:51:14-0300 [dmoz.org] INFO: Spider opened
|
||||
2008-08-20 03:51:14-0300 [dmoz.org] DEBUG: Crawled <http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: <None>)
|
||||
2008-08-20 03:51:14-0300 [dmoz.org] DEBUG: Crawled <http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: <None>)
|
||||
2008-08-20 03:51:14-0300 [dmoz.org] INFO: Spider closed (finished)
|
||||
2008-08-20 03:51:13-0300 [tutorial] INFO: Enabled extensions: ...
|
||||
2008-08-20 03:51:13-0300 [tutorial] INFO: Enabled downloader middlewares: ...
|
||||
2008-08-20 03:51:13-0300 [tutorial] INFO: Enabled spider middlewares: ...
|
||||
2008-08-20 03:51:13-0300 [tutorial] INFO: Enabled item pipelines: ...
|
||||
2008-08-20 03:51:14-0300 [dmoz] INFO: Spider opened
|
||||
2008-08-20 03:51:14-0300 [dmoz] DEBUG: Crawled <http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: <None>)
|
||||
2008-08-20 03:51:14-0300 [dmoz] DEBUG: Crawled <http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: <None>)
|
||||
2008-08-20 03:51:14-0300 [dmoz] INFO: Spider closed (finished)
|
||||
|
||||
Pay attention to the lines containing ``[dmoz.org]``, which corresponds to
|
||||
our spider (identified by the domain ``"dmoz.org"``). You can see a log line
|
||||
for each URL defined in ``start_urls``. Because these URLs are the starting
|
||||
ones, they have no referrers, which is shown at the end of the log line,
|
||||
where it says ``(referer: <None>)``.
|
||||
Pay attention to the lines containing ``[dmoz]``, which corresponds to our
|
||||
spider. You can see a log line for each URL defined in ``start_urls``. Because
|
||||
these URLs are the starting ones, they have no referrers, which is shown at the
|
||||
end of the log line, where it says ``(referer: <None>)``.
|
||||
|
||||
But more interesting, as our ``parse`` method instructs, two files have been
|
||||
created: *Books* and *Resources*, with the content of both URLs.
|
||||
|
|
@ -342,7 +338,7 @@ Let's add this code to our spider::
|
|||
from scrapy.selector import HtmlXPathSelector
|
||||
|
||||
class DmozSpider(BaseSpider):
|
||||
name = "dmoz.org"
|
||||
name = "dmoz"
|
||||
allowed_domains = ["dmoz.org"]
|
||||
start_urls = [
|
||||
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
|
||||
|
|
@ -361,7 +357,7 @@ Let's add this code to our spider::
|
|||
Now try crawling the dmoz.org domain again and you'll see sites being printed
|
||||
in your output, run::
|
||||
|
||||
scrapy crawl dmoz.org
|
||||
scrapy crawl dmoz
|
||||
|
||||
Using our item
|
||||
--------------
|
||||
|
|
@ -376,16 +372,16 @@ standard dict syntax like::
|
|||
'Example title'
|
||||
|
||||
Spiders are expected to return their scraped data inside
|
||||
:class:`~scrapy.item.Item` objects, so to actually return the data we've
|
||||
scraped so far, the code for our Spider should be like this::
|
||||
:class:`~scrapy.item.Item` objects. So, in order to returnthe data we've
|
||||
scraped so far, the final code for our Spider would be like this::
|
||||
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
|
||||
from dmoz.items import DmozItem
|
||||
from tutorial.items import DmozItem
|
||||
|
||||
class DmozSpider(BaseSpider):
|
||||
name = "dmoz.org"
|
||||
name = "dmoz"
|
||||
allowed_domains = ["dmoz.org"]
|
||||
start_urls = [
|
||||
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
|
||||
|
|
@ -404,10 +400,19 @@ scraped so far, the code for our Spider should be like this::
|
|||
items.append(item)
|
||||
return items
|
||||
|
||||
.. note:: You can find a fully-functional variant of this spider in the dirbot_
|
||||
project available at https://github.com/scrapy/dirbot
|
||||
|
||||
Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s::
|
||||
|
||||
[dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in <http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
|
||||
[dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], link=[u'http://www.informit.com/store/product.aspx?isbn=0130211192'], title=[u'XML Processing with Python']) in <http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
|
||||
[dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
|
||||
{'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n],
|
||||
'link': [u'http://gnosis.cx/TPiP/'],
|
||||
'title': [u'Text Processing in Python']}
|
||||
[dmoz] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
|
||||
{'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'],
|
||||
'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'],
|
||||
'title': [u'XML Processing with Python']}
|
||||
|
||||
Storing the scraped data
|
||||
========================
|
||||
|
|
@ -415,7 +420,7 @@ Storing the scraped data
|
|||
The simplest way to store the scraped data is by using the :ref:`Feed exports
|
||||
<topics-feed-exports>`, with the following command::
|
||||
|
||||
scrapy crawl dmoz.org --set FEED_URI=items.json --set FEED_FORMAT=json
|
||||
scrapy crawl dmoz --set FEED_URI=items.json --set FEED_FORMAT=json
|
||||
|
||||
That will generate a ``items.json`` file containing all scraped items,
|
||||
serialized in `JSON`_.
|
||||
|
|
@ -424,14 +429,16 @@ In small projects (like the one in this tutorial), that should be enough.
|
|||
However, if you want to perform more complex things with the scraped items, you
|
||||
can write an :ref:`Item Pipeline <topics-item-pipeline>`. As with Items, a
|
||||
placeholder file for Item Pipelines has been set up for you when the project is
|
||||
created, in ``dmoz/pipelines.py``. Though you don't need to implement any item
|
||||
created, in ``tutorial/pipelines.py``. Though you don't need to implement any item
|
||||
pipeline if you just want to store the scraped items.
|
||||
|
||||
Finale
|
||||
======
|
||||
Next steps
|
||||
==========
|
||||
|
||||
This tutorial covers only the basics of Scrapy, but there's a lot of other
|
||||
features not mentioned here. We recommend you continue reading the section
|
||||
:ref:`topics-index`.
|
||||
features not mentioned here. We recommend you continue by playing with an
|
||||
example project (see :ref:`intro-examples`), and then continue with the section
|
||||
:ref:`section-basics`.
|
||||
|
||||
.. _JSON: http://en.wikipedia.org/wiki/JSON
|
||||
.. _dirbot: https://github.com/scrapy/dirbot
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB After Width: | Height: | Size: 90 KiB |
|
|
@ -77,14 +77,6 @@ requests). They provide a convenient mechanism for extending Scrapy
|
|||
functionality by plugging custom code. For more information see
|
||||
:ref:`topics-spider-middleware`.
|
||||
|
||||
Scheduler middlewares
|
||||
---------------------
|
||||
|
||||
Scheduler middlewares are specific hooks that sit between the Engine and the
|
||||
Scheduler and process requests when they pass from the Engine to the Scheduler
|
||||
and vice-versa. They provide a convenient mechanism for extending Scrapy
|
||||
functionality by plugging custom code.
|
||||
|
||||
Data flow
|
||||
=========
|
||||
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ Project-only commands:
|
|||
|
||||
* :command:`crawl`
|
||||
* :command:`list`
|
||||
* :command:`edit`
|
||||
* :command:`parse`
|
||||
* :command:`genspider`
|
||||
* :command:`server`
|
||||
|
|
@ -217,22 +218,16 @@ Usage example::
|
|||
crawl
|
||||
-----
|
||||
|
||||
* Syntax: ``scrapy crawl <spider|url>``
|
||||
* Syntax: ``scrapy crawl <spider>``
|
||||
* Requires project: *yes*
|
||||
|
||||
Start crawling a spider. If a URL is passed instead of a spider, it will start
|
||||
from that URL instead of the spider start urls.
|
||||
Start crawling a spider.
|
||||
|
||||
Usage examples::
|
||||
|
||||
$ scrapy crawl example.com
|
||||
[ ... example.com spider starts crawling ... ]
|
||||
|
||||
$ scrapy crawl myspider
|
||||
[ ... myspider starts crawling ... ]
|
||||
|
||||
$ scrapy crawl http://example.com/some/page.html
|
||||
[ ... spider that handles example.com starts crawling from that url ... ]
|
||||
|
||||
.. command:: server
|
||||
|
||||
|
|
@ -269,6 +264,25 @@ Usage example::
|
|||
spider1
|
||||
spider2
|
||||
|
||||
.. command:: edit
|
||||
|
||||
edit
|
||||
----
|
||||
|
||||
* Syntax: ``scrapy edit <spider>``
|
||||
* Requires project: *yes*
|
||||
|
||||
Edit the given spider using the editor defined in the :setting:`EDITOR`
|
||||
setting.
|
||||
|
||||
This command is provided only as a convenient shortcut for the most common
|
||||
case, the developer is of course free to choose any tool or IDE to write and
|
||||
debug his spiders.
|
||||
|
||||
Usage example::
|
||||
|
||||
$ scrapy edit spider1
|
||||
|
||||
.. command:: fetch
|
||||
|
||||
fetch
|
||||
|
|
|
|||
|
|
@ -158,6 +158,8 @@ middleware, see the :ref:`downloader middleware usage guide
|
|||
For a list of the components enabled by default (and their orders) see the
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
|
||||
|
||||
.. _cookies-mw:
|
||||
|
||||
CookiesMiddleware
|
||||
-----------------
|
||||
|
||||
|
|
@ -166,7 +168,48 @@ CookiesMiddleware
|
|||
|
||||
.. class:: CookiesMiddleware
|
||||
|
||||
This middleware enables working with sites that need cookies.
|
||||
This middleware enables working with sites that require cookies, such as
|
||||
those that use sessions. It keeps track of cookies sent by web servers, and
|
||||
send them back on subsequent requests (from that spider), just like web
|
||||
browsers do.
|
||||
|
||||
The following settings can be used to configure the cookie middleware:
|
||||
|
||||
* :setting:`COOKIES_ENABLED`
|
||||
* :setting:`COOKIES_DEBUG`
|
||||
|
||||
.. setting:: COOKIES_ENABLED
|
||||
|
||||
COOKIES_ENABLED
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether to enable the cookies middleware. If disabled, no cookies will be sent
|
||||
to web servers.
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
COOKIES_DEBUG
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie``
|
||||
header) and all cookies received in responses (ie. ``Set-Cookie`` header).
|
||||
|
||||
Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
|
||||
|
||||
2011-04-06 14:35:10-0300 [diningcity] INFO: Spider opened
|
||||
2011-04-06 14:35:10-0300 [diningcity] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
|
||||
Cookie: clientlanguage_nl=en_EN
|
||||
2011-04-06 14:35:14-0300 [diningcity] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
|
||||
Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
|
||||
Set-Cookie: ip_isocode=US
|
||||
Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
|
||||
2011-04-06 14:49:50-0300 [diningcity] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
|
||||
[...]
|
||||
|
||||
|
||||
DefaultHeadersMiddleware
|
||||
------------------------
|
||||
|
|
@ -233,8 +276,18 @@ HttpCacheMiddleware
|
|||
downloads every time) and for trying your spider offline, when an Internet
|
||||
connection is not available.
|
||||
|
||||
File system storage
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
Scrapy ships with two storage backends for the HTTP cache middleware:
|
||||
|
||||
* :ref:`httpcache-fs-backend`
|
||||
* :ref:`httpcache-dbm-backend`
|
||||
|
||||
You can change the storage backend with the :setting:`HTTPCACHE_STORAGE`
|
||||
setting. Or you can also implement your own backend.
|
||||
|
||||
.. _httpcache-fs-backend:
|
||||
|
||||
File system backend (default)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
By default, the :class:`HttpCacheMiddleware` uses a file system storage with the following structure:
|
||||
|
||||
|
|
@ -257,11 +310,22 @@ inefficient in many file systems). An example directory could be::
|
|||
|
||||
/path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
|
||||
|
||||
The cache storage backend can be changed with the :setting:`HTTPCACHE_STORAGE`
|
||||
setting, but no other backend is provided with Scrapy yet.
|
||||
.. _httpcache-dbm-backend:
|
||||
|
||||
Settings
|
||||
~~~~~~~~
|
||||
DBM storage backend
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
A DBM_ storage backend is also available for the HTTP cache middleware. To use
|
||||
it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE`
|
||||
to ``scrapy.contrib.httpcache.DbmCacheStorage``.
|
||||
|
||||
By default, it uses the anydbm_ module, but you can change it with the
|
||||
:setting:`HTTPCACHE_DBM_MODULE` setting.
|
||||
|
||||
HTTPCache middleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The :class:`HttpCacheMiddleware` can be configured through the following
|
||||
settings:
|
||||
|
|
@ -346,6 +410,18 @@ Default: ``'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage
|
|||
|
||||
The class which implements the cache storage backend.
|
||||
|
||||
.. setting:: HTTPCACHE_DBM_MODULE
|
||||
|
||||
HTTPCACHE_DBM_MODULE
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``'anydbm'``
|
||||
|
||||
The database module to use in the :ref:`DBM storage backend
|
||||
<httpcache-dbm-backend>`. This setting is specific to the DBM backend.
|
||||
|
||||
|
||||
HttpCompressionMiddleware
|
||||
-------------------------
|
||||
|
|
@ -358,6 +434,16 @@ HttpCompressionMiddleware
|
|||
This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites.
|
||||
|
||||
ChunkedTransferMiddleware
|
||||
-------------------------
|
||||
|
||||
.. module:: scrapy.contrib.downloadermiddleware.chunked
|
||||
:synopsis: Chunked Transfer Middleware
|
||||
|
||||
.. class:: ChunkedTransferMiddleware
|
||||
|
||||
This middleware adds support for `chunked transfer encoding`_
|
||||
|
||||
HttpProxyMiddleware
|
||||
-------------------
|
||||
|
||||
|
|
@ -400,15 +486,48 @@ in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
|
|||
The :class:`RedirectMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`REDIRECT_MAX_METAREFRESH_DELAY` - Maximum meta-refresh delay that a page is allowed to have for redirection.
|
||||
* :setting:`REDIRECT_MAX_TIMES` - Maximum number of redirects to perform on a request.
|
||||
* :setting:`REDIRECT_PRIORITY_ADJUST` - Adjusts the redirected request priority by this amount.
|
||||
* :setting:`REDIRECT_ENABLED`
|
||||
* :setting:`REDIRECT_MAX_TIMES`
|
||||
* :setting:`REDIRECT_MAX_METAREFRESH_DELAY`
|
||||
|
||||
.. reqmeta:: dont_redirect
|
||||
|
||||
If :attr:`Request.meta <scrapy.http.Request.meta>` contains the
|
||||
``dont_redirect`` key, the request will be ignored by this middleware.
|
||||
|
||||
|
||||
RedirectMiddleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. setting:: REDIRECT_ENABLED
|
||||
|
||||
REDIRECT_ENABLED
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether the Redirect middleware will be enabled.
|
||||
|
||||
.. setting:: REDIRECT_MAX_TIMES
|
||||
|
||||
REDIRECT_MAX_TIMES
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``20``
|
||||
|
||||
The maximum number of redirections that will be follow for a single request.
|
||||
|
||||
.. setting:: REDIRECT_MAX_METAREFRESH_DELAY
|
||||
|
||||
REDIRECT_MAX_METAREFRESH_DELAY
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``100``
|
||||
|
||||
The maximum meta-refresh delay (in seconds) to follow the redirection.
|
||||
|
||||
RetryMiddleware
|
||||
---------------
|
||||
|
||||
|
|
@ -428,12 +547,13 @@ Once there are no more failed pages to retry, this middleware sends a signal
|
|||
The :class:`RetryMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`RETRY_TIMES` - how many times to retry a failed page
|
||||
* :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry
|
||||
* :setting:`RETRY_ENABLED`
|
||||
* :setting:`RETRY_TIMES`
|
||||
* :setting:`RETRY_HTTP_CODES`
|
||||
|
||||
About HTTP errors to consider:
|
||||
|
||||
You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the
|
||||
You may want to remove 400 from :setting:`RETRY_HTTP_CODES`, if you stick to the
|
||||
HTTP protocol. It's included by default because it's a common code used
|
||||
to indicate server overload, which would be something we want to retry.
|
||||
|
||||
|
|
@ -442,6 +562,39 @@ to indicate server overload, which would be something we want to retry.
|
|||
If :attr:`Request.meta <scrapy.http.Request.meta>` contains the ``dont_retry``
|
||||
key, the request will be ignored by this middleware.
|
||||
|
||||
RetryMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. setting:: RETRY_ENABLED
|
||||
|
||||
RETRY_ENABLED
|
||||
^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether the Retry middleware will be enabled.
|
||||
|
||||
.. setting:: RETRY_TIMES
|
||||
|
||||
RETRY_TIMES
|
||||
^^^^^^^^^^^
|
||||
|
||||
Default: ``2``
|
||||
|
||||
Maximum number of times to retry, in addition to the first download.
|
||||
|
||||
.. setting:: RETRY_HTTP_CODES
|
||||
|
||||
RETRY_HTTP_CODES
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``[500, 503, 504, 400, 408]``
|
||||
|
||||
Which HTTP response codes to retry. Other errors (DNS lookup issues,
|
||||
connections lost, etc) are always retried.
|
||||
|
||||
.. _topics-dlmw-robots:
|
||||
|
||||
RobotsTxtMiddleware
|
||||
|
|
@ -491,3 +644,7 @@ UserAgentMiddleware
|
|||
In order for a spider to override the default user agent, its `user_agent`
|
||||
attribute must be set.
|
||||
|
||||
|
||||
.. _DBM: http://en.wikipedia.org/wiki/Dbm
|
||||
.. _anydbm: http://docs.python.org/library/anydbm.html
|
||||
.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
|
|
|
|||
|
|
@ -22,6 +22,23 @@ DropItem
|
|||
The exception that must be raised by item pipeline stages to stop processing an
|
||||
Item. For more information see :ref:`topics-item-pipeline`.
|
||||
|
||||
CloseSpider
|
||||
-----------
|
||||
|
||||
.. exception:: CloseSpider(reason='cancelled')
|
||||
|
||||
This exception can be raised from a spider callback to request the spider to be
|
||||
closed/stopped. Supported arguments:
|
||||
|
||||
:param reason: the reason for closing
|
||||
:type reason: str
|
||||
|
||||
For example::
|
||||
|
||||
def parse_page(self, response):
|
||||
if 'Bandwidth exceeded' in response.body:
|
||||
raise CloseSpider('bandwidth_exceeded')
|
||||
|
||||
IgnoreRequest
|
||||
-------------
|
||||
|
||||
|
|
|
|||
|
|
@ -264,7 +264,7 @@ XmlItemExporter
|
|||
CsvItemExporter
|
||||
---------------
|
||||
|
||||
.. class:: CsvItemExporter(file, include_headers_line=True, \**kwargs)
|
||||
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
|
||||
|
||||
Exports Items in CSV format to the given file-like object. If the
|
||||
:attr:`fields_to_export` attribute is set, it will be used to define the
|
||||
|
|
@ -278,6 +278,10 @@ CsvItemExporter
|
|||
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
|
||||
:type include_headers_line: boolean
|
||||
|
||||
:param join_multivalued: The char (or chars) that will be used for joining
|
||||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor, and the leftover arguments to the
|
||||
`csv.writer`_ constructor, so you can use any `csv.writer` constructor
|
||||
|
|
|
|||
|
|
@ -178,10 +178,20 @@ Built-in extensions reference
|
|||
General purpose extensions
|
||||
--------------------------
|
||||
|
||||
Log Stats extension
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. module:: scrapy.contrib.logstats
|
||||
:synopsis: Basic stats logging
|
||||
|
||||
.. class:: LogStats
|
||||
|
||||
Log basic stats like crawled pages and scraped items.
|
||||
|
||||
Core Stats extension
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. module:: scrapy.contrib.corestats.corestats
|
||||
.. module:: scrapy.contrib.corestats
|
||||
:synopsis: Core stats collection
|
||||
|
||||
.. class:: CoreStats
|
||||
|
|
@ -256,11 +266,14 @@ Memory debugger extension
|
|||
|
||||
.. class:: scrapy.contrib.memdebug.MemoryDebugger
|
||||
|
||||
A memory debugger which collects some info about objects uncollected by the
|
||||
garbage collector and libxml2 memory leaks. To enable this extension, turn on
|
||||
the :setting:`MEMDEBUG_ENABLED` setting. The report will be printed to standard
|
||||
output. If the :setting:`MEMDEBUG_NOTIFY` setting contains a list of e-mails the
|
||||
report will also be sent to those addresses.
|
||||
An extension for debugging memory usage. It collects information about:
|
||||
|
||||
* objects uncollected by the Python garbage collector
|
||||
* libxml2 memory leaks
|
||||
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
|
||||
|
||||
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
|
||||
info will be stored in the stats.
|
||||
|
||||
Close spider extension
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -277,7 +290,7 @@ The conditions for closing a spider can be configured through the following
|
|||
settings:
|
||||
|
||||
* :setting:`CLOSESPIDER_TIMEOUT`
|
||||
* :setting:`CLOSESPIDER_ITEMPASSED`
|
||||
* :setting:`CLOSESPIDER_ITEMCOUNT`
|
||||
* :setting:`CLOSESPIDER_PAGECOUNT`
|
||||
* :setting:`CLOSESPIDER_ERRORCOUNT`
|
||||
|
||||
|
|
@ -293,16 +306,16 @@ more than that number of second, it will be automatically closed with the
|
|||
reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
|
||||
timeout.
|
||||
|
||||
.. setting:: CLOSESPIDER_ITEMPASSED
|
||||
.. setting:: CLOSESPIDER_ITEMCOUNT
|
||||
|
||||
CLOSESPIDER_ITEMPASSED
|
||||
""""""""""""""""""""""
|
||||
CLOSESPIDER_ITEMCOUNT
|
||||
"""""""""""""""""""""
|
||||
|
||||
Default: ``0``
|
||||
|
||||
An integer which specifies a number of items. If the spider scrapes more than
|
||||
that amount if items and those items are passed by the item pipeline, the
|
||||
spider will be closed with the reason ``closespider_itempassed``. If zero (or
|
||||
spider will be closed with the reason ``closespider_itemcount``. If zero (or
|
||||
non set), spiders won't be closed by number of passed items.
|
||||
|
||||
.. setting:: CLOSESPIDER_PAGECOUNT
|
||||
|
|
@ -357,15 +370,25 @@ Stack trace dump extension
|
|||
|
||||
.. class:: scrapy.contrib.debug.StackTraceDump
|
||||
|
||||
Dumps the stack trace of a runnning Scrapy process when a `SIGUSR2`_ signal is
|
||||
received. After the stack trace is dumped, the Scrapy process continues running
|
||||
normally.
|
||||
Dumps the stack trace and Scrapy engine status of a runnning process when a
|
||||
`SIGQUIT`_ or `SIGUSR2`_ signal is received. After the stack trace and engine
|
||||
status is dumped, the Scrapy process continues running normally.
|
||||
|
||||
The stack trace is sent to standard output.
|
||||
The dump is sent to standard output.
|
||||
|
||||
This extension only works on POSIX-compliant platforms (ie. not Windows).
|
||||
|
||||
There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
|
||||
|
||||
1. By pressing Ctrl-\ while a Scrapy process is running (Linux only?)
|
||||
|
||||
2. By running this command (assuming ``<pid>`` is the process id of the Scrapy
|
||||
process)::
|
||||
|
||||
kill -QUIT <pid>
|
||||
|
||||
.. _SIGUSR2: http://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2
|
||||
.. _SIGQUIT: http://en.wikipedia.org/wiki/SIGQUIT
|
||||
|
||||
Debugger extension
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
|
|
|||
|
|
@ -64,6 +64,15 @@ XML
|
|||
* :setting:`FEED_FORMAT`: ``xml``
|
||||
* Exporter used: :class:`~scrapy.contrib.exporter.XmlItemExporter`
|
||||
|
||||
.. _topics-feed-format-marshal:
|
||||
|
||||
Marshal
|
||||
-------
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``marshal``
|
||||
* Exporter used: :class:`~scrapy.contrib.exporter.MarshalItemExporter`
|
||||
|
||||
|
||||
.. _topics-feed-storage:
|
||||
|
||||
Storages
|
||||
|
|
@ -267,6 +276,7 @@ Default::
|
|||
'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter',
|
||||
'csv': 'scrapy.contrib.exporter.CsvItemExporter',
|
||||
'xml': 'scrapy.contrib.exporter.XmlItemExporter',
|
||||
'marshal': 'scrapy.contrib.exporter.MarshalItemExporter',
|
||||
}
|
||||
|
||||
A dict containing the built-in feed exporters supported by Scrapy.
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ objects. Here is an example::
|
|||
class Product(Item):
|
||||
name = Field()
|
||||
price = Field()
|
||||
stock = Field(default=0)
|
||||
last_updated = Field()
|
||||
stock = Field()
|
||||
last_updated = Field(serializer=str)
|
||||
|
||||
.. note:: Those familiar with `Django`_ will notice that Scrapy Items are
|
||||
declared similar to `Django Models`_, except that Scrapy Items are much
|
||||
|
|
@ -46,8 +46,8 @@ Item Fields
|
|||
===========
|
||||
|
||||
:class:`Field` objects are used to specify metadata for each field. For
|
||||
example, the default value for the ``stock`` field illustrated in the example
|
||||
above.
|
||||
example, the serializer function for the ``last_updated`` field illustrated in
|
||||
the example above.
|
||||
|
||||
You can specify any kind of metadata for each field. There is no restriction on
|
||||
the values accepted by :class:`Field` objects. For this same
|
||||
|
|
@ -95,10 +95,7 @@ Getting field values
|
|||
>>> product['price']
|
||||
1000
|
||||
|
||||
>>> product['stock'] # getting field with default value
|
||||
0
|
||||
|
||||
>>> product['last_updated'] # getting field with no default value
|
||||
>>> product['last_updated']
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
KeyError: 'last_updated'
|
||||
|
|
@ -175,28 +172,6 @@ Creating items from dicts::
|
|||
...
|
||||
KeyError: 'Product does not support field: lala'
|
||||
|
||||
Default values
|
||||
==============
|
||||
|
||||
The only field metadata key supported by Items themselves is ``default``, which
|
||||
specifies the default value to return when trying to access a field which
|
||||
wasn't populated before.
|
||||
|
||||
So, for the ``Product`` item declared above::
|
||||
|
||||
>>> product = Product()
|
||||
|
||||
>>> product['stock'] # field with default value
|
||||
0
|
||||
|
||||
>>> product['name'] # field with no default value
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
KeyError: 'name'
|
||||
|
||||
>>> product.get('name') is None
|
||||
True
|
||||
|
||||
Extending Items
|
||||
===============
|
||||
|
||||
|
|
@ -206,16 +181,16 @@ fields) by declaring a subclass of your original Item.
|
|||
For example::
|
||||
|
||||
class DiscountedProduct(Product):
|
||||
discount_percent = Field(default=0)
|
||||
discount_percent = Field(serializer=str)
|
||||
discount_expiration_date = Field()
|
||||
|
||||
You can also extend field metadata by using the previous field metadata and
|
||||
appending more values, or changing existing values, like this::
|
||||
|
||||
class SpecificProduct(Product):
|
||||
name = Field(Product.fields['name'], default='product')
|
||||
name = Field(Product.fields['name'], serializer=my_serializer)
|
||||
|
||||
That adds (or replaces) the ``default`` metadata key for the ``name`` field,
|
||||
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
|
||||
keeping all the previously existing metadata values.
|
||||
|
||||
Item objects
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ All available link extractors classes bundled with Scrapy are provided in the
|
|||
SgmlLinkExtractor
|
||||
-----------------
|
||||
|
||||
.. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None)
|
||||
.. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None)
|
||||
|
||||
The SgmlLinkExtractor extends the base :class:`BaseSgmlLinkExtractor` by
|
||||
providing additional filters that you can specify to extract links,
|
||||
|
|
@ -59,15 +59,21 @@ SgmlLinkExtractor
|
|||
that the (absolute) urls must match in order to be excluded (ie. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type allow: a regular expression (or list of)
|
||||
:type deny: a regular expression (or list of)
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
:type allow: str or list
|
||||
:type allow_domains: str or list
|
||||
|
||||
:param deny_domains: a single value or a list of strings containing
|
||||
domains which won't be considered for extracting the links
|
||||
:type allow: str or list
|
||||
:type deny_domains: str or list
|
||||
|
||||
:param deny_extensions: a list of extensions that should be ignored when
|
||||
extracting links. If not given, it will default to the
|
||||
``IGNORED_EXTENSIONS`` list defined in the `scrapy.linkextractor`_
|
||||
module.
|
||||
:type deny_extensions: list
|
||||
|
||||
:param restrict_xpaths: is a XPath (or list of XPath's) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
|
|
@ -145,3 +151,4 @@ BaseSgmlLinkExtractor
|
|||
|
||||
:type process_value: callable
|
||||
|
||||
.. _scrapy.linkextractor: http://dev.scrapy.org/browser/scrapy/linkextractor.py
|
||||
|
|
|
|||
|
|
@ -68,6 +68,8 @@ Request objects
|
|||
request_with_cookies = Request(url="http://www.example.com",
|
||||
cookies={currency: 'USD', country: 'UY'},
|
||||
meta={'dont_merge_cookies': True})
|
||||
|
||||
For more info see :ref:`cookies-mw`.
|
||||
:type cookies: dict
|
||||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ in your Ubuntu servers.
|
|||
So, if you plan to deploy Scrapyd on a Ubuntu server, just add the Ubuntu
|
||||
repositories as described in :ref:`topics-ubuntu` and then run::
|
||||
|
||||
aptitude install scrapyd-0.12
|
||||
aptitude install scrapyd-0.13
|
||||
|
||||
This will install Scrapyd in your Ubuntu server creating a ``scrapy`` user
|
||||
which Scrapyd will run as. It will also create some directories and files that
|
||||
|
|
|
|||
|
|
@ -242,15 +242,39 @@ Default: ``100``
|
|||
Maximum number of concurrent items (per response) to process in parallel in the
|
||||
Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_SPIDER
|
||||
.. setting:: CONCURRENT_REQUESTS
|
||||
|
||||
CONCURRENT_REQUESTS_PER_SPIDER
|
||||
CONCURRENT_REQUESTS
|
||||
-------------------
|
||||
|
||||
Default: ``16``
|
||||
|
||||
The maximum number of concurrent (ie. simultaneous) requests that will be
|
||||
performed by the Scrapy downloader.
|
||||
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
------------------------------
|
||||
|
||||
Default: ``8``
|
||||
|
||||
Specifies how many concurrent (ie. simultaneous) requests will be performed per
|
||||
open spider.
|
||||
The maximum number of concurrent (ie. simultaneous) requests that will be
|
||||
performed to any single domain.
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_IP
|
||||
|
||||
CONCURRENT_REQUESTS_PER_IP
|
||||
--------------------------
|
||||
|
||||
Default: ``0``
|
||||
|
||||
The maximum number of concurrent (ie. simultaneous) requests that will be
|
||||
performed to any single IP. If non-zero, the
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is
|
||||
used instead. In other words, concurrency limits will be applied per IP, not
|
||||
per domain.
|
||||
|
||||
.. setting:: CONCURRENT_SPIDERS
|
||||
|
||||
|
|
@ -261,15 +285,6 @@ Default: ``8``
|
|||
|
||||
Maximum number of spiders to scrape in parallel.
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
COOKIES_DEBUG
|
||||
-------------
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Enable debugging message of Cookies Downloader Middleware.
|
||||
|
||||
.. setting:: DEFAULT_ITEM_CLASS
|
||||
|
||||
DEFAULT_ITEM_CLASS
|
||||
|
|
@ -316,6 +331,22 @@ Default: ``0``
|
|||
The maximum depth that will be allowed to crawl for any site. If zero, no limit
|
||||
will be imposed.
|
||||
|
||||
.. setting:: DEPTH_PRIORITY
|
||||
|
||||
DEPTH_PRIORITY
|
||||
--------------
|
||||
|
||||
Default: ``1``
|
||||
|
||||
An integer that is used to set the request priority based on request depth.
|
||||
|
||||
To crawl in `breath-first order`_, set :setting:`DEPTH_PRIORITY` to ``1``.
|
||||
|
||||
To crawl in `depth-first order`_, set :setting:`DEPTH_PRIORITY` to ``-1``.
|
||||
|
||||
To disable any priority adjustment based on depth, set
|
||||
:setting:`DEPTH_PRIORITY` to ``0``.
|
||||
|
||||
.. setting:: DEPTH_STATS
|
||||
|
||||
DEPTH_STATS
|
||||
|
|
@ -323,7 +354,26 @@ DEPTH_STATS
|
|||
|
||||
Default: ``True``
|
||||
|
||||
Whether to collect depth stats.
|
||||
Whether to collect maximum depth stats.
|
||||
|
||||
.. setting:: DEPTH_STATS_VERBOSE
|
||||
|
||||
DEPTH_STATS_VERBOSE
|
||||
-------------------
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Whether to collect verbose depth stats. If this is enabled, the number of
|
||||
requests for each depth is collected in the stats.
|
||||
|
||||
.. setting:: DNSCACHE_ENABLED
|
||||
|
||||
DNSCACHE_ENABLED
|
||||
----------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether to enable DNS in-memory cache.
|
||||
|
||||
.. setting:: DOWNLOADER_DEBUG
|
||||
|
||||
|
|
@ -354,6 +404,7 @@ Default::
|
|||
{
|
||||
'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
|
||||
'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
|
||||
'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350,
|
||||
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
|
||||
'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
|
||||
'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
|
||||
|
|
@ -361,6 +412,7 @@ Default::
|
|||
'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
|
||||
'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
|
||||
'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
|
||||
'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
|
||||
'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
|
||||
'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
|
||||
}
|
||||
|
|
@ -442,12 +494,21 @@ The amount of time (in secs) that the downloader will wait before timing out.
|
|||
DUPEFILTER_CLASS
|
||||
----------------
|
||||
|
||||
Default: ``'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter'``
|
||||
Default: ``'scrapy.dupefilter.RFPDupeFilter'``
|
||||
|
||||
The class used to detect and filter duplicate requests.
|
||||
|
||||
The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint
|
||||
(using ``scrapy.utils.request.request_fingerprint``) and grouping per domain.
|
||||
The default (``RFPDupeFilter``) filters based on request fingerprint using
|
||||
the ``scrapy.utils.request.request_fingerprint`` function.
|
||||
|
||||
.. setting:: EDITOR
|
||||
|
||||
EDITOR
|
||||
------
|
||||
|
||||
The editor to use for editing spiders with the :command:`edit` command. It
|
||||
defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults
|
||||
to ``vi`` (on Unix systems) or the IDLE editor (on Windows).
|
||||
|
||||
.. setting:: ENCODING_ALIASES
|
||||
|
||||
|
|
@ -526,7 +587,7 @@ A dict containing the extensions enabled in your project, and their orders.
|
|||
EXTENSIONS_BASE
|
||||
---------------
|
||||
|
||||
Default::
|
||||
Default::
|
||||
|
||||
{
|
||||
'scrapy.contrib.corestats.CoreStats': 0,
|
||||
|
|
@ -534,7 +595,10 @@ Default::
|
|||
'scrapy.telnet.TelnetConsole': 0,
|
||||
'scrapy.contrib.memusage.MemoryUsage': 0,
|
||||
'scrapy.contrib.memdebug.MemoryDebugger': 0,
|
||||
'scrapy.contrib.closedomain.CloseDomain': 0,
|
||||
'scrapy.contrib.closespider.CloseSpider': 0,
|
||||
'scrapy.contrib.feedexport.FeedExporter': 0,
|
||||
'scrapy.contrib.spidercontext.SpiderContext': 0,
|
||||
'scrapy.contrib.logstats.LogStats': 0,
|
||||
}
|
||||
|
||||
The list of available extensions. Keep in mind that some of them need to
|
||||
|
|
@ -687,7 +751,7 @@ Default: ``False``
|
|||
|
||||
Scope: ``scrapy.contrib.memusage``
|
||||
|
||||
Whether to send a memory usage report after each domain has been closed.
|
||||
Whether to send a memory usage report after each spider has been closed.
|
||||
|
||||
See :ref:`topics-extensions-ref-memusage`.
|
||||
|
||||
|
|
@ -789,51 +853,6 @@ Default: ``'scrapy.core.scheduler.Scheduler'``
|
|||
|
||||
The scheduler to use for crawling.
|
||||
|
||||
.. setting:: SCHEDULER_ORDER
|
||||
|
||||
SCHEDULER_ORDER
|
||||
---------------
|
||||
|
||||
Default: ``'DFO'``
|
||||
|
||||
Scope: ``scrapy.core.scheduler``
|
||||
|
||||
The order to use for the crawling scheduler. Available orders are:
|
||||
|
||||
* ``'BFO'``: `Breadth-first order`_ - typically consumes more memory but
|
||||
reaches most relevant pages earlier.
|
||||
|
||||
* ``'DFO'``: `Depth-first order`_ - typically consumes less memory than
|
||||
but takes longer to reach most relevant pages.
|
||||
|
||||
.. _Breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search
|
||||
.. _Depth-first order: http://en.wikipedia.org/wiki/Depth-first_search
|
||||
|
||||
.. setting:: SCHEDULER_MIDDLEWARES
|
||||
|
||||
SCHEDULER_MIDDLEWARES
|
||||
---------------------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing the scheduler middlewares enabled in your project, and their
|
||||
orders.
|
||||
|
||||
.. setting:: SCHEDULER_MIDDLEWARES_BASE
|
||||
|
||||
SCHEDULER_MIDDLEWARES_BASE
|
||||
--------------------------
|
||||
|
||||
Default::
|
||||
|
||||
SCHEDULER_MIDDLEWARES_BASE = {
|
||||
'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware': 500,
|
||||
}
|
||||
|
||||
A dict containing the scheduler middlewares enabled by default in Scrapy. You
|
||||
should never modify this setting in your project, modify
|
||||
:setting:`SCHEDULER_MIDDLEWARES` instead.
|
||||
|
||||
.. setting:: SPIDER_MIDDLEWARES
|
||||
|
||||
SPIDER_MIDDLEWARES
|
||||
|
|
@ -853,7 +872,6 @@ Default::
|
|||
|
||||
{
|
||||
'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
|
||||
'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
|
||||
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
|
||||
'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
|
||||
'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
|
||||
|
|
@ -905,11 +923,13 @@ or subclass the StatsCollector class).
|
|||
STATS_DUMP
|
||||
----------
|
||||
|
||||
Default: ``False``
|
||||
Default: ``True``
|
||||
|
||||
Dump (to log) domain-specific stats collected when a domain is closed, and all
|
||||
global stats when the Scrapy process finishes (ie. when the engine is
|
||||
shutdown).
|
||||
Dump (to the Scrapy log) the :ref:`Scrapy stats <topics-stats>` collected
|
||||
during the crawl. The spider-specific stats are logged when the spider is
|
||||
closed, while the global stats are dumped when the Scrapy process finishes.
|
||||
|
||||
For more info see: :ref:`topics-stats`.
|
||||
|
||||
.. setting:: STATS_ENABLED
|
||||
|
||||
|
|
@ -927,7 +947,7 @@ STATSMAILER_RCPTS
|
|||
|
||||
Default: ``[]`` (empty list)
|
||||
|
||||
Send Scrapy stats after domains finish scraping. See
|
||||
Send Scrapy stats after spiders finish scraping. See
|
||||
:class:`~scrapy.contrib.statsmailer.StatsMailer` for more info.
|
||||
|
||||
.. setting:: TELNETCONSOLE_ENABLED
|
||||
|
|
@ -983,3 +1003,5 @@ Default: ``"%s/%s" % (BOT_NAME, BOT_VERSION)``
|
|||
The default User-Agent to use when crawling, unless overridden.
|
||||
|
||||
.. _Amazon web services: http://aws.amazon.com/
|
||||
.. _breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search
|
||||
.. _depth-first order: http://en.wikipedia.org/wiki/Depth-first_search
|
||||
|
|
|
|||
|
|
@ -43,11 +43,14 @@ engine_started
|
|||
.. signal:: engine_started
|
||||
.. function:: engine_started()
|
||||
|
||||
Sent when the Scrapy engine is started (for example, when a crawling
|
||||
process has started).
|
||||
Sent when the Scrapy engine has started crawling.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
|
||||
.. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
|
||||
depending on how the spider was started. So **don't** rely on this signal
|
||||
getting fired before :signal:`spider_opened`.
|
||||
|
||||
engine_stopped
|
||||
--------------
|
||||
|
||||
|
|
@ -63,45 +66,22 @@ item_scraped
|
|||
------------
|
||||
|
||||
.. signal:: item_scraped
|
||||
.. function:: item_scraped(item, spider, response)
|
||||
.. function:: item_scraped(item, response, spider)
|
||||
|
||||
Sent when the engine receives a new scraped item from the spider, and right
|
||||
before the item is sent to the :ref:`topics-item-pipeline`.
|
||||
Sent when an item has been scraped, after it has passed all the
|
||||
:ref:`topics-item-pipeline` stages (without being dropped).
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
|
||||
:param item: is the item scraped
|
||||
:param item: the item scraped
|
||||
:type item: :class:`~scrapy.item.Item` object
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
:param response: the response from which the item was scraped
|
||||
:param response: the response from where the item was scraped
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
item_passed
|
||||
-----------
|
||||
|
||||
.. signal:: item_passed
|
||||
.. function:: item_passed(item, spider, original_item)
|
||||
|
||||
Sent after an item has passed all the :ref:`topics-item-pipeline` stages
|
||||
without being dropped. Same as :func:`item_scraped` if there are no
|
||||
pipelines enabled.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
|
||||
:param item: the item which passed the pipeline
|
||||
:type item: :class:`~scrapy.item.Item` object
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
:param original_item: the input of the item pipeline. This is typically the
|
||||
same :class:`~scrapy.item.Item` object received in the ``item``
|
||||
parameter, unless some pipeline stage created a new item.
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
item_dropped
|
||||
------------
|
||||
|
||||
|
|
@ -186,6 +166,24 @@ spider_idle
|
|||
:param spider: the spider which has gone idle
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
spider_error
|
||||
------------
|
||||
|
||||
.. signal:: spider_error
|
||||
.. function:: spider_error(failure, response, spider)
|
||||
|
||||
Sent when a spider callback generates an error (ie. raises an exception).
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
|
||||
:param response: the response being processed when the exception was raised
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
|
||||
request_received
|
||||
----------------
|
||||
|
||||
|
|
@ -241,3 +239,4 @@ response_downloaded
|
|||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
.. _Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ single Python class that defines one or more of the following methods:
|
|||
|
||||
.. method:: process_spider_exception(response, exception, spider)
|
||||
|
||||
This method is called when when a spider or :meth:process_spider_input:
|
||||
This method is called when when a spider or :meth:`process_spider_input`
|
||||
method (from other spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
|
|
@ -164,6 +164,8 @@ DepthMiddleware
|
|||
* :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
|
||||
crawl for any site. If zero, no limit will be imposed.
|
||||
* :setting:`DEPTH_STATS` - Whether to collect depth stats.
|
||||
* :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
|
||||
their depth, to crawl in breadh-first or depth-first order.
|
||||
|
||||
HttpErrorMiddleware
|
||||
-------------------
|
||||
|
|
|
|||
|
|
@ -45,8 +45,13 @@ We will talk about those types here.
|
|||
Built-in spiders reference
|
||||
==========================
|
||||
|
||||
For the examples used in the following spiders reference, we'll assume we have a
|
||||
``TestItem`` declared in a ``myproject.items`` module, in your project::
|
||||
Scrapy comes with some useful generic spiders that you can use, to subclass
|
||||
your spiders from. Their aim is to provide convenient functionality for a few
|
||||
common scraping cases, like following all links on a site based on certain
|
||||
rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed.
|
||||
|
||||
For the examples used in the following spiders, we'll assume you have a project
|
||||
with a ``TestItem`` declared in a ``myproject.items`` module::
|
||||
|
||||
from scrapy.item import Item
|
||||
|
||||
|
|
@ -228,6 +233,7 @@ CrawlSpider
|
|||
|
||||
Crawling rules
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
|
||||
|
||||
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
|
||||
|
|
@ -262,7 +268,7 @@ Crawling rules
|
|||
filter out the request).
|
||||
|
||||
CrawlSpider example
|
||||
-------------------
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Let's now take a look at an example CrawlSpider with rules::
|
||||
|
||||
|
|
@ -465,3 +471,126 @@ Let's see an example similar to the previous one, but using a
|
|||
item['name'] = row['name']
|
||||
item['description'] = row['description']
|
||||
return item
|
||||
|
||||
|
||||
SitemapSpider
|
||||
-------------
|
||||
|
||||
.. class:: SitemapSpider
|
||||
|
||||
SitemapSpider allows you to crawl a site by discovering the URLs using
|
||||
`Sitemaps`_.
|
||||
|
||||
It supports nested sitemaps and discovering sitemap urls from
|
||||
`robots.txt`_.
|
||||
|
||||
.. attribute:: sitemap_urls
|
||||
|
||||
A list of urls pointing to the sitemaps whose urls you want to crawl.
|
||||
|
||||
You can also point to a `robots.txt`_ and it will be parsed to extract
|
||||
sitemap urls from it.
|
||||
|
||||
.. attribute:: sitemap_rules
|
||||
|
||||
A list of tuples ``(regex, callback)`` where:
|
||||
|
||||
* ``regex`` is a regular expression to match urls extracted from sitemaps.
|
||||
``regex`` can be either a str or a compiled regex object.
|
||||
|
||||
* callback is the callback to use for processing the urls that match
|
||||
the regular expression. ``callback`` can be a string (indicating the
|
||||
name of a spider method) or a callable.
|
||||
|
||||
For example::
|
||||
|
||||
sitemap_rules = [('/product/', 'parse_product')]
|
||||
|
||||
Rules are applied in order, and only the first one that matches will be
|
||||
used.
|
||||
|
||||
If you omit this attribute, all urls found in sitemaps will be
|
||||
processed with the ``parse`` callback.
|
||||
|
||||
.. attribute:: sitemap_follow
|
||||
|
||||
A list of regexes of sitemap that should be followed. This is is only
|
||||
for sites that use `Sitemap index files`_ that point to other sitemap
|
||||
files.
|
||||
|
||||
By default, all sitemaps are followed.
|
||||
|
||||
|
||||
SitemapSpider examples
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Simplest example: process all urls discovered through sitemaps using the
|
||||
``parse`` callback::
|
||||
|
||||
from scrapy.contrib.spiders import SitemapSpider
|
||||
|
||||
class MySpider(SitemapSpider):
|
||||
sitemap_urls = ['http://www.example.com/sitemap.xml']
|
||||
|
||||
def parse(self, response):
|
||||
pass # ... scrape item here ...
|
||||
|
||||
Process some urls with certain callback and other urls with a different
|
||||
callback::
|
||||
|
||||
from scrapy.contrib.spiders import SitemapSpider
|
||||
|
||||
class MySpider(SitemapSpider):
|
||||
sitemap_urls = ['http://www.example.com/sitemap.xml']
|
||||
sitemap_rules = [
|
||||
('/product/', 'parse_product'),
|
||||
('/category/', 'parse_category'),
|
||||
]
|
||||
|
||||
def parse_product(self, response):
|
||||
pass # ... scrape product ...
|
||||
|
||||
def parse_category(self, response):
|
||||
pass # ... scrape category ...
|
||||
|
||||
Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps
|
||||
whose url contains ``/sitemap_shop``::
|
||||
|
||||
from scrapy.contrib.spiders import SitemapSpider
|
||||
|
||||
class MySpider(SitemapSpider):
|
||||
sitemap_urls = ['http://www.example.com/robots.txt']
|
||||
sitemap_rules = [
|
||||
('/shop/', 'parse_shop'),
|
||||
]
|
||||
sitemap_follow = ['/sitemap_shops']
|
||||
|
||||
def parse_shop(self, response):
|
||||
pass # ... scrape shop here ...
|
||||
|
||||
Combine SitemapSpider with other sources of urls::
|
||||
|
||||
from scrapy.contrib.spiders import SitemapSpider
|
||||
|
||||
class MySpider(SitemapSpider):
|
||||
sitemap_urls = ['http://www.example.com/robots.txt']
|
||||
sitemap_rules = [
|
||||
('/shop/', 'parse_shop'),
|
||||
]
|
||||
|
||||
other_urls = ['http://www.example.com/about']
|
||||
|
||||
def start_requests(self):
|
||||
requests = list(super(MySpider, self).start_requests())
|
||||
requests += [Request(x, callback=self.parse_other) for x in self.other_urls]
|
||||
return requests
|
||||
|
||||
def parse_shop(self, response):
|
||||
pass # ... scrape shop here ...
|
||||
|
||||
def parse_other(self, response):
|
||||
pass # ... scrape other here ...
|
||||
|
||||
.. _Sitemaps: http://www.sitemaps.org
|
||||
.. _Sitemap index files: http://www.sitemaps.org/protocol.php#index
|
||||
.. _robots.txt: http://www.robotstxt.org/
|
||||
|
|
|
|||
|
|
@ -185,6 +185,10 @@ class (which they all inherit from).
|
|||
for this spider can be accessed. This method is called automatically on
|
||||
the :signal:`spider_closed` signal.
|
||||
|
||||
.. method:: engine_stopped()
|
||||
|
||||
Called after the engine is stopped, to dump or persist global stats.
|
||||
|
||||
Available Stats Collectors
|
||||
==========================
|
||||
|
||||
|
|
@ -223,65 +227,6 @@ DummyStatsCollector
|
|||
Stats Collector used when stats are disabled (through the
|
||||
:setting:`STATS_ENABLED` setting).
|
||||
|
||||
SimpledbStatsCollector
|
||||
----------------------
|
||||
|
||||
.. module:: scrapy.contrib.statscol
|
||||
:synopsis: Additional Stats Collectors
|
||||
|
||||
.. class:: SimpledbStatsCollector
|
||||
|
||||
A Stats collector which persists stats to `Amazon SimpleDB`_, using one
|
||||
SimpleDB item per scraping run (ie. it keeps history of all scraping runs).
|
||||
The data is persisted to the SimpleDB domain specified by the
|
||||
:setting:`STATS_SDB_DOMAIN` setting. The domain will be created if it
|
||||
doesn't exist.
|
||||
|
||||
In addition to the existing stats keys, the following keys are added at
|
||||
persitance time:
|
||||
|
||||
* ``spider``: the spider name (so you can use it later for querying stats
|
||||
for that spider)
|
||||
* ``timestamp``: the timestamp when the stats were persisted
|
||||
|
||||
Both the ``spider`` and ``timestamp`` are used to generate the SimpleDB
|
||||
item name in order to avoid overwriting stats of previous scraping runs.
|
||||
|
||||
As `required by SimpleDB`_, datetimes are stored in ISO 8601 format and
|
||||
numbers are zero-padded to 16 digits. Negative numbers are not currently
|
||||
supported.
|
||||
|
||||
This Stats Collector requires the `boto`_ library.
|
||||
|
||||
.. _Amazon SimpleDB: http://aws.amazon.com/simpledb/
|
||||
.. _required by SimpleDB: http://docs.amazonwebservices.com/AmazonSimpleDB/2009-04-15/DeveloperGuide/ZeroPadding.html
|
||||
.. _boto: http://code.google.com/p/boto/
|
||||
|
||||
This Stats Collector can be configured through the following settings:
|
||||
|
||||
.. setting:: STATS_SDB_DOMAIN
|
||||
|
||||
STATS_SDB_DOMAIN
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``'scrapy_stats'``
|
||||
|
||||
A string containing the SimpleDB domain to use in the
|
||||
:class:`SimpledbStatsCollector`.
|
||||
|
||||
.. setting:: STATS_SDB_ASYNC
|
||||
|
||||
STATS_SDB_ASYNC
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If ``True``, communication with SimpleDB will be performed asynchronously. If
|
||||
``False`` blocking IO will be used instead. This is the default as using
|
||||
asynchronous communication can result in the stats not being persisted if the
|
||||
Scrapy engine is shut down in the middle (for example, when you run only one
|
||||
spider in a process and then exit).
|
||||
|
||||
Stats signals
|
||||
=============
|
||||
|
||||
|
|
|
|||
|
|
@ -43,9 +43,13 @@ convenience:
|
|||
+----------------+-------------------------------------------------------------------+
|
||||
| Shortcut | Description |
|
||||
+================+===================================================================+
|
||||
| ``manager`` | the Project Crawler object (``scrapy.project.crawler``) |
|
||||
| ``crawler`` | the Scrapy Crawler object (``scrapy.crawler``) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``engine`` | the Scrapy engine object (``scrapy.project.crawler.engine``) |
|
||||
| ``engine`` | the Scrapy Engine object (``scrapy.core.engine``) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``spider`` | the spider object (only if there is a single spider opened) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``slot`` | the engine slot (only if there is a single spider opened) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``extensions`` | the Extension Manager (``scrapy.project.crawler.extensions``) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
|
|
@ -79,30 +83,29 @@ using the telnet console::
|
|||
>>> est()
|
||||
Execution engine status
|
||||
|
||||
datetime.now()-self.start_time : 0:00:09.051588
|
||||
self.is_idle() : False
|
||||
self.scheduler.is_idle() : False
|
||||
len(self.scheduler.pending_requests) : 1
|
||||
self.downloader.is_idle() : False
|
||||
len(self.downloader.sites) : 1
|
||||
self.downloader.has_capacity() : True
|
||||
self.pipeline.is_idle() : False
|
||||
len(self.pipeline.domaininfo) : 1
|
||||
len(self._scraping) : 1
|
||||
time()-engine.start_time : 21.3188259602
|
||||
engine.is_idle() : False
|
||||
engine.has_capacity() : True
|
||||
engine.scheduler.is_idle() : False
|
||||
len(engine.scheduler.pending_requests) : 1
|
||||
engine.downloader.is_idle() : False
|
||||
len(engine.downloader.slots) : 1
|
||||
engine.scraper.is_idle() : False
|
||||
len(engine.scraper.slots) : 1
|
||||
|
||||
example.com
|
||||
self.domain_is_idle(domain) : False
|
||||
self.closing.get(domain) : None
|
||||
self.scheduler.domain_has_pending_requests(domain) : True
|
||||
len(self.scheduler.pending_requests[domain]) : 97
|
||||
len(self.downloader.sites[domain].queue) : 17
|
||||
len(self.downloader.sites[domain].active) : 25
|
||||
len(self.downloader.sites[domain].transferring) : 8
|
||||
self.downloader.sites[domain].closing : False
|
||||
self.downloader.sites[domain].lastseen : 2009-06-23 15:20:16.563675
|
||||
self.pipeline.domain_is_idle(domain) : True
|
||||
len(self.pipeline.domaininfo[domain]) : 0
|
||||
len(self._scraping[domain]) : 0
|
||||
Spider: <GayotSpider 'gayotcom' at 0x2dc2b10>
|
||||
engine.spider_is_idle(spider) : False
|
||||
engine.slots[spider].closing : False
|
||||
len(engine.scheduler.pending_requests[spider]) : 11504
|
||||
len(engine.downloader.slots[spider].queue) : 9
|
||||
len(engine.downloader.slots[spider].active) : 17
|
||||
len(engine.downloader.slots[spider].transferring) : 8
|
||||
engine.downloader.slots[spider].lastseen : 1311311093.61
|
||||
len(engine.scraper.slots[spider].queue) : 0
|
||||
len(engine.scraper.slots[spider].active) : 0
|
||||
engine.scraper.slots[spider].active_size : 0
|
||||
engine.scraper.slots[spider].itemproc_size : 0
|
||||
engine.scraper.slots[spider].needs_backout() : False
|
||||
|
||||
|
||||
Pause, resume and stop the Scrapy engine
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ latest bug fixes.
|
|||
|
||||
To use the packages, just add the following line to your
|
||||
``/etc/apt/sources.list``, and then run ``aptitude update`` and ``aptitude
|
||||
install scrapy-0.12``::
|
||||
install scrapy-0.13``::
|
||||
|
||||
deb http://archive.scrapy.org/ubuntu DISTRO main
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
.. _versioning:
|
||||
|
||||
============================
|
||||
Versioning and API Stability
|
||||
============================
|
||||
|
||||
Versioning
|
||||
==========
|
||||
|
||||
Scrapy uses the `odd-numbered versions for development releases`_.
|
||||
|
||||
There are 4 numbers in a Scrapy version: *A.B.C.D*
|
||||
|
||||
* *A* is the major version. This will rarely change and will signify very
|
||||
large changes. So far, only zero is available for *A*
|
||||
* *B* is the release number. This will include many changes including features
|
||||
and things that possible break backwards compatibility. Even Bs will be
|
||||
stable branches, and odd Bs will be development.
|
||||
* *C* is the bugfix release number, but it has been recently deprecated of
|
||||
favor of using the revision number (*D*)
|
||||
* *D* is the revision number of the Mercurial repo from where the release was
|
||||
taken
|
||||
|
||||
For example:
|
||||
|
||||
* *0.12.0.2542* is the stable release *12* at revision *2542* (safe to use in
|
||||
production)
|
||||
* *0.13.0.2691* is the development release *13* at revision *2691* (use with
|
||||
care in production)
|
||||
|
||||
API Stability
|
||||
=============
|
||||
|
||||
API stability is one of Scrapy major goals for the *1.0* release, which doesn't
|
||||
have a due date scheduled yet.
|
||||
|
||||
Methods or functions that start with a single dash (``_``) are private and
|
||||
should never be relied as stable. Besides those, the plan is to stabilize and
|
||||
document the entire API, as we approach the 1.0 release.
|
||||
|
||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||
new methods or functionality but the existing methods should keep working the
|
||||
same way.
|
||||
|
||||
|
||||
.. _odd-numbered versions for development releases: http://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
|
||||
|
||||
|
|
@ -1 +0,0 @@
|
|||
# googledir project
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# http://doc.scrapy.org/topics/items.html
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
class GoogledirItem(Item):
|
||||
|
||||
name = Field(default='')
|
||||
url = Field(default='')
|
||||
description = Field(default='')
|
||||
|
||||
def __str__(self):
|
||||
return "Google Category: name=%s url=%s" \
|
||||
% (self['name'], self['url'])
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: http://doc.scrapy.org/topics/item-pipeline.html
|
||||
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
class FilterWordsPipeline(object):
|
||||
"""
|
||||
A pipeline for filtering out items which contain certain
|
||||
words in their description
|
||||
"""
|
||||
|
||||
# put all words in lowercase
|
||||
words_to_filter = ['politics', 'religion']
|
||||
|
||||
def process_item(self, item, spider):
|
||||
for word in self.words_to_filter:
|
||||
if word in unicode(item['description']).lower():
|
||||
raise DropItem("Contains forbidden word: %s" % word)
|
||||
else:
|
||||
return item
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Scrapy settings for googledir project
|
||||
#
|
||||
# For simplicity, this file contains only the most important settings by
|
||||
# default. All the other settings are documented here:
|
||||
#
|
||||
# http://doc.scrapy.org/topics/settings.html
|
||||
|
||||
BOT_NAME = 'googledir'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['googledir.spiders']
|
||||
NEWSPIDER_MODULE = 'googledir.spiders'
|
||||
DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline']
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# To create the first spider for your project use this command:
|
||||
#
|
||||
# scrapy genspider myspider myspider-domain.com
|
||||
#
|
||||
# For more info see:
|
||||
# http://doc.scrapy.org/topics/spiders.html
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.contrib.loader import XPathItemLoader
|
||||
from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule
|
||||
|
||||
from googledir.items import GoogledirItem
|
||||
|
||||
class GoogleDirectorySpider(CrawlSpider):
|
||||
|
||||
name = 'google_directory'
|
||||
allowed_domains = ['directory.google.com']
|
||||
start_urls = ['http://directory.google.com/']
|
||||
|
||||
rules = (
|
||||
# search for categories pattern and follow links
|
||||
Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True),
|
||||
)
|
||||
|
||||
def parse_category(self, response):
|
||||
# The main selector we're using to extract data from the page
|
||||
main_selector = HtmlXPathSelector(response)
|
||||
|
||||
# The XPath to website links in the directory page
|
||||
xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font'
|
||||
|
||||
# Get a list of (sub) selectors to each website node pointed by the XPath
|
||||
sub_selectors = main_selector.select(xpath)
|
||||
|
||||
# Iterate over the sub-selectors to extract data for each website
|
||||
for selector in sub_selectors:
|
||||
item = GoogledirItem()
|
||||
|
||||
l = XPathItemLoader(item=item, selector=selector)
|
||||
l.add_xpath('name', 'a/text()')
|
||||
l.add_xpath('url', 'a/@href')
|
||||
l.add_xpath('description', 'font[2]/text()')
|
||||
|
||||
# Here we populate the item and yield it
|
||||
yield l.load_item()
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[settings]
|
||||
default = googledir.settings
|
||||
|
|
@ -1 +0,0 @@
|
|||
# package
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# http://doc.scrapy.org/topics/items.html
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
class ImdbItem(Item):
|
||||
# define the fields for your item here like:
|
||||
# name = Field()
|
||||
title = Field()
|
||||
url = Field()
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: http://doc.scrapy.org/topics/item-pipeline.html
|
||||
|
||||
class ImdbPipeline(object):
|
||||
def process_item(self, item, spider):
|
||||
return item
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
# Scrapy settings for imdb project
|
||||
#
|
||||
# For simplicity, this file contains only the most important settings by
|
||||
# default. All the other settings are documented here:
|
||||
#
|
||||
# http://doc.scrapy.org/topics/settings.html
|
||||
|
||||
BOT_NAME = 'imdb'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['imdb.spiders']
|
||||
NEWSPIDER_MODULE = 'imdb.spiders'
|
||||
DEFAULT_ITEM_CLASS = 'imdb.items.ImdbItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# To create the first spider for your project use this command:
|
||||
#
|
||||
# scrapy genspider myspider myspider-domain.com
|
||||
#
|
||||
# For more info see:
|
||||
# http://doc.scrapy.org/topics/spiders.html
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
from scrapy.http import Request
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.contrib.loader import XPathItemLoader
|
||||
from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule
|
||||
from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor
|
||||
from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize, \
|
||||
FilterDupes, FilterUrl
|
||||
from scrapy.utils.url import urljoin_rfc
|
||||
|
||||
from imdb.items import ImdbItem, Field
|
||||
|
||||
from itertools import chain, imap, izip
|
||||
|
||||
class UsaOpeningWeekMovie(ImdbItem):
|
||||
pass
|
||||
|
||||
class UsaTopWeekMovie(ImdbItem):
|
||||
pass
|
||||
|
||||
class Top250Movie(ImdbItem):
|
||||
rank = Field()
|
||||
rating = Field()
|
||||
year = Field()
|
||||
votes = Field()
|
||||
|
||||
class MovieItem(ImdbItem):
|
||||
release_date = Field()
|
||||
tagline = Field()
|
||||
|
||||
|
||||
class ImdbSiteSpider(CrawlSpider):
|
||||
name = 'imdb.com'
|
||||
allowed_domains = ['imdb.com']
|
||||
start_urls = ['http://www.imdb.com/']
|
||||
|
||||
# extract requests using this classes from urls matching 'follow' flag
|
||||
request_extractors = [
|
||||
SgmlRequestExtractor(tags=['a'], attrs=['href']),
|
||||
]
|
||||
|
||||
# process requests using this classes from urls matching 'follow' flag
|
||||
request_processors = [
|
||||
Canonicalize(),
|
||||
FilterDupes(),
|
||||
FilterUrl(deny=r'/tt\d+/$'), # deny movie url as we will dispatch
|
||||
# manually the movie requests
|
||||
]
|
||||
|
||||
# include domain bit for demo purposes
|
||||
rules = (
|
||||
# these two rules expects requests from start url
|
||||
Rule(r'imdb.com/nowplaying/$', 'parse_now_playing'),
|
||||
Rule(r'imdb.com/chart/top$', 'parse_top_250'),
|
||||
# this rule will parse requests manually dispatched
|
||||
Rule(r'imdb.com/title/tt\d+/$', 'parse_movie_info'),
|
||||
)
|
||||
|
||||
def parse_now_playing(self, response):
|
||||
"""Scrapes USA openings this week and top 10 in week"""
|
||||
self.log("Parsing USA Top Week")
|
||||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
_urljoin = lambda url: self._urljoin(response, url)
|
||||
|
||||
#
|
||||
# openings this week
|
||||
#
|
||||
openings = hxs.select('//table[@class="movies"]//a[@class="title"]')
|
||||
boxoffice = hxs.select('//table[@class="boxoffice movies"]//a[@class="title"]')
|
||||
|
||||
opening_titles = openings.select('text()').extract()
|
||||
opening_urls = imap(_urljoin, openings.select('@href').extract())
|
||||
|
||||
box_titles = boxoffice.select('text()').extract()
|
||||
box_urls = imap(_urljoin, boxoffice.select('@href').extract())
|
||||
|
||||
# items
|
||||
opening_items = (UsaOpeningWeekMovie(title=title, url=url)
|
||||
for (title, url)
|
||||
in izip(opening_titles, opening_urls))
|
||||
|
||||
box_items = (UsaTopWeekMovie(title=title, url=url)
|
||||
for (title, url)
|
||||
in izip(box_titles, box_urls))
|
||||
|
||||
# movie requests
|
||||
requests = imap(self.make_requests_from_url,
|
||||
chain(opening_urls, box_urls))
|
||||
|
||||
return chain(opening_items, box_items, requests)
|
||||
|
||||
def parse_top_250(self, response):
|
||||
"""Scrapes movies from top 250 list"""
|
||||
self.log("Parsing Top 250")
|
||||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
# scrap each row in the table
|
||||
rows = hxs.select('//div[@id="main"]/table/tr//a/ancestor::tr')
|
||||
for row in rows:
|
||||
fields = row.select('td//text()').extract()
|
||||
url, = row.select('td//a/@href').extract()
|
||||
url = self._urljoin(response, url)
|
||||
|
||||
item = Top250Movie()
|
||||
item['title'] = fields[2]
|
||||
item['url'] = url
|
||||
item['rank'] = fields[0]
|
||||
item['rating'] = fields[1]
|
||||
item['year'] = fields[3]
|
||||
item['votes'] = fields[4]
|
||||
|
||||
# scrapped top250 item
|
||||
yield item
|
||||
# fetch movie
|
||||
yield self.make_requests_from_url(url)
|
||||
|
||||
def parse_movie_info(self, response):
|
||||
"""Scrapes movie information"""
|
||||
self.log("Parsing Movie Info")
|
||||
hxs = HtmlXPathSelector(response)
|
||||
selector = hxs.select('//div[@class="maindetails"]')
|
||||
|
||||
item = MovieItem()
|
||||
# set url
|
||||
item['url'] = response.url
|
||||
|
||||
# use item loader for other attributes
|
||||
l = XPathItemLoader(item=item, selector=selector)
|
||||
l.add_xpath('title', './/h1/text()')
|
||||
l.add_xpath('release_date', './/h5[text()="Release Date:"]'
|
||||
'/following-sibling::div/text()')
|
||||
l.add_xpath('tagline', './/h5[text()="Tagline:"]'
|
||||
'/following-sibling::div/text()')
|
||||
|
||||
yield l.load_item()
|
||||
|
||||
def _urljoin(self, response, url):
|
||||
"""Helper to convert relative urls to absolute"""
|
||||
return urljoin_rfc(response.url, url, response.encoding)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[settings]
|
||||
default = imdb.settings
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
from scrapy.item import Item, Field
|
||||
|
||||
class GoogledirItem(Item):
|
||||
|
||||
name = Field()
|
||||
url = Field()
|
||||
description = Field()
|
||||
|
||||
def __str__(self):
|
||||
return "Google Category: name=%s url=%s" % (self['name'], self['url'])
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
from scrapy.exceptions import DropItem
|
||||
|
||||
class FilterWordsPipeline(object):
|
||||
"""A pipeline for filtering out items which contain certain words in their
|
||||
description"""
|
||||
|
||||
# put all words in lowercase
|
||||
words_to_filter = ['politics', 'religion']
|
||||
|
||||
def process_item(self, item, spider):
|
||||
for word in self.words_to_filter:
|
||||
if word in unicode(item['description']).lower():
|
||||
raise DropItem("Contains forbidden word: %s" % word)
|
||||
else:
|
||||
return item
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
# - Scrapy settings for googledir project
|
||||
|
||||
import googledir
|
||||
|
||||
BOT_NAME = 'googledir'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['googledir.spiders']
|
||||
NEWSPIDER_MODULE = 'googledir.spiders'
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.Item'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline']
|
||||
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Place here all your scrapy spiders
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
|
||||
from scrapy.contrib.spiders import CrawlSpider, Rule
|
||||
from scrapy.contrib.loader import XPathItemLoader
|
||||
from googledir.items import GoogledirItem
|
||||
|
||||
class GoogleDirectorySpider(CrawlSpider):
|
||||
|
||||
name = 'directory.google.com'
|
||||
allowed_domains = ['directory.google.com']
|
||||
start_urls = ['http://directory.google.com/']
|
||||
|
||||
rules = (
|
||||
Rule(SgmlLinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$'),
|
||||
'parse_category',
|
||||
follow=True,
|
||||
),
|
||||
)
|
||||
|
||||
def parse_category(self, response):
|
||||
# The main selector we're using to extract data from the page
|
||||
main_selector = HtmlXPathSelector(response)
|
||||
|
||||
# The XPath to website links in the directory page
|
||||
xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font'
|
||||
|
||||
# Get a list of (sub) selectors to each website node pointed by the XPath
|
||||
sub_selectors = main_selector.select(xpath)
|
||||
|
||||
# Iterate over the sub-selectors to extract data for each website
|
||||
for selector in sub_selectors:
|
||||
item = GoogledirItem()
|
||||
|
||||
l = XPathItemLoader(item=item, selector=selector)
|
||||
l.add_xpath('name', 'a/text()')
|
||||
l.add_xpath('url', 'a/@href')
|
||||
l.add_xpath('description', 'font[2]/text()')
|
||||
|
||||
# Here we populate the item and yield it
|
||||
yield l.load_item()
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[settings]
|
||||
default = googledir.settings
|
||||
|
|
@ -16,7 +16,9 @@ def build(suffix):
|
|||
with open(ifn, 'w') as of:
|
||||
of.write(s)
|
||||
|
||||
check_call('debchange -m -D unstable --force-distribution -v $(python setup.py --version)-r$(hg tip --template "{rev}")+$(date +%s) "Automatic build"', shell=True)
|
||||
env={'SCRAPY_VERSION_FROM_HG': '1'}
|
||||
check_call('debchange -m -D unstable --force-distribution -v $(python setup.py --version)+$(date +%s) "Automatic build"', \
|
||||
shell=True, env=env)
|
||||
check_call('debuild -us -uc -b', shell=True)
|
||||
|
||||
def clean(suffix):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ _scrapy_completion() {
|
|||
cmd=${COMP_WORDS[1]}
|
||||
cur=${COMP_WORDS[2]}
|
||||
case "$cmd" in
|
||||
crawl)
|
||||
crawl|edit)
|
||||
spiders=$(scrapy list 2>/dev/null) || spiders=""
|
||||
COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$spiders" -- "$cur"))
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -1,370 +0,0 @@
|
|||
from collections import deque, defaultdict
|
||||
from heapq import heappush, heappop
|
||||
import time
|
||||
from itertools import chain
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue1(object):
|
||||
"""heapq
|
||||
|
||||
A simple priority queue
|
||||
"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.items = []
|
||||
|
||||
def push(self, item, priority=0):
|
||||
heappush(self.items, (priority, time.time(), item))
|
||||
|
||||
def pop(self):
|
||||
priority, _, item = heappop(self.items)
|
||||
return item, priority
|
||||
|
||||
def __len__(self):
|
||||
return len(self.items)
|
||||
|
||||
def __iter__(self):
|
||||
return ((item, priority) for priority, _, item in self.items)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.items)
|
||||
|
||||
|
||||
class PriorityQueue1b(PriorityQueue1):
|
||||
"""heapq+int
|
||||
|
||||
A simple priority queue using incremental integer instead of time.time()
|
||||
"""
|
||||
time = 0
|
||||
|
||||
def push(self, item, priority=0):
|
||||
self.time += 1
|
||||
heappush(self.items, (priority, time, item))
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue2(object):
|
||||
"""dict+deque
|
||||
|
||||
@author: Federico Feroldi <federico@cloudify.me>
|
||||
"""
|
||||
def __init__(self, size=1):
|
||||
self.items = defaultdict(deque)
|
||||
|
||||
def push(self, item, priority=0):
|
||||
self.items[priority].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
priorities = self.items.keys()
|
||||
priorities.sort()
|
||||
|
||||
for priority in priorities:
|
||||
if len(self.items[priority]) > 0:
|
||||
return (self.items[priority].pop(), priority)
|
||||
|
||||
raise IndexError
|
||||
|
||||
def __len__(self):
|
||||
totlen = 0
|
||||
for q in self.items.values():
|
||||
totlen += len(q)
|
||||
return totlen
|
||||
|
||||
def __iter__(self):
|
||||
priorities = self.items.keys()
|
||||
priorities.sort()
|
||||
|
||||
for priority in priorities:
|
||||
for i in self.items[priority]:
|
||||
yield (i, priority)
|
||||
|
||||
def __nonzero__(self):
|
||||
for q in self.items.values():
|
||||
if len(q) > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue3(object):
|
||||
"""deque+heapq"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.negitems = []
|
||||
self.pzero = deque()
|
||||
self.positems = []
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
heappush(self.negitems, (priority, time.time(), item))
|
||||
else:
|
||||
heappush(self.positems, (priority, time.time(), item))
|
||||
|
||||
def pop(self):
|
||||
if self.negitems:
|
||||
priority, _, item = heappop(self.negitems)
|
||||
return item, priority
|
||||
elif self.pzero:
|
||||
return (self.pzero.pop(), 0)
|
||||
else:
|
||||
priority, _, item = heappop(self.positems)
|
||||
return item, priority
|
||||
|
||||
def __len__(self):
|
||||
return len(self.negitems) + len(self.pzero) + len(self.positems)
|
||||
|
||||
def __iter__(self):
|
||||
for priority, _, item in self.negitems:
|
||||
yield (item, priority)
|
||||
for item in self.pzero:
|
||||
yield (item, 0)
|
||||
for priority, _, item in self.positems:
|
||||
yield (item, priority)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.negitems and self.pzero and self.positems)
|
||||
|
||||
|
||||
class PriorityQueue3b(PriorityQueue3):
|
||||
"""deque+heapq+int"""
|
||||
|
||||
left_time = 0
|
||||
right_time = 0
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
self.left_time += 1
|
||||
heappush(self.negitems, (priority, self.left_time, item))
|
||||
else:
|
||||
self.right_time += 1
|
||||
heappush(self.positems, (priority, self.right_time, item))
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue4(object):
|
||||
"""deque+defaultdict+deque"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.negitems = defaultdict(deque)
|
||||
self.pzero = deque()
|
||||
self.positems = defaultdict(deque)
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
self.negitems[priority].appendleft(item)
|
||||
else:
|
||||
self.positems[priority].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
if self.negitems:
|
||||
priorities = self.negitems.keys()
|
||||
priorities.sort()
|
||||
for priority in priorities:
|
||||
deq = self.negitems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
del self.negitems[priority]
|
||||
return t
|
||||
elif self.pzero:
|
||||
return (self.pzero.pop(), 0)
|
||||
else:
|
||||
priorities = self.positems.keys()
|
||||
priorities.sort()
|
||||
for priority in priorities:
|
||||
deq = self.positems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
del self.positems[priority]
|
||||
return t
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
def __len__(self):
|
||||
total = sum(len(v) for v in self.negitems.values()) + \
|
||||
len(self.pzero) + \
|
||||
sum(len(v) for v in self.positems.values())
|
||||
return total
|
||||
|
||||
def __iter__(self):
|
||||
gen_negs = ((i, priority)
|
||||
for priority in sorted(self.negitems.keys())
|
||||
for i in reversed(self.negitems[priority]))
|
||||
gen_zeros = ((item,0) for item in self.pzero)
|
||||
gen_pos = ((i, priority)
|
||||
for priority in sorted(self.positems.keys())
|
||||
for i in reversed(self.positems[priority]))
|
||||
return chain(gen_negs, gen_zeros, gen_pos)
|
||||
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.negitems or self.pzero or self.positems)
|
||||
|
||||
class PriorityQueue4b(object):
|
||||
"""deque+defaultdict+deque+cache"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
self.negitems = defaultdict(deque)
|
||||
self.pzero = deque()
|
||||
self.positems = defaultdict(deque)
|
||||
self._sort_neg = False
|
||||
self._sort_pos = False
|
||||
self._cached_sorted_negitems = []
|
||||
self._cached_sorted_positems = []
|
||||
|
||||
def push(self, item, priority=0):
|
||||
if priority == 0:
|
||||
self.pzero.appendleft(item)
|
||||
elif priority < 0:
|
||||
if priority not in self.negitems:
|
||||
self._sort_neg = True
|
||||
self.negitems[priority].appendleft(item)
|
||||
else:
|
||||
if priority not in self.positems:
|
||||
self._sort_pos = True
|
||||
self.positems[priority].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
if self.negitems:
|
||||
if self._sort_neg:
|
||||
priorities = self.negitems.keys()
|
||||
priorities.sort()
|
||||
self._cached_sorted_negitems = priorities
|
||||
self._sort_neg = False
|
||||
else:
|
||||
priorities = self._cached_sorted_negitems
|
||||
|
||||
for priority in priorities:
|
||||
deq = self.negitems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
self._sort_neg = True
|
||||
del self.negitems[priority]
|
||||
return t
|
||||
elif self.pzero:
|
||||
return (self.pzero.pop(), 0)
|
||||
else:
|
||||
if self._sort_pos:
|
||||
priorities = self.positems.keys()
|
||||
priorities.sort()
|
||||
self._cached_sorted_positems = priorities
|
||||
self._sort_pos = False
|
||||
else:
|
||||
priorities = self._cached_sorted_positems
|
||||
|
||||
priorities = self.positems.keys()
|
||||
priorities.sort()
|
||||
for priority in priorities:
|
||||
deq = self.positems[priority]
|
||||
if deq:
|
||||
t = (deq.pop(), priority)
|
||||
if not deq:
|
||||
self._sort_pos = True
|
||||
del self.positems[priority]
|
||||
return t
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
def __len__(self):
|
||||
total = sum(len(v) for v in self.negitems.values()) + \
|
||||
len(self.pzero) + \
|
||||
sum(len(v) for v in self.positems.values())
|
||||
return total
|
||||
|
||||
def __iter__(self):
|
||||
gen_negs = ((i, priority)
|
||||
for priority in sorted(self.negitems.keys())
|
||||
for i in reversed(self.negitems[priority]))
|
||||
gen_zeros = ((item,0) for item in self.pzero)
|
||||
gen_pos = ((i, priority)
|
||||
for priority in sorted(self.positems.keys())
|
||||
for i in reversed(self.positems[priority]))
|
||||
return chain(gen_negs, gen_zeros, gen_pos)
|
||||
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.negitems or self.pzero or self.positems)
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
class PriorityQueue5(object):
|
||||
"""list+deque"""
|
||||
|
||||
def __init__(self, size=1):
|
||||
# preallocate deques for a fixed number of priorities
|
||||
size = size if size % 2 else size + 1
|
||||
self.zero = size // 2
|
||||
self.priolist = [deque() for _ in range(size)]
|
||||
|
||||
def push(self, item, priority=0):
|
||||
self.priolist[priority + self.zero].appendleft(item)
|
||||
|
||||
def pop(self):
|
||||
for prio, queue in enumerate(self.priolist):
|
||||
if len(queue):
|
||||
final = prio - self.zero
|
||||
return (queue.pop(), final)
|
||||
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
def __len__(self):
|
||||
return sum(len(v) for v in self.priolist)
|
||||
|
||||
def __iter__(self):
|
||||
for prio, queue in enumerate(self.priolist):
|
||||
final = prio - self.zero
|
||||
for i in reversed(queue):
|
||||
yield (i, final)
|
||||
|
||||
def __nonzero__(self):
|
||||
return any(d for d in self.priolist)
|
||||
|
||||
class PriorityQueue5b(PriorityQueue5):
|
||||
"""list+deque+cache"""
|
||||
index = 0
|
||||
|
||||
def push(self, item, priority=0):
|
||||
i = priority + self.zero
|
||||
self.priolist[i].appendleft(item)
|
||||
self.index = min(self.index, i)
|
||||
|
||||
def pop(self):
|
||||
cached = self.priolist[self.index]
|
||||
if cached:
|
||||
return (cached.pop(), self.index - self.zero)
|
||||
|
||||
for prio, queue in enumerate(self.priolist[self.index:]):
|
||||
if queue:
|
||||
self.index += prio
|
||||
return (queue.pop(), self.index - self.zero)
|
||||
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
|
||||
from itertools import islice
|
||||
class PriorityQueue5c(PriorityQueue5b):
|
||||
"""list+deque+cache+islice"""
|
||||
|
||||
def pop(self):
|
||||
cached = self.priolist[self.index]
|
||||
if cached:
|
||||
return (cached.pop(), self.index - self.zero)
|
||||
|
||||
for prio, queue in enumerate(islice(self.priolist, self.index, None), self.index):
|
||||
if queue:
|
||||
self.index = prio
|
||||
return (queue.pop(), prio - self.zero)
|
||||
|
||||
raise IndexError("pop from an empty queue")
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
__all__ = [name for name in globals().keys() if name.startswith('PriorityQueue')]
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
from __future__ import with_statement
|
||||
import os
|
||||
import timeit
|
||||
import random
|
||||
from optparse import OptionParser
|
||||
from tempfile import mktemp
|
||||
from pq_classes import *
|
||||
|
||||
TESTID = os.getpid()
|
||||
|
||||
TESTCASES = (
|
||||
("heapq", PriorityQueue1),
|
||||
#("heapq+int", PriorityQueue1b),
|
||||
("dict+deque", PriorityQueue2),
|
||||
("deque+heapq", PriorityQueue3),
|
||||
#("deque+heapq+int", PriorityQueue3b),
|
||||
("deque+defaultdict+deque", PriorityQueue4),
|
||||
("deque+defaultdict+deque+cache", PriorityQueue4b),
|
||||
#('list+deque', PriorityQueue5),
|
||||
('list+deque+cache', PriorityQueue5b),
|
||||
#('list+deque+cache+islice', PriorityQueue5c),
|
||||
)
|
||||
|
||||
|
||||
stmt_fmt = """
|
||||
for n, prio in enumerate(randomprio):
|
||||
q.push(n, prio)
|
||||
|
||||
try:
|
||||
while True:
|
||||
q.pop()
|
||||
except IndexError:
|
||||
pass
|
||||
"""
|
||||
|
||||
setup_fmt = """
|
||||
from collections import deque
|
||||
from __main__ import %(PriorityClass)s as PriorityQueue
|
||||
q = PriorityQueue(%(priorities)i)
|
||||
|
||||
randomprio = deque()
|
||||
for line in open('%(samplefile)s'):
|
||||
prio = int(line.strip())
|
||||
randomprio.append(prio)
|
||||
"""
|
||||
|
||||
|
||||
def _distribution(priorities, distribution):
|
||||
half = priorities // 2
|
||||
prio = -priorities
|
||||
while not (-half <= prio <= half):
|
||||
prio = round(distribution())
|
||||
return min(max(prio, -half), half)
|
||||
|
||||
def normal_priority(priorities):
|
||||
sigma = priorities / 4.0
|
||||
dist = lambda: random.normalvariate(mu=0, sigma=sigma)
|
||||
return _distribution(priorities, dist)
|
||||
|
||||
def gauss_priority(priorities):
|
||||
sigma = priorities / 4.0
|
||||
dist = lambda: random.gauss(mu=0, sigma=sigma)
|
||||
return _distribution(priorities, dist)
|
||||
|
||||
def triangular_priority(priorities):
|
||||
half = priorities // 2
|
||||
return random.triangular(-half-1, half+1, 0)
|
||||
|
||||
def uniform_priority(priorities):
|
||||
return int(random.random() * priorities) - (priorities / 2)
|
||||
|
||||
|
||||
PRIORITY_DISTRIBUTIONS = {
|
||||
'uniform': uniform_priority,
|
||||
'normal': normal_priority,
|
||||
'gauss': gauss_priority,
|
||||
'triangular': triangular_priority,
|
||||
}
|
||||
|
||||
|
||||
def gen_samples(count, priorities, priority_distribution=uniform_priority):
|
||||
fn = '/tmp/pq-%i-%i-%i' % (TESTID, priorities, count)
|
||||
|
||||
with open(fn, 'w') as samplefile:
|
||||
for n in xrange(count):
|
||||
prio = priority_distribution(priorities)
|
||||
samplefile.write('%i\n' % prio)
|
||||
return fn
|
||||
|
||||
def runtests(pushpops=50*1000, times=30, priorities=1, samplefile=None, priority_distribution=uniform_priority):
|
||||
samplefile = samplefile or gen_samples(pushpops, priorities, priority_distribution)
|
||||
|
||||
print "\n== With %s priorities (%s) ==\n" % (priorities, samplefile)
|
||||
print "pushpops = %s, times = %s" % (pushpops, times)
|
||||
|
||||
|
||||
stmt = stmt_fmt
|
||||
for name, cls in TESTCASES:
|
||||
setup = setup_fmt % {
|
||||
'PriorityClass': cls.__name__,
|
||||
'priorities': priorities,
|
||||
'samplefile': samplefile,
|
||||
}
|
||||
t = timeit.Timer(stmt, setup)
|
||||
print "%s implementation: %s" % (name, t.timeit(number=times))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
o = OptionParser()
|
||||
o.add_option('-n', '--samples-count', type='int', default=50000, metavar='NUMBER',
|
||||
help='The max number or samples to generate')
|
||||
o.add_option('-r', '--retry-times', type='int', default=30, metavar='NUMBER',
|
||||
help='the times to retry each test')
|
||||
o.add_option('-s', '--samplefile', default=None, metavar='FILENAME',
|
||||
help='load samples from file, default: use sample generator')
|
||||
o.add_option('-p', '--priorities', default='1,3,5,10,100', metavar='CSV_PRIOLIST',
|
||||
help='a comma separated list of priorities to test')
|
||||
o.add_option('-d', '--priority-distribution', default='uniform', metavar='DISTRIBUTION',
|
||||
help='distribution used for random priority generator, default: uniform. possibles: %s' \
|
||||
% ','.join(PRIORITY_DISTRIBUTIONS.keys()))
|
||||
|
||||
opt, args = o.parse_args()
|
||||
|
||||
priolist = map(int, opt.priorities.split(','))
|
||||
distribution = PRIORITY_DISTRIBUTIONS[opt.priority_distribution]
|
||||
for prio in priolist:
|
||||
runtests(pushpops=opt.samples_count, priorities=prio, times=opt.retry_times,
|
||||
samplefile=opt.samplefile, priority_distribution=distribution)
|
||||
|
||||
# Results (in seconds, on an intel core2 2.16ghz):
|
||||
# == Without priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 7.7959010601
|
||||
# dict+deque implementation: 5.6420109272
|
||||
# deque+heapq implementation: 3.57563900948
|
||||
|
||||
# == With 5 priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 9.83902192116
|
||||
# dict+deque implementation: 9.21094298363
|
||||
# deque+heapq implementation: 9.05321097374
|
||||
|
||||
# == With 10 priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 9.97831392288
|
||||
# dict+deque implementation: 11.9721341133
|
||||
# deque+heapq implementation: 9.79048800468
|
||||
|
||||
# == With 100 priorities ==
|
||||
|
||||
# pushpops = 50000, times = 30
|
||||
# heapq implementation: 10.4782910347
|
||||
# dict+deque implementation: 64.6989660263
|
||||
# deque+heapq implementation: 10.858932972
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
import sys
|
||||
import unittest
|
||||
from pq_classes import *
|
||||
|
||||
# (ITEM, PRIORITY)
|
||||
INPUT = [(1, -5), (30, -1), (80, -3), (4, 1), (6, 3), (20, 0), (50, -1)]
|
||||
OUTPUT = [(1, -5), (80, -3), (30, -1), (50, -1), (20, 0), (4, 1), (6, 3)]
|
||||
|
||||
PRIOSIZE = reduce(max, (abs(i[1]) for i in INPUT + OUTPUT)) * 2
|
||||
|
||||
class TestPriorityQueue(object):
|
||||
|
||||
def test_popping(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
for item, pr in INPUT:
|
||||
pq.push(item, pr)
|
||||
l = []
|
||||
while pq:
|
||||
l.append(pq.pop())
|
||||
self.assertEquals(l, OUTPUT)
|
||||
|
||||
def test_iter(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
for item, pr in INPUT:
|
||||
pq.push(item, pr)
|
||||
result = [x for x in pq]
|
||||
self.assertEquals(result, OUTPUT)
|
||||
|
||||
def test_nonzero(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
pq.push(80, -1)
|
||||
pq.push(20, 0)
|
||||
pq.push(30, 1)
|
||||
|
||||
pq.pop()
|
||||
self.assertEquals(bool(pq), True)
|
||||
pq.pop()
|
||||
self.assertEquals(bool(pq), True)
|
||||
pq.pop()
|
||||
self.assertEquals(bool(pq), False)
|
||||
|
||||
def test_len(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
pq.push(80, -1)
|
||||
pq.push(20, 0)
|
||||
pq.push(30, 1)
|
||||
|
||||
self.assertEquals(len(pq), 3)
|
||||
pq.pop()
|
||||
self.assertEquals(len(pq), 2)
|
||||
pq.pop()
|
||||
self.assertEquals(len(pq), 1)
|
||||
pq.pop()
|
||||
self.assertEquals(len(pq), 0)
|
||||
|
||||
def test_indexerror(self):
|
||||
pq = self.PriorityQueue(PRIOSIZE)
|
||||
self.assertRaises(IndexError, pq.pop)
|
||||
|
||||
|
||||
# automatically test any PriorityQueueN class defined at pq_classes
|
||||
prioclasses = [(name, cls) for name, cls in globals().items() if name.startswith('PriorityQueue')]
|
||||
for name, cls in prioclasses:
|
||||
testname = '%sTestCase' % name
|
||||
testcase = type(testname, (TestPriorityQueue, unittest.TestCase), {'PriorityQueue': cls})
|
||||
setattr(sys.modules[__name__], testname, testcase)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print "\n== Unit testing for every implementation =="
|
||||
unittest.main()
|
||||
|
|
@ -2,8 +2,8 @@
|
|||
Scrapy - a screen scraping framework written in Python
|
||||
"""
|
||||
|
||||
version_info = (0, 12, 0)
|
||||
__version__ = "0.12.0"
|
||||
version_info = (0, 13, 0)
|
||||
__version__ = "0.13.0"
|
||||
|
||||
import sys, os, warnings
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.crawler import CrawlerProcess
|
|||
from scrapy.xlib import lsprofcalltree
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project
|
||||
|
||||
|
|
@ -82,7 +82,7 @@ def _check_deprecated_scrapy_ctl(argv, inproject):
|
|||
return
|
||||
import warnings
|
||||
warnings.warn("`scrapy-ctl.py` command-line tool is deprecated and will be removed in Scrapy 0.11, use `scrapy` instead",
|
||||
DeprecationWarning, stacklevel=3)
|
||||
ScrapyDeprecationWarning, stacklevel=3)
|
||||
if inproject:
|
||||
projpath = os.path.abspath(os.path.dirname(os.path.dirname(settings.settings_module.__file__)))
|
||||
cfg_path = os.path.join(projpath, 'scrapy.cfg')
|
||||
|
|
|
|||
|
|
@ -1,33 +1,21 @@
|
|||
import sys
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.url import is_url
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
|
||||
def syntax(self):
|
||||
return "[options] <spider|url> ..."
|
||||
return "[options] <spider>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Start crawling from a spider or URL"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None, \
|
||||
help="always use this spider when arguments are urls")
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
|
||||
help="don't follow links (for use with URLs only)")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -35,49 +23,13 @@ class Command(ScrapyCommand):
|
|||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.nofollow:
|
||||
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
|
||||
|
||||
def run(self, args, opts):
|
||||
q = self.crawler.queue
|
||||
urls, names = self._split_urls_and_names(args)
|
||||
for name in names:
|
||||
q.append_spider_name(name, **opts.spargs)
|
||||
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = self.crawler.spiders.create(opts.spider, **opts.spargs)
|
||||
for url in urls:
|
||||
q.append_url(url, spider)
|
||||
except KeyError:
|
||||
log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
|
||||
else:
|
||||
for name, urls in self._group_urls_by_spider(urls):
|
||||
spider = self.crawler.spiders.create(name, **opts.spargs)
|
||||
for url in urls:
|
||||
q.append_url(url, spider)
|
||||
if len(args) < 1:
|
||||
raise UsageError()
|
||||
elif len(args) > 1:
|
||||
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
|
||||
for spname in args:
|
||||
spider = self.crawler.spiders.create(spname, **opts.spargs)
|
||||
self.crawler.crawl(spider)
|
||||
self.crawler.start()
|
||||
|
||||
def _group_urls_by_spider(self, urls):
|
||||
spider_urls = defaultdict(list)
|
||||
for url in urls:
|
||||
spider_names = self.crawler.spiders.find_by_request(Request(url))
|
||||
if not spider_names:
|
||||
log.msg('Could not find spider that handles url: %s' % url,
|
||||
log.ERROR)
|
||||
elif len(spider_names) > 1:
|
||||
log.msg('More than one spider can handle url: %s - %s' % \
|
||||
(url, ", ".join(spider_names)), log.ERROR)
|
||||
else:
|
||||
spider_urls[spider_names[0]].append(url)
|
||||
return spider_urls.items()
|
||||
|
||||
def _split_urls_and_names(self, args):
|
||||
urls = []
|
||||
names = []
|
||||
for arg in args:
|
||||
if is_url(arg):
|
||||
urls.append(arg)
|
||||
else:
|
||||
names.append(arg)
|
||||
return urls, names
|
||||
|
|
|
|||
|
|
@ -11,10 +11,11 @@ import netrc
|
|||
from urlparse import urlparse, urljoin
|
||||
from subprocess import Popen, PIPE, check_call
|
||||
|
||||
from w3lib.form import encode_multipart
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.py26 import json
|
||||
from scrapy.utils.multipart import encode_multipart
|
||||
from scrapy.utils.http import basic_auth_header
|
||||
from scrapy.utils.conf import get_config, closest_scrapy_cfg
|
||||
|
||||
|
|
@ -57,6 +58,8 @@ class Command(ScrapyCommand):
|
|||
help="list available projects on TARGET")
|
||||
parser.add_option("--egg", metavar="FILE",
|
||||
help="use the given egg, instead of building it")
|
||||
parser.add_option("--build-egg", metavar="FILE",
|
||||
help="only build the egg, don't deploy it")
|
||||
|
||||
def run(self, args, opts):
|
||||
try:
|
||||
|
|
@ -75,18 +78,26 @@ class Command(ScrapyCommand):
|
|||
projects = json.loads(f.read())['projects']
|
||||
print os.linesep.join(projects)
|
||||
return
|
||||
target_name = _get_target_name(args)
|
||||
target = _get_target(target_name)
|
||||
project = _get_project(target, opts)
|
||||
version = _get_version(target, opts)
|
||||
|
||||
tmpdir = None
|
||||
if opts.egg:
|
||||
_log("Using egg: %s" % opts.egg)
|
||||
egg = opts.egg
|
||||
else:
|
||||
_log("Building egg of %s-%s" % (project, version))
|
||||
|
||||
if opts.build_egg: # build egg only
|
||||
egg, tmpdir = _build_egg()
|
||||
_upload_egg(target, egg, project, version)
|
||||
_log("Writing egg to %s" % opts.build_egg)
|
||||
shutil.copyfile(egg, opts.build_egg)
|
||||
else: # buld egg and deploy
|
||||
target_name = _get_target_name(args)
|
||||
target = _get_target(target_name)
|
||||
project = _get_project(target, opts)
|
||||
version = _get_version(target, opts)
|
||||
if opts.egg:
|
||||
_log("Using egg: %s" % opts.egg)
|
||||
egg = opts.egg
|
||||
else:
|
||||
_log("Building egg of %s-%s" % (project, version))
|
||||
egg, tmpdir = _build_egg()
|
||||
_upload_egg(target, egg, project, version)
|
||||
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
import sys, os
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
default_settings = {'LOG_ENABLED': False}
|
||||
|
||||
def syntax(self):
|
||||
return "<spider>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Edit spider"
|
||||
|
||||
def long_desc(self):
|
||||
return "Edit a spider using the editor defined in EDITOR setting"
|
||||
|
||||
def _err(self, msg):
|
||||
sys.stderr.write(msg + os.linesep)
|
||||
self.exitcode = 1
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1:
|
||||
raise UsageError()
|
||||
editor = self.crawler.settings['EDITOR']
|
||||
try:
|
||||
spider = self.crawler.spiders.create(args[0])
|
||||
except KeyError:
|
||||
return self._err("Spider not found: %s" % args[0])
|
||||
|
||||
sfile = sys.modules[spider.__module__].__file__
|
||||
sfile = sfile.replace('.pyc', '.py')
|
||||
self.exitcode = os.system('%s "%s"' % (editor, sfile))
|
||||
|
|
@ -1,11 +1,10 @@
|
|||
import pprint
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.url import is_url
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.spider import create_spider_for_request
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
|
|
@ -28,9 +27,16 @@ class Command(ScrapyCommand):
|
|||
parser.add_option("--headers", dest="headers", action="store_true", \
|
||||
help="print response HTTP headers instead of body")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
for value in values:
|
||||
print '%s %s: %s' % (prefix, key, value)
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
if opts.headers:
|
||||
pprint.pprint(response.headers)
|
||||
self._print_headers(response.request.headers, '>')
|
||||
print '>'
|
||||
self._print_headers(response.headers, '<')
|
||||
else:
|
||||
print response.body
|
||||
|
||||
|
|
@ -39,15 +45,14 @@ class Command(ScrapyCommand):
|
|||
raise UsageError()
|
||||
cb = lambda x: self._print_response(x, opts)
|
||||
request = Request(args[0], callback=cb, dont_filter=True)
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
|
||||
spider = None
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = self.crawler.spiders.create(opts.spider)
|
||||
except KeyError:
|
||||
log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
|
||||
|
||||
self.crawler.queue.append_request(request, spider, \
|
||||
default_spider=BaseSpider('default'))
|
||||
spider = self.crawler.spiders.create(opts.spider)
|
||||
else:
|
||||
spider = create_spider_for_request(self.crawler.spiders, request, \
|
||||
default_spider=BaseSpider('default'))
|
||||
self.crawler.crawl(spider, [request])
|
||||
self.crawler.start()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import os
|
||||
import shutil
|
||||
import string
|
||||
from os import listdir
|
||||
from os.path import join, dirname, abspath, exists, splitext
|
||||
|
||||
import scrapy
|
||||
|
|
@ -38,6 +38,8 @@ class Command(ScrapyCommand):
|
|||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="crawl",
|
||||
|
|
@ -72,6 +74,8 @@ class Command(ScrapyCommand):
|
|||
template_file = self._find_template(opts.template)
|
||||
if template_file:
|
||||
self._genspider(module, name, domain, opts.template, template_file)
|
||||
if opts.edit:
|
||||
self.exitcode = os.system('scrapy edit "%s"' % name)
|
||||
|
||||
def _genspider(self, module, name, domain, template_name, template_file):
|
||||
"""Generate the spider module, based on the given template"""
|
||||
|
|
@ -102,7 +106,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _list_templates(self):
|
||||
print "Available templates:"
|
||||
for filename in sorted(listdir(self.templates_dir)):
|
||||
for filename in sorted(os.listdir(self.templates_dir)):
|
||||
if filename.endswith('.tmpl'):
|
||||
print " %s" % splitext(filename)[0]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import os
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.conf import settings
|
||||
|
|
@ -15,4 +13,5 @@ class Command(ScrapyCommand):
|
|||
def run(self, args, opts):
|
||||
spman_cls = load_object(settings['SPIDER_MANAGER_CLASS'])
|
||||
spiders = spman_cls.from_settings(settings)
|
||||
print os.linesep.join(spiders.list())
|
||||
for s in spiders.list():
|
||||
print s
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from w3lib.url import is_url
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.utils import display
|
||||
from scrapy.utils.spider import iterate_spider_output, create_spider_for_request
|
||||
from scrapy.utils.url import is_url
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy import log
|
||||
|
||||
|
|
@ -84,7 +84,7 @@ class Command(ScrapyCommand):
|
|||
spider = self.get_spider(request, opts)
|
||||
if not spider:
|
||||
return None, None
|
||||
self.crawler.queue.append_request(request, spider)
|
||||
self.crawler.crawl(spider, [request])
|
||||
self.crawler.start()
|
||||
if not responses:
|
||||
log.msg('No response downloaded for: %s' % request, log.ERROR, \
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
from twisted.internet import reactor, threads
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.commands import runserver
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
|
||||
class Command(runserver.Command):
|
||||
|
||||
requires_project = True
|
||||
default_settings = {'LOG_LEVEL': 'WARNING'}
|
||||
|
||||
def syntax(self):
|
||||
return "[options] <list|clear|count|add spider1 ..>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Deprecated command. See Scrapyd documentation."
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", "--arg", dest="spargs", action="append", default=[], \
|
||||
help="set spider argument (may be repeated)")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) < 1:
|
||||
raise UsageError()
|
||||
cmd = args[0]
|
||||
|
||||
import warnings
|
||||
warnings.warn("Scrapy queue command is deprecated - use Scrapyd instead.", \
|
||||
DeprecationWarning)
|
||||
|
||||
q = self.crawler.queue._queue
|
||||
|
||||
if cmd == 'add' and len(args) < 2:
|
||||
raise UsageError()
|
||||
|
||||
d = threads.deferToThread(self._run_in_thread, args, opts, q, cmd)
|
||||
d.addBoth(lambda _: reactor.stop())
|
||||
from scrapy import log
|
||||
log.start()
|
||||
reactor.run()
|
||||
|
||||
def _run_in_thread(self, args, opts, q, cmd):
|
||||
if cmd == 'add':
|
||||
for x in args[1:]:
|
||||
self._call(q.add, x, **opts.spargs)
|
||||
print "Added: name=%s args=%s" % (x, opts.spargs)
|
||||
elif cmd == 'list':
|
||||
x = self._call(q.list)
|
||||
print "\n".join(map(str, x))
|
||||
elif cmd == 'count':
|
||||
print self._call(q.count)
|
||||
elif cmd == 'clear':
|
||||
self._call(q.clear)
|
||||
else:
|
||||
raise UsageError()
|
||||
|
||||
def _call(self, f, *a, **kw):
|
||||
return threads.blockingCallFromThread(reactor, f, *a, **kw)
|
||||
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
default_settings = {'KEEP_ALIVE': True}
|
||||
|
||||
def short_desc(self):
|
||||
return "Deprecated command. Use 'server' command instead"
|
||||
|
||||
def run(self, args, opts):
|
||||
import warnings
|
||||
warnings.warn("Scrapy queue command is deprecated - use 'server' command instead.", \
|
||||
DeprecationWarning)
|
||||
self.crawler.start()
|
||||
|
|
@ -4,6 +4,7 @@ import os
|
|||
from scrapy.utils.spider import iter_spider_classes
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
|
||||
def _import_file(filepath):
|
||||
abspath = os.path.abspath(filepath)
|
||||
|
|
@ -33,8 +34,17 @@ class Command(ScrapyCommand):
|
|||
def long_desc(self):
|
||||
return "Run the spider defined in the given file"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
|
||||
help="set spider argument (may be repeated)")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1:
|
||||
|
|
@ -49,7 +59,7 @@ class Command(ScrapyCommand):
|
|||
spclasses = list(iter_spider_classes(module))
|
||||
if not spclasses:
|
||||
raise UsageError("No spider found in file: %s\n" % filename)
|
||||
spider = spclasses.pop()()
|
||||
# schedule spider and start engine
|
||||
self.crawler.queue.append_spider(spider)
|
||||
spider = spclasses.pop()(**opts.spargs)
|
||||
|
||||
self.crawler.crawl(spider)
|
||||
self.crawler.start()
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy import log
|
|||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
default_settings = {'KEEP_ALIVE': True}
|
||||
default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}
|
||||
|
||||
def syntax(self):
|
||||
return "[url|file]"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ conditions are met.
|
|||
See documentation in docs/topics/extensions.rst
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -11,14 +12,19 @@ from twisted.python import log as txlog
|
|||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals, log
|
||||
from scrapy.project import crawler
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CloseSpider(object):
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, crawler):
|
||||
self.crawler = crawler
|
||||
self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
|
||||
self.itempassed = settings.getint('CLOSESPIDER_ITEMPASSED')
|
||||
self.itemcount = settings.getint('CLOSESPIDER_ITEMCOUNT')
|
||||
# XXX: legacy support - remove for future releases
|
||||
if settings.getint('CLOSESPIDER_ITEMPASSED'):
|
||||
warnings.warn("CLOSESPIDER_ITEMPASSED setting is deprecated, use CLOSESPIDER_ITEMCOUNT instead", ScrapyDeprecationWarning)
|
||||
self.pagecount = settings.getint('CLOSESPIDER_ITEMPASSED')
|
||||
self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT')
|
||||
self.errorcount = settings.getint('CLOSESPIDER_ERRORCOUNT')
|
||||
|
||||
|
|
@ -33,32 +39,36 @@ class CloseSpider(object):
|
|||
dispatcher.connect(self.page_count, signal=signals.response_received)
|
||||
if self.timeout:
|
||||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
if self.itempassed:
|
||||
dispatcher.connect(self.item_passed, signal=signals.item_passed)
|
||||
if self.itemcount:
|
||||
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def catch_log(self, event):
|
||||
if event.get('logLevel') == log.ERROR:
|
||||
spider = event.get('spider')
|
||||
if spider:
|
||||
self.errorcounts[spider] += 1
|
||||
if self.errorcounts[spider] == self.errorcount:
|
||||
crawler.engine.close_spider(spider, 'closespider_errorcount')
|
||||
self.crawler.engine.close_spider(spider, 'closespider_errorcount')
|
||||
|
||||
def page_count(self, response, request, spider):
|
||||
self.pagecounts[spider] += 1
|
||||
if self.pagecounts[spider] == self.pagecount:
|
||||
crawler.engine.close_spider(spider, 'closespider_pagecount')
|
||||
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.tasks[spider] = reactor.callLater(self.timeout, \
|
||||
crawler.engine.close_spider, spider=spider, \
|
||||
self.crawler.engine.close_spider, spider=spider, \
|
||||
reason='closespider_timeout')
|
||||
|
||||
def item_passed(self, item, spider):
|
||||
def item_scraped(self, item, spider):
|
||||
self.counts[spider] += 1
|
||||
if self.counts[spider] == self.itempassed:
|
||||
crawler.engine.close_spider(spider, 'closespider_itempassed')
|
||||
if self.counts[spider] == self.itemcount:
|
||||
self.crawler.engine.close_spider(spider, 'closespider_itemcount')
|
||||
|
||||
def spider_closed(self, spider):
|
||||
self.counts.pop(spider, None)
|
||||
|
|
|
|||
|
|
@ -1,52 +1,32 @@
|
|||
"""
|
||||
Scrapy extension for collecting scraping stats
|
||||
Extension for collecting core stats like items scraped and start/finish times
|
||||
"""
|
||||
import os
|
||||
import getpass
|
||||
import socket
|
||||
import datetime
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CoreStats(object):
|
||||
"""Scrapy core stats collector"""
|
||||
|
||||
def __init__(self):
|
||||
stats.set_value('envinfo/user', getpass.getuser())
|
||||
stats.set_value('envinfo/host', socket.gethostname())
|
||||
stats.set_value('envinfo/logfile', settings['LOG_FILE'])
|
||||
stats.set_value('envinfo/pid', os.getpid())
|
||||
|
||||
dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened)
|
||||
dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing)
|
||||
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
|
||||
dispatcher.connect(self.item_passed, signal=signals.item_passed)
|
||||
dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
|
||||
|
||||
def stats_spider_opened(self, spider):
|
||||
stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
|
||||
stats.set_value('envinfo/host', stats.get_value('envinfo/host'), spider=spider)
|
||||
stats.inc_value('spider_count/opened')
|
||||
|
||||
def stats_spider_closing(self, spider, reason):
|
||||
stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
|
||||
stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, spider=spider)
|
||||
stats.inc_value('spider_count/%s' % reason, spider=spider)
|
||||
stats.set_value('finish_reason', reason, spider=spider)
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
stats.inc_value('item_scraped_count', spider=spider)
|
||||
stats.inc_value('item_scraped_count')
|
||||
|
||||
def item_passed(self, item, spider):
|
||||
stats.inc_value('item_passed_count', spider=spider)
|
||||
stats.inc_value('item_passed_count')
|
||||
|
||||
def item_dropped(self, item, spider, exception):
|
||||
reason = exception.__class__.__name__
|
||||
stats.inc_value('item_dropped_count', spider=spider)
|
||||
stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
|
||||
stats.inc_value('item_dropped_count')
|
||||
|
|
|
|||
|
|
@ -4,22 +4,30 @@ Extensions for debugging Scrapy
|
|||
See documentation in docs/topics/extensions.rst
|
||||
"""
|
||||
|
||||
import os
|
||||
import signal
|
||||
import traceback
|
||||
from pdb import Pdb
|
||||
|
||||
from scrapy.utils.engine import format_engine_status
|
||||
from scrapy import log
|
||||
|
||||
|
||||
class StackTraceDump(object):
|
||||
def __init__(self):
|
||||
try:
|
||||
signal.signal(signal.SIGUSR2, self.dump_stacktrace)
|
||||
signal.signal(signal.SIGQUIT, self.dump_stacktrace)
|
||||
except AttributeError:
|
||||
# win32 platforms don't support SIGUSR signals
|
||||
pass
|
||||
|
||||
def dump_stacktrace(self, signum, frame):
|
||||
print "Got signal. Dumping stack trace..."
|
||||
traceback.print_stack(frame)
|
||||
msg = "Dumping stack trace and engine status" + os.linesep
|
||||
msg += "".join(traceback.format_stack(frame))
|
||||
msg += os.linesep
|
||||
msg += format_engine_status()
|
||||
log.msg(msg)
|
||||
|
||||
|
||||
class Debugger(object):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
from scrapy.utils.http import decode_chunked_transfer
|
||||
|
||||
|
||||
class ChunkedTransferMiddleware(object):
|
||||
"""This middleware adds support for chunked transfer encoding, as
|
||||
documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
"""
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if response.headers.get('Transfer-Encoding') == 'chunked':
|
||||
body = decode_chunked_transfer(response.body)
|
||||
return response.replace(body=body)
|
||||
return response
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
import os
|
||||
from collections import defaultdict
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.conf import settings
|
||||
|
|
@ -13,6 +15,8 @@ class CookiesMiddleware(object):
|
|||
debug = settings.getbool('COOKIES_DEBUG')
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('COOKIES_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.jars = defaultdict(CookieJar)
|
||||
dispatcher.connect(self.spider_closed, signals.spider_closed)
|
||||
|
||||
|
|
@ -28,7 +32,7 @@ class CookiesMiddleware(object):
|
|||
# set Cookie header
|
||||
request.headers.pop('Cookie', None)
|
||||
jar.add_cookie_header(request)
|
||||
self._debug_cookie(request)
|
||||
self._debug_cookie(request, spider)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if 'dont_merge_cookies' in request.meta:
|
||||
|
|
@ -37,31 +41,28 @@ class CookiesMiddleware(object):
|
|||
# extract cookies from Set-Cookie and drop invalid/expired cookies
|
||||
jar = self.jars[spider]
|
||||
jar.extract_cookies(response, request)
|
||||
self._debug_set_cookie(response)
|
||||
self._debug_set_cookie(response, spider)
|
||||
|
||||
return response
|
||||
|
||||
def spider_closed(self, spider):
|
||||
self.jars.pop(spider, None)
|
||||
|
||||
def _debug_cookie(self, request):
|
||||
"""log Cookie header for request"""
|
||||
def _debug_cookie(self, request, spider):
|
||||
if self.debug:
|
||||
c = request.headers.get('Cookie')
|
||||
c = c and [p.split('=')[0] for p in c.split(';')]
|
||||
log.msg('Cookie: %s for %s' % (c, request.url), level=log.DEBUG)
|
||||
cl = request.headers.getlist('Cookie')
|
||||
if cl:
|
||||
msg = "Sending cookies to: %s" % request + os.linesep
|
||||
msg += os.linesep.join("Cookie: %s" % c for c in cl)
|
||||
log.msg(msg, spider=spider, level=log.DEBUG)
|
||||
|
||||
def _debug_set_cookie(self, response):
|
||||
"""log Set-Cookies headers but exclude cookie values"""
|
||||
def _debug_set_cookie(self, response, spider):
|
||||
if self.debug:
|
||||
cl = response.headers.getlist('Set-Cookie')
|
||||
res = []
|
||||
for c in cl:
|
||||
kv, tail = c.split(';', 1)
|
||||
k = kv.split('=', 1)[0]
|
||||
res.append('%s %s' % (k, tail))
|
||||
log.msg('Set-Cookie: %s from %s' % (res, response.url))
|
||||
|
||||
if cl:
|
||||
msg = "Received cookies from: %s" % response + os.linesep
|
||||
msg += os.linesep.join("Set-Cookie: %s" % c for c in cl)
|
||||
log.msg(msg, spider=spider, level=log.DEBUG)
|
||||
|
||||
def _get_request_cookies(self, jar, request):
|
||||
headers = {'Set-Cookie': ['%s=%s;' % (k, v) for k, v in request.cookies.iteritems()]}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ HTTP basic auth downloader middleware
|
|||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
|
||||
from scrapy.utils.http import basic_auth_header
|
||||
from w3lib.http import basic_auth_header
|
||||
from scrapy.utils.python import WeakKeyCache
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,13 +5,14 @@ from os.path import join, exists
|
|||
from time import time
|
||||
import cPickle as pickle
|
||||
|
||||
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy import signals
|
||||
from scrapy.http import Headers
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.project import data_path
|
||||
|
|
@ -80,8 +81,7 @@ class FilesystemCacheStorage(object):
|
|||
body = f.read()
|
||||
with open(join(rpath, 'response_headers'), 'rb') as f:
|
||||
rawheaders = f.read()
|
||||
# We failback to metadata['url'] to support old generated caches. TODO: remove for Scrapy 0.11
|
||||
url = metadata.get('response_url') or metadata['url']
|
||||
url = metadata.get('response_url')
|
||||
status = metadata['status']
|
||||
headers = Headers(headers_raw_to_dict(rawheaders))
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import zlib
|
||||
from gzip import GzipFile
|
||||
from cStringIO import StringIO
|
||||
|
||||
from scrapy.utils.gz import gunzip
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
|
||||
class HttpCompressionMiddleware(object):
|
||||
|
|
@ -11,7 +10,7 @@ class HttpCompressionMiddleware(object):
|
|||
sent/received from web sites"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
|
||||
request.headers.setdefault('Accept-Encoding', 'x-gzip,gzip,deflate')
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if isinstance(response, Response):
|
||||
|
|
@ -33,8 +32,8 @@ class HttpCompressionMiddleware(object):
|
|||
return response
|
||||
|
||||
def _decode(self, body, encoding):
|
||||
if encoding == 'gzip':
|
||||
body = GzipFile(fileobj=StringIO(body)).read()
|
||||
if encoding == 'gzip' or encoding == 'x-gzip':
|
||||
body = gunzip(body)
|
||||
|
||||
if encoding == 'deflate':
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from w3lib.url import urljoin_rfc
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.utils.url import urljoin_rfc
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
|
|
@ -10,6 +11,8 @@ class RedirectMiddleware(object):
|
|||
"""Handle redirection of requests based on response status and meta-refresh html tag"""
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('REDIRECT_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY')
|
||||
self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
|
||||
self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
|
||||
|
|
|
|||
|
|
@ -20,23 +20,29 @@ About HTTP errors to consider:
|
|||
|
||||
from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.defer import TimeoutError as UserTimeoutError
|
||||
from twisted.web.client import PartialDownloadError
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.response import response_status_message
|
||||
from scrapy.conf import settings
|
||||
|
||||
class RetryMiddleware(object):
|
||||
|
||||
# IOError is raised by the HttpCompression middleware when trying to
|
||||
# decompress an empty response
|
||||
EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError,
|
||||
ConnectionLost, PartialDownloadError)
|
||||
ConnectionLost, PartialDownloadError, TCPTimedOutError,
|
||||
IOError)
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('RETRY_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.max_retry_times = settings.getint('RETRY_TIMES')
|
||||
self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES'))
|
||||
self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
|
||||
self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
|
|
@ -64,6 +70,6 @@ class RetryMiddleware(object):
|
|||
retryreq.priority = request.priority + self.priority_adjust
|
||||
return retryreq
|
||||
else:
|
||||
log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason),
|
||||
log.msg("Gave up retrying %s (failed %d times): %s" % (request, retries, reason),
|
||||
spider=spider, level=log.DEBUG)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,25 +9,28 @@ import robotparser
|
|||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals, log
|
||||
from scrapy.project import crawler
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.conf import settings
|
||||
|
||||
class RobotsTxtMiddleware(object):
|
||||
DOWNLOAD_PRIORITY = 1000
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('ROBOTSTXT_OBEY'):
|
||||
def __init__(self, crawler):
|
||||
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
|
||||
raise NotConfigured
|
||||
|
||||
self.crawler = crawler
|
||||
self._parsers = {}
|
||||
self._spider_netlocs = {}
|
||||
self._useragents = {}
|
||||
dispatcher.connect(self.spider_opened, signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signals.spider_closed)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
useragent = self._useragents[spider]
|
||||
rp = self.robot_parser(request, spider)
|
||||
|
|
@ -42,7 +45,7 @@ class RobotsTxtMiddleware(object):
|
|||
self._parsers[netloc] = None
|
||||
robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
|
||||
robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
|
||||
dfd = crawler.engine.download(robotsreq, spider)
|
||||
dfd = self.crawler.engine.download(robotsreq, spider)
|
||||
dfd.addCallback(self._parse_robots)
|
||||
self._spider_netlocs[spider].add(netloc)
|
||||
return self._parsers[netloc]
|
||||
|
|
|
|||
|
|
@ -11,24 +11,19 @@ class DownloaderStats(object):
|
|||
raise NotConfigured
|
||||
|
||||
def process_request(self, request, spider):
|
||||
stats.inc_value('downloader/request_count')
|
||||
stats.inc_value('downloader/request_count', spider=spider)
|
||||
stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
|
||||
reqlen = len(request_httprepr(request))
|
||||
stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
|
||||
stats.inc_value('downloader/request_bytes', reqlen)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
stats.inc_value('downloader/response_count')
|
||||
stats.inc_value('downloader/response_count', spider=spider)
|
||||
stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
|
||||
reslen = len(response_httprepr(response))
|
||||
stats.inc_value('downloader/response_bytes', reslen, spider=spider)
|
||||
stats.inc_value('downloader/response_bytes', reslen)
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
|
||||
stats.inc_value('downloader/exception_count')
|
||||
stats.inc_value('downloader/exception_count', spider=spider)
|
||||
stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
"""
|
||||
Dupe Filter classes implement a mechanism for filtering duplicate requests.
|
||||
They must implement the following methods:
|
||||
|
||||
* open_spider(spider)
|
||||
open a spider for tracking duplicates (typically used to reserve resources)
|
||||
|
||||
* close_spider(spider)
|
||||
close a spider (typically used for freeing resources)
|
||||
|
||||
* request_seen(spider, request, dont_record=False)
|
||||
return ``True`` if the request was seen before, or ``False`` otherwise. If
|
||||
``dont_record`` is ``True`` the request must not be recorded as seen.
|
||||
|
||||
"""
|
||||
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
class NullDupeFilter(dict):
|
||||
def open_spider(self, spider):
|
||||
pass
|
||||
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
def request_seen(self, spider, request, dont_record=False):
|
||||
return False
|
||||
|
||||
|
||||
class RequestFingerprintDupeFilter(object):
|
||||
"""Duplicate filter using scrapy.utils.request.request_fingerprint"""
|
||||
|
||||
def __init__(self):
|
||||
self.fingerprints = {}
|
||||
|
||||
def open_spider(self, spider):
|
||||
self.fingerprints[spider] = set()
|
||||
|
||||
def close_spider(self, spider):
|
||||
del self.fingerprints[spider]
|
||||
|
||||
def request_seen(self, spider, request, dont_record=False):
|
||||
fp = request_fingerprint(request)
|
||||
if fp in self.fingerprints[spider]:
|
||||
return True
|
||||
if not dont_record:
|
||||
self.fingerprints[spider].add(fp)
|
||||
return False
|
||||
|
|
@ -4,6 +4,7 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
|
||||
import csv
|
||||
import pprint
|
||||
import marshal
|
||||
from cPickle import Pickler
|
||||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ from scrapy.utils.py26 import json
|
|||
|
||||
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \
|
||||
'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \
|
||||
'JsonItemExporter']
|
||||
'JsonItemExporter', 'MarshalItemExporter']
|
||||
|
||||
class BaseItemExporter(object):
|
||||
|
||||
|
|
@ -143,11 +144,20 @@ class XmlItemExporter(BaseItemExporter):
|
|||
|
||||
class CsvItemExporter(BaseItemExporter):
|
||||
|
||||
def __init__(self, file, include_headers_line=True, **kwargs):
|
||||
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
|
||||
self._configure(kwargs, dont_fail=True)
|
||||
self.include_headers_line = include_headers_line
|
||||
self.csv_writer = csv.writer(file, **kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
|
||||
def _to_str_if_unicode(self, value):
|
||||
if isinstance(value, (list, tuple)):
|
||||
try:
|
||||
value = self._join_multivalued.join(value)
|
||||
except TypeError: # list in value may not contain strings
|
||||
pass
|
||||
return super(CsvItemExporter, self)._to_str_if_unicode(value)
|
||||
|
||||
def export_item(self, item):
|
||||
if self._headers_not_written:
|
||||
|
|
@ -176,6 +186,16 @@ class PickleItemExporter(BaseItemExporter):
|
|||
self.pickler.dump(dict(self._get_serialized_fields(item)))
|
||||
|
||||
|
||||
class MarshalItemExporter(BaseItemExporter):
|
||||
|
||||
def __init__(self, file, **kwargs):
|
||||
self._configure(kwargs)
|
||||
self.file = file
|
||||
|
||||
def export_item(self, item):
|
||||
marshal.dump(dict(self._get_serialized_fields(item)), self.file)
|
||||
|
||||
|
||||
class PprintItemExporter(BaseItemExporter):
|
||||
|
||||
def __init__(self, file, **kwargs):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from scrapy.contrib.exporter import JsonLinesItemExporter
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
import warnings
|
||||
warnings.warn("Module `scrapy.contrib.exporter.jsonlines` is deprecated - use `scrapy.contrib.exporter` instead",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ from ftplib import FTP
|
|||
from shutil import copyfileobj
|
||||
|
||||
from zope.interface import Interface, implements
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
from w3lib.url import file_uri_to_path
|
||||
|
||||
from scrapy import log, signals
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.utils.ftp import ftp_makedirs_cwd
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.url import file_uri_to_path
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
|
|
@ -136,7 +136,7 @@ class FeedExporter(object):
|
|||
self.slots = {}
|
||||
dispatcher.connect(self.open_spider, signals.spider_opened)
|
||||
dispatcher.connect(self.close_spider, signals.spider_closed)
|
||||
dispatcher.connect(self.item_passed, signals.item_passed)
|
||||
dispatcher.connect(self.item_scraped, signals.item_scraped)
|
||||
|
||||
def open_spider(self, spider):
|
||||
file = TemporaryFile(prefix='feed-')
|
||||
|
|
@ -163,7 +163,7 @@ class FeedExporter(object):
|
|||
d.addBoth(lambda _: slot.file.close())
|
||||
return d
|
||||
|
||||
def item_passed(self, item, spider):
|
||||
def item_scraped(self, item, spider):
|
||||
slot = self.slots[spider]
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import os
|
||||
from time import time
|
||||
import cPickle as pickle
|
||||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy import conf
|
||||
|
||||
|
||||
class DbmCacheStorage(object):
|
||||
|
||||
def __init__(self, settings=conf.settings):
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'])
|
||||
self.dbs = {}
|
||||
|
||||
def open_spider(self, spider):
|
||||
dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
|
||||
self.dbs[spider] = self.dbmodule.open(dbpath, 'c')
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.dbs[spider].close()
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
url = data['url']
|
||||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
body = data['body']
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
'headers': dict(response.headers),
|
||||
'body': response.body,
|
||||
}
|
||||
self.dbs[spider]['%s_data' % key] = pickle.dumps(data, protocol=2)
|
||||
self.dbs[spider]['%s_time' % key] = str(time())
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
db = self.dbs[spider]
|
||||
tkey = '%s_time' % key
|
||||
if not db.has_key(tkey):
|
||||
return # not found
|
||||
ts = db[tkey]
|
||||
if 0 < self.expiration_secs < time() - float(ts):
|
||||
return # expired
|
||||
return pickle.loads(db['%s_data' % key])
|
||||
|
||||
def _request_key(self, request):
|
||||
return request_fingerprint(request)
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
"""
|
||||
This contrib implements an automatic extraction library based on an Instance
|
||||
Based Learning (IBL) algorithm, as described in the following papers:
|
||||
|
||||
A hierarchical approach to wrapper induction
|
||||
http://portal.acm.org/citation.cfm?id=301191
|
||||
|
||||
Extracting web data using instance based learning
|
||||
http://portal.acm.org/citation.cfm?id=1265174
|
||||
|
||||
This code requires the numpy library.
|
||||
"""
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
"""
|
||||
Extended types for IBL extraction
|
||||
"""
|
||||
from itertools import chain
|
||||
|
||||
from scrapy.contrib.ibl.extractors import text
|
||||
|
||||
class FieldDescriptor(object):
|
||||
"""description of a scraped attribute"""
|
||||
__slots__ = ('name', 'description', 'extractor', 'required', 'allow_markup')
|
||||
|
||||
def __init__(self, name, description, extractor=text, required=False,
|
||||
allow_markup=False):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.extractor = extractor
|
||||
self.required = required
|
||||
self.allow_markup = allow_markup
|
||||
|
||||
@classmethod
|
||||
def from_field(cls, name, field):
|
||||
return cls(name, field.get('description'), \
|
||||
field.get('ibl_extractor', text), field.get('required', False), \
|
||||
field.get('allow_markup', False))
|
||||
|
||||
def __str__(self):
|
||||
return "FieldDescriptor(%s)" % self.name
|
||||
|
||||
class ItemDescriptor(object):
|
||||
"""Simple auto scraping item descriptor.
|
||||
|
||||
This used to describe type-specific operations and may be overridden where
|
||||
necessary.
|
||||
"""
|
||||
|
||||
def __init__(self, name, description, attribute_descriptors):
|
||||
self.name = name
|
||||
self.attribute_map = dict((d.name, d) for d in attribute_descriptors)
|
||||
self._required_attributes = [d.name for d in attribute_descriptors \
|
||||
if d.required]
|
||||
|
||||
@classmethod
|
||||
def from_item(cls, name, description, item):
|
||||
a = [FieldDescriptor.from_field(n, f) for n, f in item.fields.items()]
|
||||
return cls(name, description, a)
|
||||
|
||||
def validated(self, data):
|
||||
"""Only return the items in the data that are valid"""
|
||||
return [d for d in data if self._item_validates(d)]
|
||||
|
||||
def _item_validates(self, item):
|
||||
"""simply checks that all mandatory attributes are present"""
|
||||
variant_attrs = set(chain(*
|
||||
[v.keys() for v in item.get('variants', [])]))
|
||||
return all([(name in item or name in variant_attrs) \
|
||||
for name in self._required_attributes])
|
||||
|
||||
def get_required_attributes(self):
|
||||
return self._required_attributes
|
||||
|
||||
def __str__(self):
|
||||
return "ItemDescriptor(%s)" % self.name
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
"""
|
||||
IBL module
|
||||
|
||||
This contains an extraction algorithm based on the paper Extracting Web Data
|
||||
Using Instance-Based Learning by Yanhong Zhai and Bing Liu.
|
||||
|
||||
It defines the InstanceBasedLearningExtractor class, which implements this
|
||||
extraction algorithm.
|
||||
|
||||
Main departures from the original algorithm:
|
||||
* there is no limit in prefix or suffix size
|
||||
* we have "attribute adaptors" that allow generic post processing and may
|
||||
affect the extraction process. For example, a price field may require a
|
||||
numeric value to be present.
|
||||
* tags can be inserted to extract regions not wrapped by html tags. These
|
||||
regions are then identified using the longest unique character prefix and
|
||||
suffix.
|
||||
"""
|
||||
from operator import itemgetter
|
||||
from .regionextract import build_extraction_tree
|
||||
from .pageparsing import parse_template, parse_extraction_page
|
||||
from .pageobjects import TokenDict
|
||||
|
||||
class InstanceBasedLearningExtractor(object):
|
||||
"""Implementation of the instance based learning algorithm to
|
||||
extract data from web pages.
|
||||
"""
|
||||
|
||||
def __init__(self, templates, type_descriptor=None, trace=False):
|
||||
"""Initialise this extractor
|
||||
|
||||
templates should contain a sequence of strings, each containing
|
||||
annotated html that will be used as templates for extraction.
|
||||
|
||||
Tags surrounding areas to be extracted must contain a
|
||||
'data-scrapy-annotate' attribute and the value must be the name
|
||||
of the attribute. If the tag was inserted and was not present in the
|
||||
original page, the data-scrapy-generated attribute must be present.
|
||||
|
||||
type_descriptor may contain a type descriptor describing the item
|
||||
to be extracted.
|
||||
|
||||
if trace is true, the returned extracted data will have a 'trace'
|
||||
property that contains a trace of the extraction execution.
|
||||
"""
|
||||
self.token_dict = TokenDict()
|
||||
parsed_plus_templates = [(parse_template(self.token_dict, t), t) for t in templates]
|
||||
parsed_plus_epages = [(p, parse_extraction_page(self.token_dict, t)) for p, t \
|
||||
in parsed_plus_templates if _annotation_count(p)]
|
||||
parsed_templates = map(itemgetter(0), parsed_plus_epages)
|
||||
|
||||
# templates with more attributes are considered first
|
||||
sorted_templates = sorted(parsed_templates, key=_annotation_count, reverse=True)
|
||||
self.extraction_trees = [build_extraction_tree(t, type_descriptor,
|
||||
trace) for t in sorted_templates]
|
||||
self.validated = type_descriptor.validated if type_descriptor else \
|
||||
self._filter_not_none
|
||||
|
||||
def extract(self, html, pref_template_id=None, useone=False):
|
||||
"""extract data from an html page
|
||||
|
||||
If pref_template_url is specified, the template with that url will be
|
||||
used first.
|
||||
if useone is True and no data was extracted, no additional template will
|
||||
be tried. If False and no data was extracted, try with rest of item templates
|
||||
"""
|
||||
extraction_page = parse_extraction_page(self.token_dict, html)
|
||||
if pref_template_id is not None:
|
||||
if useone:
|
||||
extraction_trees = [x for x in self.extraction_trees if x.template.id == pref_template_id]
|
||||
else:
|
||||
extraction_trees = sorted(self.extraction_trees,
|
||||
key=lambda x: x.template.id != pref_template_id)
|
||||
else:
|
||||
extraction_trees = self.extraction_trees
|
||||
|
||||
for extraction_tree in extraction_trees:
|
||||
extracted = extraction_tree.extract(extraction_page)
|
||||
correctly_extracted = self.validated(extracted)
|
||||
extra_required = extraction_tree.template.extra_required_attrs
|
||||
correctly_extracted = [c for c in correctly_extracted if \
|
||||
extra_required.intersection(c.keys()) == extra_required ]
|
||||
if len(correctly_extracted) > 0:
|
||||
return correctly_extracted, extraction_tree.template
|
||||
return None, None
|
||||
|
||||
def __str__(self):
|
||||
return "InstanceBasedLearningExtractor[\n%s\n]" % \
|
||||
(',\n'.join(map(str, self.extraction_trees)))
|
||||
|
||||
@staticmethod
|
||||
def _filter_not_none(items):
|
||||
return [d for d in items if d is not None]
|
||||
|
||||
def _annotation_count(template):
|
||||
return len(template.annotations)
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
"""
|
||||
Page objects
|
||||
|
||||
This module contains objects representing pages and parts of pages (e.g. tokens
|
||||
and annotations) used in the instance based learning algorithm.
|
||||
"""
|
||||
from numpy import array, ndarray
|
||||
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlTagType
|
||||
|
||||
class TokenType(object):
|
||||
"""constants for token types"""
|
||||
WORD = 0
|
||||
OPEN_TAG = HtmlTagType.OPEN_TAG
|
||||
CLOSE_TAG = HtmlTagType.CLOSE_TAG
|
||||
NON_PAIRED_TAG = HtmlTagType.UNPAIRED_TAG
|
||||
|
||||
class TokenDict(object):
|
||||
"""Mapping from parse tokens to integers
|
||||
|
||||
>>> d = TokenDict()
|
||||
>>> d.tokenid('i')
|
||||
0
|
||||
>>> d.tokenid('b')
|
||||
1
|
||||
>>> d.tokenid('i')
|
||||
0
|
||||
|
||||
Tokens can be searched for by id
|
||||
>>> d.find_token(1)
|
||||
'b'
|
||||
|
||||
The lower 24 bits store the token reference and the higher bits the type.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.token_ids = {}
|
||||
|
||||
def tokenid(self, token, token_type=TokenType.WORD):
|
||||
"""create an integer id from the token and token type passed"""
|
||||
tid = self.token_ids.setdefault(token, len(self.token_ids))
|
||||
return tid | (token_type << 24)
|
||||
|
||||
@staticmethod
|
||||
def token_type(token):
|
||||
"""extract the token type from the token id passed"""
|
||||
return token >> 24
|
||||
|
||||
def find_token(self, tid):
|
||||
"""Search for a tag with the given ID
|
||||
|
||||
This is O(N) and is only intended for debugging
|
||||
"""
|
||||
tid &= 0xFFFFFF
|
||||
if tid >= len(self.token_ids) or tid < 0:
|
||||
raise ValueError("tag id %s out of range" % tid)
|
||||
|
||||
for (token, token_id) in self.token_ids.items():
|
||||
if token_id == tid:
|
||||
return token
|
||||
assert False, "token dictionary is corrupt"
|
||||
|
||||
def token_string(self, tid):
|
||||
"""create a string representation of a token
|
||||
|
||||
This is O(N).
|
||||
"""
|
||||
templates = ["%s", "<%s>", "</%s>", "<%s/>"]
|
||||
return templates[tid >> 24] % self.find_token(tid)
|
||||
|
||||
class Page(object):
|
||||
"""Basic representation of a page. This consists of a reference to a
|
||||
dictionary of tokens and an array of raw token ids
|
||||
"""
|
||||
|
||||
__slots__ = ('token_dict', 'page_tokens')
|
||||
|
||||
def __init__(self, token_dict, page_tokens):
|
||||
self.token_dict = token_dict
|
||||
# use a numpy array becuase we can index/slice easily and efficiently
|
||||
if not isinstance(page_tokens, ndarray):
|
||||
page_tokens = array(page_tokens)
|
||||
self.page_tokens = page_tokens
|
||||
|
||||
class TemplatePage(Page):
|
||||
__slots__ = ('annotations', 'id', 'ignored_regions', 'extra_required_attrs')
|
||||
|
||||
def __init__(self, token_dict, page_tokens, annotations, template_id=None, \
|
||||
ignored_regions=None, extra_required=None):
|
||||
Page.__init__(self, token_dict, page_tokens)
|
||||
# ensure order is the same as start tag order in the original page
|
||||
annotations = sorted(annotations, key=lambda x: x.end_index, reverse=True)
|
||||
self.annotations = sorted(annotations, key=lambda x: x.start_index)
|
||||
self.id = template_id
|
||||
self.ignored_regions = ignored_regions or []
|
||||
self.extra_required_attrs = set(extra_required or [])
|
||||
|
||||
def __str__(self):
|
||||
summary = []
|
||||
for index, token in enumerate(self.page_tokens):
|
||||
text = "%s: %s" % (index, self.token_dict.find_token(token))
|
||||
summary.append(text)
|
||||
return "TemplatePage\n============\nTokens: (index, token)\n%s\nAnnotations: %s\n" % \
|
||||
('\n'.join(summary), '\n'.join(map(str, self.annotations)))
|
||||
|
||||
class ExtractionPage(Page):
|
||||
"""Parsed data belonging to a web page upon which we wish to perform
|
||||
extraction.
|
||||
"""
|
||||
__slots__ = ('text',
|
||||
'token_start_indexes', # index in text of the start of a token
|
||||
'token_follow_indexes', # index in text of data following token
|
||||
'tag_attributes' # a map from token index to tag attributes
|
||||
)
|
||||
|
||||
def __init__(self, text, token_dict, page_tokens, token_start_indexes,
|
||||
token_follow_indexes, tag_attributes):
|
||||
Page.__init__(self, token_dict, page_tokens)
|
||||
self.text = text
|
||||
self.token_start_indexes = token_start_indexes
|
||||
self.token_follow_indexes = token_follow_indexes
|
||||
self.tag_attributes = tag_attributes
|
||||
|
||||
def token_html(self, token_index):
|
||||
"""The raw html for a page token at the given index in the page_tokens
|
||||
list
|
||||
"""
|
||||
text_start = self.token_start_indexes[token_index]
|
||||
text_end = self.token_follow_indexes[token_index]
|
||||
return self.text[text_start:text_end]
|
||||
|
||||
def html_between_tokens(self, start_token_index, end_token_index):
|
||||
"""The raw html between the tokens at the specified indexes in the
|
||||
page_tokens list
|
||||
|
||||
This assumes start_token_index <= end_token_index
|
||||
"""
|
||||
text_start = self.token_follow_indexes[start_token_index]
|
||||
text_end = self.token_start_indexes[end_token_index]
|
||||
return self.text[text_start:text_end]
|
||||
|
||||
def text_between_tokens(self, start_token_index, end_token_index,
|
||||
tag_replacement=u' '):
|
||||
"""The text between the the tokens at the specified indexes in the
|
||||
page_tokens list. Tags are replaced by tag_replacement (default one space
|
||||
character)
|
||||
"""
|
||||
return tag_replacement.join([self.text[
|
||||
self.token_follow_indexes[i]:self.token_start_indexes[i+1]] \
|
||||
for i in xrange(start_token_index, end_token_index)])
|
||||
|
||||
def tag_attribute(self, token_index, attribute):
|
||||
"""The value of a tag attribute. The tag is identified by its
|
||||
corresponding token index.
|
||||
|
||||
If the tag or attribute is not present, None is returned
|
||||
"""
|
||||
return self.tag_attributes.get(token_index, {}).get(attribute)
|
||||
|
||||
def __str__(self):
|
||||
summary = []
|
||||
for (token, start, follow) in zip(self.page_tokens, self.token_start_indexes,
|
||||
self.token_follow_indexes):
|
||||
text = "%s %s-%s (%s)" % (self.token_dict.find_token(token), start, follow,
|
||||
self.text[start:follow])
|
||||
summary.append(text)
|
||||
return "ExtractionPage\n==============\nTokens: %s\n\nRaw text: %s\n\n" \
|
||||
"Tag attributes: %s\n" % ('\n'.join(summary), self.text,
|
||||
self.tag_attributes)
|
||||
|
||||
class AnnotationText(object):
|
||||
__slots__ = ('start_text', 'follow_text')
|
||||
|
||||
def __init__(self, start_text=None, follow_text=None):
|
||||
self.start_text = start_text
|
||||
self.follow_text = follow_text
|
||||
|
||||
def __str__(self):
|
||||
return "AnnotationText(%s..%s)" % \
|
||||
(repr(self.start_text), repr(self.follow_text))
|
||||
|
||||
class AnnotationTag(object):
|
||||
"""A tag that annotates part of the document
|
||||
|
||||
It has the following properties:
|
||||
start_index - index of the token for the opening tag
|
||||
end_index - index of the token for the closing tag
|
||||
surrounds_attribute - the attribute name surrounded by this tag
|
||||
tag_attributes - list of (tag attribute, extracted attribute) tuples
|
||||
for each item to be extracted from a tag attribute
|
||||
annotation_text - text prefix and suffix for the attribute to be extracted
|
||||
metadata - dict with annotation data not used by IBL extractor
|
||||
"""
|
||||
__slots__ = ('surrounds_attribute', 'start_index', 'end_index',
|
||||
'tag_attributes', 'annotation_text', 'variant_id',
|
||||
'metadata')
|
||||
|
||||
def __init__(self, start_index, end_index, surrounds_attribute=None,
|
||||
annotation_text=None, tag_attributes=None, variant_id=None):
|
||||
self.start_index = start_index
|
||||
self.end_index = end_index
|
||||
self.surrounds_attribute = surrounds_attribute
|
||||
self.annotation_text = annotation_text
|
||||
self.tag_attributes = tag_attributes or []
|
||||
self.variant_id = variant_id
|
||||
self.metadata = {}
|
||||
|
||||
def __str__(self):
|
||||
return "AnnotationTag(%s)" % ", ".join(
|
||||
["%s=%s" % (s, getattr(self, s)) \
|
||||
for s in self.__slots__ if getattr(self, s)])
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
class LabelledRegion(object):
|
||||
__slots__ = ('start_index', 'end_index')
|
||||
|
||||
def __init__(self, start, end):
|
||||
self.start_index = start
|
||||
self.end_index = end
|
||||
|
||||
def __str__(self):
|
||||
return "LabelledRegion (%s, %s)" % (self.start_index, self.end_index)
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
|
|
@ -1,319 +0,0 @@
|
|||
"""
|
||||
Page parsing
|
||||
|
||||
Parsing of web pages for extraction task.
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from numpy import array
|
||||
|
||||
from scrapy.utils.py26 import json
|
||||
|
||||
from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
|
||||
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
|
||||
TemplatePage, ExtractionPage, AnnotationText, TokenDict)
|
||||
|
||||
def parse_strings(template_html, extraction_html):
|
||||
"""Create a template and extraction page from raw strings
|
||||
|
||||
this is useful for testing purposes
|
||||
"""
|
||||
t = TokenDict()
|
||||
template_page = HtmlPage(body=template_html)
|
||||
extraction_page = HtmlPage(body=extraction_html)
|
||||
return (parse_template(t, template_page),
|
||||
parse_extraction_page(t, extraction_page))
|
||||
|
||||
def parse_template(token_dict, template_html):
|
||||
"""Create an TemplatePage object by parsing the annotated html"""
|
||||
parser = TemplatePageParser(token_dict)
|
||||
parser.feed(template_html)
|
||||
return parser.to_template()
|
||||
|
||||
def parse_extraction_page(token_dict, page_html):
|
||||
"""Create an ExtractionPage object by parsing the html"""
|
||||
parser = ExtractionPageParser(token_dict)
|
||||
parser.feed(page_html)
|
||||
return parser.to_extraction_page()
|
||||
|
||||
class InstanceLearningParser(object):
|
||||
"""Base parser for instance based learning algorithm
|
||||
|
||||
This does not require correct HTML and the parsing method should not alter
|
||||
the original tag order. It is important that parsing results do not vary.
|
||||
"""
|
||||
def __init__(self, token_dict):
|
||||
self.token_dict = token_dict
|
||||
self.token_list = []
|
||||
|
||||
def _add_token(self, token, token_type, start, end):
|
||||
tid = self.token_dict.tokenid(token, token_type)
|
||||
self.token_list.append(tid)
|
||||
|
||||
def feed(self, html_page):
|
||||
self.html_page = html_page
|
||||
self.previous_element_class = None
|
||||
for data in html_page.parsed_body:
|
||||
if isinstance(data, HtmlTag):
|
||||
self._add_token(data.tag, data.tag_type, data.start, data.end)
|
||||
self.handle_tag(data)
|
||||
else:
|
||||
self.handle_data(data)
|
||||
self.previous_element_class = data.__class__
|
||||
|
||||
def handle_data(self, html_data_fragment):
|
||||
pass
|
||||
|
||||
def handle_tag(self, html_tag):
|
||||
pass
|
||||
|
||||
_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]
|
||||
|
||||
class TemplatePageParser(InstanceLearningParser):
|
||||
"""Template parsing for instance based learning algorithm"""
|
||||
|
||||
def __init__(self, token_dict):
|
||||
InstanceLearningParser.__init__(self, token_dict)
|
||||
self.annotations = []
|
||||
self.ignored_regions = []
|
||||
self.extra_required_attrs = []
|
||||
self.ignored_tag_stacks = defaultdict(list)
|
||||
# tag names that have not been completed
|
||||
self.labelled_tag_stacks = defaultdict(list)
|
||||
self.replacement_stacks = defaultdict(list)
|
||||
self.unpairedtag_stack = []
|
||||
self.variant_stack = []
|
||||
self.prev_data = None
|
||||
self.last_text_region = None
|
||||
self.next_tag_index = 0
|
||||
|
||||
def handle_tag(self, html_tag):
|
||||
if self.last_text_region:
|
||||
self._process_text('')
|
||||
|
||||
if html_tag.tag_type == HtmlTagType.OPEN_TAG:
|
||||
self._handle_open_tag(html_tag)
|
||||
elif html_tag.tag_type == HtmlTagType.CLOSE_TAG:
|
||||
self._handle_close_tag(html_tag)
|
||||
else:
|
||||
# the tag is not paired, it can contain only attribute annotations
|
||||
self._handle_unpaired_tag(html_tag)
|
||||
|
||||
@staticmethod
|
||||
def _read_template_annotation(html_tag):
|
||||
template_attr = html_tag.attributes.get('data-scrapy-annotate')
|
||||
if template_attr is None:
|
||||
return None
|
||||
unescaped = template_attr.replace('"', '"')
|
||||
return json.loads(unescaped)
|
||||
|
||||
@staticmethod
|
||||
def _read_bool_template_attribute(html_tag, attribute):
|
||||
return html_tag.attributes.get("data-scrapy-" + attribute) == "true"
|
||||
|
||||
def _close_unpaired_tag(self):
|
||||
self.unpairedtag_stack[0].end_index = self.next_tag_index
|
||||
self.unpairedtag_stack = []
|
||||
|
||||
def _handle_unpaired_tag(self, html_tag):
|
||||
if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
|
||||
self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
|
||||
elif self._read_bool_template_attribute(html_tag, "ignore-beneath"):
|
||||
self.ignored_regions.append((self.next_tag_index, None))
|
||||
jannotation = self._read_template_annotation(html_tag)
|
||||
if jannotation:
|
||||
if self.unpairedtag_stack:
|
||||
self._close_unpaired_tag()
|
||||
|
||||
annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
|
||||
attribute_annotations = jannotation.pop('annotations', {}).items()
|
||||
for extract_attribute, tag_value in attribute_annotations:
|
||||
if extract_attribute == 'content':
|
||||
annotation.surrounds_attribute = tag_value
|
||||
self.unpairedtag_stack.append(annotation)
|
||||
else:
|
||||
annotation.tag_attributes.append((extract_attribute, tag_value))
|
||||
self.annotations.append(annotation)
|
||||
|
||||
self.extra_required_attrs.extend(jannotation.pop('required', []))
|
||||
annotation.metadata = jannotation
|
||||
|
||||
self.next_tag_index += 1
|
||||
|
||||
def _handle_open_tag(self, html_tag):
|
||||
if self._read_bool_template_attribute(html_tag, "ignore"):
|
||||
if html_tag.tag == "img":
|
||||
self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
|
||||
else:
|
||||
self.ignored_regions.append((self.next_tag_index, None))
|
||||
self.ignored_tag_stacks[html_tag.tag].append(html_tag)
|
||||
|
||||
elif self.ignored_tag_stacks.get(html_tag.tag):
|
||||
self.ignored_tag_stacks[html_tag.tag].append(None)
|
||||
if self._read_bool_template_attribute(html_tag, "ignore-beneath"):
|
||||
self.ignored_regions.append((self.next_tag_index, None))
|
||||
|
||||
replacement = html_tag.attributes.pop("data-scrapy-replacement", None)
|
||||
if replacement:
|
||||
self.token_list.pop()
|
||||
self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
|
||||
self.replacement_stacks[html_tag.tag].append(replacement)
|
||||
elif html_tag.tag in self.replacement_stacks:
|
||||
self.replacement_stacks[html_tag.tag].append(None)
|
||||
|
||||
if self.unpairedtag_stack:
|
||||
if html_tag.tag in _END_UNPAIREDTAG_TAGS:
|
||||
self._close_unpaired_tag()
|
||||
else:
|
||||
self.unpairedtag_stack.append(html_tag.tag)
|
||||
|
||||
# can't be a p inside another p. Also, an open p element closes
|
||||
# a previous open p element.
|
||||
if html_tag.tag == "p" and html_tag.tag in self.labelled_tag_stacks:
|
||||
annotation = self.labelled_tag_stacks.pop(html_tag.tag)[0]
|
||||
annotation.end_index = self.next_tag_index
|
||||
self.annotations.append(annotation)
|
||||
|
||||
jannotation = self._read_template_annotation(html_tag)
|
||||
if not jannotation:
|
||||
if html_tag.tag in self.labelled_tag_stacks:
|
||||
# add this tag to the stack to match correct end tag
|
||||
self.labelled_tag_stacks[html_tag.tag].append(None)
|
||||
self.next_tag_index += 1
|
||||
return
|
||||
|
||||
annotation = AnnotationTag(self.next_tag_index, None)
|
||||
if jannotation.pop('generated', False):
|
||||
self.token_list.pop()
|
||||
annotation.start_index -= 1
|
||||
if self.previous_element_class == HtmlTag:
|
||||
annotation.annotation_text = AnnotationText('')
|
||||
else:
|
||||
annotation.annotation_text = AnnotationText(self.prev_data)
|
||||
if self._read_bool_template_attribute(html_tag, "ignore") \
|
||||
or self._read_bool_template_attribute(html_tag, "ignore-beneath"):
|
||||
ignored = self.ignored_regions.pop()
|
||||
self.ignored_regions.append((ignored[0]-1, ignored[1]))
|
||||
|
||||
self.extra_required_attrs.extend(jannotation.pop('required', []))
|
||||
|
||||
attribute_annotations = jannotation.pop('annotations', {}).items()
|
||||
for extract_attribute, tag_value in attribute_annotations:
|
||||
if extract_attribute == 'content':
|
||||
annotation.surrounds_attribute = tag_value
|
||||
else:
|
||||
annotation.tag_attributes.append((extract_attribute, tag_value))
|
||||
|
||||
variant_id = jannotation.pop('variant', 0)
|
||||
if variant_id > 0:
|
||||
if annotation.surrounds_attribute is not None:
|
||||
self.variant_stack.append(variant_id)
|
||||
else:
|
||||
annotation.variant_id = variant_id
|
||||
|
||||
annotation.metadata = jannotation
|
||||
|
||||
if annotation.annotation_text is None:
|
||||
self.next_tag_index += 1
|
||||
if self.variant_stack and annotation.variant_id is None:
|
||||
variant_id = self.variant_stack[-1]
|
||||
if variant_id == '0':
|
||||
variant_id = None
|
||||
annotation.variant_id = variant_id
|
||||
|
||||
# look for a closing tag if the content is important
|
||||
if annotation.surrounds_attribute:
|
||||
self.labelled_tag_stacks[html_tag.tag].append(annotation)
|
||||
else:
|
||||
annotation.end_index = annotation.start_index + 1
|
||||
self.annotations.append(annotation)
|
||||
|
||||
def _handle_close_tag(self, html_tag):
|
||||
|
||||
if self.unpairedtag_stack:
|
||||
if html_tag.tag == self.unpairedtag_stack[-1]:
|
||||
self.unpairedtag_stack.pop()
|
||||
else:
|
||||
self._close_unpaired_tag()
|
||||
ignored_tags = self.ignored_tag_stacks.get(html_tag.tag)
|
||||
if ignored_tags is not None:
|
||||
tag = ignored_tags.pop()
|
||||
if isinstance(tag, HtmlTag):
|
||||
for i in range(-1, -len(self.ignored_regions) - 1, -1):
|
||||
if self.ignored_regions[i][1] is None:
|
||||
self.ignored_regions[i] = (self.ignored_regions[i][0], self.next_tag_index)
|
||||
break
|
||||
if len(ignored_tags) == 0:
|
||||
del self.ignored_tag_stacks[html_tag.tag]
|
||||
|
||||
if html_tag.tag in self.replacement_stacks:
|
||||
replacement = self.replacement_stacks[html_tag.tag].pop()
|
||||
if replacement:
|
||||
self.token_list.pop()
|
||||
self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
|
||||
if len(self.replacement_stacks[html_tag.tag]) == 0:
|
||||
del self.replacement_stacks[html_tag.tag]
|
||||
|
||||
labelled_tags = self.labelled_tag_stacks.get(html_tag.tag)
|
||||
if labelled_tags is None:
|
||||
self.next_tag_index += 1
|
||||
return
|
||||
annotation = labelled_tags.pop()
|
||||
if annotation is None:
|
||||
self.next_tag_index += 1
|
||||
else:
|
||||
annotation.end_index = self.next_tag_index
|
||||
self.annotations.append(annotation)
|
||||
if annotation.annotation_text is not None:
|
||||
self.token_list.pop()
|
||||
self.last_text_region = annotation
|
||||
else:
|
||||
self.next_tag_index += 1
|
||||
if len(labelled_tags) == 0:
|
||||
del self.labelled_tag_stacks[html_tag.tag]
|
||||
if annotation.variant_id and self.variant_stack:
|
||||
prev = self.variant_stack.pop()
|
||||
if prev != annotation.variant_id:
|
||||
raise ValueError("unbalanced variant annotation tags")
|
||||
|
||||
def handle_data(self, html_data_fragment):
|
||||
fragment_text = self.html_page.fragment_data(html_data_fragment)
|
||||
self._process_text(fragment_text)
|
||||
|
||||
def _process_text(self, text):
|
||||
if self.last_text_region is not None:
|
||||
self.last_text_region.annotation_text.follow_text = text
|
||||
self.last_text_region = None
|
||||
self.prev_data = text
|
||||
|
||||
def to_template(self):
|
||||
"""create a TemplatePage from the data fed to this parser"""
|
||||
return TemplatePage(self.token_dict, self.token_list, self.annotations,
|
||||
self.html_page.page_id, self.ignored_regions, self.extra_required_attrs)
|
||||
|
||||
class ExtractionPageParser(InstanceLearningParser):
|
||||
"""Parse an HTML page for extraction using the instance based learning
|
||||
algorithm
|
||||
|
||||
This needs to extract the tokens in a similar way to LabelledPageParser,
|
||||
it needs to also maintain a mapping from token index to the original content
|
||||
so that once regions are identified, the original content can be extracted.
|
||||
"""
|
||||
def __init__(self, token_dict):
|
||||
InstanceLearningParser.__init__(self, token_dict)
|
||||
self.page_data = []
|
||||
self.token_start_index = []
|
||||
self.token_follow_index = []
|
||||
self.tag_attrs = {}
|
||||
|
||||
def _add_token(self, token, token_type, start, end):
|
||||
InstanceLearningParser._add_token(self, token, token_type, start, end)
|
||||
self.token_start_index.append(start)
|
||||
self.token_follow_index.append(end)
|
||||
|
||||
def handle_tag(self, html_tag):
|
||||
if html_tag.attributes:
|
||||
self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes
|
||||
|
||||
def to_extraction_page(self):
|
||||
return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list),
|
||||
self.token_start_index, self.token_follow_index, self.tag_attrs)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue