mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into tutorial-upgrades
This commit is contained in:
Elias Dorneles
commit
8975371a57
|
|
@ -8,7 +8,7 @@ branches:
|
|||
- /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
|
||||
env:
|
||||
- TOXENV=py27
|
||||
- TOXENV=precise
|
||||
- TOXENV=jessie
|
||||
- TOXENV=py33
|
||||
- TOXENV=py35
|
||||
- TOXENV=docs
|
||||
|
|
|
|||
|
|
@ -13,13 +13,15 @@ Having trouble? We'd like to help!
|
|||
|
||||
* Try the :doc:`FAQ <faq>` -- it's got answers to some common questions.
|
||||
* Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`.
|
||||
* Ask or search questions in `StackOverflow using the scrapy tag`_,
|
||||
* Search for information in the `archives of the scrapy-users mailing list`_, or
|
||||
`post a question`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
|
||||
.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
|
||||
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
|
||||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
|
||||
|
|
|
|||
|
|
@ -19,74 +19,69 @@ Walk-through of an example spider
|
|||
In order to show you what Scrapy brings to the table, we'll walk you through an
|
||||
example of a Scrapy Spider using the simplest way to run a spider.
|
||||
|
||||
So, here's the code for a spider that follows the links to the top
|
||||
voted questions on StackOverflow and scrapes some data from each page::
|
||||
Here's the code for a spider that scrapes famous quotes from website
|
||||
http://quotes.toscrape.com, following the pagination::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class StackOverflowSpider(scrapy.Spider):
|
||||
name = 'stackoverflow'
|
||||
start_urls = ['http://stackoverflow.com/questions?sort=votes']
|
||||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/tag/humor/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
for href in response.css('.question-summary h3 a::attr(href)'):
|
||||
full_url = response.urljoin(href.extract())
|
||||
yield scrapy.Request(full_url, callback=self.parse_question)
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.xpath('span/small/text()').extract_first(),
|
||||
}
|
||||
|
||||
def parse_question(self, response):
|
||||
yield {
|
||||
'title': response.css('h1 a::text').extract_first(),
|
||||
'votes': response.css('.question .vote-count-post::text').extract_first(),
|
||||
'body': response.css('.question .post-text').extract_first(),
|
||||
'tags': response.css('.question .post-tag::text').extract(),
|
||||
'link': response.url,
|
||||
}
|
||||
next_page = response.css('li.next a::attr("href")').extract_first()
|
||||
if next_page is not None:
|
||||
next_page = response.urljoin(next_page)
|
||||
yield scrapy.Request(next_page, callback=self.parse)
|
||||
|
||||
|
||||
Put this in a file, name it to something like ``stackoverflow_spider.py``
|
||||
Put this in a text file, name it to something like ``quotes_spider.py``
|
||||
and run the spider using the :command:`runspider` command::
|
||||
|
||||
scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json
|
||||
scrapy runspider quotes_spider.py -o quotes.json
|
||||
|
||||
|
||||
When this finishes you will have in the ``top-stackoverflow-questions.json`` file
|
||||
a list of the most upvoted questions in StackOverflow in JSON format, containing the
|
||||
title, link, number of upvotes, a list of the tags and the question content in HTML,
|
||||
looking like this (reformatted for easier reading)::
|
||||
When this finishes you will have in the ``quotes.json`` file a list of the
|
||||
quotes in JSON format, containing text and author, looking like this (reformatted
|
||||
here for better readability)::
|
||||
|
||||
[{
|
||||
"body": "... LONG HTML HERE ...",
|
||||
"link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",
|
||||
"tags": ["java", "c++", "performance", "optimization"],
|
||||
"title": "Why is processing a sorted array faster than an unsorted array?",
|
||||
"votes": "9924"
|
||||
"author": "Jane Austen",
|
||||
"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
|
||||
},
|
||||
{
|
||||
"body": "... LONG HTML HERE ...",
|
||||
"link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",
|
||||
"tags": ["git", "git-submodules"],
|
||||
"title": "How do I remove a Git submodule?",
|
||||
"votes": "1764"
|
||||
"author": "Groucho Marx",
|
||||
"text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
|
||||
},
|
||||
{
|
||||
"author": "Steve Martin",
|
||||
"text": "\u201cA day without sunshine is like, you know, night.\u201d"
|
||||
},
|
||||
...]
|
||||
|
||||
|
||||
|
||||
What just happened?
|
||||
-------------------
|
||||
|
||||
When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a
|
||||
When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked for a
|
||||
Spider definition inside it and ran it through its crawler engine.
|
||||
|
||||
The crawl started by making requests to the URLs defined in the ``start_urls``
|
||||
attribute (in this case, only the URL for StackOverflow top questions page)
|
||||
attribute (in this case, only the URL for quotes in *humor* category)
|
||||
and called the default callback method ``parse``, passing the response object as
|
||||
an argument. In the ``parse`` callback we extract the links to the
|
||||
question pages using a CSS Selector with a custom extension that allows to get
|
||||
the value for an attribute. Then we yield a few more requests to be sent,
|
||||
registering the method ``parse_question`` as the callback to be called for each
|
||||
of them as they finish.
|
||||
an argument. In the ``parse`` callback, we loop through the quote elements
|
||||
using a CSS Selector, yield a Python dict with the extracted quote text and author,
|
||||
look for a link to the next page and schedule another request using the same
|
||||
``parse`` method as callback.
|
||||
|
||||
Here you notice one of the main advantages about Scrapy: requests are
|
||||
:ref:`scheduled and processed asynchronously <topics-architecture>`. This
|
||||
|
|
@ -103,10 +98,6 @@ each request, limiting amount of concurrent requests per domain or per IP, and
|
|||
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
||||
to figure out these automatically.
|
||||
|
||||
Finally, the ``parse_question`` callback scrapes the question data for each
|
||||
page yielding a dict, which Scrapy then collects and writes to a JSON file as
|
||||
requested in the command line.
|
||||
|
||||
.. note::
|
||||
|
||||
This is using :ref:`feed exports <topics-feed-exports>` to generate the
|
||||
|
|
@ -145,12 +136,13 @@ scraping easy and efficient, such as:
|
|||
:ref:`pipelines <topics-item-pipeline>`).
|
||||
|
||||
* Wide range of built-in extensions and middlewares for handling:
|
||||
* cookies and session handling
|
||||
* HTTP features like compression, authentication, caching
|
||||
* user-agent spoofing
|
||||
* robots.txt
|
||||
* crawl depth restriction
|
||||
* and more
|
||||
|
||||
- cookies and session handling
|
||||
- HTTP features like compression, authentication, caching
|
||||
- user-agent spoofing
|
||||
- robots.txt
|
||||
- crawl depth restriction
|
||||
- and more
|
||||
|
||||
* A :ref:`Telnet console <topics-telnetconsole>` for hooking into a Python
|
||||
console running inside your Scrapy process, to introspect and debug your
|
||||
|
|
@ -165,8 +157,8 @@ What's next?
|
|||
============
|
||||
|
||||
The next steps for you are to :ref:`install Scrapy <intro-install>`,
|
||||
:ref:`follow through the tutorial <intro-tutorial>` to learn how to organize
|
||||
your code in Scrapy projects and `join the community`_. Thanks for your
|
||||
:ref:`follow through the tutorial <intro-tutorial>` to learn how to create
|
||||
a full-blown Scrapy project and `join the community`_. Thanks for your
|
||||
interest!
|
||||
|
||||
.. _join the community: http://scrapy.org/community/
|
||||
|
|
|
|||
|
|
@ -16,20 +16,71 @@ components and an outline of the data flow that takes place inside the system
|
|||
below with links for more detailed information about them. The data flow is
|
||||
also described below.
|
||||
|
||||
.. _data-flow:
|
||||
|
||||
Data flow
|
||||
=========
|
||||
|
||||
.. image:: _images/scrapy_architecture_02.png
|
||||
:width: 700
|
||||
:height: 470
|
||||
:alt: Scrapy architecture
|
||||
|
||||
The data flow in Scrapy is controlled by the execution engine, and goes like
|
||||
this:
|
||||
|
||||
1. The :ref:`Engine <component-engine>` gets the first URLs to crawl from the
|
||||
:ref:`Spider <component-spiders>`.
|
||||
|
||||
2. The :ref:`Engine <component-engine>` schedules the URLs in the
|
||||
:ref:`Scheduler <component-scheduler>` as Requests and asks for the
|
||||
next URLs to crawl.
|
||||
|
||||
3. The :ref:`Scheduler <component-scheduler>` returns the next URLs to crawl
|
||||
to the :ref:`Engine <component-engine>`.
|
||||
|
||||
4. The :ref:`Engine <component-engine>` sends the URLs to the
|
||||
:ref:`Downloader <component-downloader>`, passing through the
|
||||
:ref:`Downloader Middleware <component-downloader-middleware>`
|
||||
(request direction).
|
||||
|
||||
5. Once the page finishes downloading the
|
||||
:ref:`Downloader <component-downloader>` generates a Response (with
|
||||
that page) and sends it to the Engine, passing through the
|
||||
:ref:`Downloader Middleware <component-downloader-middleware>`
|
||||
(response direction).
|
||||
|
||||
6. The :ref:`Engine <component-engine>` receives the Response from the
|
||||
:ref:`Downloader <component-downloader>` and sends it to the
|
||||
:ref:`Spider <component-spiders>` for processing, passing
|
||||
through the :ref:`Spider Middleware <component-spider-middleware>`
|
||||
(input direction).
|
||||
|
||||
7. The :ref:`Spider <component-spiders>` processes the Response and returns
|
||||
scraped items and new Requests (to follow) to the
|
||||
:ref:`Engine <component-engine>`, passing through the
|
||||
:ref:`Spider Middleware <component-spider-middleware>` (output direction).
|
||||
|
||||
8. The :ref:`Engine <component-engine>` sends processed items to
|
||||
:ref:`Item Pipelines <component-pipelines>` and processed Requests to
|
||||
the :ref:`Scheduler <component-scheduler>`.
|
||||
|
||||
9. The process repeats (from step 1) until there are no more requests from the
|
||||
:ref:`Scheduler <component-scheduler>`.
|
||||
|
||||
Components
|
||||
==========
|
||||
|
||||
.. _component-engine:
|
||||
|
||||
Scrapy Engine
|
||||
-------------
|
||||
|
||||
The engine is responsible for controlling the data flow between all components
|
||||
of the system, and triggering events when certain actions occur. See the Data
|
||||
Flow section below for more details.
|
||||
of the system, and triggering events when certain actions occur. See the
|
||||
:ref:`Data Flow <data-flow>` section above for more details.
|
||||
|
||||
.. _component-scheduler:
|
||||
|
||||
Scheduler
|
||||
---------
|
||||
|
|
@ -37,12 +88,16 @@ Scheduler
|
|||
The Scheduler receives requests from the engine and enqueues them for feeding
|
||||
them later (also to the engine) when the engine requests them.
|
||||
|
||||
.. _component-downloader:
|
||||
|
||||
Downloader
|
||||
----------
|
||||
|
||||
The Downloader is responsible for fetching web pages and feeding them to the
|
||||
engine which, in turn, feeds them to the spiders.
|
||||
|
||||
.. _component-spiders:
|
||||
|
||||
Spiders
|
||||
-------
|
||||
|
||||
|
|
@ -50,6 +105,8 @@ Spiders are custom classes written by Scrapy users to parse responses and
|
|||
extract items (aka scraped items) from them or additional URLs (requests) to
|
||||
follow. For more information see :ref:`topics-spiders`.
|
||||
|
||||
.. _component-pipelines:
|
||||
|
||||
Item Pipeline
|
||||
-------------
|
||||
|
||||
|
|
@ -58,6 +115,8 @@ extracted (or scraped) by the spiders. Typical tasks include cleansing,
|
|||
validation and persistence (like storing the item in a database). For more
|
||||
information see :ref:`topics-item-pipeline`.
|
||||
|
||||
.. _component-downloader-middleware:
|
||||
|
||||
Downloader middlewares
|
||||
----------------------
|
||||
|
||||
|
|
@ -76,6 +135,8 @@ Use a Downloader middleware if you need to do one of the following:
|
|||
|
||||
For more information see :ref:`topics-downloader-middleware`.
|
||||
|
||||
.. _component-spider-middleware:
|
||||
|
||||
Spider middlewares
|
||||
------------------
|
||||
|
||||
|
|
@ -93,39 +154,6 @@ Use a Spider middleware if you need to
|
|||
|
||||
For more information see :ref:`topics-spider-middleware`.
|
||||
|
||||
Data flow
|
||||
=========
|
||||
|
||||
The data flow in Scrapy is controlled by the execution engine, and goes like
|
||||
this:
|
||||
|
||||
1. The Engine gets the first URLs to crawl from the Spider.
|
||||
|
||||
2. The Engine schedules the URLs in the Scheduler as Requests and asks for the
|
||||
next URLs to crawl.
|
||||
|
||||
3. The Scheduler returns the next URLs to crawl to the Engine.
|
||||
|
||||
4. The Engine sends the URLs to the Downloader, passing through the
|
||||
Downloader Middleware (request direction).
|
||||
|
||||
5. Once the page finishes downloading the Downloader generates a Response (with
|
||||
that page) and sends it to the Engine, passing through the Downloader
|
||||
Middleware (response direction).
|
||||
|
||||
6. The Engine receives the Response from the Downloader and sends it to the
|
||||
Spider for processing, passing through the Spider Middleware (input direction).
|
||||
|
||||
7. The Spider processes the Response and returns scraped items and new Requests
|
||||
(to follow) to the Engine, passing through the Spider Middleware
|
||||
(output direction).
|
||||
|
||||
8. The Engine sends processed items to Item Pipelines and processed Requests to
|
||||
the Scheduler.
|
||||
|
||||
9. The process repeats (from step 1) until there are no more requests from the
|
||||
Scheduler.
|
||||
|
||||
Event-driven networking
|
||||
=======================
|
||||
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ class Command(ScrapyCommand):
|
|||
def get_callback_from_rules(self, spider, response):
|
||||
if getattr(spider, 'rules', None):
|
||||
for rule in spider.rules:
|
||||
if rule.link_extractor.matches(response.url) and rule.callback:
|
||||
return rule.callback
|
||||
if rule.link_extractor.matches(response.url):
|
||||
return rule.callback or "parse"
|
||||
else:
|
||||
logger.error('No CrawlSpider rules found in spider %(spider)r, '
|
||||
'please specify a callback to use for parsing',
|
||||
|
|
@ -166,6 +166,11 @@ class Command(ScrapyCommand):
|
|||
if not cb:
|
||||
if opts.rules and self.first_response == response:
|
||||
cb = self.get_callback_from_rules(spider, response)
|
||||
|
||||
if not cb:
|
||||
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
|
||||
{'url': response.url, 'spider': spider.name})
|
||||
return
|
||||
else:
|
||||
cb = 'parse'
|
||||
|
||||
|
|
|
|||
|
|
@ -89,8 +89,8 @@ class Scheduler(object):
|
|||
msg = ("Unable to serialize request: %(request)s - reason:"
|
||||
" %(reason)s - no more unserializable requests will be"
|
||||
" logged (stats being collected)")
|
||||
logger.error(msg, {'request': request, 'reason': e},
|
||||
exc_info=True, extra={'spider': self.spider})
|
||||
logger.warning(msg, {'request': request, 'reason': e},
|
||||
exc_info=True, extra={'spider': self.spider})
|
||||
self.logunser = False
|
||||
self.stats.inc_value('scheduler/unserializable',
|
||||
spider=self.spider)
|
||||
|
|
|
|||
|
|
@ -170,6 +170,7 @@ class FeedExporter(object):
|
|||
if not self._exporter_supported(self.format):
|
||||
raise NotConfigured
|
||||
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
|
||||
self._exporting = False
|
||||
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
|
||||
uripar = settings['FEED_URI_PARAMS']
|
||||
self._uripar = load_object(uripar) if uripar else lambda x, y: None
|
||||
|
|
@ -188,14 +189,18 @@ class FeedExporter(object):
|
|||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(file, fields_to_export=self.export_fields,
|
||||
encoding=self.export_encoding)
|
||||
exporter.start_exporting()
|
||||
if self.store_empty:
|
||||
exporter.start_exporting()
|
||||
self._exporting = True
|
||||
self.slot = SpiderSlot(file, exporter, storage, uri)
|
||||
|
||||
def close_spider(self, spider):
|
||||
slot = self.slot
|
||||
if not slot.itemcount and not self.store_empty:
|
||||
return
|
||||
slot.exporter.finish_exporting()
|
||||
if self._exporting:
|
||||
slot.exporter.finish_exporting()
|
||||
self._exporting = False
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': self.format,
|
||||
'itemcount': slot.itemcount,
|
||||
|
|
@ -210,6 +215,9 @@ class FeedExporter(object):
|
|||
|
||||
def item_scraped(self, item, spider):
|
||||
slot = self.slot
|
||||
if not self._exporting:
|
||||
slot.exporter.start_exporting()
|
||||
self._exporting = True
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
return item
|
||||
|
|
|
|||
|
|
@ -233,7 +233,8 @@ class FilesPipeline(MediaPipeline):
|
|||
cls_name = "FilesPipeline"
|
||||
self.store = self._get_store(store_uri)
|
||||
resolve = functools.partial(self._key_for_pipe,
|
||||
base_class_name=cls_name)
|
||||
base_class_name=cls_name,
|
||||
settings=settings)
|
||||
self.expires = settings.getint(
|
||||
resolve('FILES_EXPIRES'), self.EXPIRES
|
||||
)
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@ class ImagesPipeline(FilesPipeline):
|
|||
settings = Settings(settings)
|
||||
|
||||
resolve = functools.partial(self._key_for_pipe,
|
||||
base_class_name="ImagesPipeline")
|
||||
base_class_name="ImagesPipeline",
|
||||
settings=settings)
|
||||
self.expires = settings.getint(
|
||||
resolve("IMAGES_EXPIRES"), self.EXPIRES
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ class MediaPipeline(object):
|
|||
self.download_func = download_func
|
||||
|
||||
|
||||
def _key_for_pipe(self, key, base_class_name=None):
|
||||
def _key_for_pipe(self, key, base_class_name=None,
|
||||
settings=None):
|
||||
"""
|
||||
>>> MediaPipeline()._key_for_pipe("IMAGES")
|
||||
'IMAGES'
|
||||
|
|
@ -38,9 +39,11 @@ class MediaPipeline(object):
|
|||
'MYPIPE_IMAGES'
|
||||
"""
|
||||
class_name = self.__class__.__name__
|
||||
if class_name == base_class_name or not base_class_name:
|
||||
formatted_key = "{}_{}".format(class_name.upper(), key)
|
||||
if class_name == base_class_name or not base_class_name \
|
||||
or (settings and not settings.get(formatted_key)):
|
||||
return key
|
||||
return "{}_{}".format(class_name.upper(), key)
|
||||
return formatted_key
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
|
|
|
|||
|
|
@ -115,6 +115,9 @@ class Shell(object):
|
|||
self.populate_vars(response, request, spider)
|
||||
|
||||
def populate_vars(self, response=None, request=None, spider=None):
|
||||
import scrapy
|
||||
|
||||
self.vars['scrapy'] = scrapy
|
||||
self.vars['crawler'] = self.crawler
|
||||
self.vars['item'] = self.item_class()
|
||||
self.vars['settings'] = self.crawler.settings
|
||||
|
|
@ -136,6 +139,7 @@ class Shell(object):
|
|||
def get_help(self):
|
||||
b = []
|
||||
b.append("Available Scrapy objects:")
|
||||
b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)")
|
||||
for k, v in sorted(self.vars.items()):
|
||||
if self._is_relevant(v):
|
||||
b.append(" %-10s %s" % (k, v))
|
||||
|
|
|
|||
|
|
@ -145,6 +145,10 @@ class StreamLogger(object):
|
|||
for line in buf.rstrip().splitlines():
|
||||
self.logger.log(self.log_level, line.rstrip())
|
||||
|
||||
def flush(self):
|
||||
for h in self.logger.handlers:
|
||||
h.flush()
|
||||
|
||||
|
||||
class LogCounterHandler(logging.Handler):
|
||||
"""Record log levels count into a crawler stats"""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
from os.path import join, abspath
|
||||
from twisted.trial import unittest
|
||||
from twisted.internet import defer
|
||||
from scrapy.utils.testsite import SiteTest
|
||||
from scrapy.utils.testproc import ProcessTest
|
||||
from scrapy.utils.python import to_native_str
|
||||
from tests.test_commands import CommandTest
|
||||
|
||||
|
||||
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
|
||||
command = 'parse'
|
||||
|
||||
def setUp(self):
|
||||
super(ParseCommandTest, self).setUp()
|
||||
self.spider_name = 'parse_spider'
|
||||
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
|
||||
with open(fname, 'w') as f:
|
||||
f.write("""
|
||||
import scrapy
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.spiders import CrawlSpider, Rule
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = '{0}'
|
||||
|
||||
def parse(self, response):
|
||||
if getattr(self, 'test_arg', None):
|
||||
self.logger.debug('It Works!')
|
||||
return [scrapy.Item(), dict(foo='bar')]
|
||||
|
||||
|
||||
class MyGoodCrawlSpider(CrawlSpider):
|
||||
name = 'goodcrawl{0}'
|
||||
|
||||
rules = (
|
||||
Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
|
||||
Rule(LinkExtractor(allow=r'/text'), follow=True),
|
||||
)
|
||||
|
||||
def parse_item(self, response):
|
||||
return [scrapy.Item(), dict(foo='bar')]
|
||||
|
||||
def parse(self, response):
|
||||
return [scrapy.Item(), dict(nomatch='default')]
|
||||
|
||||
|
||||
class MyBadCrawlSpider(CrawlSpider):
|
||||
'''Spider which doesn't define a parse_item callback while using it in a rule.'''
|
||||
name = 'badcrawl{0}'
|
||||
|
||||
rules = (
|
||||
Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),
|
||||
)
|
||||
|
||||
def parse(self, response):
|
||||
return [scrapy.Item(), dict(foo='bar')]
|
||||
""".format(self.spider_name))
|
||||
|
||||
fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
|
||||
with open(fname, 'w') as f:
|
||||
f.write("""
|
||||
import logging
|
||||
|
||||
class MyPipeline(object):
|
||||
component_name = 'my_pipeline'
|
||||
|
||||
def process_item(self, item, spider):
|
||||
logging.info('It Works!')
|
||||
return item
|
||||
""")
|
||||
|
||||
fname = abspath(join(self.proj_mod_path, 'settings.py'))
|
||||
with open(fname, 'a') as f:
|
||||
f.write("""
|
||||
ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
||||
""" % self.project_name)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_spider_arguments(self):
|
||||
_, _, stderr = yield self.execute(['--spider', self.spider_name,
|
||||
'-a', 'test_arg=1',
|
||||
'-c', 'parse',
|
||||
self.url('/html')])
|
||||
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_pipelines(self):
|
||||
_, _, stderr = yield self.execute(['--spider', self.spider_name,
|
||||
'--pipelines',
|
||||
'-c', 'parse',
|
||||
self.url('/html')])
|
||||
self.assertIn("INFO: It Works!", to_native_str(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_parse_items(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
|
||||
)
|
||||
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_parse_items_no_callback_passed(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', self.spider_name, self.url('/html')]
|
||||
)
|
||||
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_wrong_callback_passed(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
|
||||
)
|
||||
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
|
||||
self.assertIn("""Cannot find callback""", to_native_str(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_matching_rule_callback_set(self):
|
||||
"""If a rule matches the URL, use it's defined callback."""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
|
||||
)
|
||||
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_matching_rule_default_callback(self):
|
||||
"""If a rule match but it has no callback set, use the 'parse' callback."""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
|
||||
)
|
||||
self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_spider_with_no_rules_attribute(self):
|
||||
"""Using -r with a spider with no rule should not produce items."""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', self.spider_name, '-r', self.url('/html')]
|
||||
)
|
||||
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
|
||||
self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_missing_callback(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
|
||||
)
|
||||
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_no_matching_rule(self):
|
||||
"""The requested URL has no matching rule, so no items should be scraped"""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
|
||||
)
|
||||
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
|
||||
self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr))
|
||||
|
|
@ -56,6 +56,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
errcode, out, _ = yield self.execute(['-c', code.format(url)])
|
||||
self.assertEqual(errcode, 0, out)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_scrapy_import(self):
|
||||
url = self.url('/text')
|
||||
code = "fetch(scrapy.Request('{0}'))"
|
||||
errcode, out, _ = yield self.execute(['-c', code.format(url)])
|
||||
self.assertEqual(errcode, 0, out)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_local_file(self):
|
||||
filepath = join(tests_datadir, 'test_site/index.html')
|
||||
|
|
|
|||
|
|
@ -246,71 +246,6 @@ class BadSpider(scrapy.Spider):
|
|||
self.assertIn("start_requests", log)
|
||||
self.assertIn("badspider.py", log)
|
||||
|
||||
|
||||
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
|
||||
command = 'parse'
|
||||
|
||||
def setUp(self):
|
||||
super(ParseCommandTest, self).setUp()
|
||||
self.spider_name = 'parse_spider'
|
||||
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
|
||||
with open(fname, 'w') as f:
|
||||
f.write("""
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = '{0}'
|
||||
|
||||
def parse(self, response):
|
||||
if getattr(self, 'test_arg', None):
|
||||
self.logger.debug('It Works!')
|
||||
return [scrapy.Item(), dict(foo='bar')]
|
||||
""".format(self.spider_name))
|
||||
|
||||
fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
|
||||
with open(fname, 'w') as f:
|
||||
f.write("""
|
||||
import logging
|
||||
|
||||
class MyPipeline(object):
|
||||
component_name = 'my_pipeline'
|
||||
|
||||
def process_item(self, item, spider):
|
||||
logging.info('It Works!')
|
||||
return item
|
||||
""")
|
||||
|
||||
fname = abspath(join(self.proj_mod_path, 'settings.py'))
|
||||
with open(fname, 'a') as f:
|
||||
f.write("""
|
||||
ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
||||
""" % self.project_name)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_spider_arguments(self):
|
||||
_, _, stderr = yield self.execute(['--spider', self.spider_name,
|
||||
'-a', 'test_arg=1',
|
||||
'-c', 'parse',
|
||||
self.url('/html')])
|
||||
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_pipelines(self):
|
||||
_, _, stderr = yield self.execute(['--spider', self.spider_name,
|
||||
'--pipelines',
|
||||
'-c', 'parse',
|
||||
self.url('/html')])
|
||||
self.assertIn("INFO: It Works!", to_native_str(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_parse_items(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
|
||||
)
|
||||
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
|
||||
|
||||
|
||||
|
||||
class BenchCommandTest(CommandTest):
|
||||
|
||||
def test_run(self):
|
||||
|
|
|
|||
|
|
@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase):
|
|||
data = yield self.run_and_export(TestSpider, settings)
|
||||
defer.returnValue(data)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def exported_no_data(self, settings):
|
||||
"""
|
||||
Return exported data which a spider yielding no ``items`` would return.
|
||||
"""
|
||||
class TestSpider(scrapy.Spider):
|
||||
name = 'testspider'
|
||||
start_urls = ['http://localhost:8998/']
|
||||
|
||||
def parse(self, response):
|
||||
pass
|
||||
|
||||
data = yield self.run_and_export(TestSpider, settings)
|
||||
defer.returnValue(data)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
|
||||
settings = settings or {}
|
||||
|
|
@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase):
|
|||
header = self.MyItem.fields.keys()
|
||||
yield self.assertExported(items, header, rows, ordered=False)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_no_items_not_store_empty(self):
|
||||
formats = ('json',
|
||||
'jsonlines',
|
||||
'xml',
|
||||
'csv',)
|
||||
|
||||
for fmt in formats:
|
||||
settings = {'FEED_FORMAT': fmt}
|
||||
data = yield self.exported_no_data(settings)
|
||||
self.assertEqual(data, b'')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_no_items_store_empty(self):
|
||||
formats = (
|
||||
('json', b'[\n\n]'),
|
||||
('jsonlines', b''),
|
||||
('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
|
||||
('csv', b''),
|
||||
)
|
||||
|
||||
for fmt, expctd in formats:
|
||||
settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}
|
||||
data = yield self.exported_no_data(settings)
|
||||
self.assertEqual(data, expctd)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_multiple_item_classes(self):
|
||||
|
||||
|
|
@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase):
|
|||
def test_export_encoding(self):
|
||||
items = [dict({'foo': u'Test\xd6'})]
|
||||
header = ['foo']
|
||||
|
||||
|
||||
formats = {
|
||||
'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),
|
||||
'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
|
||||
'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
|
||||
'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
|
||||
}
|
||||
|
||||
|
||||
for format in formats:
|
||||
settings = {'FEED_FORMAT': format}
|
||||
data = yield self.exported_data(items, settings)
|
||||
self.assertEqual(formats[format], data)
|
||||
|
||||
|
||||
formats = {
|
||||
'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
|
||||
'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
|
||||
'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
|
||||
'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
|
||||
}
|
||||
|
||||
|
||||
for format in formats:
|
||||
settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}
|
||||
data = yield self.exported_data(items, settings)
|
||||
|
|
|
|||
|
|
@ -255,16 +255,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
|
||||
def test_subclass_attrs_preserved_custom_settings(self):
|
||||
"""
|
||||
If file settings are defined but they are not defined for subclass class attributes
|
||||
should be preserved.
|
||||
If file settings are defined but they are not defined for subclass
|
||||
settings should be preserved.
|
||||
"""
|
||||
pipeline_cls = self._generate_fake_pipeline()
|
||||
settings = self._generate_fake_settings()
|
||||
pipeline = pipeline_cls.from_settings(Settings(settings))
|
||||
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
|
||||
value = getattr(pipeline, pipe_ins_attr)
|
||||
setting_value = settings.get(settings_attr)
|
||||
self.assertNotEqual(value, self.default_cls_settings[pipe_attr])
|
||||
self.assertEqual(value, getattr(pipeline, pipe_attr))
|
||||
self.assertEqual(value, setting_value)
|
||||
|
||||
def test_no_custom_settings_for_subclasses(self):
|
||||
"""
|
||||
|
|
@ -321,6 +322,24 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
self.assertEqual(pipeline.files_urls_field, "that")
|
||||
|
||||
|
||||
def test_user_defined_subclass_default_key_names(self):
|
||||
"""Test situation when user defines subclass of FilesPipeline,
|
||||
but uses attribute names for default pipeline (without prefixing
|
||||
them with pipeline class name).
|
||||
"""
|
||||
settings = self._generate_fake_settings()
|
||||
|
||||
class UserPipe(FilesPipeline):
|
||||
pass
|
||||
|
||||
pipeline_cls = UserPipe.from_settings(Settings(settings))
|
||||
|
||||
for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
|
||||
expected_value = settings.get(settings_attr)
|
||||
self.assertEqual(getattr(pipeline_cls, pipe_inst_attr),
|
||||
expected_value)
|
||||
|
||||
|
||||
class TestS3FilesStore(unittest.TestCase):
|
||||
@defer.inlineCallbacks
|
||||
def test_persist(self):
|
||||
|
|
|
|||
|
|
@ -309,17 +309,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
|
||||
def test_subclass_attrs_preserved_custom_settings(self):
|
||||
"""
|
||||
If image settings are defined but they are not defined for subclass class attributes
|
||||
should be preserved.
|
||||
If image settings are defined but they are not defined for subclass default
|
||||
values taken from settings should be preserved.
|
||||
"""
|
||||
pipeline_cls = self._generate_fake_pipeline_subclass()
|
||||
settings = self._generate_fake_settings()
|
||||
pipeline = pipeline_cls.from_settings(Settings(settings))
|
||||
for pipe_attr, settings_attr in self.img_cls_attribute_names:
|
||||
# Instance attribute (lowercase) must be equal to class attribute (uppercase).
|
||||
# Instance attribute (lowercase) must be equal to
|
||||
# value defined in settings.
|
||||
value = getattr(pipeline, pipe_attr.lower())
|
||||
self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr])
|
||||
self.assertEqual(value, getattr(pipeline, pipe_attr))
|
||||
setings_value = settings.get(settings_attr)
|
||||
self.assertEqual(value, setings_value)
|
||||
|
||||
def test_no_custom_settings_for_subclasses(self):
|
||||
"""
|
||||
|
|
@ -370,11 +372,26 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
class UserDefinedImagePipeline(ImagesPipeline):
|
||||
DEFAULT_IMAGES_URLS_FIELD = "something"
|
||||
DEFAULT_IMAGES_RESULT_FIELD = "something_else"
|
||||
|
||||
pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
|
||||
self.assertEqual(pipeline.images_result_field, "something_else")
|
||||
self.assertEqual(pipeline.images_urls_field, "something")
|
||||
|
||||
def test_user_defined_subclass_default_key_names(self):
|
||||
"""Test situation when user defines subclass of ImagePipeline,
|
||||
but uses attribute names for default pipeline (without prefixing
|
||||
them with pipeline class name).
|
||||
"""
|
||||
settings = self._generate_fake_settings()
|
||||
|
||||
class UserPipe(ImagesPipeline):
|
||||
pass
|
||||
|
||||
pipeline_cls = UserPipe.from_settings(Settings(settings))
|
||||
|
||||
for pipe_attr, settings_attr in self.img_cls_attribute_names:
|
||||
expected_value = settings.get(settings_attr)
|
||||
self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()),
|
||||
expected_value)
|
||||
|
||||
def _create_image(format, *a, **kw):
|
||||
buf = TemporaryFile()
|
||||
|
|
|
|||
14
tox.ini
14
tox.ini
|
|
@ -33,6 +33,20 @@ deps =
|
|||
zope.interface==3.6.1
|
||||
-rtests/requirements.txt
|
||||
|
||||
[testenv:jessie]
|
||||
# https://packages.debian.org/en/jessie/python/
|
||||
# https://packages.debian.org/en/jessie/zope/
|
||||
basepython = python2.7
|
||||
deps =
|
||||
pyOpenSSL==0.14
|
||||
lxml==3.4.0
|
||||
Twisted==14.0.2
|
||||
boto==2.34.0
|
||||
Pillow==2.6.1
|
||||
cssselect==0.9.1
|
||||
zope.interface==4.1.1
|
||||
-rtests/requirements.txt
|
||||
|
||||
[testenv:trunk]
|
||||
basepython = python2.7
|
||||
commands =
|
||||
|
|
|
|||
Loading…
Reference in New Issue