diff --git a/AUTHORS b/AUTHORS index a0fbe722f..1392aa71f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,28 +1,25 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and -improved by Insophia (insophia.com), with the sponsorship of By Design (the -company behind Mydeco). +improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to +bootstrap the project. -Here is the list of the primary authors & contributors, along with their user -name (in Scrapy trac/subversion). Emails are intentionally left out to avoid -spam. +Here is the list of the primary authors & contributors: - * Pablo Hoffman (pablo) - * Daniel Graña (daniel) - * Martin Olveyra (olveyra) - * Gabriel García (elpolilla) - * Michael Cetrulo (samus_) - * Artem Bogomyagkov (artem) - * Damian Canabal (calarval) - * Andres Moreira (andres) - * Ismael Carnales (ismael) - * Matías Aguirre (omab) - * German Hoffman (german) - * Anibal Pacheco (anibal) + * Pablo Hoffman + * Daniel Graña + * Martin Olveyra + * Gabriel García + * Michael Cetrulo + * Artem Bogomyagkov + * Damian Canabal + * Andres Moreira + * Ismael Carnales + * Matías Aguirre + * German Hoffmann + * Anibal Pacheco * Bruno Deferrari * Shane Evans - -And here is the list of people who have helped to put the Scrapy homepage live: - - * Ezequiel Rivero (ezequiel) + * Ezequiel Rivero + * Patrick Mezard + * Rolando Espinoza diff --git a/bin/scrapy.tac b/bin/scrapy.tac new file mode 100644 index 000000000..35e25c56f --- /dev/null +++ b/bin/scrapy.tac @@ -0,0 +1,5 @@ +from twisted.application.service import Application +from scrapy.service import ScrapyService + +application = Application("Scrapy") +ScrapyService().setServiceParent(application) diff --git a/docs/experimental/crawlspider-v2.rst b/docs/experimental/crawlspider-v2.rst new file mode 100644 index 000000000..a1ea09c48 --- /dev/null +++ b/docs/experimental/crawlspider-v2.rst @@ -0,0 +1,128 @@ +.. _topics-crawlspider-v2: + +============== +CrawlSpider v2 +============== + +Introduction +============ + +TODO: introduction + +Rules Matching +============== + +TODO: describe purpose of rules + +Request Extractors & Processors +=============================== + +TODO: describe purpose of extractors & processors + +Examples +======== + +TODO: plenty of examples + + +.. module:: scrapy.contrib_exp.crawlspider.spider + :synopsis: CrawlSpider + + +Reference +========= + +CrawlSpider +----------- + +TODO: describe crawlspider + +.. class:: CrawlSpider + + TODO: describe class + + +.. module:: scrapy.contrib_exp.crawlspider.rules + :synopsis: Rules + +Rules +----- + +TODO: describe spider rules + +.. class:: Rule + + TODO: describe Rules class + + +.. module:: scrapy.contrib_exp.crawlspider.reqext + :synopsis: Request Extractors + +Request Extractors +------------------ + +TODO: describe extractors purpose + +.. class:: BaseSgmlRequestExtractor + + TODO: describe base extractor + +.. class:: SgmlRequestExtractor + + TODO: describe sgml extractor + +.. class:: XPathRequestExtractor + + TODO: describe xpath request extractor + + +.. module:: scrapy.contrib_exp.crawlspider.reqproc + :synopsis: Request Processors + +Request Processors +------------------ + +TODO: describe request processors + +.. class:: Canonicalize + + TODO: describe proc + +.. class:: Unique + + TODO: describe unique + +.. class:: FilterDomain + + TODO: describe filter domain + +.. class:: FilterUrl + + TODO: describe filter url + + +.. module:: scrapy.contrib_exp.crawlspider.matchers + :synopsis: Matchers + +Request/Response Matchers +------------------------- + +TODO: describe matchers + +.. class:: BaseMatcher + + TODO: describe base matcher + +.. class:: UrlMatcher + + TODO: describe url matcher + +.. class:: UrlRegexMatcher + + TODO: describe UrlListMatcher + +.. class:: UrlListMatcher + + TODO: describe url list matcher + + diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 47f4ee4ac..f63ceddd9 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -21,3 +21,4 @@ it's properly merged) . Use at your own risk. djangoitems scheduler-middleware + crawlspider-v2 diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7eec036cf..5dd61100e 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -128,7 +128,8 @@ Finally, here's the spider code:: class MininovaSpider(CrawlSpider): - domain_name = 'mininova.org' + name = 'mininova.org' + allowed_domains = ['mininova.org'] start_urls = ['http://www.mininova.org/today'] rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index e8aad451b..8fb6c7db5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -102,8 +102,8 @@ to parse the contents of those pages to extract :ref:`items `. To create a Spider, you must subclass :class:`scrapy.spider.BaseSpider`, and define the three main, mandatory, attributes: -* :attr:`~scrapy.spider.BaseSpider.domain_name`: identifies the Spider. It must - be unique, that is, you can't set the same domain name for different Spiders. +* :attr:`~scrapy.spider.BaseSpider.name`: identifies the Spider. It must be + unique, that is, you can't set the same name for different Spiders. * :attr:`~scrapy.spider.BaseSpider.start_urls`: is a list of URLs where the Spider will begin to crawl from. So, the first pages downloaded will be those @@ -128,7 +128,8 @@ This is the code for our first Spider, save it in a file named from scrapy.spider import BaseSpider class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -354,7 +355,8 @@ Let's add this code to our spider:: from scrapy.selector import HtmlXPathSelector class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -398,7 +400,8 @@ scraped so far, the code for our Spider should be like this:: from dmoz.items import DmozItem class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -420,8 +423,8 @@ scraped so far, the code for our Spider should be like this:: Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: - [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'Text Processing in Python'], 'link': [u'http://gnosis.cx/TPiP/'], 'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n']}) in - [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'XML Processing with Python'], 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], 'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n']}) in + [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in + [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], link=[u'http://www.informit.com/store/product.aspx?isbn=0130211192'], title=[u'XML Processing with Python']) in Storing the data (using an Item Pipeline) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ec3679649..dfd30d3b2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -199,7 +199,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' - domain_name = 'intranet.example.com' + name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e8f6bef75..cbde79421 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -52,7 +52,7 @@ Exporter to export scraped items to different files, one per spider:: self.files = {} def spider_opened(self, spider): - file = open('%s_products.xml' % spider.domain_name, 'w+b') + file = open('%s_products.xml' % spider.name, 'w+b') self.files[spider] = file self.exporter = XmlItemExporter(file) self.exporter.start_exporting() diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index fc7367bac..972d6ea54 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -105,10 +105,10 @@ every time a domain/spider is opened and closed:: dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def spider_opened(self, spider): - log.msg("opened spider %s" % spider.domain_name) + log.msg("opened spider %s" % spider.name) def spider_closed(self, spider): - log.msg("closed spider %s" % spider.domain_name) + log.msg("closed spider %s" % spider.name) .. _topics-extensions-ref-manager: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 3e3cd94d0..649b2667e 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -79,7 +79,8 @@ This is how the spider would look so far:: from scrapy.contrib.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allowed_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index d4e185aa1..ba69c93b3 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -98,10 +98,10 @@ spider returns multiples items with the same id:: del self.duplicates[spider] def process_item(self, spider, item): - if item.id in self.duplicates[spider]: + if item['id'] in self.duplicates[spider]: raise DropItem("Duplicate item found: %s" % item) else: - self.duplicates[spider].add(item.id) + self.duplicates[spider].add(item['id']) return item Built-in Item Pipelines reference diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index f57128287..5e1a0e4bb 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -129,3 +129,14 @@ scrapy.log module Log level for debugging messages (recommended level for development) +Logging settings +================ + +These settings can be used to configure the logging: + +* :setting:`LOG_ENABLED` +* :setting:`LOG_ENCODING` +* :setting:`LOG_FILE` +* :setting:`LOG_LEVEL` +* :setting:`LOG_STDOUT` + diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index cffcfdee4..084108164 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -321,7 +321,7 @@ user name and password. You can use the :meth:`FormRequest.from_response` method for this job. Here's an example spider which uses it:: class LoginSpider(BaseSpider): - domain_name = 'example.com' + name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] def parse(self, response): @@ -466,12 +466,14 @@ TextResponse objects .. attribute:: TextResponse.encoding - A string with the encoding of this response. The encoding is resolved in the - following order: + A string with the encoding of this response. The encoding is resolved by + trying the following mechanisms, in order: 1. the encoding passed in the constructor `encoding` argument - 2. the encoding declared in the Content-Type HTTP header + 2. the encoding declared in the Content-Type HTTP header. If this + encoding is not valid (ie. unknown), it is ignored and the next + resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class doesn't provide any special functionality for this. However, the @@ -483,23 +485,11 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.headers_encoding() - - Returns a string with the encoding declared in the headers (ie. the - Content-Type HTTP header). - - .. method:: TextResponse.body_encoding() - - Returns a string with the encoding of the body, either declared or inferred - from its contents. The body encoding declaration is implemented in - :class:`TextResponse` subclasses such as: :class:`HtmlResponse` or - :class:`XmlResponse`. - .. method:: TextResponse.body_as_unicode() Returns the body of the response as unicode. This is equivalent to:: - response.body.encode(response.encoding) + response.body.decode(response.encoding) But **not** equivalent to:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b3dbc78f..e64ee329b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -340,16 +340,6 @@ Default: ``True`` Whether to collect depth stats. -.. setting:: DOMAIN_SCHEDULER - -SPIDER_SCHEDULER ----------------- - -Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` - -The Spider Scheduler to use. The spider scheduler returns the next spider to -scrape. - .. setting:: DOWNLOADER_DEBUG DOWNLOADER_DEBUG @@ -418,6 +408,15 @@ supported. Example:: DOWNLOAD_DELAY = 0.25 # 250 ms of delay +This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting (which is enabled by default). By default, Scrapy doesn't wait a fixed +amount of time between requests, but uses a random interval between 0.5 and 1.5 +* :setting:`DOWNLOAD_DELAY`. + +Another way to change the download delay (per spider, instead of globally) is +by using the ``download_delay`` spider attribute, which takes more precedence +than this setting. + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -439,6 +438,69 @@ The class used to detect and filter duplicate requests. The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint (using ``scrapy.utils.request.request_fingerprint``) and grouping per domain. +.. setting:: ENCODING_ALIASES + +ENCODING_ALIASES +---------------- + +Default: ``{}`` + +A mapping of custom encoding aliases for your project, where the keys are the +aliases (and must be lower case) and the values are the encodings they map to. + +This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which +contains some default mappings. + +.. setting:: ENCODING_ALIASES_BASE + +ENCODING_ALIASES_BASE +--------------------- + +Default:: + + { + # gb2312 is superseded by gb18030 + 'gb2312': 'gb18030', + 'chinese': 'gb18030', + 'csiso58gb231280': 'gb18030', + 'euc- cn': 'gb18030', + 'euccn': 'gb18030', + 'eucgb2312-cn': 'gb18030', + 'gb2312-1980': 'gb18030', + 'gb2312-80': 'gb18030', + 'iso- ir-58': 'gb18030', + # gbk is superseded by gb18030 + 'gbk': 'gb18030', + '936': 'gb18030', + 'cp936': 'gb18030', + 'ms936': 'gb18030', + # latin_1 is a subset of cp1252 + 'latin_1': 'cp1252', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'l1': 'cp1252', + # others + 'zh-cn': 'gb18030', + 'win-1251': 'cp1251', + 'macintosh' : 'mac_roman', + 'x-sjis': 'shift_jis', + } + +The default encoding aliases defined in Scrapy. Don't override this setting in +your project, override :setting:`ENCODING_ALIASES` instead. + +The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is +due to a well known browser hack. For more information see: `Character +encodings in HTML`_. + +.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 +.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 +.. _Character encodings in HTML: http://en.wikipedia.org/wiki/Character_encodings_in_HTML + .. setting:: EXTENSIONS EXTENSIONS @@ -517,7 +579,16 @@ LOG_ENABLED Default: ``True`` -Enable logging. +Whether to enable logging. + +.. setting:: LOG_ENCODING + +LOG_ENCODING +------------ + +Default: ``'utf-8'`` + +The encoding to use for logging. .. setting:: LOG_FILE @@ -677,6 +748,27 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' +.. setting:: RANDOMIZE_DOWNLOAD_DELAY + +RANDOMIZE_DOWNLOAD_DELAY +------------------------ + +Default: ``True`` + +If enabled, Scrapy will wait a random amount of time (between 0.5 and 1.5 +* :setting:`DOWNLOAD_DELAY`) while fetching requests from the same +spider. + +This randomization decreases the chance of the crawler being detected (and +subsequently blocked) by sites which analyze requests looking for statistically +significant similarities in the time between their times. + +The randomization policy is the same used by `wget`_ ``--random-wait`` option. + +If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. + +.. _wget: http://www.gnu.org/software/wget/manual/wget.html + .. setting:: REDIRECT_MAX_TIMES REDIRECT_MAX_TIMES @@ -773,7 +865,7 @@ The scheduler to use for crawling. SCHEDULER_ORDER --------------- -Default: ``'BFO'`` +Default: ``'DFO'`` Scope: ``scrapy.core.scheduler`` @@ -858,6 +950,16 @@ Example:: SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev'] +.. setting:: SPIDER_SCHEDULER + +SPIDER_SCHEDULER +---------------- + +Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` + +The Spider Scheduler to use. The spider scheduler returns the next spider to +scrape. + .. setting:: STATS_CLASS STATS_CLASS diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 6e3de9bc1..45d9cf0c3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -163,7 +163,7 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function. Here's an example of how you would call it from your spider:: class MySpider(BaseSpider): - domain_name = 'example.com' + ... def parse(self, response): if response.url == 'http://www.example.com/products.php': diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c0e57ed06..0c5dcc675 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -210,11 +210,8 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. - This middleware filters out every request whose host names don't match - :attr:`~scrapy.spider.BaseSpider.domain_name`, or the spider - :attr:`~scrapy.spider.BaseSpider.domain_name` prefixed by "www.". - Spider can add more domains to exclude using - :attr:`~scrapy.spider.BaseSpider.extra_domain_names` attribute. + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e60549098..b69f9a445 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -70,20 +70,22 @@ BaseSpider requests the given ``start_urls``/``start_requests``, and calls the spider's method ``parse`` for each of the resulting responses. - .. attribute:: domain_name + .. attribute:: name - A string which defines the domain name for this spider, which will also be - the unique identifier for this spider (which means you can't have two - spider with the same ``domain_name``). This is the most important spider - attribute and it's required, and it's the name by which Scrapy will known - the spider. + A string which defines the name for this spider. The spider name is how + the spider is located (and instantiated) by Scrapy, so it must be + unique. However, nothing prevents you from instantiating more than one + instance of the same spider. This is the most important spider attribute + and it's required. - .. attribute:: extra_domain_names + Is recommended to name your spiders after the domain that their crawl. - An optional list of strings containing additional domains that this - spider is allowed to crawl. Requests for URLs not belonging to the - domain name specified in :attr:`domain_name` or this list won't be - followed. + .. attribute:: allowed_domains + + An optional list of strings containing domains that this spider is + allowed to crawl. Requests for URLs not belonging to the domain names + specified in this list won't be followed if + :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls @@ -144,7 +146,7 @@ BaseSpider .. method:: log(message, [level, component]) Log a message using the :func:`scrapy.log.msg` function, automatically - populating the domain argument with the :attr:`domain_name` of this + populating the spider argument with the :attr:`name` of this spider. For more information see :ref:`topics-logging`. @@ -157,7 +159,8 @@ Let's see an example:: from scrapy.spider import BaseSpider class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -177,7 +180,8 @@ Another example returning multiples Requests and Items from a single callback:: from myproject.items import MyItem class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -254,7 +258,8 @@ Let's now take a look at an example CrawlSpider with rules:: from scrapy.item import Item class MySpider(CrawlSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com'] rules = ( @@ -378,7 +383,8 @@ These spiders are pretty easy to use, let's have at one example:: from myproject.items import TestItem class MySpider(XMLFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.xml'] iterator = 'iternodes' # This is actually unnecesary, since it's the default value itertag = 'item' @@ -435,7 +441,8 @@ Let's see an example similar to the previous one, but using a from myproject.items import TestItem class MySpider(CSVFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.csv'] delimiter = ';' headers = ['id', 'name', 'description'] diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 971a33364..34a3c8b7c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -204,15 +204,15 @@ MemoryStatsCollector A simple stats collector that keeps the stats of the last scraping run (for each spider) in memory, after they're closed. The stats can be accessed - through the :attr:`domain_stats` attribute, which is a dict keyed by spider + through the :attr:`spider_stats` attribute, which is a dict keyed by spider domain name. This is the default Stats Collector used in Scrapy. - .. attribute:: domain_stats + .. attribute:: spider_stats - A dict of dicts (keyed by spider domain name) containing the stats of - the last scraping run for each domain. + A dict of dicts (keyed by spider name) containing the stats of the last + scraping run for each spider. DummyStatsCollector ------------------- @@ -240,11 +240,11 @@ SimpledbStatsCollector In addition to the existing stats keys the following keys are added at persitance time: - * ``domain``: the spider domain (so you can use it later for querying stats - for that domain) + * ``spider``: the spider name (so you can use it later for querying stats + for that spider) * ``timestamp``: the timestamp when the stats were persisited - Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB + Both the ``spider`` and ``timestamp`` are used for generating the SimpleDB item name in order to avoid overwriting stats of previous scraping runs. As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and diff --git a/examples/experimental/googledir/googledir/__init__.py b/examples/experimental/googledir/googledir/__init__.py new file mode 100644 index 000000000..3104ef709 --- /dev/null +++ b/examples/experimental/googledir/googledir/__init__.py @@ -0,0 +1 @@ +# googledir project diff --git a/examples/experimental/googledir/googledir/items.py b/examples/experimental/googledir/googledir/items.py new file mode 100644 index 000000000..decc2c9ba --- /dev/null +++ b/examples/experimental/googledir/googledir/items.py @@ -0,0 +1,16 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class GoogledirItem(Item): + + name = Field(default='') + url = Field(default='') + description = Field(default='') + + def __str__(self): + return "Google Category: name=%s url=%s" \ + % (self['name'], self['url']) diff --git a/examples/experimental/googledir/googledir/pipelines.py b/examples/experimental/googledir/googledir/pipelines.py new file mode 100644 index 000000000..f775b254c --- /dev/null +++ b/examples/experimental/googledir/googledir/pipelines.py @@ -0,0 +1,22 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +from scrapy.core.exceptions import DropItem + +class FilterWordsPipeline(object): + """ + A pipeline for filtering out items which contain certain + words in their description + """ + + # put all words in lowercase + words_to_filter = ['politics', 'religion'] + + def process_item(self, spider, item): + for word in self.words_to_filter: + if word in unicode(item['description']).lower(): + raise DropItem("Contains forbidden word: %s" % word) + else: + return item diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py new file mode 100644 index 000000000..4e3c11163 --- /dev/null +++ b/examples/experimental/googledir/googledir/settings.py @@ -0,0 +1,21 @@ +# Scrapy settings for googledir project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'googledir' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['googledir.spiders'] +NEWSPIDER_MODULE = 'googledir.spiders' +DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + +ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline'] diff --git a/examples/experimental/googledir/googledir/spiders/__init__.py b/examples/experimental/googledir/googledir/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py new file mode 100644 index 000000000..2ed7c52a6 --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -0,0 +1,41 @@ +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule + +from googledir.items import GoogledirItem + +class GoogleDirectorySpider(CrawlSpider): + + name = 'google_directory' + allowed_domains = ['directory.google.com'] + start_urls = ['http://directory.google.com/'] + + rules = ( + # search for categories pattern and follow links + Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True), + ) + + def parse_category(self, response): + # The main selector we're using to extract data from the page + main_selector = HtmlXPathSelector(response) + + # The XPath to website links in the directory page + xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font' + + # Get a list of (sub) selectors to each website node pointed by the XPath + sub_selectors = main_selector.select(xpath) + + # Iterate over the sub-selectors to extract data for each website + for selector in sub_selectors: + item = GoogledirItem() + + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('name', 'a/text()') + l.add_xpath('url', 'a/@href') + l.add_xpath('description', 'font[2]/text()') + + # Here we populate the item and yield it + yield l.load_item() + +SPIDER = GoogleDirectorySpider() + diff --git a/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py new file mode 100644 index 000000000..552421ac3 --- /dev/null +++ b/examples/experimental/googledir/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings') + +from scrapy.command.cmdline import execute +execute() diff --git a/examples/experimental/imdb/imdb/__init__.py b/examples/experimental/imdb/imdb/__init__.py new file mode 100644 index 000000000..5bb534f79 --- /dev/null +++ b/examples/experimental/imdb/imdb/__init__.py @@ -0,0 +1 @@ +# package diff --git a/examples/experimental/imdb/imdb/items.py b/examples/experimental/imdb/imdb/items.py new file mode 100644 index 000000000..03bb5c2c3 --- /dev/null +++ b/examples/experimental/imdb/imdb/items.py @@ -0,0 +1,12 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class ImdbItem(Item): + # define the fields for your item here like: + # name = Field() + title = Field() + url = Field() diff --git a/examples/experimental/imdb/imdb/pipelines.py b/examples/experimental/imdb/imdb/pipelines.py new file mode 100644 index 000000000..e60714159 --- /dev/null +++ b/examples/experimental/imdb/imdb/pipelines.py @@ -0,0 +1,8 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +class ImdbPipeline(object): + def process_item(self, spider, item): + return item diff --git a/examples/experimental/imdb/imdb/settings.py b/examples/experimental/imdb/imdb/settings.py new file mode 100644 index 000000000..de026dc14 --- /dev/null +++ b/examples/experimental/imdb/imdb/settings.py @@ -0,0 +1,20 @@ +# Scrapy settings for imdb project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'imdb' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['imdb.spiders'] +NEWSPIDER_MODULE = 'imdb.spiders' +DEFAULT_ITEM_CLASS = 'imdb.items.ImdbItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + diff --git a/examples/experimental/imdb/imdb/spiders/__init__.py b/examples/experimental/imdb/imdb/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/imdb/imdb/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py new file mode 100644 index 000000000..8c2ebcd01 --- /dev/null +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -0,0 +1,141 @@ +from scrapy.http import Request +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize, \ + FilterDupes, FilterUrl +from scrapy.utils.url import urljoin_rfc + +from imdb.items import ImdbItem, Field + +from itertools import chain, imap, izip + +class UsaOpeningWeekMovie(ImdbItem): + pass + +class UsaTopWeekMovie(ImdbItem): + pass + +class Top250Movie(ImdbItem): + rank = Field() + rating = Field() + year = Field() + votes = Field() + +class MovieItem(ImdbItem): + release_date = Field() + tagline = Field() + + +class ImdbSiteSpider(CrawlSpider): + name = 'imdb.com' + allowed_domains = ['imdb.com'] + start_urls = ['http://www.imdb.com/'] + + # extract requests using this classes from urls matching 'follow' flag + request_extractors = [ + SgmlRequestExtractor(tags=['a'], attrs=['href']), + ] + + # process requests using this classes from urls matching 'follow' flag + request_processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(deny=r'/tt\d+/$'), # deny movie url as we will dispatch + # manually the movie requests + ] + + # include domain bit for demo purposes + rules = ( + # these two rules expects requests from start url + Rule(r'imdb.com/nowplaying/$', 'parse_now_playing'), + Rule(r'imdb.com/chart/top$', 'parse_top_250'), + # this rule will parse requests manually dispatched + Rule(r'imdb.com/title/tt\d+/$', 'parse_movie_info'), + ) + + def parse_now_playing(self, response): + """Scrapes USA openings this week and top 10 in week""" + self.log("Parsing USA Top Week") + hxs = HtmlXPathSelector(response) + + _urljoin = lambda url: self._urljoin(response, url) + + # + # openings this week + # + openings = hxs.select('//table[@class="movies"]//a[@class="title"]') + boxoffice = hxs.select('//table[@class="boxoffice movies"]//a[@class="title"]') + + opening_titles = openings.select('text()').extract() + opening_urls = imap(_urljoin, openings.select('@href').extract()) + + box_titles = boxoffice.select('text()').extract() + box_urls = imap(_urljoin, boxoffice.select('@href').extract()) + + # items + opening_items = (UsaOpeningWeekMovie(title=title, url=url) + for (title, url) + in izip(opening_titles, opening_urls)) + + box_items = (UsaTopWeekMovie(title=title, url=url) + for (title, url) + in izip(box_titles, box_urls)) + + # movie requests + requests = imap(self.make_requests_from_url, + chain(opening_urls, box_urls)) + + return chain(opening_items, box_items, requests) + + def parse_top_250(self, response): + """Scrapes movies from top 250 list""" + self.log("Parsing Top 250") + hxs = HtmlXPathSelector(response) + + # scrap each row in the table + rows = hxs.select('//div[@id="main"]/table/tr//a/ancestor::tr') + for row in rows: + fields = row.select('td//text()').extract() + url, = row.select('td//a/@href').extract() + url = self._urljoin(response, url) + + item = Top250Movie() + item['title'] = fields[2] + item['url'] = url + item['rank'] = fields[0] + item['rating'] = fields[1] + item['year'] = fields[3] + item['votes'] = fields[4] + + # scrapped top250 item + yield item + # fetch movie + yield self.make_requests_from_url(url) + + def parse_movie_info(self, response): + """Scrapes movie information""" + self.log("Parsing Movie Info") + hxs = HtmlXPathSelector(response) + selector = hxs.select('//div[@class="maindetails"]') + + item = MovieItem() + # set url + item['url'] = response.url + + # use item loader for other attributes + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('title', './/h1/text()') + l.add_xpath('release_date', './/h5[text()="Release Date:"]' + '/following-sibling::div/text()') + l.add_xpath('tagline', './/h5[text()="Tagline:"]' + '/following-sibling::div/text()') + + yield l.load_item() + + def _urljoin(self, response, url): + """Helper to convert relative urls to absolute""" + return urljoin_rfc(response.url, url, response.encoding) + +SPIDER = ImdbSiteSpider() diff --git a/examples/experimental/imdb/scrapy-ctl.py b/examples/experimental/imdb/scrapy-ctl.py new file mode 100644 index 000000000..df57621b3 --- /dev/null +++ b/examples/experimental/imdb/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'imdb.settings') + +from scrapy.command.cmdline import execute +execute() diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index 054cef022..2af52bf1f 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -6,7 +6,8 @@ from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allow_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/examples/scripts/count_and_follow_links.py b/examples/scripts/count_and_follow_links.py deleted file mode 100644 index 4ead870fc..000000000 --- a/examples/scripts/count_and_follow_links.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Simple script to follow links from a start url. The links are followed in no -particular order. - -Usage: -count_and_follow_links.py - -Example: -count_and_follow_links.py http://scrapy.org/ 20 - -For each page visisted, this script will print the page body size and the -number of links found. -""" - -import sys -from urlparse import urljoin - -from scrapy.crawler import Crawler -from scrapy.selector import HtmlXPathSelector -from scrapy.http import Request, HtmlResponse - -links_followed = 0 - -def parse(response): - global links_followed - links_followed += 1 - if links_followed >= links_to_follow: - crawler.stop() - - # ignore non-HTML responses - if not isinstance(response, HtmlResponse): - return - - links = HtmlXPathSelector(response).select('//a/@href').extract() - abslinks = [urljoin(response.url, l) for l in links] - - print "page %2d/%d: %s" % (links_followed, links_to_follow, response.url) - print " size : %d bytes" % len(response.body) - print " links: %d" % len(links) - print - - return [Request(l, callback=parse) for l in abslinks] - -if len(sys.argv) != 3: - print __doc__ - sys.exit(2) - -start_url, links_to_follow = sys.argv[1], int(sys.argv[2]) -request = Request(start_url, callback=parse) -crawler = Crawler() -crawler.crawl(request) diff --git a/extras/sql/scraping.sql b/extras/sql/scraping.sql deleted file mode 100644 index c3314f5a1..000000000 --- a/extras/sql/scraping.sql +++ /dev/null @@ -1,72 +0,0 @@ -DROP TABLE IF EXISTS `url_history`; -DROP TABLE IF EXISTS `version`; -DROP TABLE IF EXISTS `url_status`; -DROP TABLE IF EXISTS `ticket`; -DROP TABLE IF EXISTS `domain_stats`; -DROP TABLE IF EXISTS `domain_stats_history`; -DROP TABLE IF EXISTS `domain_data_history`; - -CREATE TABLE `ticket` ( - `guid` char(40) NOT NULL, - `domain` varchar(255) default NULL, - `url` varchar(2048) default NULL, - `url_hash` char(40) default NULL, -- so we can join to url_status - PRIMARY KEY (`guid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `version` ( - `id` bigint(20) NOT NULL auto_increment, - `guid` char(40) NOT NULL, - `version` char(40) NOT NULL, - `seen` datetime NOT NULL, - PRIMARY KEY (`id`), - FOREIGN KEY (`guid`) REFERENCES ticket(guid) ON UPDATE CASCADE ON DELETE CASCADE, - UNIQUE KEY (`version`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_status` ( - -- see http://support.microsoft.com/kb/q208427/ for explanation of 2048 - `url_hash` char(40) NOT NULL, -- for faster searches - `url` varchar(2048) NOT NULL, - `parent_hash` char(40) default NULL, -- the url that was followed to this one - for reporting - `last_version` char(40) default NULL, -- can be null if it generated an error the last time is was checked - `last_checked` datetime NOT NULL, - PRIMARY KEY (`url_hash`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_history` ( - `url_hash` char(40) NOT NULL, - `version` char(40) NOT NULL, - `postdata_hash` char(40) default NULL, - `created` datetime NOT NULL, - PRIMARY KEY (`version`), - FOREIGN KEY (`url_hash`) REFERENCES url_status(url_hash) ON UPDATE CASCADE ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats` ( - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` text, - PRIMARY KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats_history` ( - `id` bigint(20) NOT NULL auto_increment, - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` varchar(2048) NOT NULL, - `stored` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`), - KEY `stored` (`stored`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_data_history` ( - `domain` varchar(255) NOT NULL, - `stored` datetime NOT NULL, - `data` text, - KEY `domain_stored` (`domain`, `stored`), - KEY `domain` (`domain`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 2ecc7749c..184b51cb1 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 8, 0, '', 0) -__version__ = "0.8" +version_info = (0, 9, 0, 'dev') +__version__ = "0.9-dev" import sys, os, warnings @@ -17,11 +17,6 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches -# add some common encoding aliases not included by default in Python -from scrapy.utils.encoding import add_encoding_alias -add_encoding_alias('gb2312', 'zh-cn') -add_encoding_alias('cp1251', 'win-1251') - # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index 25f531cea..1a190549d 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -7,20 +7,14 @@ import cProfile import scrapy from scrapy import log -from scrapy.spider import spiders from scrapy.xlib import lsprofcalltree from scrapy.conf import settings from scrapy.command.models import ScrapyCommand +from scrapy.utils.signal import send_catch_log -# This dict holds information about the executed command for later use -command_executed = {} - -def _save_command_executed(cmdname, cmd, args, opts): - """Save command executed info for later reference""" - command_executed['name'] = cmdname - command_executed['class'] = cmd - command_executed['args'] = args[:] - command_executed['opts'] = opts.__dict__.copy() +# Signal that carries information about the command which was executed +# args: cmdname, cmdobj, args, opts +command_executed = object() def _find_commands(dir): try: @@ -127,7 +121,8 @@ def execute(argv=None): sys.exit(2) del args[0] # remove command name from args - _save_command_executed(cmdname, cmd, args, opts) + send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \ + args=args, opts=opts) from scrapy.core.manager import scrapymanager scrapymanager.configure(control_reactor=True) ret = _run_command(cmd, args, opts) @@ -136,23 +131,25 @@ def execute(argv=None): def _run_command(cmd, args, opts): if opts.profile or opts.lsprof: - if opts.profile: - log.msg("writing cProfile stats to %r" % opts.profile) - if opts.lsprof: - log.msg("writing lsprof stats to %r" % opts.lsprof) - loc = locals() - p = cProfile.Profile() - p.runctx('ret = cmd.run(args, opts)', globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) - k = lsprofcalltree.KCacheGrind(p) - if opts.lsprof: - with open(opts.lsprof, 'w') as f: - k.output(f) - ret = loc['ret'] + return _run_command_profiled(cmd, args, opts) else: - ret = cmd.run(args, opts) - return ret + return cmd.run(args, opts) + +def _run_command_profiled(cmd, args, opts): + if opts.profile: + log.msg("writing cProfile stats to %r" % opts.profile) + if opts.lsprof: + log.msg("writing lsprof stats to %r" % opts.lsprof) + loc = locals() + p = cProfile.Profile() + p.runctx('ret = cmd.run(args, opts)', globals(), loc) + if opts.profile: + p.dump_stats(opts.profile) + k = lsprofcalltree.KCacheGrind(p) + if opts.lsprof: + with open(opts.lsprof, 'w') as f: + k.output(f) + return loc['ret'] if __name__ == '__main__': execute() diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index 3a04423fc..9c02b08e3 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -1,20 +1,27 @@ +from scrapy import log from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.conf import settings +from scrapy.http import Request +from scrapy.spider import spiders +from scrapy.utils.url import is_url +from collections import defaultdict class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] ..." + return "[options] ..." def short_desc(self): - return "Start crawling a domain or URL" + return "Start crawling from a spider or URL" def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider when arguments are urls") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \ help="don't follow links (for use with URLs only)") @@ -24,4 +31,45 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - scrapymanager.runonce(*args) + urls, names = self._split_urls_and_names(args) + for name in names: + scrapymanager.crawl_spider_name(name) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + for url in urls: + scrapymanager.crawl_url(url, spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + else: + for name, urls in self._group_urls_by_spider(urls): + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) + + scrapymanager.start() + + def _group_urls_by_spider(self, urls): + spider_urls = defaultdict(list) + for url in urls: + spider_names = spiders.find_by_request(Request(url)) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + return spider_urls.items() + + def _split_urls_and_names(self, args): + urls = [] + names = [] + for arg in args: + if is_url(arg): + urls.append(arg) + else: + names.append(arg) + return urls, names diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index bbc2efa9e..bb2df1400 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -1,7 +1,11 @@ import pprint +from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager +from scrapy.http import Request +from scrapy.spider import BaseSpider, spiders +from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -19,17 +23,33 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", + help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") def run(self, args, opts): - if len(args) != 1: - print "One URL is required" - return + if len(args) != 1 or not is_url(args[0]): + return False + responses = [] # to collect downloaded responses + request = Request(args[0], callback=responses.append, dont_filter=True) - responses = fetch(args) + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg("Could not find spider: %s" % opts.spider, log.ERROR) + else: + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default')) + + scrapymanager.crawl_request(request, spider) + scrapymanager.start() + + # display response if responses: if opts.headers: pprint.pprint(responses[0].headers) else: print responses[0].body + diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index f39d52d14..2289fee1a 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -15,10 +15,11 @@ SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders') def sanitize_module_name(module_name): - """Sanitize the given module name, by replacing dashes with underscores and - prefixing it with a letter if it doesn't start with one + """Sanitize the given module name, by replacing dashes and points + with underscores and prefixing it with a letter if it doesn't start + with one """ - module_name = module_name.replace('-', '_') + module_name = module_name.replace('-', '_').replace('.', '_') if module_name[0] not in string.ascii_letters: module_name = "a" + module_name return module_name @@ -28,7 +29,7 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] " + return "[options] " def short_desc(self): return "Generate new spider based on template passed with -t or --template" @@ -54,28 +55,37 @@ class Command(ScrapyCommand): print template.read() return - if len(args) < 2: + if len(args) != 2: return False - module = sanitize_module_name(args[0]) + name = args[0] domain = args[1] - spider = spiders.fromdomain(domain) - if spider and not opts.force: - print "Spider '%s' already exists in module:" % domain - print " %s" % spider.__module__ - sys.exit(1) + + module = sanitize_module_name(name) + + # if spider already exists and not force option then halt + try: + spider = spiders.create(name) + except KeyError: + pass + else: + if not opts.force: + print "Spider '%s' already exists in module:" % name + print " %s" % spider.__module__ + sys.exit(1) template_file = self._find_template(opts.template) if template_file: - self._genspider(module, domain, opts.template, template_file) + self._genspider(module, name, domain, opts.template, template_file) - def _genspider(self, module, domain, template_name, template_file): + def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" tvars = { 'project_name': settings.get('BOT_NAME'), 'ProjectName': string_camelcase(settings.get('BOT_NAME')), 'module': module, - 'site': domain, + 'name': name, + 'domain': domain, 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } @@ -86,7 +96,7 @@ class Command(ScrapyCommand): shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print "Created spider %r using template %r in module:" % (domain, \ + print "Created spider %r using template %r in module:" % (name, \ template_name) print " %s.%s" % (spiders_module.__name__, module) diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 5d5143f08..9aabe94ff 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -1,11 +1,15 @@ from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager from scrapy.http import Request from scrapy.item import BaseItem from scrapy.spider import spiders from scrapy.utils import display +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.url import is_url from scrapy import log +from collections import defaultdict + class Command(ScrapyCommand): requires_project = True @@ -18,6 +22,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider") parser.add_option("--nolinks", dest="nolinks", action="store_true", \ help="don't show extracted links") parser.add_option("--noitems", dest="noitems", action="store_true", \ @@ -37,18 +43,13 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for url: %s' % response.url, level=log.ERROR) - return (), () - if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) if not callback_fcn: - log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name)) + log.msg('Cannot find callback %s in %s spider' % (callback, spider.name)) return (), () - result = callback_fcn(response) + result = iterate_spider_output(callback_fcn(response)) links = [i for i in result if isinstance(i, Request)] items = [self.pipeline_process(i, spider, opts) for i in result if \ isinstance(i, BaseItem)] @@ -71,36 +72,68 @@ class Command(ScrapyCommand): display.pprint(list(links)) def run(self, args, opts): - if not args: - print "An URL is required" + if not len(args) == 1 or not is_url(args[0]): + return False + + request = Request(args[0]) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + return + else: + spider = scrapymanager._create_spider_for_request(request, \ + log_none=True, log_multiple=True) + + if not spider: return - for response in fetch(args): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for "%s"' % response.url) - continue + responses = [] # to collect downloaded responses + request = request.replace(callback=responses.append) - if self.callbacks: - for callback in self.callbacks: - items, links = self.run_callback(spider, response, callback, args, opts) - self.print_results(items, links, callback, opts) + scrapymanager.crawl_request(request, spider) + scrapymanager.start() - elif opts.rules: - rules = getattr(spider, 'rules', None) - if rules: - items, links = [], [] - for rule in rules: - if rule.callback and rule.link_extractor.matches(response.url): - items, links = self.run_callback(spider, response, rule.callback, args, opts) - self.print_results(items, links, rule.callback, opts) - break - else: - log.msg('No rules found for spider "%s", please specify a callback for parsing' \ - % spider.domain_name) - continue + if not responses: + log.msg('No response returned', log.ERROR, spider=spider) + return + # now process response + # - if callbacks defined then call each one print results + # - if --rules option given search for matching spider's rule + # - default print result using default 'parse' spider's callback + response = responses[0] + + if self.callbacks: + # apply each callback + for callback in self.callbacks: + items, links = self.run_callback(spider, response, + callback, args, opts) + self.print_results(items, links, callback, opts) + elif opts.rules: + # search for matching spider's rule + if hasattr(spider, 'rules') and spider.rules: + items, links = [], [] + for rule in spider.rules: + if rule.link_extractor.matches(response.url) \ + and rule.callback: + + items, links = self.run_callback(spider, + response, rule.callback, + args, opts) + self.print_results(items, links, + rule.callback, opts) + # first-match rule breaks rules loop + break else: - items, links = self.run_callback(spider, response, 'parse', args, opts) - self.print_results(items, links, 'parse', opts) + log.msg('No rules found for spider "%s", ' \ + 'please specify a callback for parsing' \ + % spider.name, log.ERROR) + else: + # default callback 'parse' + items, links = self.run_callback(spider, response, + 'parse', args, opts) + self.print_results(items, links, 'parse', opts) diff --git a/scrapy/command/commands/runspider.py b/scrapy/command/commands/runspider.py index e18f13cdd..2bfcd87f9 100644 --- a/scrapy/command/commands/runspider.py +++ b/scrapy/command/commands/runspider.py @@ -52,6 +52,10 @@ class Command(ScrapyCommand): dispatcher.connect(exporter.export_item, signal=signals.item_passed) exporter.start_exporting() module = _import_file(args[0]) - scrapymanager.runonce(module.SPIDER) + + # schedule spider and start engine + scrapymanager.crawl_spider(module.SPIDER) + scrapymanager.start() + if opts.output: exporter.finish_exporting() diff --git a/scrapy/command/commands/start.py b/scrapy/command/commands/start.py index 032f12bcc..2c7304787 100644 --- a/scrapy/command/commands/start.py +++ b/scrapy/command/commands/start.py @@ -9,4 +9,4 @@ class Command(ScrapyCommand): return "Start the Scrapy manager but don't run any spider (idle mode)" def run(self, args, opts): - scrapymanager.start(*args) + scrapymanager.start(keep_alive=True) diff --git a/scrapy/command/commands/startproject.py b/scrapy/command/commands/startproject.py index eed8ae260..ee44026ce 100644 --- a/scrapy/command/commands/startproject.py +++ b/scrapy/command/commands/startproject.py @@ -7,7 +7,7 @@ from os.path import join, exists import scrapy from scrapy.command import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase -from scrapy.utils.python import ignore_patterns, copytree +from scrapy.utils.py26 import ignore_patterns, copytree TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project') diff --git a/scrapy/command/models.py b/scrapy/command/models.py index b019c3e52..049365ebb 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -57,8 +57,6 @@ class ScrapyCommand(object): help="log level (default: %s)" % settings['LOGLEVEL']) group.add_option("--nolog", action="store_true", dest="nolog", \ help="disable logging completely") - group.add_option("--spider", dest="spider", default=None, \ - help="always use this spider when arguments are urls") group.add_option("--profile", dest="profile", metavar="FILE", default=None, \ help="write python cProfile stats to FILE") group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \ @@ -99,10 +97,6 @@ class ScrapyCommand(object): if opts.nolog: settings.overrides['LOG_ENABLED'] = False - if opts.spider: - from scrapy.spider import spiders - spiders.force_domain = opts.spider - if opts.pidfile: with open(opts.pidfile, "w") as f: f.write(str(os.getpid())) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 8892e41b4..b80e89385 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -71,6 +71,40 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' +ENCODING_ALIASES = {} + +ENCODING_ALIASES_BASE = { + # gb2312 is superseded by gb18030 + 'gb2312': 'gb18030', + 'chinese': 'gb18030', + 'csiso58gb231280': 'gb18030', + 'euc- cn': 'gb18030', + 'euccn': 'gb18030', + 'eucgb2312-cn': 'gb18030', + 'gb2312-1980': 'gb18030', + 'gb2312-80': 'gb18030', + 'iso- ir-58': 'gb18030', + # gbk is superseded by gb18030 + 'gbk': 'gb18030', + '936': 'gb18030', + 'cp936': 'gb18030', + 'ms936': 'gb18030', + # latin_1 is a subset of cp1252 + 'latin_1': 'cp1252', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'l1': 'cp1252', + # others + 'zh-cn': 'gb18030', + 'win-1251': 'cp1251', + 'macintosh' : 'mac_roman', + 'x-sjis': 'shift_jis', +} + EXTENSIONS = {} EXTENSIONS_BASE = { @@ -101,6 +135,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' ITEM_PIPELINES = [] LOG_ENABLED = True +LOG_ENCODING = 'utf-8' LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' @@ -122,6 +157,8 @@ MYSQL_CONNECTION_SETTINGS = {} NEWSPIDER_MODULE = '' +RANDOMIZE_DOWNLOAD_DELAY = True + REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 @@ -150,7 +187,7 @@ SCHEDULER_MIDDLEWARES_BASE = { 'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware': 500, } -SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO +SCHEDULER_ORDER = 'DFO' SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager' diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 1f54b7634..b62c9e6b3 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -5,13 +5,13 @@ because Amazon Web Service use timestamps for authentication. """ import os -import time - -from scrapy.utils.httpobj import urlparse_cached +from time import strftime, gmtime from scrapy.utils.aws import sign_request from scrapy.conf import settings + class AWSMiddleware(object): + def __init__(self): self.access_key = settings['AWS_ACCESS_KEY_ID'] or \ os.environ.get('AWS_ACCESS_KEY_ID') @@ -19,9 +19,6 @@ class AWSMiddleware(object): os.environ.get('AWS_SECRET_ACCESS_KEY') def process_request(self, request, spider): - hostname = urlparse_cached(request).hostname - if spider.domain_name == 's3.amazonaws.com' \ - or (hostname and hostname.endswith('s3.amazonaws.com')): - request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \ - time.gmtime()) + if request.meta.get('sign_s3_request'): + request.headers['Date'] = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime()) sign_request(request, self.access_key, self.secret_key) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 239c05bc7..341e9b5a1 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -108,7 +108,7 @@ class FilesystemCacheStorage(object): def _get_request_path(self, spider, request): key = request_fingerprint(request) - return join(self.cachedir, spider.domain_name, key[0:2], key) + return join(self.cachedir, spider.name, key[0:2], key) def _read_meta(self, spider, request): rpath = self._get_request_path(spider, request) diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 249862824..1c6297b49 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,5 @@ from scrapy import log +from scrapy.http import HtmlResponse from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh from scrapy.core.exceptions import IgnoreRequest @@ -24,10 +25,11 @@ class RedirectMiddleware(object): redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) - interval, url = get_meta_refresh(response) - if url and interval < self.max_metarefresh_delay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + if isinstance(response, HtmlResponse): + interval, url = get_meta_refresh(response) + if url and interval < self.max_metarefresh_delay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index 4bd1b46b3..d26e9607b 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -1,9 +1,5 @@ from scrapy.contrib.exporter import BaseItemExporter - -try: - import json -except ImportError: - import simplejson as json +from scrapy.utils.py26 import json class JsonLinesItemExporter(BaseItemExporter): diff --git a/scrapy/contrib/groupsettings.py b/scrapy/contrib/groupsettings.py deleted file mode 100644 index 4bad4100b..000000000 --- a/scrapy/contrib/groupsettings.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Extensions to override scrapy settings with per-group settings according to the -group the spider belongs to. It only overrides the settings when running the -crawl command with *only one domain as argument*. -""" - -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured -from scrapy.command.cmdline import command_executed - -class GroupSettings(object): - - def __init__(self): - if not settings.getbool("GROUPSETTINGS_ENABLED"): - raise NotConfigured - - if command_executed and command_executed['name'] == 'crawl': - mod = __import__(settings['GROUPSETTINGS_MODULE'], {}, {}, ['']) - args = command_executed['args'] - if len(args) == 1 and not args[0].startswith('http://'): - domain = args[0] - settings.overrides.update(mod.default_settings) - for group, domains in mod.group_spiders.iteritems(): - if domain in domains: - settings.overrides.update(mod.group_settings.get(group, {})) - diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index 6021bbf5a..9f8666467 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -1,6 +1,6 @@ """ This module provides a mechanism for collecting one (or more) sample items per -domain. +spider. The items are collected in a dict of guid->item and persisted by pickling that dict into a file. @@ -8,7 +8,7 @@ dict into a file. This can be useful for testing changes made to the framework or other common code that affects several spiders. -It uses the scrapy stats service to keep track of which domains are already +It uses the scrapy stats service to keep track of which spiders are already sampled. Settings that affect this module: @@ -48,7 +48,7 @@ class ItemSamplerPipeline(object): raise NotConfigured self.items = {} self.spiders_count = 0 - self.empty_domains = set() + self.empty_spiders = set() dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -66,21 +66,21 @@ class ItemSamplerPipeline(object): def engine_stopped(self): with open(self.filename, 'w') as f: pickle.dump(self.items, f) - if self.empty_domains: - log.msg("No products sampled for: %s" % " ".join(self.empty_domains), \ + if self.empty_spiders: + log.msg("No products sampled for: %s" % " ".join(self.empty_spiders), \ level=log.WARNING) def spider_closed(self, spider, reason): if reason == 'finished' and not stats.get_value("items_sampled", spider=spider): - self.empty_domains.add(spider.domain_name) + self.empty_spiders.add(spider.name) self.spiders_count += 1 - log.msg("Sampled %d domains so far (%d empty)" % (self.spiders_count, \ - len(self.empty_domains)), level=log.INFO) + log.msg("Sampled %d spiders so far (%d empty)" % (self.spiders_count, \ + len(self.empty_spiders)), level=log.INFO) class ItemSamplerMiddleware(object): - """This middleware drops items and requests (when domain sampling has been - completed) to accelerate the processing of remaining domains""" + """This middleware drops items and requests (when spider sampling has been + completed) to accelerate the processing of remaining spiders""" def __init__(self): if not settings['ITEMSAMPLER_FILE']: diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index 2714fb562..fb3fd661b 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -26,7 +26,7 @@ class HtmlParserLinkExtractor(HTMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 711623b81..88dd78577 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -3,7 +3,6 @@ This module implements the HtmlImageLinkExtractor for extracting image links only. """ -import urlparse from scrapy.link import Link from scrapy.utils.url import canonicalize_url, urljoin_rfc @@ -25,13 +24,13 @@ class HTMLImageLinkExtractor(object): self.unique = unique self.canonicalize = canonicalize - def extract_from_selector(self, selector, parent=None): + def extract_from_selector(self, selector, encoding, parent=None): ret = [] def _add_link(url_sel, alt_sel=None): url = flatten([url_sel.extract()]) alt = flatten([alt_sel.extract()]) if alt_sel else (u'', ) if url: - ret.append(Link(unicode_to_str(url[0]), alt[0])) + ret.append(Link(unicode_to_str(url[0], encoding), alt[0])) if selector.xmlNode.type == 'element': if selector.xmlNode.name == 'img': @@ -41,7 +40,7 @@ class HTMLImageLinkExtractor(object): children = selector.select('child::*') if len(children): for child in children: - ret.extend(self.extract_from_selector(child, parent=selector)) + ret.extend(self.extract_from_selector(child, encoding, parent=selector)) elif selector.xmlNode.name == 'a' and not parent: _add_link(selector.select('@href'), selector.select('@title')) else: @@ -52,7 +51,7 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) base_url = xs.select('//base/@href').extract() - base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) + base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url links = [] for location in self.locations: @@ -64,7 +63,7 @@ class HTMLImageLinkExtractor(object): continue for selector in selectors: - links.extend(self.extract_from_selector(selector)) + links.extend(self.extract_from_selector(selector, response.encoding)) seen, ret = set(), [] for link in links: diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py index 390e3a304..27cd0697a 100644 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ b/scrapy/contrib/linkextractors/lxmlparser.py @@ -29,7 +29,7 @@ class LxmlLinkExtractor(object): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 08e1e4526..1de044df3 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -16,8 +16,9 @@ def clean_link(link_text): class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" + def _extract_links(self, response_text, response_url, response_encoding): - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip() diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index d548626ab..9ec664bda 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -28,7 +28,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/pipeline/fileexport.py b/scrapy/contrib/pipeline/fileexport.py index d20b83c2a..6e2109d01 100644 --- a/scrapy/contrib/pipeline/fileexport.py +++ b/scrapy/contrib/pipeline/fileexport.py @@ -8,6 +8,7 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.exceptions import NotConfigured from scrapy.contrib import exporter +from scrapy.contrib.exporter import jsonlines from scrapy.conf import settings class FileExportPipeline(object): @@ -48,7 +49,6 @@ class FileExportPipeline(object): elif format == 'pickle': exp = exporter.PickleItemExporter(file, **exp_kwargs) elif format == 'json': - from scrapy.contrib.exporter import jsonlines exp = jsonlines.JsonLinesItemExporter(file, **exp_kwargs) else: raise NotConfigured("Unsupported export format: %s" % format) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 3a93553dd..640cc7179 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -47,7 +47,7 @@ class FSImagesStore(object): dispatcher.connect(self.spider_closed, signals.spider_closed) def spider_closed(self, spider): - self.created_directories.pop(spider.domain_name, None) + self.created_directories.pop(spider.name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) @@ -92,7 +92,7 @@ class _S3AmazonAWSSpider(BaseSpider): It means that a spider that uses download_delay or alike is not going to be delayed even more because it is uploading images to s3. """ - domain_name = "s3.amazonaws.com" + name = "s3.amazonaws.com" start_urls = ['http://s3.amazonaws.com/'] max_concurrent_requests = 100 @@ -143,7 +143,7 @@ class S3ImagesStore(object): def _build_request(self, key, method, body=None, headers=None): url = 'http://%s.s3.amazonaws.com/%s%s' % (self.bucket, self.prefix, key) return Request(url, method=method, body=body, headers=headers, \ - priority=self.request_priority) + meta={'sign_s3_request': True}, priority=self.request_priority) def _download_request(self, request, info): """This method is used for HEAD and PUT requests sent to amazon S3 diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 229301c2c..1c41b625d 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -4,7 +4,6 @@ spiders """ import sys -import urlparse from twisted.plugin import getCache from twisted.python.rebuild import rebuild @@ -19,42 +18,38 @@ class TwistedPluginSpiderManager(object): def __init__(self): self.loaded = False - self.force_domain = None - self._invaliddict = {} self._spiders = {} - def fromdomain(self, domain): - return self._spiders.get(domain) + def create(self, spider_name, **spider_kwargs): + """Returns a Spider instance for the given spider name, using the given + spider arguments. If the sipder name is not found, it raises a + KeyError. + """ + spider = self._spiders[spider_name] + spider.__dict__.update(spider_kwargs) + return spider - def fromurl(self, url): - if self.force_domain: - return self._spiders.get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self._spiders: # try first locating by domain - return self._spiders[domain] - else: # else search spider by spider - plist = self._spiders.values() - for p in plist: - if url_is_from_spider(url, p): - return p + def find_by_request(self, request): + """Returns list of spiders names that match the given Request""" + return [name for name, spider in self._spiders.iteritems() + if url_is_from_spider(request.url, spider)] def list(self): + """Returns list of spiders available.""" return self._spiders.keys() def load(self, spider_modules=None): + """Load spiders from module directory.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._invaliddict = {} self._spiders = {} modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] for module in modules: for spider in self._getspiders(ISpider, module): ISpider.validateInvariants(spider) - self._spiders[spider.domain_name] = spider + self._spiders[spider.name] = spider self.loaded = True def _getspiders(self, interface, package): @@ -77,14 +72,14 @@ class TwistedPluginSpiderManager(object): """Reload spider module to release any resources held on to by the spider """ - domain = spider.domain_name - if domain not in self._spiders: + name = spider.name + if name not in self._spiders: return - spider = self._spiders[domain] + spider = self._spiders[name] module_name = spider.__module__ module = sys.modules[module_name] if hasattr(module, 'SPIDER'): log.msg("Reloading module %s" % module_name, spider=spider, \ level=log.DEBUG) new_module = rebuild(module, doLog=0) - self._spiders[domain] = new_module.SPIDER + self._spiders[name] = new_module.SPIDER diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 5e40d9915..f28b0d53d 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -47,8 +47,7 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - domains = [spider.domain_name] + spider.extra_domain_names - self.host_regexes[spider] = self.get_host_regex(domains) + self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains) self.domains_seen[spider] = set() def spider_closed(self, spider): diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 648765b06..2e74dab4e 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -59,9 +59,9 @@ class CrawlSpider(InitSpider): """ rules = () - def __init__(self): + def __init__(self, *a, **kw): """Constructor takes care of compiling rules""" - super(CrawlSpider, self).__init__() + super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() def parse(self, response): diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py index b37591ca5..f759fd81f 100644 --- a/scrapy/contrib/spiders/init.py +++ b/scrapy/contrib/spiders/init.py @@ -3,8 +3,8 @@ from scrapy.spider import BaseSpider class InitSpider(BaseSpider): """Base Spider with initialization facilities""" - def __init__(self): - super(InitSpider, self).__init__() + def __init__(self, *a, **kw): + super(InitSpider, self).__init__(*a, **kw) self._postinit_reqs = [] self._init_complete = False self._init_started = False diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index c2185e0a7..fc76ccfbf 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -23,6 +23,6 @@ class StatsMailer(object): mail = MailSender() body = "Global stats\n\n" body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.domain_name + body += "\n\n%s stats\n\n" % spider.name body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - mail.send(self.recipients, "Scrapy stats for: %s" % spider.domain_name, body) + mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 22afc20c1..2e61e8bc3 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -60,7 +60,7 @@ class LiveStats(object): runtime = datetime.now() - stats.started s += '%s%d%d%d%d%d%d%s%s\n' % \ - (spider.domain_name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) + (spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) totdomains += 1 totscraped += stats.scraped diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 9719d12f0..ce3fdffd6 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -25,18 +25,18 @@ class Spiderctl(object): dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) def spider_opened(self, spider): - self.running[spider.domain_name] = spider + self.running[spider.name] = spider def spider_closed(self, spider): - del self.running[spider.domain_name] - self.finished.add(spider.domain_name) + del self.running[spider.name] + self.finished.add(spider.name) def webconsole_render(self, wc_request): if wc_request.args: changes = self.webconsole_control(wc_request) - self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders] - self.idle = [d for d in self.enabled_domains if d not in self.scheduled + self.scheduled = [s.name for s in scrapyengine.spider_scheduler._pending_spiders] + self.idle = [d for d in self.enabled_spiders if d not in self.scheduled and d not in self.running and d not in self.finished] @@ -53,9 +53,9 @@ class Spiderctl(object): # idle s += "\n" s += '
\n' - s += '\n' + for name in sorted(self.idle): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -65,9 +65,9 @@ class Spiderctl(object): # scheduled s += "\n" s += '\n' - s += '\n' + for name in self.scheduled: + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -78,9 +78,9 @@ class Spiderctl(object): # running s += "\n" s += '\n' - s += '\n' + for name in sorted(self.running): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -90,9 +90,9 @@ class Spiderctl(object): # finished s += "\n" s += '\n' - s += '\n' + for name in sorted(self.finished): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -114,42 +114,42 @@ class Spiderctl(object): args = wc_request.args s = "
\n" - if "stop_running_domains" in args: + if "stop_running_spiders" in args: s += "

" - stopped_domains = [] - for domain in args["stop_running_domains"]: - if domain in self.running: - scrapyengine.close_spider(self.running[domain]) - stopped_domains.append(domain) - s += "Stopped spiders:

  • %s
" % "
  • ".join(stopped_domains) + stopped_spiders = [] + for name in args["stop_running_spiders"]: + if name in self.running: + scrapyengine.close_spider(self.running[name]) + stopped_spiders.append(name) + s += "Stopped spiders:
    • %s
    " % "
  • ".join(stopped_spiders) s += "

    " - if "remove_pending_domains" in args: + if "remove_pending_spiders" in args: removed = [] - for domain in args["remove_pending_domains"]: - if scrapyengine.spider_scheduler.remove_pending_domain(domain): - removed.append(domain) + for name in args["remove_pending_spiders"]: + if scrapyengine.spider_scheduler.remove_pending_spider(name): + removed.append(name) if removed: s += "

    " - s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_domains"]) + s += "Removed scheduled spiders:
    • %s
    " % "
  • ".join(args["remove_pending_spiders"]) s += "

    " - if "add_pending_domains" in args: - for domain in args["add_pending_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) + if "add_pending_spiders" in args: + for name in args["add_pending_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) s += "

    " - s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_domains"]) + s += "Scheduled spiders:
    • %s
    " % "
  • ".join(args["add_pending_spiders"]) s += "

    " - if "rerun_finished_domains" in args: - for domain in args["rerun_finished_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) - self.finished.remove(domain) + if "rerun_finished_spiders" in args: + for name in args["rerun_finished_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) + self.finished.remove(name) s += "

    " - s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) + s += "Re-scheduled finished spiders:
    • %s
    " % "
  • ".join(args["rerun_finished_spiders"]) s += "

    " return s def webconsole_discover_module(self): - self.enabled_domains = spiders.list() + self.enabled_spiders = spiders.list() return self diff --git a/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py index 837cbcf81..e8c707f64 100644 --- a/scrapy/contrib/webconsole/stats.py +++ b/scrapy/contrib/webconsole/stats.py @@ -23,7 +23,7 @@ class StatsDump(object): s += "

    Global stats

    \n" s += stats_html_table(stats.get_stats()) for spider, spider_stats in stats.iter_spider_stats(): - s += "

    %s

    \n" % spider.domain_name + s += "

    %s

    \n" % spider.name s += stats_html_table(spider_stats) s += "\n" s += "\n" diff --git a/scrapy/contrib_exp/crawlspider/__init__.py b/scrapy/contrib_exp/crawlspider/__init__.py new file mode 100644 index 000000000..03173eb38 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/__init__.py @@ -0,0 +1,4 @@ +"""CrawlSpider v2""" + +from .rules import Rule +from .spider import CrawlSpider diff --git a/scrapy/contrib_exp/crawlspider/matchers.py b/scrapy/contrib_exp/crawlspider/matchers.py new file mode 100644 index 000000000..3ef259c67 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/matchers.py @@ -0,0 +1,61 @@ +""" +Request/Response Matchers + +Perform evaluation to Request or Response attributes +""" + +import re + +class BaseMatcher(object): + """Base matcher. Returns True by default.""" + + def matches_request(self, request): + """Performs Request Matching""" + return True + + def matches_response(self, response): + """Performs Response Matching""" + return True + + +class UrlMatcher(BaseMatcher): + """Matches URL attribute""" + + def __init__(self, url): + """Initialize url attribute""" + self._url = url + + def matches_url(self, url): + """Returns True if given url is equal to matcher's url""" + return self._url == url + + def matches_request(self, request): + """Returns True if Request's url matches initial url""" + return self.matches_url(request.url) + + def matches_response(self, response): + """Returns True if Response's url matches initial url""" + return self.matches_url(response.url) + + +class UrlRegexMatcher(UrlMatcher): + """Matches URL using regular expression""" + + def __init__(self, regex, flags=0): + """Initialize regular expression""" + self._regex = re.compile(regex, flags) + + def matches_url(self, url): + """Returns True if url matches regular expression""" + return self._regex.search(url) is not None + + +class UrlListMatcher(UrlMatcher): + """Matches if URL is in List""" + + def __init__(self, urls): + self._urls = urls + + def matches_url(self, url): + """Returns True if url is in urls list""" + return url in self._urls diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py new file mode 100644 index 000000000..e23e78082 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqext.py @@ -0,0 +1,117 @@ +"""Request Extractors""" +from scrapy.http import Request +from scrapy.selector import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import FixedSGMLParser, str_to_unicode +from scrapy.utils.url import safe_url_string, urljoin_rfc + +from itertools import ifilter + + +class BaseSgmlRequestExtractor(FixedSGMLParser): + """Base SGML Request Extractor""" + + def __init__(self, tag='a', attr='href'): + """Initialize attributes""" + FixedSGMLParser.__init__(self) + + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.current_request = None + + def extract_requests(self, response): + """Returns list of requests extracted from response""" + return self._extract_requests(response.body, response.url, + response.encoding) + + def _extract_requests(self, response_text, response_url, response_encoding): + """Extract requests with absolute urls""" + self.reset() + self.feed(response_text) + self.close() + + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url + self._make_absolute_urls(base_url, response_encoding) + self._fix_link_text_encoding(response_encoding) + + return self.requests + + def _make_absolute_urls(self, base_url, encoding): + """Makes all request's urls absolute""" + for req in self.requests: + url = req.url + # make absolute url + url = urljoin_rfc(base_url, url, encoding) + url = safe_url_string(url, encoding) + # replace in-place request's url + req.url = url + + def _fix_link_text_encoding(self, encoding): + """Convert link_text to unicode for each request""" + for req in self.requests: + req.meta.setdefault('link_text', '') + req.meta['link_text'] = str_to_unicode(req.meta['link_text'], + encoding) + + def reset(self): + """Reset state""" + FixedSGMLParser.reset(self) + self.requests = [] + self.base_url = None + + def unknown_starttag(self, tag, attrs): + """Process unknown start tag""" + if 'base' == tag: + self.base_url = dict(attrs).get('href') + + _matches = lambda (attr, value): self.scan_attr(attr) \ + and value is not None + if self.scan_tag(tag): + for attr, value in ifilter(_matches, attrs): + req = Request(url=value) + self.requests.append(req) + self.current_request = req + + def unknown_endtag(self, tag): + """Process unknown end tag""" + self.current_request = None + + def handle_data(self, data): + """Process data""" + current = self.current_request + if current and not 'link_text' in current.meta: + current.meta['link_text'] = data.strip() + + +class SgmlRequestExtractor(BaseSgmlRequestExtractor): + """SGML Request Extractor""" + + def __init__(self, tags=None, attrs=None): + """Initialize with custom tag & attribute function checkers""" + # defaults + tags = tuple(tags) if tags else ('a', 'area') + attrs = tuple(attrs) if attrs else ('href', ) + + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) + +# TODO: move to own file +class XPathRequestExtractor(SgmlRequestExtractor): + """SGML Request Extractor with XPath restriction""" + + def __init__(self, restrict_xpaths, tags=None, attrs=None): + """Initialize XPath restrictions""" + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + SgmlRequestExtractor.__init__(self, tags, attrs) + + def extract_requests(self, response): + """Restrict to XPath regions""" + hxs = HtmlXPathSelector(response) + fragments = (''.join( + html_frag for html_frag in hxs.select(xpath).extract() + ) for xpath in self.restrict_xpaths) + html_slice = ''.join(html_frag for html_frag in fragments) + return self._extract_requests(html_slice, response.url, + response.encoding) + diff --git a/scrapy/contrib_exp/crawlspider/reqgen.py b/scrapy/contrib_exp/crawlspider/reqgen.py new file mode 100644 index 000000000..3858fbcf7 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqgen.py @@ -0,0 +1,27 @@ +"""Request Generator""" +from itertools import imap + +class RequestGenerator(object): + """Extracto and process requests from response""" + + def __init__(self, req_extractors, req_processors, callback, spider=None): + """Initialize attributes""" + self._request_extractors = req_extractors + self._request_processors = req_processors + #TODO: resolve callback? + self._callback = callback + + def generate_requests(self, response): + """Extract and process new requests from response. + Attach callback to each request as default callback.""" + requests = [] + for ext in self._request_extractors: + requests.extend(ext.extract_requests(response)) + + for proc in self._request_processors: + requests = proc(requests) + + # return iterator + # @@@ creates new Request object with callback + return imap(lambda r: r.replace(callback=self._callback), requests) + diff --git a/scrapy/contrib_exp/crawlspider/reqproc.py b/scrapy/contrib_exp/crawlspider/reqproc.py new file mode 100644 index 000000000..d39399b20 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqproc.py @@ -0,0 +1,111 @@ +"""Request Processors""" +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.url import canonicalize_url, url_is_from_any_domain + +from itertools import ifilter, imap + +import re + +class Canonicalize(object): + """Canonicalize Request Processor""" + def _replace_url(self, req): + # replace in-place + req.url = canonicalize_url(req.url) + return req + + def __call__(self, requests): + """Canonicalize all requests' urls""" + return imap(self._replace_url, requests) + + +class FilterDupes(object): + """Filter duplicate Requests""" + + def __init__(self, *attributes): + """Initialize comparison attributes""" + self._attributes = tuple(attributes) if attributes \ + else tuple(['url']) + + def _equal_attr(self, obj1, obj2, attr): + return getattr(obj1, attr) == getattr(obj2, attr) + + def _requests_equal(self, req1, req2): + """Attribute comparison helper""" + # look for not equal attribute + _not_equal = lambda attr: not self._equal_attr(req1, req2, attr) + for attr in ifilter(_not_equal, self._attributes): + return False + # all attributes equal + return True + + def _request_in(self, request, requests_seen): + """Check if request is in given requests seen list""" + _req_seen = lambda r: self._requests_equal(r, request) + for seen in ifilter(_req_seen, requests_seen): + return True + # request not seen + return False + + def __call__(self, requests): + """Filter seen requests""" + # per-call duplicates filter + self.requests_seen = set() + _not_seen = lambda r: not self._request_in(r, self.requests_seen) + for req in ifilter(_not_seen, requests): + yield req + # registry seen request + self.requests_seen.add(req) + + +class FilterDomain(object): + """Filter request's domain""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + self.allow = tuple(arg_to_iter(allow)) + self.deny = tuple(arg_to_iter(deny)) + + def __call__(self, requests): + """Filter domains""" + processed = (req for req in requests) + + if self.allow: + processed = (req for req in requests + if url_is_from_any_domain(req.url, self.allow)) + if self.deny: + processed = (req for req in requests + if not url_is_from_any_domain(req.url, self.deny)) + + return processed + + +class FilterUrl(object): + """Filter request's url""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + _re_type = type(re.compile('', 0)) + + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + def __call__(self, requests): + """Filter request's url based on allow/deny rules""" + #TODO: filter valid urls here? + processed = (req for req in requests) + + if self.allow_res: + processed = (req for req in requests + if self._matches(req.url, self.allow_res)) + if self.deny_res: + processed = (req for req in requests + if not self._matches(req.url, self.deny_res)) + + return processed + + def _matches(self, url, regexs): + """Returns True if url matches any regex in given list""" + return any(r.search(url) for r in regexs) + diff --git a/scrapy/contrib_exp/crawlspider/rules.py b/scrapy/contrib_exp/crawlspider/rules.py new file mode 100644 index 000000000..ff1691ad0 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/rules.py @@ -0,0 +1,100 @@ +"""Crawler Rules""" +from scrapy.http import Request +from scrapy.http import Response + +from functools import partial +from itertools import ifilter + +from .matchers import BaseMatcher +# default strint-to-matcher class +from .matchers import UrlRegexMatcher + +class CompiledRule(object): + """Compiled version of Rule""" + def __init__(self, matcher, callback=None, follow=False): + """Initialize attributes checking type""" + assert isinstance(matcher, BaseMatcher) + assert callback is None or callable(callback) + assert isinstance(follow, bool) + + self.matcher = matcher + self.callback = callback + self.follow = follow + + +class Rule(object): + """Crawler Rule""" + def __init__(self, matcher=None, callback=None, follow=False, **kwargs): + """Store attributes""" + self.matcher = matcher + self.callback = callback + self.cb_kwargs = kwargs if kwargs else {} + self.follow = True if follow else False + + if self.callback is None and self.follow is False: + raise ValueError("Rule must either have a callback or " + "follow=True: %r" % self) + + def __repr__(self): + return "Rule(matcher=%r, callback=%r, follow=%r, **%r)" \ + % (self.matcher, self.callback, self.follow, self.cb_kwargs) + + +class RulesManager(object): + """Rules Manager""" + def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): + """Initialize rules using spider and default matcher""" + self._rules = tuple() + + # compile absolute/relative-to-spider callbacks""" + for rule in rules: + # prepare matcher + if rule.matcher is None: + # instance BaseMatcher by default + matcher = BaseMatcher() + elif isinstance(rule.matcher, BaseMatcher): + matcher = rule.matcher + else: + # matcher not BaseMatcher, check for string + if isinstance(rule.matcher, basestring): + # instance default matcher + matcher = default_matcher(rule.matcher) + else: + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) + + # prepare callback + if callable(rule.callback): + callback = rule.callback + elif not rule.callback is None: + # callback from spider + callback = getattr(spider, rule.callback) + + if not callable(callback): + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) + else: + callback = None + + if rule.cb_kwargs: + # build partial callback + callback = partial(callback, **rule.cb_kwargs) + + # append compiled rule to rules list + crule = CompiledRule(matcher, callback, follow=rule.follow) + self._rules += (crule, ) + + def get_rule_from_request(self, request): + """Returns first rule that matches given Request""" + _matches = lambda r: r.matcher.matches_request(request) + for rule in ifilter(_matches, self._rules): + # return first match of iterator + return rule + + def get_rule_from_response(self, response): + """Returns first rule that matches given Response""" + _matches = lambda r: r.matcher.matches_response(response) + for rule in ifilter(_matches, self._rules): + # return first match of iterator + return rule + diff --git a/scrapy/contrib_exp/crawlspider/spider.py b/scrapy/contrib_exp/crawlspider/spider.py new file mode 100644 index 000000000..e2bf1a34b --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/spider.py @@ -0,0 +1,69 @@ +"""CrawlSpider v2""" +from scrapy.spider import BaseSpider +from scrapy.utils.spider import iterate_spider_output + +from .matchers import UrlListMatcher +from .rules import Rule, RulesManager +from .reqext import SgmlRequestExtractor +from .reqgen import RequestGenerator +from .reqproc import Canonicalize, FilterDupes + +class CrawlSpider(BaseSpider): + """CrawlSpider v2""" + + request_extractors = None + request_processors = None + rules = [] + + def __init__(self, *a, **kw): + """Initialize dispatcher""" + super(CrawlSpider, self).__init__(*a, **kw) + + # auto follow start urls + if self.start_urls: + _matcher = UrlListMatcher(self.start_urls) + # append new rule using type from current self.rules + rules = self.rules + type(self.rules)([ + Rule(_matcher, follow=True) + ]) + else: + rules = self.rules + + # set defaults if not set + if self.request_extractors is None: + # default link extractor. Extracts all links from response + self.request_extractors = [ SgmlRequestExtractor() ] + + if self.request_processors is None: + # default proccessor. Filter duplicates requests + self.request_processors = [ FilterDupes() ] + + + # wrap rules + self._rulesman = RulesManager(rules, spider=self) + # generates new requests with given callback + self._reqgen = RequestGenerator(self.request_extractors, + self.request_processors, + callback=self.parse) + + def parse(self, response): + """Dispatch callback and generate requests""" + # get rule for response + rule = self._rulesman.get_rule_from_response(response) + + if rule: + # dispatch callback if set + if rule.callback: + output = iterate_spider_output(rule.callback(response)) + for req_or_item in output: + yield req_or_item + + if rule.follow: + for req in self._reqgen.generate_requests(response): + # only dispatch request if has matching rule + if self._rulesman.get_rule_from_request(req): + yield req + else: + self.log("No rule for response %s" % response, level=log.WARNING) + + diff --git a/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py deleted file mode 100644 index 869c62c58..000000000 --- a/scrapy/contrib_exp/pipeline/shoveitem.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -A pipeline to persist objects using shove. - -Shove is a "new generation" shelve. For more information see: -http://pypi.python.org/pypi/shove -""" - -from string import Template - -from shove import Shove -from scrapy.xlib.pydispatch import dispatcher - -from scrapy import log -from scrapy.core import signals -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured - -class ShoveItemPipeline(object): - - def __init__(self): - self.uritpl = settings['SHOVEITEM_STORE_URI'] - if not self.uritpl: - raise NotConfigured - self.opts = settings['SHOVEITEM_STORE_OPT'] or {} - self.stores = {} - - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_item(self, spider, item): - guid = str(item.guid) - - if guid in self.stores[spider]: - if self.stores[spider][guid] == item: - status = 'old' - else: - status = 'upd' - else: - status = 'new' - - if not status == 'old': - self.stores[spider][guid] = item - self.log(spider, item, status) - return item - - def spider_opened(self, spider): - uri = Template(self.uritpl).substitute(domain=spider.domain_name) - self.stores[spider] = Shove(uri, **self.opts) - - def spider_closed(self, spider): - self.stores[spider].sync() - - def log(self, spider, item, status): - log.msg("Shove (%s): Item guid=%s" % (status, item.guid), level=log.DEBUG, \ - spider=spider) diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index b53db5811..aec17b05e 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -2,6 +2,7 @@ Download web pages using asynchronous IO """ +import random from time import time from twisted.internet import reactor, defer @@ -20,15 +21,21 @@ class SpiderInfo(object): def __init__(self, download_delay=None, max_concurrent_requests=None): if download_delay is None: - self.download_delay = settings.getfloat('DOWNLOAD_DELAY') + self._download_delay = settings.getfloat('DOWNLOAD_DELAY') else: - self.download_delay = download_delay - if self.download_delay: + self._download_delay = float(download_delay) + if self._download_delay: self.max_concurrent_requests = 1 elif max_concurrent_requests is None: self.max_concurrent_requests = settings.getint('CONCURRENT_REQUESTS_PER_SPIDER') else: self.max_concurrent_requests = max_concurrent_requests + if self._download_delay and settings.getbool('RANDOMIZE_DOWNLOAD_DELAY'): + # same policy as wget --random-wait + self.random_delay_interval = (0.5*self._download_delay, \ + 1.5*self._download_delay) + else: + self.random_delay_interval = None self.active = set() self.queue = [] @@ -44,6 +51,12 @@ class SpiderInfo(object): # use self.active to include requests in the downloader middleware return len(self.active) > 2 * self.max_concurrent_requests + def download_delay(self): + if self.random_delay_interval: + return random.uniform(*self.random_delay_interval) + else: + return self._download_delay + def cancel_request_calls(self): for call in self.next_request_calls: call.cancel() @@ -99,8 +112,9 @@ class Downloader(object): # Delay queue processing if a download_delay is configured now = time() - if site.download_delay: - penalty = site.download_delay - now + site.lastseen + delay = site.download_delay() + if delay: + penalty = delay - now + site.lastseen if penalty > 0: d = defer.Deferred() d.addCallback(self._process_queue) diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index 2078aaa95..b1dc3754f 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -1,5 +1,4 @@ import signal -from collections import defaultdict from twisted.internet import reactor @@ -7,54 +6,13 @@ from scrapy.extension import extensions from scrapy import log from scrapy.http import Request from scrapy.core.engine import scrapyengine -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import spiders from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import is_url from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -def _get_spider_requests(*args): - """Collect requests and spiders from the given arguments. Returns a dict of - spider -> list of requests - """ - spider_requests = defaultdict(list) - for arg in args: - if isinstance(arg, tuple): - request, spider = arg - spider_requests[spider] = request - elif isinstance(arg, Request): - spider = spiders.fromurl(arg.url) or BaseSpider('default') - if spider: - spider_requests[spider] += [arg] - else: - log.msg('Could not find spider for request: %s' % arg, log.ERROR) - elif isinstance(arg, BaseSpider): - spider_requests[arg] += arg.start_requests() - elif is_url(arg): - spider = spiders.fromurl(arg) or BaseSpider('default') - if spider: - for req in arg_to_iter(spider.make_requests_from_url(arg)): - spider_requests[spider] += [req] - else: - log.msg('Could not find spider for url: %s' % arg, log.ERROR) - elif isinstance(arg, basestring): - spider = spiders.fromdomain(arg) - if spider: - spider_requests[spider] += spider.start_requests() - else: - log.msg('Could not find spider for domain: %s' % arg, log.ERROR) - else: - raise TypeError("Unsupported argument: %r" % arg) - return spider_requests - class ExecutionManager(object): - """Process a list of sites or urls. - This class should be used in a main for process a list of sites/urls. - - It extracts products and could be used to store results in a database or - just for testing spiders. - """ def __init__(self): self.interrupted = False self.configured = False @@ -78,24 +36,46 @@ class ExecutionManager(object): scrapyengine.configure() self.configured = True - def crawl(self, *args): - """Schedule the given args for crawling. args is a list of urls or domains""" + def crawl_url(self, url, spider=None): + """Schedule given url for crawling.""" + if spider is None: + spider = self._create_spider_for_request(Request(url), log_none=True, \ + log_multiple=True) + if spider: + requests = arg_to_iter(spider.make_requests_from_url(url)) + self._crawl_requests(requests, spider) + + def crawl_request(self, request, spider=None): + """Schedule request for crawling.""" assert self.configured, "Scrapy Manager not yet configured" - spider_requests = _get_spider_requests(*args) - for spider, requests in spider_requests.iteritems(): - for request in requests: - scrapyengine.crawl(request, spider) + if spider is None: + spider = self._create_spider_for_request(request, log_none=True, \ + log_multiple=True) + if spider: + scrapyengine.crawl(request, spider) - def runonce(self, *args): - """Run the engine until it finishes scraping all domains and then exit""" - self.crawl(*args) - scrapyengine.start() - if self.control_reactor: - reactor.run(installSignalHandlers=False) + def crawl_spider_name(self, name): + """Schedule given spider by name for crawling.""" + try: + spider = spiders.create(name) + except KeyError: + log.msg('Could not find spider: %s' % name, log.ERROR) + else: + self.crawl_spider(spider) - def start(self): + def crawl_spider(self, spider): + """Schedule spider for crawling.""" + requests = spider.start_requests() + self._crawl_requests(requests, spider) + + def _crawl_requests(self, requests, spider): + """Shortcut to schedule a list of requests""" + for req in requests: + self.crawl_request(req, spider) + + def start(self, keep_alive=False): """Start the scrapy server, without scheduling any domains""" - scrapyengine.keep_alive = True + scrapyengine.keep_alive = keep_alive scrapyengine.start() if self.control_reactor: reactor.run(installSignalHandlers=False) @@ -105,6 +85,17 @@ class ExecutionManager(object): self.interrupted = True scrapyengine.stop() + def _create_spider_for_request(self, request, default=None, log_none=False, \ + log_multiple=False): + spider_names = spiders.find_by_request(request) + if len(spider_names) == 1: + return spiders.create(spider_names[0]) + if len(spider_names) > 1 and log_multiple: + log.msg('More than one spider found for: %s' % request, log.ERROR) + if len(spider_names) == 0 and log_none: + log.msg('Could not find spider for: %s' % request, log.ERROR) + return default + def _signal_shutdown(self, signum, _): signame = signal_names[signum] log.msg("Received %s, shutting down gracefully. Send again to force " \ diff --git a/scrapy/crawler.py b/scrapy/crawler.py deleted file mode 100644 index e793d9125..000000000 --- a/scrapy/crawler.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Crawler class - -The Crawler class can be used to crawl pages using the Scrapy crawler from -outside a Scrapy project, for example, from a standalone script. - -To use it, instantiate it and call the "crawl" method with one (or more) -requests. For example: - - >>> from scrapy.crawler import Crawler - >>> from scrapy.http import Request - >>> def parse_response(response): - ... print "Visited: %s" % response.url - ... - >>> request = Request('http://scrapy.org', callback=parse_response) - >>> crawler = Crawler() - >>> crawler.crawl(request) - Visited: http://scrapy.org - >>> - -Request callbacks follow the same API of spiders callback, which means that all -requests returned from the callbacks will be followed. - -See examples/scripts/count_and_follow_links.py for a more detailed example. - -WARNING: The Crawler class currently has a big limitation - it cannot be used -more than once in the same Python process. This is due to the fact that Twisted -reactors cannot be restarted. Hopefully, this limitation will be removed in the -future. -""" - -from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.conf import settings as scrapy_settings -from scrapy import log - -class Crawler(object): - - def __init__(self, enable_log=False, stop_on_error=False, silence_errors=False, \ - settings=None): - self.stop_on_error = stop_on_error - self.silence_errors = silence_errors - # disable offsite middleware (by default) because it prevents free crawling - if settings is not None: - settings.overrides.update(settings) - scrapy_settings.overrides['SPIDER_MIDDLEWARES'] = { - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None} - scrapy_settings.overrides['LOG_ENABLED'] = enable_log - scrapymanager.configure() - dispatcher.connect(self._logmessage_received, signal=log.logmessage_received) - - def crawl(self, *args): - scrapymanager.runonce(*args) - - def stop(self): - scrapyengine.stop() - log.log_level = log.SILENT - scrapyengine.kill() - - def _logmessage_received(self, message, level): - if level <= log.ERROR: - if not self.silence_errors: - print "Crawler error: %s" % message - if self.stop_on_error: - self.stop() diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 187238b1c..4dce92880 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -96,20 +96,12 @@ class Request(object_ref): """Return a copy of this Request""" return self.replace() - def replace(self, url=None, callback=None, method=None, headers=None, body=None, \ - cookies=None, meta=None, encoding=None, priority=None, \ - dont_filter=None, errback=None): + def replace(self, *args, **kwargs): """Create a new Request with the same attributes except for those given new values. """ - return self.__class__(url=self.url if url is None else url, - callback=callback, - method=self.method if method is None else method, - headers=copy.deepcopy(self.headers) if headers is None else headers, - body=self.body if body is None else body, - cookies=self.cookies if cookies is None else cookies, - meta=self.meta if meta is None else meta, - encoding=self.encoding if encoding is None else encoding, - priority=self.priority if priority is None else priority, - dont_filter=self.dont_filter if dont_filter is None else dont_filter, - errback=errback) + for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', \ + 'encoding', 'priority', 'dont_filter']: + kwargs.setdefault(x, getattr(self, x)) + cls = kwargs.pop('cls', self.__class__) + return cls(*args, **kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f632ea117..251e77ad7 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -71,18 +71,11 @@ class Response(object_ref): """Return a copy of this Response""" return self.replace() - def replace(self, url=None, status=None, headers=None, body=None, meta=None, \ - flags=None, cls=None, **kwargs): + def replace(self, *args, **kwargs): """Create a new Response with the same attributes except for those given new values. """ - if cls is None: - cls = self.__class__ - new = cls(url=self.url if url is None else url, - status=self.status if status is None else status, - headers=copy.deepcopy(self.headers) if headers is None else headers, - body=self.body if body is None else body, - meta=self.meta if meta is None else meta, - flags=self.flags if flags is None else flags, - **kwargs) - return new + for x in ['url', 'status', 'headers', 'body', 'meta', 'flags']: + kwargs.setdefault(x, getattr(self, x)) + cls = kwargs.pop('cls', self.__class__) + return cls(*args, **kwargs) diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index f1557e6f7..dc812ac0e 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -23,9 +23,6 @@ class HtmlResponse(TextResponse): METATAG_RE = re.compile(r'[\w-]+)') XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) - def body_encoding(self): - return self._body_declared_encoding() or super(XmlResponse, self).body_encoding() - @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] diff --git a/scrapy/log.py b/scrapy/log.py index 71897ffee..c8e969928 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -29,8 +29,9 @@ BOT_NAME = settings['BOT_NAME'] # args: message, level, spider logmessage_received = object() -# default logging level +# default values log_level = DEBUG +log_encoding = 'utf-8' started = False @@ -47,11 +48,12 @@ def _get_log_level(level_name_or_id=None): def start(logfile=None, loglevel=None, logstdout=None): """Initialize and start logging facility""" - global log_level, started + global log_level, log_encoding, started if started or not settings.getbool('LOG_ENABLED'): return log_level = _get_log_level(loglevel) + log_encoding = settings['LOG_ENCODING'] started = True # set log observer @@ -73,8 +75,8 @@ def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): "use 'spider' argument instead", DeprecationWarning, stacklevel=2) dispatcher.send(signal=logmessage_received, message=message, level=level, \ spider=spider) - system = domain or (spider.domain_name if spider else component) - msg_txt = unicode_to_str("%s: %s" % (level_names[level], message)) + system = domain or (spider.name if spider else component) + msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding) log.msg(msg_txt, system=system) def exc(message, level=ERROR, component=BOT_NAME, domain=None, spider=None): @@ -91,7 +93,7 @@ def err(_stuff=None, _why=None, **kwargs): import warnings warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \ "use 'spider' argument instead", DeprecationWarning, stacklevel=2) - kwargs['system'] = domain or (spider.domain_name if spider else component) + kwargs['system'] = domain or (spider.name if spider else component) if _why: - _why = unicode_to_str("ERROR: %s" % _why) + _why = unicode_to_str("ERROR: %s" % _why, log_encoding) log.err(_stuff, _why, **kwargs) diff --git a/scrapy/mail.py b/scrapy/mail.py index 8275e75cb..87825ed9c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -47,34 +47,26 @@ class MailSender(object): part = MIMEBase(*mimetype.split('/')) part.set_payload(f.read()) Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="%s"' % attach_name) + part.add_header('Content-Disposition', 'attachment; filename="%s"' \ + % attach_name) msg.attach(part) else: msg.set_payload(body) - # FIXME --------------------------------------------------------------------- - # There seems to be a problem with sending emails using deferreds when - # the last thing left to do is sending the mail, cause the engine stops - # the reactor and the email don't get send. we need to fix this. until - # then, we'll revert to use Python standard (IO-blocking) smtplib. - - #dfd = self._sendmail(self.smtphost, self.mailfrom, rcpts, msg.as_string()) - #dfd.addCallbacks(self._sent_ok, self._sent_failed, - # callbackArgs=[to, cc, subject, len(attachs)], - # errbackArgs=[to, cc, subject, len(attachs)]) - import smtplib - smtp = smtplib.SMTP(self.smtphost) - smtp.sendmail(self.mailfrom, rcpts, msg.as_string()) - log.msg('Mail sent: To=%s Cc=%s Subject="%s"' % (to, cc, subject)) - smtp.close() - # --------------------------------------------------------------------------- + dfd = self._sendmail(self.smtphost, self.mailfrom, rcpts, msg.as_string()) + dfd.addCallbacks(self._sent_ok, self._sent_failed, + callbackArgs=[to, cc, subject, len(attachs)], + errbackArgs=[to, cc, subject, len(attachs)]) + reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) def _sent_ok(self, result, to, cc, subject, nattachs): - log.msg('Mail sent OK: To=%s Cc=%s Subject="%s" Attachs=%d' % (to, cc, subject, nattachs)) + log.msg('Mail sent OK: To=%s Cc=%s Subject="%s" Attachs=%d' % \ + (to, cc, subject, nattachs)) def _sent_failed(self, failure, to, cc, subject, nattachs): errstr = str(failure.value) - log.msg('Unable to send mail: To=%s Cc=%s Subject="%s" Attachs=%d - %s' % (to, cc, subject, nattachs, errstr), level=log.ERROR) + log.msg('Unable to send mail: To=%s Cc=%s Subject="%s" Attachs=%d - %s' % \ + (to, cc, subject, nattachs, errstr), level=log.ERROR) def _sendmail(self, smtphost, from_addr, to_addrs, msg, port=25): """ This is based on twisted.mail.smtp.sendmail except that it diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 37f86d431..bf2dc52c7 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -29,8 +29,8 @@ class XPathSelector(object_ref): self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc elif text: - response = TextResponse(url='about:blank', body=unicode_to_str(text), \ - encoding='utf-8') + response = TextResponse(url='about:blank', \ + body=unicode_to_str(text, 'utf-8'), encoding='utf-8') self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc self.expr = expr diff --git a/scrapy/service.py b/scrapy/service.py new file mode 100644 index 000000000..7247fd0ca --- /dev/null +++ b/scrapy/service.py @@ -0,0 +1,60 @@ +import sys, os + +from twisted.python import log +from twisted.internet import reactor, protocol, error +from twisted.application.service import Service + +from scrapy.utils.py26 import cpu_count +from scrapy.conf import settings + + +class ScrapyService(Service): + + def startService(self): + reactor.callWhenRunning(self.start_processes) + + def start_processes(self): + for i in range(cpu_count()): + self.start_process(i+1) + + def start_process(self, id): + args = [sys.executable, '-m', 'scrapy.service'] + env = os.environ.copy() + self.set_log_file(env, id) + pp = ScrapyProcessProtocol(self, id, env.get('SCRAPY_LOG_FILE')) + reactor.spawnProcess(pp, sys.executable, args=args, env=env) + + def set_log_file(self, env, suffix): + logfile = settings['LOG_FILE'] + if logfile: + file, ext = os.path.splitext(logfile) + env['SCRAPY_LOG_FILE'] = "%s-%s%s" % (file, suffix, ext) + + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + + def __init__(self, service, id, logfile): + self.service = service + self.id = id + self.logfile = logfile + self.pid = None + + def connectionMade(self): + self.pid = self.transport.pid + log.msg("Process %r started: pid=%r logfile=%r" % (self.id, self.pid, \ + self.logfile)) + + def processEnded(self, status): + if isinstance(status.value, error.ProcessDone): + log.msg("Process %r finished: pid=%r logfile=%r" % (self.id, \ + self.pid, self.logfile)) + else: + log.msg("Process %r died: exitstatus=%r pid=%r logfile=%r" % \ + (self.id, status.value.exitCode, self.pid, self.logfile)) + reactor.callLater(5, self.service.start_process, self.id) + + +if __name__ == '__main__': + from scrapy.core.manager import scrapymanager + scrapymanager.configure() + scrapymanager.start(keep_alive=True) diff --git a/scrapy/shell.py b/scrapy/shell.py index 8af6af006..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -35,6 +35,7 @@ def parse_url(url): u = urlparse.urlparse(url) return url + class Shell(object): requires_project = False @@ -52,18 +53,21 @@ class Shell(object): else: url = parse_url(request_or_url) request = Request(url) - spider = spiders.fromurl(url) or BaseSpider('default') + + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default'), log_multiple=True) + print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ request, spider) if response: - self.populate_vars(url, response, request) + self.populate_vars(url, response, request, spider) if print_help: self.print_help() else: print "Done - use shelp() to see available objects" - def populate_vars(self, url=None, response=None, request=None): + def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class() self.vars['item'] = item if url: @@ -73,7 +77,7 @@ class Shell(object): self.vars['url'] = url self.vars['response'] = response self.vars['request'] = request - self.vars['spider'] = spiders.fromurl(url) + self.vars['spider'] = spider if not self.nofetch: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser @@ -104,7 +108,7 @@ class Shell(object): signal.signal(signal.SIGINT, signal.SIG_IGN) reactor.callInThread(self._console_thread, url) - scrapymanager.start() + scrapymanager.start(keep_alive=True) def inspect_response(self, response): print diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index a9b64995e..c1ac2c1d4 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -3,6 +3,9 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + +import warnings + from zope.interface import Interface, Attribute, invariant, implements from twisted.plugin import IPlugin @@ -11,17 +14,9 @@ from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref -def _valid_domain_name(obj): - """Check the domain name specified is valid""" - if not obj.domain_name: - raise ValueError("Spider 'domain_name' attribute is required") - class ISpider(Interface, IPlugin) : - """Interface to be implemented by site-specific web spiders""" - - domain_name = Attribute("The domain name of the site to be scraped.") - - invariant(_valid_domain_name) + """Interface used by TwistedPluginSpiderManager to discover spiders""" + pass class BaseSpider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -31,19 +26,37 @@ class BaseSpider(object_ref): implements(ISpider) # XXX: class attributes kept for backwards compatibility - domain_name = None + name = None start_urls = [] - extra_domain_names = [] + allowed_domains = [] - def __init__(self, domain_name=None): - if domain_name is not None: - self.domain_name = domain_name + def __init__(self, name=None, **kwargs): + self.__dict__.update(kwargs) + # XXX: SEP-12 backward compatibility (remove for 0.10) + if hasattr(self, 'domain_name'): + warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead and Spider.allowed_domains", \ + DeprecationWarning, stacklevel=4) + self.name = self.domain_name + self.allowed_domains = [self.name] + if hasattr(self, 'extra_domain_names'): + warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \ + DeprecationWarning, stacklevel=4) + self.allowed_domains += list(self.extra_domain_names) + + if name is not None: + self.name = name # XXX: create instance attributes (class attributes were kept for # backwards compatibility) if not self.start_urls: self.start_urls = [] - if not self.extra_domain_names: - self.extra_domain_names = [] + if not self.allowed_domains: + self.allowed_domains = [] + if not self.name: + raise ValueError("%s must have a name" % type(self).__name__) + + # XXX: SEP-12 forward compatibility (remove for 0.10) + self.domain_name = self.name + self.extra_domain_names = self.allowed_domains def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this @@ -67,6 +80,6 @@ class BaseSpider(object_ref): pass def __str__(self): - return "<%s %r>" % (type(self).__name__, self.domain_name) + return "<%s %r>" % (type(self).__name__, self.name) __repr__ = __str__ diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index f41916818..0acf36073 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -76,11 +76,11 @@ class MemoryStatsCollector(StatsCollector): def __init__(self): super(MemoryStatsCollector, self).__init__() - self.domain_stats = {} + self.spider_stats = {} def _persist_stats(self, stats, spider=None): if spider is not None: - self.domain_stats[spider.domain_name] = stats + self.spider_stats[spider.name] = stats class DummyStatsCollector(StatsCollector): diff --git a/scrapy/stats/collector/mysql.py b/scrapy/stats/collector/mysql.py deleted file mode 100644 index 2cf145347..000000000 --- a/scrapy/stats/collector/mysql.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -A Stats collector for persisting stats (pickled) to a MySQL db -""" - -import cPickle as pickle -from datetime import datetime - -from scrapy.stats.collector import StatsCollector -from scrapy.utils.mysql import mysql_connect -from scrapy.conf import settings - -class MysqlStatsCollector(StatsCollector): - - def __init__(self): - super(MysqlStatsCollector, self).__init__() - mysqluri = settings['STATS_MYSQL_URI'] - self._mysql_conn = mysql_connect(mysqluri, use_unicode=False) if mysqluri else None - - def _persist_stats(self, stats, spider=None): - if spider is None: # only store spider-specific stats - return - if self._mysql_conn is None: - return - stored = datetime.utcnow() - datas = pickle.dumps(stats) - table = 'domain_data_history' - - c = self._mysql_conn.cursor() - c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % table, \ - (spider.domain_name, stored, datas)) - self._mysql_conn.commit() diff --git a/scrapy/stats/collector/simpledb.py b/scrapy/stats/collector/simpledb.py index 850558442..d521c5e40 100644 --- a/scrapy/stats/collector/simpledb.py +++ b/scrapy/stats/collector/simpledb.py @@ -36,9 +36,9 @@ class SimpledbStatsCollector(StatsCollector): def _persist_to_sdb(self, spider, stats): ts = self._get_timestamp(spider).isoformat() - sdb_item_id = "%s_%s" % (spider.domain_name, ts) + sdb_item_id = "%s_%s" % (spider.name, ts) sdb_item = dict((k, self._to_sdb_value(v, k)) for k, v in stats.iteritems()) - sdb_item['domain'] = spider.domain_name + sdb_item['spider'] = spider.name sdb_item['timestamp'] = self._to_sdb_value(ts) connect_sdb().put_attributes(self._sdbdomain, sdb_item_id, sdb_item) diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index fa6f5ea6f..e3f89342d 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -4,5 +4,5 @@ # See: http://doc.scrapy.org/topics/item-pipeline.html class ${ProjectName}Pipeline(object): - def process_item(self, domain, item): + def process_item(self, spider, item): return item diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 2e3baf992..246015466 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,9 +1,10 @@ from scrapy.spider import BaseSpider class $classname(BaseSpider): - domain_name = "$site" + name = "$name" + allowed_domains = ["$domain"] start_urls = ( - 'http://www.$site/', + 'http://www.$domain/', ) def parse(self, response): diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 2d4f7fce6..578779c06 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -6,19 +6,20 @@ from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item class $classname(CrawlSpider): - domain_name = '$site' - start_urls = ['http://www.$site/'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/'] rules = ( - Rule(SgmlLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), + Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), ) def parse_item(self, response): - xs = HtmlXPathSelector(response) + hxs = HtmlXPathSelector(response) i = ${ProjectName}Item() - #i['site_id'] = xs.select('//input[@id="sid"]/@value').extract() - #i['name'] = xs.select('//div[@id="name"]').extract() - #i['description'] = xs.select('//div[@id="description"]').extract() + #i['domain_id'] = hxs.select('//input[@id="sid"]/@value').extract() + #i['name'] = hxs.select('//div[@id="name"]').extract() + #i['description'] = hxs.select('//div[@id="description"]').extract() return i SPIDER = $classname() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 794288570..c9a723000 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item class $classname(CSVFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.csv'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.csv'] # headers = ['id', 'name', 'description', 'image_link'] # delimiter = '\t' diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index f249b4537..f5ecbd707 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item class $classname(XMLFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.xml'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.xml'] def parse_item(self, response, selector): i = ${ProjectName}Item() diff --git a/scrapy/tests/__init__.py b/scrapy/tests/__init__.py index c77772af5..afd463740 100644 --- a/scrapy/tests/__init__.py +++ b/scrapy/tests/__init__.py @@ -6,9 +6,6 @@ To run all Scrapy unittests go to Scrapy main dir and type: bin/runtests.sh If you're in windows use runtests.bat instead. - -Keep in mind that some tests may be skipped if you don't have some (optional) -modules available like MySQLdb or simplejson, but that's not a problem. """ import os diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index cb4e5ced1..6eed6d22a 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -59,10 +59,18 @@ class CommandTest(ProjectTest): class GenspiderCommandTest(CommandTest): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + self.assertEqual(0, self.call('genspider', 'test_name')) + assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + # pass two arguments . spider script should be created + self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) + assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + def test_template_default(self, *args): - self.assertEqual(0, self.call('genspider', 'testspider', 'test.com', *args)) - assert exists(join(self.proj_mod_path, 'spiders', 'testspider.py')) - self.assertEqual(1, self.call('genspider', 'otherspider', 'test.com')) + self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args)) + assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com')) def test_template_basic(self): self.test_template_default('--template=basic') diff --git a/scrapy/tests/test_contrib_exp_crawlspider_matchers.py b/scrapy/tests/test_contrib_exp_crawlspider_matchers.py new file mode 100644 index 000000000..4cb832aa1 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_matchers.py @@ -0,0 +1,94 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import Response + +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlListMatcher + +import re + +class MatchersTest(unittest.TestCase): + + def setUp(self): + pass + + def test_base_matcher(self): + matcher = BaseMatcher() + + request = Request('http://example.com') + response = Response('http://example.com') + + self.assertTrue(matcher.matches_request(request)) + self.assertTrue(matcher.matches_response(response)) + + def test_url_matcher(self): + matcher = UrlMatcher('http://example.com') + + request = Request('http://example.com') + response = Response('http://example.com') + + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_request(response)) + + request = Request('http://example2.com') + response = Response('http://example2.com') + + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_request(response)) + + def test_url_regex_matcher(self): + matcher = UrlRegexMatcher(r'sample') + urls = ( + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample4.html', + ) + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + matcher = UrlRegexMatcher(r'sample_fail') + for url in urls: + request, response = Request(url), Response(url) + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_response(response)) + + matcher = UrlRegexMatcher(r'SAMPLE\d+', re.IGNORECASE) + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + def test_url_list_matcher(self): + urls = ( + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample4.html', + ) + urls2 = ( + 'http://example.com/sample5.html', + 'http://example.com/sample6.html', + 'http://example.com/sample7.html', + 'http://example.com/sample8.html', + 'http://example.com/', + ) + matcher = UrlListMatcher(urls) + + # match urls + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + # non-match urls + for url in urls2: + request, response = Request(url), Response(url) + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_response(response)) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py new file mode 100644 index 000000000..0259b30fb --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py @@ -0,0 +1,156 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.tests import get_testdata + +from scrapy.contrib_exp.crawlspider.reqext import BaseSgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqext import XPathRequestExtractor + +class AbstractRequestExtractorTest(unittest.TestCase): + + def _requests_equals(self, list1, list2): + """Compares request's urls and link_text""" + for (r1, r2) in zip(list1, list2): + if r1.url != r2.url: + return False + if r1.meta['link_text'] != r2.meta['link_text']: + return False + # all equal + return True + + +class RequestExtractorTest(AbstractRequestExtractorTest): + + def test_basic(self): + base_url = 'http://example.org/somepage/index.html' + html = """Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + requests = [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/about.html', + meta={'link_text': 'About us'}), + Request('http://example.org/othercat.html', + meta={'link_text': 'Other category'}), + Request('http://example.org/', + meta={'link_text': ''}), + ] + + response = HtmlResponse(base_url, body=html) + reqx = BaseSgmlRequestExtractor() # default: tag=a, attr=href + + self.failUnless( + self._requests_equals(requests, reqx.extract_requests(response)) + ) + + def test_base_url(self): + reqx = BaseSgmlRequestExtractor() + + html = """<html><head><title>Page title<title> + <base href="http://otherdomain.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('http://otherdomain.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title> + <base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://example.org/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + # base url has no scheme + html = """<html><head><title>Page title<title> + <base href="//noscheme.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://noscheme.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + def test_extraction_encoding(self): + #TODO: use own fixtures + body = get_testdata('link_extractor', 'linkextractor_noenc.html') + response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, + headers={'Content-Type': ['text/html; charset=utf-8']}) + response_noenc = HtmlResponse(url='http://example.com/noenc', + body=body) + body = get_testdata('link_extractor', 'linkextractor_latin1.html') + response_latin1 = HtmlResponse(url='http://example.com/latin1', + body=body) + + reqx = BaseSgmlRequestExtractor() + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_utf8), + [ Request(url='http://example.com/sample_%C3%B1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E2%82%AC.html', + meta={'link_text': + 'sample \xe2\x82\xac text'.decode('utf-8')}) ] + ) + ) + + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_noenc), + [ Request(url='http://example.com/sample_%C3%B1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E2%82%AC.html', + meta={'link_text': + 'sample \xe2\x82\xac text'.decode('utf-8')}) ] + ) + ) + + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_latin1), + [ Request(url='http://example.com/sample_%F1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E1.html', + meta={'link_text': + 'sample \xe1 text'.decode('latin1')}) ] + ) + ) + + +class SgmlRequestExtractorTest(AbstractRequestExtractorTest): + pass + + +class XPathRequestExtractorTest(AbstractRequestExtractorTest): + + def setUp(self): + # TODO: use own fixtures + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + + def test_restrict_xpaths(self): + reqx = XPathRequestExtractor('//div[@id="subwrapper"]') + self.failUnless( + self._requests_equals( + reqx.extract_requests(self.response), + [ Request(url='http://example.com/sample1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample2.html', + meta={'link_text': 'sample 2'}) ] + ) + ) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py b/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py new file mode 100644 index 000000000..67aca2387 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py @@ -0,0 +1,128 @@ +from twisted.internet import defer +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.utils.python import equal_attributes + +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqgen import RequestGenerator +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDomain +from scrapy.contrib_exp.crawlspider.reqproc import FilterUrl +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + +class RequestGeneratorTest(unittest.TestCase): + + def setUp(self): + url = 'http://example.org/somepage/index.html' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + self.response = HtmlResponse(url, body=html) + self.deferred = defer.Deferred() + self.requests = [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/about.html', + meta={'link_text': 'About us'}), + Request('http://example.org/othercat.html', + meta={'link_text': 'Other category'}), + Request('http://example.org/', + meta={'link_text': ''}), + ] + + def _equal_requests_list(self, list1, list2): + list1 = list(list1) + list2 = list(list2) + if not len(list1) == len(list2): + return False + + for (req1, req2) in zip(list1, list2): + if not equal_attributes(req1, req2, ['url']): + return False + return True + + def test_basic(self): + reqgen = RequestGenerator([], [], callback=self.deferred) + # returns generator + requests = reqgen.generate_requests(self.response) + self.failUnlessEqual(list(requests), []) + + def test_request_extractor(self): + extractors = [ + SgmlRequestExtractor() + ] + + # extract all requests + reqgen = RequestGenerator(extractors, [], callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnless(self._equal_requests_list(requests, self.requests)) + + for req in requests: + # check callback + self.failUnlessEqual(req.deferred, self.deferred) + + def test_request_processor(self): + extractors = [ + SgmlRequestExtractor() + ] + + processors = [ + Canonicalize(), + FilterDupes(), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnless(self._equal_requests_list(requests, self.requests)) + + # filter domain + processors = [ + Canonicalize(), + FilterDupes(), + FilterDomain(deny='example.org'), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnlessEqual(list(requests), []) + + # filter url + processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(deny=(r'about', r'othercat')), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + + self.failUnless(self._equal_requests_list(requests, [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/', + meta={'link_text': ''}), + ])) + + processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(allow=r'/somepage/'), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + + self.failUnless(self._equal_requests_list(requests, [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + ])) + + + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py b/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py new file mode 100644 index 000000000..da5db67b2 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py @@ -0,0 +1,144 @@ +from twisted.trial import unittest + +from scrapy.http import Request + +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDomain +from scrapy.contrib_exp.crawlspider.reqproc import FilterUrl +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + +import copy + +class RequestProcessorsTest(unittest.TestCase): + + def test_canonicalize_requests(self): + urls = [ + 'http://example.com/do?&b=1&a=2&c=3', + 'http://example.com/do?123,&q=a space', + ] + urls_after = [ + 'http://example.com/do?a=2&b=1&c=3', + 'http://example.com/do?123%2C=&q=a+space', + ] + + proc = Canonicalize() + results = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(results, urls_after) + + def test_unique_requests(self): + urls = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + ] + urls_unique = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + ] + + proc = FilterDupes() + results = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(results, urls_unique) + + # Check custom attributes + requests = [ + Request('http://example.com', method='GET'), + Request('http://example.com', method='POST'), + ] + proc = FilterDupes('url', 'method') + self.failUnlessEqual(len(list(proc(requests))), 2) + + proc = FilterDupes('url') + self.failUnlessEqual(len(list(proc(requests))), 1) + + def test_filter_domain(self): + urls = [ + 'http://blah1.com/index', + 'http://blah2.com/index', + 'http://blah1.com/section', + 'http://blah2.com/section', + ] + + proc = FilterDomain(allow=('blah1.com'), deny=('blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, [ + 'http://blah1.com/index', + 'http://blah1.com/section', + ]) + + proc = FilterDomain(deny=('blah1.com', 'blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, []) + + proc = FilterDomain(allow=('blah1.com', 'blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, urls) + + def test_filter_url(self): + urls = [ + 'http://blah1.com/index', + 'http://blah2.com/index', + 'http://blah1.com/section', + 'http://blah2.com/section', + ] + + proc = FilterUrl(allow=(r'blah1'), deny=(r'blah2')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, [ + 'http://blah1.com/index', + 'http://blah1.com/section', + ]) + + proc = FilterUrl(deny=('blah1', 'blah2')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, []) + + proc = FilterUrl(allow=('index$', 'section$')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, urls) + + + + def test_all_processors(self): + urls = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/do?&b=1&a=2&c=3', + 'http://example.com/do?123,&q=a space', + ] + urls_processed = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/do?a=2&b=1&c=3', + 'http://example.com/do?123%2C=&q=a+space', + ] + + processors = [ + Canonicalize(), + FilterDupes(), + ] + + def _process(requests): + """Apply all processors""" + # copy list + processed = [copy.copy(req) for req in requests] + for proc in processors: + processed = proc(processed) + return processed + + # empty requests + results1 = [r.url for r in _process([])] + self.failUnlessEquals(results1, []) + + # try urls + requests = (Request(url) for url in urls) + results2 = [r.url for r in _process(requests)] + self.failUnlessEquals(results2, urls_processed) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_rules.py b/scrapy/tests/test_contrib_exp_crawlspider_rules.py new file mode 100644 index 000000000..e04eb7b06 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_rules.py @@ -0,0 +1,262 @@ +from twisted.trial import unittest + +from scrapy.http import HtmlResponse +from scrapy.spider import BaseSpider +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher + +from scrapy.contrib_exp.crawlspider.rules import CompiledRule +from scrapy.contrib_exp.crawlspider.rules import Rule +from scrapy.contrib_exp.crawlspider.rules import RulesManager + +from functools import partial + +class RuleInitializationTest(unittest.TestCase): + + def test_fail_if_rule_null(self): + # fail on empty rule + self.failUnlessRaises(ValueError, Rule) + self.failUnlessRaises(ValueError, Rule, + **dict(callback=None, follow=None)) + self.failUnlessRaises(ValueError, Rule, + **dict(callback=None, follow=False)) + + def test_minimal_arguments_to_instantiation(self): + # not fail if callback set + self.failUnless(Rule(callback=lambda: True)) + # not fail if follow set + self.failUnless(Rule(follow=True)) + + def test_validate_default_attributes(self): + # test null Rule + rule = Rule(follow=True) + self.failUnlessEqual(None, rule.matcher) + self.failUnlessEqual(None, rule.callback) + self.failUnlessEqual({}, rule.cb_kwargs) + # follow default False + self.failUnlessEqual(True, rule.follow) + + def test_validate_attributes_set(self): + matcher = BaseMatcher() + callback = lambda: True + rule = Rule(matcher, callback, True, a=1) + # test attributes + self.failUnlessEqual(matcher, rule.matcher) + self.failUnlessEqual(callback, rule.callback) + self.failUnlessEqual({'a': 1}, rule.cb_kwargs) + self.failUnlessEqual(True, rule.follow) + +class CompiledRuleInitializationTest(unittest.TestCase): + + def test_fail_on_invalid_matcher(self): + # pass with valid matcher + self.failUnless(CompiledRule(BaseMatcher()), + "Failed CompiledRule instantiation") + + # at least needs valid matcher + self.assertRaises(AssertionError, CompiledRule, None) + self.assertRaises(AssertionError, CompiledRule, False) + self.assertRaises(AssertionError, CompiledRule, True) + + def test_fail_on_invalid_callback(self): + # pass with valid callback + callback = lambda: True + self.failUnless(CompiledRule(BaseMatcher(), callback)) + # pass with callback none + self.failUnless(CompiledRule(BaseMatcher(), None)) + + # assert on invalid callback + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + 'myfunc') + + # numeric variable + var = 123 + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + var) + + class A: + pass + + # random instance + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + A()) + + + def test_fail_on_invalid_follow_value(self): + callback = lambda: True + matcher = BaseMatcher() + # pass bool + self.failUnless(CompiledRule(matcher, callback, True)) + self.failUnless(CompiledRule(matcher, callback, False)) + + # assert with non-bool + self.assertRaises(AssertionError, CompiledRule, matcher, + callback, None) + self.assertRaises(AssertionError, CompiledRule, matcher, + callback, 1) + + def test_validate_default_attributes(self): + callback = lambda: True + matcher = BaseMatcher() + rule = CompiledRule(matcher, callback, True) + + # test attributes + self.failUnlessEqual(matcher, rule.matcher) + self.failUnlessEqual(callback, rule.callback) + self.failUnlessEqual(True, rule.follow) + + +class RulesTest(unittest.TestCase): + def test_rules_manager_basic(self): + spider = BaseSpider('foo') + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + rulesman = RulesManager([], spider) + + # should return none + self.failIf(rulesman.get_rule_from_response(response1)) + self.failIf(rulesman.get_rule_from_response(response2)) + + # rules manager with match-all rule + rulesman = RulesManager([ + Rule(BaseMatcher(), follow=True), + ], spider) + + # returns CompiledRule + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnless(isinstance(rule1, CompiledRule)) + self.failUnless(isinstance(rule2, CompiledRule)) + self.assert_(rule1 is rule2) + self.failUnlessEqual(rule1.callback, None) + self.failUnlessEqual(rule1.follow, True) + + def test_rules_manager_empty_rule(self): + spider = BaseSpider('foo') + response = HtmlResponse('http://example.org') + + rulesman = RulesManager([Rule(follow=True)], spider) + + rule = rulesman.get_rule_from_response(response) + # default matcher if None: BaseMatcher + self.failUnless(isinstance(rule.matcher, BaseMatcher)) + + def test_rules_manager_default_matcher(self): + spider = BaseSpider('foo') + response = HtmlResponse('http://example.org') + callback = lambda x: None + + rulesman = RulesManager([ + Rule('http://example.org', callback), + ], spider, default_matcher=UrlMatcher) + + rule = rulesman.get_rule_from_response(response) + self.failUnless(isinstance(rule.matcher, UrlMatcher)) + + def test_rules_manager_matchers(self): + spider = BaseSpider('foo') + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + + urlmatcher = UrlMatcher('http://example.org') + basematcher = BaseMatcher() + # callback needed for Rule + callback = lambda x: None + + # test fail matcher resolve + self.assertRaises(ValueError, RulesManager, + [Rule(False, callback)], spider) + self.assertRaises(ValueError, RulesManager, + [Rule(spider, callback)], spider) + + rulesman = RulesManager([ + Rule(urlmatcher, callback), + Rule(basematcher, callback), + ], spider) + + # response1 matches example.org + rule1 = rulesman.get_rule_from_response(response1) + # response2 is catch by BaseMatcher() + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.matcher, urlmatcher) + self.failUnlessEqual(rule2.matcher, basematcher) + + # reverse order. BaseMatcher should match all + rulesman = RulesManager([ + Rule(basematcher, callback), + Rule(urlmatcher, callback), + ], spider) + + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.matcher, basematcher) + self.failUnlessEqual(rule2.matcher, basematcher) + self.failUnless(rule1 is rule2) + + def test_rules_manager_callbacks(self): + mycallback = lambda: True + + spider = BaseSpider('foo') + spider.parse_item = lambda: True + + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + + rulesman = RulesManager([ + Rule('example', mycallback), + Rule('othersite', 'parse_item'), + ], spider, default_matcher=UrlRegexMatcher) + + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.callback, mycallback) + self.failUnlessEqual(rule2.callback, spider.parse_item) + + # fail unknown callback + self.assertRaises(AttributeError, RulesManager, [ + Rule(BaseMatcher(), 'mycallback') + ], spider) + # fail not callable + spider.not_callable = True + self.assertRaises(AttributeError, RulesManager, [ + Rule(BaseMatcher(), 'not_callable') + ], spider) + + + def test_rules_manager_callback_with_arguments(self): + spider = BaseSpider('foo') + response = HtmlResponse('http://example.org') + + kwargs = {'a': 1} + + def myfunc(**mykwargs): + return mykwargs + + # verify return validation + self.failUnlessEquals(kwargs, myfunc(**kwargs)) + + # test callback w/o arguments + rulesman = RulesManager([ + Rule(BaseMatcher(), myfunc), + ], spider) + rule = rulesman.get_rule_from_response(response) + + # without arguments should return same callback + self.failUnlessEqual(rule.callback, myfunc) + + # test callback w/ arguments + rulesman = RulesManager([ + Rule(BaseMatcher(), myfunc, **kwargs), + ], spider) + rule = rulesman.get_rule_from_response(response) + + # with argument should return partial applied callback + self.failUnless(isinstance(rule.callback, partial)) + self.failUnlessEquals(kwargs, rule.callback()) + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_spider.py b/scrapy/tests/test_contrib_exp_crawlspider_spider.py new file mode 100644 index 000000000..21f7f71e4 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_spider.py @@ -0,0 +1,222 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.item import BaseItem +from scrapy.utils.spider import iterate_spider_output + +# basics +from scrapy.contrib_exp.crawlspider import CrawlSpider +from scrapy.contrib_exp.crawlspider import Rule + +# matchers +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlListMatcher + +# extractors +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor + +# processors +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + + +# mock items +class Item1(BaseItem): + pass + +class Item2(BaseItem): + pass + +class Item3(BaseItem): + pass + + +class CrawlSpiderTest(unittest.TestCase): + + def spider_factory(self, rules=[], + extractors=[], processors=[], + start_urls=[]): + # mock spider + class Spider(CrawlSpider): + def parse_item1(self, response): + return Item1() + + def parse_item2(self, response): + return Item2() + + def parse_item3(self, response): + return Item3() + + def parse_request1(self, response): + return Request('http://example.org/request1') + + def parse_request2(self, response): + return Request('http://example.org/request2') + + Spider.start_urls = start_urls + Spider.rules = rules + Spider.request_extractors = extractors + Spider.request_processors = processors + + return Spider('foo') + + def test_start_url_auto_rule(self): + spider = self.spider_factory() + # zero spider rules + self.failUnlessEqual(len(spider.rules), 0) + self.failUnlessEqual(len(spider._rulesman._rules), 0) + + spider = self.spider_factory(start_urls=['http://example.org']) + + self.failUnlessEqual(len(spider.rules), 0) + self.failUnlessEqual(len(spider._rulesman._rules), 1) + + def test_start_url_matcher(self): + url = 'http://example.org' + spider = self.spider_factory(start_urls=[url]) + + response = HtmlResponse(url) + + rule = spider._rulesman.get_rule_from_response(response) + self.failUnless(isinstance(rule.matcher, UrlListMatcher)) + + response = HtmlResponse(url + '/item.html') + + rule = spider._rulesman.get_rule_from_response(response) + self.failUnless(rule is None) + + # TODO: remove this block + # in previous version get_rule returns rule from response.request + response.request = Request(url) + rule = spider._rulesman.get_rule_from_response(response.request) + self.failUnless(isinstance(rule.matcher, UrlListMatcher)) + self.failUnlessEqual(rule.follow, True) + + def test_parse_callback(self): + response = HtmlResponse('http://example.org') + rules = ( + Rule(BaseMatcher(), 'parse_item1'), + ) + spider = self.spider_factory(rules) + + result = list(spider.parse(response)) + self.failUnlessEqual(len(result), 1) + self.failUnless(isinstance(result[0], Item1)) + + def test_crawling_start_url(self): + url = 'http://example.org/' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + response = HtmlResponse(url, body=html) + + extractors = (SgmlRequestExtractor(), ) + spider = self.spider_factory(start_urls=[url], + extractors=extractors) + result = list(spider.parse(response)) + + # 1 request extracted: example.org/ + # because requests returns only matching + self.failUnlessEqual(len(result), 1) + + # we will add catch-all rule to extract all + callback = lambda x: None + rules = [Rule(r'\.html$', callback=callback)] + spider = self.spider_factory(rules, start_urls=[url], + extractors=extractors) + result = list(spider.parse(response)) + + # 4 requests extracted + # 3 of .html pattern + # 1 of start url patter + self.failUnlessEqual(len(result), 4) + + def test_crawling_simple_rule(self): + url = 'http://example.org/somepage/index.html' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + response = HtmlResponse(url, body=html) + + rules = ( + # first response callback + Rule(r'index\.html', 'parse_item1'), + ) + spider = self.spider_factory(rules) + result = list(spider.parse(response)) + + # should return Item1 + self.failUnlessEqual(len(result), 1) + self.failUnless(isinstance(result[0], Item1)) + + # test request generation + rules = ( + # first response without callback and follow flag + Rule(r'index\.html', follow=True), + Rule(r'(\.html|/)$', 'parse_item1'), + ) + spider = self.spider_factory(rules) + result = list(spider.parse(response)) + + # 0 because spider does not have extractors + self.failUnlessEqual(len(result), 0) + + extractors = (SgmlRequestExtractor(), ) + + # instance spider with extractor + spider = self.spider_factory(rules, extractors) + result = list(spider.parse(response)) + # 4 requests extracted + self.failUnlessEqual(len(result), 4) + + def test_crawling_multiple_rules(self): + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + response = HtmlResponse('http://example.org/index.html', body=html) + response1 = HtmlResponse('http://example.org/1.html') + response2 = HtmlResponse('http://example.org/othercat.html') + + rules = ( + Rule(r'\d+\.html$', 'parse_item1'), + Rule(r'othercat\.html$', 'parse_item2'), + # follow-only rules + Rule(r'index\.html', 'parse_item3', follow=True) + ) + extractors = [SgmlRequestExtractor()] + spider = self.spider_factory(rules, extractors) + + result = list(spider.parse(response)) + # 1 Item 2 Requests + self.failUnlessEqual(len(result), 3) + # parse_item3 + self.failUnless(isinstance(result[0], Item3)) + only_requests = lambda r: isinstance(r, Request) + requests = filter(only_requests, result[1:]) + self.failUnlessEqual(len(requests), 2) + self.failUnless(all(requests)) + + result1 = list(spider.parse(response1)) + # parse_item1 + self.failUnlessEqual(len(result1), 1) + self.failUnless(isinstance(result1[0], Item1)) + + result2 = list(spider.parse(response2)) + # parse_item2 + self.failUnlessEqual(len(result2), 1) + self.failUnless(isinstance(result2[0], Item2)) + + diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 48cc90c46..140108b42 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,10 +1,10 @@ -import cPickle as pickle +import unittest, cPickle as pickle from cStringIO import StringIO -from twisted.trial import unittest - from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode +from scrapy.utils.py26 import json +from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter @@ -149,22 +149,10 @@ class XmlItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - def setUp(self): - try: - import json - except ImportError: - try: - import simplejson - except ImportError: - raise unittest.SkipTest("simplejson module not available") - super(JsonLinesItemExporterTest, self).setUp() - def _get_exporter(self, **kwargs): - from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - from scrapy.contrib.exporter.jsonlines import json exported = json.loads(self.output.getvalue().strip()) self.assertEqual(exported, dict(self.i)) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 4a60b8a2d..65ffad714 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -35,6 +35,20 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title><base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://example.org/item/12.html', text='Item 12')]) + + # base url has no scheme + html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + def test_extraction_encoding(self): body = get_testdata('link_extractor', 'linkextractor_noenc.html') response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py new file mode 100644 index 000000000..5b0d9e954 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -0,0 +1,62 @@ +import weakref +import unittest + +# just a hack to avoid cyclic imports of scrapy.spider when running this test +# alone +import scrapy.spider +from scrapy.contrib.spidermanager import TwistedPluginSpiderManager +from scrapy.http import Request + +class TwistedPluginSpiderManagerTest(unittest.TestCase): + + def setUp(self): + self.spiderman = TwistedPluginSpiderManager() + assert not self.spiderman.loaded + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager']) + assert self.spiderman.loaded + + def test_list(self): + self.assertEqual(set(self.spiderman.list()), + set(['spider1', 'spider2'])) + + def test_create(self): + spider1 = self.spiderman.create("spider1") + self.assertEqual(spider1.__class__.__name__, 'Spider1') + spider2 = self.spiderman.create("spider2", foo="bar") + self.assertEqual(spider2.__class__.__name__, 'Spider2') + self.assertEqual(spider2.foo, 'bar') + + def test_create_uses_cache(self): + # TwistedPluginSpiderManager uses an internal cache which is + # invalidated in close_spider() but this isn't necessarily the best + # thing to do in all cases. + spider1 = self.spiderman.create("spider1") + spider2 = self.spiderman.create("spider1") + assert spider1 is spider2 + + def test_find_by_request(self): + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), + ['spider1']) + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy2.org/test')), + ['spider2']) + self.assertEqual(set(self.spiderman.find_by_request(Request('http://scrapy3.org/test'))), + set(['spider1', 'spider2'])) + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), + []) + + def test_close_spider_remove_refs(self): + spider = self.spiderman.create("spider1") + wref = weakref.ref(spider) + assert wref() + self.spiderman.close_spider(spider) + del spider + assert not wref() + + def test_close_spider_invalidates_cache(self): + spider1 = self.spiderman.create("spider1") + self.spiderman.close_spider(spider1) + spider2 = self.spiderman.create("spider1") + assert spider1 is not spider2 + +if __name__ == '__main__': + unittest.main() diff --git a/scrapy/tests/test_contrib_spidermanager/dropin.cache b/scrapy/tests/test_contrib_spidermanager/dropin.cache new file mode 100644 index 000000000..81239e0b4 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/dropin.cache @@ -0,0 +1,71 @@ +(dp1 +S'spider2' +p2 +ccopy_reg +_reconstructor +p3 +(ctwisted.plugin +CachedDropin +p4 +c__builtin__ +object +p5 +NtRp6 +(dp7 +S'moduleName' +p8 +S'scrapy.tests.test_contrib_spidermanager.spider2' +p9 +sS'description' +p10 +NsS'plugins' +p11 +(lp12 +g3 +(ctwisted.plugin +CachedPlugin +p13 +g5 +NtRp14 +(dp15 +S'provided' +p16 +(lp17 +cscrapy.spider.models +ISpider +p18 +asS'dropin' +p19 +g6 +sS'name' +p20 +S'SPIDER' +p21 +sg10 +NsbasbsS'spider1' +p22 +g3 +(g4 +g5 +NtRp23 +(dp24 +g8 +S'scrapy.tests.test_contrib_spidermanager.spider1' +p25 +sg10 +Nsg11 +(lp26 +g3 +(g13 +g5 +NtRp27 +(dp28 +g16 +(lp29 +g18 +asg19 +g23 +sg20 +g21 +sg10 +Nsbasbs. \ No newline at end of file diff --git a/scrapy/tests/test_contrib_spidermanager/spider1.py b/scrapy/tests/test_contrib_spidermanager/spider1.py new file mode 100644 index 000000000..0a9b60989 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/spider1.py @@ -0,0 +1,7 @@ +from scrapy.spider import BaseSpider + +class Spider1(BaseSpider): + name = "spider1" + allowed_domains = ["scrapy1.org", "scrapy3.org"] + +SPIDER = Spider1() diff --git a/scrapy/tests/test_contrib_spidermanager/spider2.py b/scrapy/tests/test_contrib_spidermanager/spider2.py new file mode 100644 index 000000000..52023277f --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/spider2.py @@ -0,0 +1,7 @@ +from scrapy.spider import BaseSpider + +class Spider2(BaseSpider): + name = "spider2" + allowed_domains = ["scrapy2.org", "scrapy3.org"] + +SPIDER = Spider2() diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index a23d6c6a4..ae92e16da 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -32,11 +32,11 @@ class FileTestCase(unittest.TestCase): request = Request('file://%s' % self.tmpname + '^') assert request.url.upper().endswith('%5E') - return download_file(request, BaseSpider()).addCallback(_test) + return download_file(request, BaseSpider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = download_file(request, BaseSpider()) + d = download_file(request, BaseSpider('foo')) return self.assertFailure(d, IOError) @@ -66,20 +66,20 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_timeout_download_from_spider(self): - spider = BaseSpider() + spider = BaseSpider('foo') spider.download_timeout = 0.000001 request = Request(self.getURL('wait')) d = download_http(request, spider) @@ -91,7 +91,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -99,9 +99,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -109,14 +109,14 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d def test_broken_download(self): request = Request(self.getURL('broken')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) return self.assertFailure(d, PartialDownloadError) @@ -152,7 +152,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -161,4 +161,4 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d1b4275e8..a04efd79d 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -10,7 +10,7 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = CookiesMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index 1de53e85c..09dfdbaf8 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -21,7 +21,7 @@ class DecompressionMiddlewareTest(TestCase): def setUp(self): self.mw = DecompressionMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_known_compression_formats(self): for fmt in self.test_formats: diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index cd1624eb4..805289787 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -9,7 +9,7 @@ from scrapy.spider import BaseSpider class TestDefaultHeadersMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() self.default_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) diff --git a/scrapy/tests/test_downloadermiddleware_httpauth.py b/scrapy/tests/test_downloadermiddleware_httpauth.py index 79c815184..747ea9de1 100644 --- a/scrapy/tests/test_downloadermiddleware_httpauth.py +++ b/scrapy/tests/test_downloadermiddleware_httpauth.py @@ -18,7 +18,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): self.mw.default_useragent = 'default_useragent' - spider = TestSpider() + spider = TestSpider('foo') req = Request('http://scrapytest.org/') assert self.mw.process_request(req, spider) is None self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==') diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 9490fb40f..c8fb38d22 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -20,7 +20,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpCompressionMiddleware() def _getresponse(self, coding): diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py index dd9602661..7f6f7ee4e 100644 --- a/scrapy/tests/test_downloadermiddleware_httpproxy.py +++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py @@ -8,7 +8,7 @@ from scrapy.http import Response, Request from scrapy.spider import BaseSpider from scrapy.conf import settings -spider = BaseSpider() +spider = BaseSpider('foo') class TestDefaultHeadersMiddleware(TestCase): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index ba7daece8..8409fec6d 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -3,12 +3,12 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import BaseSpider from scrapy.core.exceptions import IgnoreRequest -from scrapy.http import Request, Response, Headers +from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RedirectMiddleware() def test_priority_adjust(self): @@ -58,7 +58,7 @@ class RedirectMiddlewareTest(unittest.TestCase): <head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head> </html>""" req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -70,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase): <head><meta http-equiv="refresh" content="1000;url=http://example.org/newpage" /></head> </html>""" req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) rsp2 = self.mw.process_response(req, rsp, self.spider) assert rsp is rsp2 @@ -81,7 +81,7 @@ class RedirectMiddlewareTest(unittest.TestCase): </html>""" req = Request(url='http://example.org', method='POST', body='test', headers={'Content-Type': 'text/plain', 'Content-length': '4'}) - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index 101bb01c0..96e7cd297 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -10,7 +10,7 @@ from scrapy.http import Request, Response class RetryTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RetryMiddleware() self.mw.max_retry_times = 2 diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index d12f2bc63..338a63a23 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -9,7 +9,7 @@ from scrapy.conf import settings class UserAgentMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UserAgentMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py index d11595a9d..fa9eef332 100644 --- a/scrapy/tests/test_dupefilter.py +++ b/scrapy/tests/test_dupefilter.py @@ -8,7 +8,7 @@ from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilt class RequestFingerprintDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = RequestFingerprintDupeFilter() filter.open_spider(spider) @@ -28,7 +28,7 @@ class RequestFingerprintDupeFilterTest(unittest.TestCase): class NullDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = NullDupeFilter() filter.open_spider(spider) diff --git a/scrapy/tests/test_encoding_aliases.py b/scrapy/tests/test_encoding_aliases.py deleted file mode 100644 index cc3a0089f..000000000 --- a/scrapy/tests/test_encoding_aliases.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -import scrapy # adds encoding aliases (if not added before) - -class EncodingAliasesTestCase(unittest.TestCase): - - def test_encoding_aliases(self): - """Test common encdoing aliases not included in Python""" - - uni = u'\u041c\u041e\u0421K\u0412\u0410' - str = uni.encode('windows-1251') - self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251')) - self.assertEqual(str.decode('windows-1251'), str.decode('win-1251')) - - text = u'\u8f6f\u4ef6\u540d\u79f0' - str = uni.encode('gb2312') - self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn')) - self.assertEqual(str.decode('gb2312'), str.decode('zh-cn')) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index f205ff264..2d7b5950c 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -22,8 +22,8 @@ class TestItem(Item): price = Field() class TestSpider(BaseSpider): - domain_name = "scrapytest.org" - extra_domain_names = ["localhost"] + name = "scrapytest.org" + allowed_domains = ["scrapytest.org", "localhost"] start_urls = ['http://localhost'] itemurl_re = re.compile("item\d+.html") @@ -68,7 +68,7 @@ def start_test_site(): class CrawlingSession(object): def __init__(self): - self.domain = 'scrapytest.org' + self.name = 'scrapytest.org' self.spider = None self.respplug = [] self.reqplug = [] @@ -97,7 +97,8 @@ class CrawlingSession(object): dispatcher.connect(self.response_downloaded, signals.response_downloaded) scrapymanager.configure() - scrapymanager.runonce(self.spider) + scrapymanager.crawl_spider(self.spider) + scrapymanager.start() self.port.stopListening() self.wasrun = True @@ -138,7 +139,7 @@ class EngineTest(unittest.TestCase): Check the spider is loaded and located properly via the SpiderLocator """ assert session.spider is not None - self.assertEqual(session.spider.domain_name, session.domain) + self.assertEqual(session.spider.name, session.name) def test_visited_urls(self): """ diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index c0ae0cab5..b37eb33e7 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -171,13 +171,6 @@ class RequestTest(unittest.TestCase): self.assertEqual(r4.meta, {}) assert r4.dont_filter is False - # __init__ and replace() signatures must be equal unles *args,**kwargs is used - i_args, i_varargs, i_varkwargs, _ = getargspec(self.request_class.__init__) - self.assertFalse(bool(i_varargs) ^ bool(i_varkwargs)) - if not i_varargs: - r_args, _, _, _ = getargspec(self.request_class.replace) - self.assertEqual(i_args, r_args) - def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" x = self.request_class('http://www.example.com') diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 3b0a144d2..437d75b98 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -2,7 +2,7 @@ import unittest import weakref from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers -from scrapy.conf import settings +from scrapy.utils.encoding import resolve_encoding class BaseResponseTest(unittest.TestCase): @@ -112,10 +112,13 @@ class BaseResponseTest(unittest.TestCase): body_str = body assert isinstance(response.body, str) - self.assertEqual(response.encoding, encoding) + self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_str) self.assertEqual(response.body_as_unicode(), body_unicode) + def _assert_response_encoding(self, response, encoding): + self.assertEqual(response.encoding, resolve_encoding(encoding)) + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): @@ -134,14 +137,14 @@ class TextResponseTest(BaseResponseTest): assert isinstance(r2, self.response_class) self.assertEqual(r2.url, "http://www.example.com/other") - self.assertEqual(r2.encoding, "cp852") + self._assert_response_encoding(r2, "cp852") self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3.encoding, "latin1") + self.assertEqual(r3._declared_encoding(), "latin1") def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) resp = self.response_class(u"http://www.example.com/") - self.assertEqual(resp.encoding, settings['DEFAULT_RESPONSE_ENCODING']) + self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') @@ -175,21 +178,64 @@ class TextResponseTest(BaseResponseTest): r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") + r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") + r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body="\xa8D") + r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body="\xa8D") - self.assertEqual(r1.headers_encoding(), "utf-8") - self.assertEqual(r2.headers_encoding(), None) - self.assertEqual(r2.encoding, 'utf-8') - self.assertEqual(r3.headers_encoding(), "iso-8859-1") - self.assertEqual(r3.encoding, 'iso-8859-1') - self.assertEqual(r4.headers_encoding(), None) - assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii' + self.assertEqual(r1._headers_encoding(), "utf-8") + self.assertEqual(r2._headers_encoding(), None) + self.assertEqual(r2._declared_encoding(), 'utf-8') + self._assert_response_encoding(r2, 'utf-8') + self.assertEqual(r3._headers_encoding(), "iso-8859-1") + self.assertEqual(r3._declared_encoding(), "iso-8859-1") + self.assertEqual(r4._headers_encoding(), None) + self.assertEqual(r5._headers_encoding(), None) + self._assert_response_encoding(r5, "utf-8") + assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3") self._assert_response_values(r3, 'iso-8859-1', u"\xa3") + self._assert_response_values(r6, 'gb18030', u"\u2015") + self._assert_response_values(r7, 'gb18030', u"\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + def test_declared_encoding_invalid(self): + """Check that unknown declared encodings are ignored""" + r = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=UKNOWN"]}, body="\xc2\xa3") + self.assertEqual(r._declared_encoding(), None) + self._assert_response_values(r, 'utf-8', u"\xa3") + + def test_utf16(self): + """Test utf-16 because UnicodeDammit is known to have problems with""" + r = self.response_class("http://www.example.com", body='\xff\xfeh\x00i\x00', encoding='utf-16') + self._assert_response_values(r, 'utf-16', u"hi") + + def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): + r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xef\xbb\xbfWORD\xe3\xab") + self.assertEqual(r6.encoding, 'utf-8') + self.assertEqual(r6.body_as_unicode(), u'\ufeffWORD\ufffd\ufffd') + + def test_replace_wrong_encoding(self): + """Test invalid chars are replaced properly""" + r = self.response_class("http://www.example.com", encoding='utf-8', body='PREFIX\xe3\xabSUFFIX') + # XXX: Policy for replacing invalid chars may suffer minor variations + # but it should always contain the unicode replacement char (u'\ufffd') + assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + + # Do not destroy html tags due to encoding bugs + r = self.response_class("http://example.com", encoding='utf-8', \ + body='\xf0<span>value</span>') + assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode()) + + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse + #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') + #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse @@ -231,7 +277,7 @@ class XmlResponseTest(TextResponseTest): body = "<xml></xml>" r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, settings['DEFAULT_RESPONSE_ENCODING'], body) + self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>""" r2 = self.response_class("http://www.example.com", body=body) diff --git a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index db59de2d5..1384c2237 100644 --- a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -10,7 +10,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = DuplicatesFilterMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw.open_spider(self.spider) def tearDown(self): diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py new file mode 100644 index 000000000..14aa78168 --- /dev/null +++ b/scrapy/tests/test_spider.py @@ -0,0 +1,101 @@ +from __future__ import with_statement + +import sys +import warnings + +from twisted.trial import unittest + +from scrapy.spider import BaseSpider +from scrapy.contrib.spiders.init import InitSpider +from scrapy.contrib.spiders.crawl import CrawlSpider +from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider + + +class BaseSpiderTest(unittest.TestCase): + + spider_class = BaseSpider + + class OldSpider(spider_class): + + domain_name = 'example.com' + extra_domain_names = ('example.org', 'example.net') + + + class OldSpiderWithoutExtradomains(spider_class): + + domain_name = 'example.com' + + + class NewSpider(spider_class): + + name = 'example.com' + allowed_domains = ('example.org', 'example.net') + + + def setUp(self): + warnings.simplefilter("always") + + def tearDown(self): + warnings.resetwarnings() + + def test_sep12_deprecation_warnings(self): + if sys.version_info[:2] < (2, 6): + # warnings.catch_warnings() was added in Python 2.6 + raise unittest.SkipTest("This test requires Python 2.6+") + with warnings.catch_warnings(record=True) as w: + spider = self.OldSpider() + self.assertEqual(len(w), 2) # one for domain_name & one for extra_domain_names + self.assert_(issubclass(w[-1].category, DeprecationWarning)) + + def test_sep12_backwards_compatibility(self): + spider = self.OldSpider() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains) + + spider = self.OldSpiderWithoutExtradomains() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + + spider = self.NewSpider() + self.assertEqual(spider.domain_name, 'example.com') + self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names) + self.assert_('example.net' in spider.extra_domain_names, spider.extra_domain_names) + + def test_base_spider(self): + spider = self.spider_class("example.com") + self.assertEqual(spider.name, 'example.com') + self.assertEqual(spider.start_urls, []) + self.assertEqual(spider.allowed_domains, []) + + def test_spider_args(self): + """Constructor arguments are assigned to spider attributes""" + spider = self.spider_class('example.com', foo='bar') + self.assertEqual(spider.foo, 'bar') + + def test_spider_without_name(self): + """Constructor arguments are assigned to spider attributes""" + self.assertRaises(ValueError, self.spider_class) + self.assertRaises(ValueError, self.spider_class, somearg='foo') + + +class InitSpiderTest(BaseSpiderTest): + + spider_class = InitSpider + +class XMLFeedSpiderTest(BaseSpiderTest): + + spider_class = XMLFeedSpider + +class CSVFeedSpiderTest(BaseSpiderTest): + + spider_class = CSVFeedSpider + +class CrawlSpiderTest(BaseSpiderTest): + + spider_class = CrawlSpider + + +if __name__ == '__main__': + unittest.main() diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index a0dfb492e..505b1a091 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware class TestHttpErrorMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpErrorMiddleware() self.req = Request('http://scrapytest.org') diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 6c71213aa..861595524 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -8,9 +8,9 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() - self.spider.domain_name = 'scrapytest.org' - self.spider.extra_domain_names = ['scrapy.org'] + self.spider = BaseSpider('foo') + self.spider.name = 'scrapytest.org' + self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] self.mw = OffsiteMiddleware() self.mw.spider_opened(self.spider) diff --git a/scrapy/tests/test_spidermiddleware_referer.py b/scrapy/tests/test_spidermiddleware_referer.py index ec11b0489..467c301ba 100644 --- a/scrapy/tests/test_spidermiddleware_referer.py +++ b/scrapy/tests/test_spidermiddleware_referer.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.referer import RefererMiddleware class TestRefererMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RefererMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urlfilter.py b/scrapy/tests/test_spidermiddleware_urlfilter.py index 21f776f17..45621599a 100644 --- a/scrapy/tests/test_spidermiddleware_urlfilter.py +++ b/scrapy/tests/test_spidermiddleware_urlfilter.py @@ -9,7 +9,7 @@ from scrapy.utils.url import canonicalize_url class TestUrlFilterMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlFilterMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/scrapy/tests/test_spidermiddleware_urllength.py index de0e2eae0..cbd80d488 100644 --- a/scrapy/tests/test_spidermiddleware_urllength.py +++ b/scrapy/tests/test_spidermiddleware_urllength.py @@ -12,7 +12,7 @@ class TestUrlLengthMiddleware(TestCase): settings.disabled = False settings.overrides['URLLENGTH_LIMIT'] = 25 - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlLengthMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index 612ee0eaa..786021898 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -9,7 +9,7 @@ from scrapy.stats.signals import stats_spider_opened, stats_spider_closing, \ class StatsCollectorTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_collector(self): stats = StatsCollector() diff --git a/scrapy/tests/test_utils_encoding.py b/scrapy/tests/test_utils_encoding.py new file mode 100644 index 000000000..b6800cd75 --- /dev/null +++ b/scrapy/tests/test_utils_encoding.py @@ -0,0 +1,25 @@ +import unittest + +from scrapy.utils.encoding import encoding_exists, resolve_encoding + +class UtilsEncodingTestCase(unittest.TestCase): + + _ENCODING_ALIASES = { + 'foo': 'cp1252', + 'bar': 'none', + } + + def test_resolve_encoding(self): + self.assertEqual(resolve_encoding('latin1', self._ENCODING_ALIASES), + 'latin1') + self.assertEqual(resolve_encoding('foo', self._ENCODING_ALIASES), + 'cp1252') + + def test_encoding_exists(self): + assert encoding_exists('latin1', self._ENCODING_ALIASES) + assert encoding_exists('foo', self._ENCODING_ALIASES) + assert not encoding_exists('bar', self._ENCODING_ALIASES) + assert not encoding_exists('none', self._ENCODING_ALIASES) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index 13d8be30c..dc46e1f91 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -1,7 +1,8 @@ +import operator import unittest from scrapy.utils.python import str_to_unicode, unicode_to_str, \ - memoizemethod_noargs, isbinarytext + memoizemethod_noargs, isbinarytext, equal_attributes class UtilsPythonTestCase(unittest.TestCase): def test_str_to_unicode(self): @@ -61,5 +62,52 @@ class UtilsPythonTestCase(unittest.TestCase): # finally some real binary bytes assert isbinarytext("\x02\xa3") + def test_equal_attributes(self): + class Obj: + pass + + a = Obj() + b = Obj() + # no attributes given return False + self.failIf(equal_attributes(a, b, [])) + # not existent attributes + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + a.x = 1 + b.x = 1 + # equal attribute + self.failUnless(equal_attributes(a, b, ['x'])) + + b.y = 2 + # obj1 has no attribute y + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + a.y = 2 + # equal attributes + self.failUnless(equal_attributes(a, b, ['x', 'y'])) + + a.y = 1 + # differente attributes + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + # test callable + a.meta = {} + b.meta = {} + self.failUnless(equal_attributes(a, b, ['meta'])) + + # compare ['meta']['a'] + a.meta['z'] = 1 + b.meta['z'] = 1 + + get_z = operator.itemgetter('z') + get_meta = operator.attrgetter('meta') + compare_z = lambda obj: get_z(get_meta(obj)) + + self.failUnless(equal_attributes(a, b, [compare_z, 'x'])) + # fail z equality + a.meta['z'] = 2 + self.failIf(equal_attributes(a, b, [compare_z, 'x'])) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 9281f4f40..71c14569e 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -1,9 +1,10 @@ import unittest +import urlparse from scrapy.xlib.BeautifulSoup import BeautifulSoup -from scrapy.http import Response, TextResponse +from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, \ - response_httprepr, get_cached_beautifulsoup + response_httprepr, get_cached_beautifulsoup, open_in_browser class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body='dummy_response') @@ -28,30 +29,46 @@ class ResponseUtilsTest(unittest.TestCase): self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode)) def test_get_base_url(self): - response = Response(url='http://example.org', body="""\ + response = HtmlResponse(url='https://example.org', body="""\ <html>\ <head><title>Dummy\ blahablsdfsal&\ """) self.assertEqual(get_base_url(response), 'http://example.org/something') + # relative url with absolute path + response = HtmlResponse(url='https://example.org', body="""\ + \ + Dummy\ + blahablsdfsal&\ + """) + self.assertEqual(get_base_url(response), 'https://example.org/absolutepath') + + # no scheme url + response = HtmlResponse(url='https://example.org', body="""\ + \ + Dummy\ + blahablsdfsal&\ + """) + self.assertEqual(get_base_url(response), 'https://noscheme.com/path') + def test_get_meta_refresh(self): body = """ Dummy blahablsdfsal& """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # refresh without url should return (None, None) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (None, None)) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # meta refresh in multiple lines @@ -59,17 +76,17 @@ class ResponseUtilsTest(unittest.TestCase): """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage')) # entities in the redirect url body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other')) # relative redirects body = """""" - response = Response(url='http://example.com/page/this.html', body=body) + response = TextResponse(url='http://example.com/page/this.html', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html')) # non-standard encodings (utf-16) @@ -80,7 +97,7 @@ class ResponseUtilsTest(unittest.TestCase): # non-ascii chars in the url (default encoding - utf8) body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) # non-ascii chars in the url (custom encoding - latin1) @@ -88,13 +105,8 @@ class ResponseUtilsTest(unittest.TestCase): response = TextResponse(url='http://example.com', body=body, encoding='latin1') self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) - # wrong encodings (possibly caused by truncated chunks) - body = """""" - response = Response(url='http://example.com', body=body) - self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT')) - # responses without refresh tag should return None None - response = Response(url='http://example.org') + response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) @@ -131,5 +143,18 @@ class ResponseUtilsTest(unittest.TestCase): assert soup1 is soup2 assert soup1 is not soup3 + def test_open_in_browser(self): + url = "http:///www.example.com/some/page.html" + body = " test page test body " + def browser_open(burl): + bbody = open(urlparse.urlparse(burl).path).read() + assert '' % url in bbody, " tag not added" + return True + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=browser_open), \ + "Browser not called" + self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \ + debug=True) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 05a22e8db..8c3d95fd5 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -136,12 +136,24 @@ class UrlUtilsTest(unittest.TestCase): 'http://rmc-offers.co.uk/productlist.asp?BCat=newvalue&CatID=60') def test_url_query_cleaner(self): - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id'), - 'product.html?id=200') - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']), - 'product.html?id=200&name=wired') - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo']), - 'product.html?id=200&foo=bar') + self.assertEqual('product.html?id=200', + url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'])) + self.assertEqual('product.html', + url_query_cleaner("product.html?foo=bar&name=wired", ['id'])) + self.assertEqual('product.html?id=200&name=wired', + url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name'])) + self.assertEqual('product.html?id', + url_query_cleaner("product.html?id&other=3&novalue=", ['id'])) + self.assertEqual('product.html?d=1&d=2&d=3', + url_query_cleaner("product.html?d=1&e=b&d=2&d=3&other=other", ['d'], unique=False)) + self.assertEqual('product.html?id=200&foo=bar', + url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo'])) + self.assertEqual('product.html?foo=bar&name=wired', + url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'], remove=True)) + self.assertEqual('product.html?name=wired', + url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'foo'], remove=True)) + self.assertEqual('product.html?foo=bar&name=wired', + url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'footo'], remove=True)) def test_canonicalize_url(self): # simplest case diff --git a/scrapy/utils/encoding.py b/scrapy/utils/encoding.py index 9eb06d942..c7d645041 100644 --- a/scrapy/utils/encoding.py +++ b/scrapy/utils/encoding.py @@ -1,11 +1,20 @@ import codecs -def add_encoding_alias(encoding, alias, overwrite=False): +from scrapy.conf import settings + +_ENCODING_ALIASES = dict(settings['ENCODING_ALIASES_BASE']) +_ENCODING_ALIASES.update(settings['ENCODING_ALIASES']) + +def encoding_exists(encoding, _aliases=_ENCODING_ALIASES): + """Returns ``True`` if encoding is valid, otherwise returns ``False``""" try: - codecs.lookup(alias) - alias_exists = True + codecs.lookup(resolve_encoding(encoding, _aliases)) except LookupError: - alias_exists = False - if overwrite or not alias_exists: - codec = codecs.lookup(encoding) - codecs.register(lambda x: codec if x == alias else None) + return False + return True + +def resolve_encoding(alias, _aliases=_ENCODING_ALIASES): + """Return the encoding the given alias maps to, or the alias as passed if + no mapping is found. + """ + return _aliases.get(alias.lower(), alias) diff --git a/scrapy/utils/fetch.py b/scrapy/utils/fetch.py deleted file mode 100644 index b489c75bf..000000000 --- a/scrapy/utils/fetch.py +++ /dev/null @@ -1,17 +0,0 @@ -from scrapy.http import Request -from scrapy.core.manager import scrapymanager - -def fetch(urls): - """Fetch a list of urls and return a list of the downloaded Scrapy - Responses. - - This is a blocking function not suitable for calling from spiders. Instead, - it is indended to be called from outside the framework such as Scrapy - commands or standalone scripts. - """ - responses = [] - requests = [Request(url, callback=responses.append, dont_filter=True) \ - for url in urls] - scrapymanager.runonce(*requests) - return responses - diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index d422fbde5..5e2e0b1c1 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -60,10 +60,10 @@ def remove_entities(text, keep=(), remove_illegal=True, encoding='utf-8'): return _ent_re.sub(convert_entity, str_to_unicode(text, encoding)) -def has_entities(text): - return bool(_ent_re.search(str_to_unicode(text))) +def has_entities(text, encoding=None): + return bool(_ent_re.search(str_to_unicode(text, encoding))) -def replace_tags(text, token=''): +def replace_tags(text, token='', encoding=None): """Replace all markup tags found in the given text by the given token. By default token is a null string so it just remove all tags. @@ -71,43 +71,44 @@ def replace_tags(text, token=''): Always returns a unicode string. """ - return _tag_re.sub(token, str_to_unicode(text)) + return _tag_re.sub(token, str_to_unicode(text, encoding)) -def remove_comments(text): +def remove_comments(text, encoding=None): """ Remove HTML Comments. """ - return re.sub('', u'', str_to_unicode(text), re.DOTALL) + return re.sub('', u'', str_to_unicode(text, encoding), re.DOTALL) -def remove_tags(text, which_ones=()): +def remove_tags(text, which_ones=(), encoding=None): """ Remove HTML Tags only. which_ones -- is a tuple of which tags we want to remove. if is empty remove all tags. """ if which_ones: - tags = ['<%s>|<%s .*?>|' % (tag,tag,tag) for tag in which_ones] + tags = ['<%s>|<%s .*?>|' % (tag, tag, tag) for tag in which_ones] regex = '|'.join(tags) else: regex = '<.*?>' retags = re.compile(regex, re.DOTALL | re.IGNORECASE) - return retags.sub(u'', str_to_unicode(text)) + return retags.sub(u'', str_to_unicode(text, encoding)) -def remove_tags_with_content(text, which_ones=()): +def remove_tags_with_content(text, which_ones=(), encoding=None): """ Remove tags and its content. which_ones -- is a tuple of which tags with its content we want to remove. if is empty do nothing. """ - text = str_to_unicode(text) + text = str_to_unicode(text, encoding) if which_ones: - tags = '|'.join(['<%s.*?' % (tag,tag) for tag in which_ones]) + tags = '|'.join(['<%s.*?' % (tag, tag) for tag in which_ones]) retags = re.compile(tags, re.DOTALL | re.IGNORECASE) text = retags.sub(u'', text) return text -def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): +def replace_escape_chars(text, which_ones=('\n', '\t', '\r'), replace_by=u'', \ + encoding=None): """ Remove escape chars. Default : \\n, \\t, \\r which_ones -- is a tuple of which escape chars we want to remove. @@ -117,10 +118,10 @@ def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): It defaults to '', so the escape chars are removed. """ for ec in which_ones: - text = text.replace(ec, str_to_unicode(replace_by)) - return str_to_unicode(text) + text = text.replace(ec, str_to_unicode(replace_by, encoding)) + return str_to_unicode(text, encoding) -def unquote_markup(text, keep=(), remove_illegal=True): +def unquote_markup(text, keep=(), remove_illegal=True, encoding=None): """ This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following: - removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA @@ -138,7 +139,7 @@ def unquote_markup(text, keep=(), remove_illegal=True): offset = match_e yield txt[offset:] - text = str_to_unicode(text) + text = str_to_unicode(text, encoding) ret_text = u'' for fragment in _get_fragments(text, _cdata_re): if isinstance(fragment, basestring): diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py new file mode 100644 index 000000000..c8a9cd7ea --- /dev/null +++ b/scrapy/utils/py26.py @@ -0,0 +1,107 @@ +""" +This module provides functions added in Python 2.6, which weren't yet available +in Python 2.5. The Python 2.6 function is used when available. +""" + +import sys +import os +import fnmatch +from shutil import copy2, copystat + +__all__ = ['cpu_count', 'copytree', 'ignore_patterns'] + +try: + import multiprocessing + cpu_count = multiprocessing.cpu_count +except ImportError: + def cpu_count(): + ''' + Returns the number of CPUs in the system + ''' + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif 'bsd' in sys.platform or sys.platform == 'darwin': + try: + num = int(os.popen('sysctl -n hw.ncpu').read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + + if num >= 1: + return num + else: + raise NotImplementedError('cannot determine number of cpus') + +if sys.version_info >= (2, 6): + from shutil import copytree, ignore_patterns +else: + try: + WindowsError + except NameError: + WindowsError = None + + class Error(EnvironmentError): + pass + + def ignore_patterns(*patterns): + def _ignore_patterns(path, names): + ignored_names = [] + for pattern in patterns: + ignored_names.extend(fnmatch.filter(names, pattern)) + return set(ignored_names) + return _ignore_patterns + + def copytree(src, dst, symlinks=False, ignore=None): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + os.makedirs(dst) + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + copytree(srcname, dstname, symlinks, ignore) + else: + copy2(srcname, dstname) + # XXX What about devices, sockets etc.? + except (IOError, os.error), why: + errors.append((srcname, dstname, str(why))) + # catch the Error from the recursive copytree so that we can + # continue with other files + except Error, err: + errors.extend(err.args[0]) + try: + copystat(src, dst) + except OSError, why: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) + if errors: + raise Error, errors + +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + import scrapy.xlib.simplejson as json diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7d43c4566..abf3625e3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,12 +6,9 @@ higher than 2.5 which is the lowest version supported by Scrapy. """ import re -import os -import fnmatch import inspect import weakref from functools import wraps -from shutil import copy2, copystat from sgmllib import SGMLParser class FixedSGMLParser(SGMLParser): @@ -64,13 +61,15 @@ def unique(list_, key=lambda x: x): return result -def str_to_unicode(text, encoding='utf-8'): +def str_to_unicode(text, encoding=None): """Return the unicode representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a unicode object without the risk of double-decoding problems (which can happen if you don't use the default 'ascii' encoding) """ + if encoding is None: + encoding = 'utf-8' if isinstance(text, str): return text.decode(encoding) elif isinstance(text, unicode): @@ -78,13 +77,15 @@ def str_to_unicode(text, encoding='utf-8'): else: raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__) -def unicode_to_str(text, encoding='utf-8'): +def unicode_to_str(text, encoding=None): """Return the str representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a str object without the risk of double-decoding problems (which can happen if you don't use the default 'ascii' encoding) """ + if encoding is None: + encoding = 'utf-8' if isinstance(text, unicode): return text.encode(encoding) elif isinstance(text, str): @@ -142,68 +143,6 @@ def isbinarytext(text): assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ return any(c in _BINARYCHARS for c in text) - -# ----- shutil.copytree function from Python 2.6 adds ignore argument ---- # - -try: - WindowsError -except NameError: - WindowsError = None - -class Error(EnvironmentError): - pass - -def ignore_patterns(*patterns): - def _ignore_patterns(path, names): - ignored_names = [] - for pattern in patterns: - ignored_names.extend(fnmatch.filter(names, pattern)) - return set(ignored_names) - return _ignore_patterns - -def copytree(src, dst, symlinks=False, ignore=None): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except (IOError, os.error), why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error, err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except OSError, why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.extend((src, dst, str(why))) - if errors: - raise Error, errors - -# ----- end of shutil.copytree function from Python 2.6 ---- # - - def get_func_args(func): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -216,3 +155,27 @@ def get_func_args(func): else: raise TypeError('%s is not callable' % type(func)) return func_args + +def equal_attributes(obj1, obj2, attributes): + """Compare two objects attributes""" + # not attributes given return False by default + if not attributes: + return False + + for attr in attributes: + # support callables like itemgetter + if callable(attr): + if not attr(obj1) == attr(obj2): + return False + else: + # check that objects has attribute + if not hasattr(obj1, attr): + return False + if not hasattr(obj2, attr): + return False + # compare object attributes + if not getattr(obj1, attr) == getattr(obj2, attr): + return False + # all attributes equal + return True + diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 6ed6f43a3..ca258ef44 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -18,7 +18,8 @@ from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse def body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ + assert isinstance(obj, (Response, basestring)), \ + "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): return obj.body_as_unicode() if unicode else obj.body elif isinstance(obj, str): @@ -26,16 +27,17 @@ def body_or_str(obj, unicode=True): else: return obj if unicode else obj.encode('utf-8') -BASEURL_RE = re.compile(r']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', re.DOTALL | re.IGNORECASE) +META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', \ + re.DOTALL | re.IGNORECASE) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv parameter of the HTML meta element from the given @@ -46,9 +48,7 @@ def get_meta_refresh(response): If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - encoding = getattr(response, 'encoding', None) or 'utf-8' - body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \ - errors='ignore')) + body_chunk = remove_entities(response.body_as_unicode()[0:4096]) match = META_REFRESH_RE.search(body_chunk) if match: interval = int(match.group('int')) @@ -92,7 +92,7 @@ def response_httprepr(response): s += response.body return s -def open_in_browser(response): +def open_in_browser(response, _openfunc=webbrowser.open): """Open the given response in a local web browser, populating the tag for external links to work """ @@ -106,4 +106,4 @@ def open_in_browser(response): fd, fname = tempfile.mkstemp('.html') os.write(fd, body) os.close(fd) - webbrowser.open("file://%s" % fname) + return _openfunc("file://%s" % fname) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 9307fc234..29432f0e9 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,4 +4,3 @@ from scrapy.utils.misc import arg_to_iter def iterate_spider_output(result): return [result] if isinstance(result, BaseItem) else arg_to_iter(result) - diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1c2fe18ae..3473fd487 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,9 +22,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - domains = [spider.domain_name] - domains.extend(spider.extra_domain_names) - return url_is_from_any_domain(url, domains) + return url_is_from_any_domain(url, spider.allowed_domains) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref @@ -87,29 +85,31 @@ def url_query_parameter(url, parameter, default=None, keep_blank_values=0): keep_blank_values=keep_blank_values) return queryparams.get(parameter, [default])[0] -def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='): - """Clean url arguments leaving only those passed in the parameterlist""" - try: - url = urlparse.urldefrag(url)[0] - base, query = url.split('?', 1) - parameters = [pair.split(kvsep, 1) for pair in query.split(sep)] - except: - base = url - query = "" - parameters = [] +def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='=', remove=False, unique=True): + """Clean url arguments leaving only those passed in the parameterlist. + + If remove is True, leave only those not in parameterlist. + If unique is False, do not remove duplicated keys + """ + url = urlparse.urldefrag(url)[0] + base, _, query = url.partition('?') + parameters = [ksv.partition(kvsep) for ksv in query.split(sep)] # unique parameters while keeping order - unique = {} + seen = set() querylist = [] - for pair in parameters: - k = pair[0] - if not unique.get(k): - querylist += [pair] - unique[k] = 1 - - query = sep.join([kvsep.join(pair) for pair in querylist if pair[0] in \ - parameterlist]) - return '?'.join([base, query]) + for k, s, v in parameters: + if unique and k in seen: + continue + elif remove and k in parameterlist: + continue + elif not remove and k not in parameterlist: + continue + else: + querylist.append([k, s, v]) + seen.add(k) + query = '?' + sep.join(''.join(ksv) for ksv in querylist) + return urlparse.urljoin(base, query) def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False): """Add or remove a parameter to a given url""" @@ -130,7 +130,8 @@ def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False) name+'='+new_value) return next_url -def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): +def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ + encoding=None): """Canonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value @@ -147,7 +148,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): For examples see the tests in scrapy.tests.test_utils_url """ - url = unicode_to_str(url) + url = unicode_to_str(url, encoding) scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) keyvals = cgi.parse_qsl(query, keep_blank_values) keyvals.sort() diff --git a/scrapy/xlib/simplejson/__init__.py b/scrapy/xlib/simplejson/__init__.py new file mode 100644 index 000000000..dcfd5413b --- /dev/null +++ b/scrapy/xlib/simplejson/__init__.py @@ -0,0 +1,437 @@ +r"""JSON (JavaScript Object Notation) is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. + +Encoding basic Python object hierarchies:: + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print json.dumps("\"foo\bar") + "\"foo\bar" + >>> print json.dumps(u'\u1234') + "\u1234" + >>> print json.dumps('\\') + "\\" + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> json.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> json.load(io)[0] == 'streaming API' + True + +Specializing JSON object decoding:: + + >>> import simplejson as json + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + >>> from decimal import Decimal + >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') + True + +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) + '[2.0, 1.0]' + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) + '[2.0, 1.0]' + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' + + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) +""" +__version__ = '2.1.1' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', + 'OrderedDict', +] + +__author__ = 'Bob Ippolito ' + +from decimal import Decimal + +from decoder import JSONDecoder, JSONDecodeError +from encoder import JSONEncoder +def _import_OrderedDict(): + import collections + try: + return collections.OrderedDict + except AttributeError: + import ordered_dict + return ordered_dict.OrderedDict +OrderedDict = _import_OrderedDict() + +def _import_c_make_encoder(): + try: + from simplejson._speedups import make_encoder + return make_encoder + except ImportError: + return None + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + use_decimal=False, +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If *indent* is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + default=default, use_decimal=use_decimal, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not use_decimal + and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, default=default, + use_decimal=use_decimal, **kw).encode(obj) + + +_default_decoder = JSONDecoder(encoding=None, object_hook=None, + object_pairs_hook=None) + + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + use_decimal=use_decimal, **kw) + + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and object_pairs_hook is None + and not use_decimal and not kw): + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + if object_pairs_hook is not None: + kw['object_pairs_hook'] = object_pairs_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant + if use_decimal: + if parse_float is not None: + raise TypeError("use_decimal=True implies parse_float=Decimal") + kw['parse_float'] = Decimal + return cls(encoding=encoding, **kw).decode(s) + + +def _toggle_speedups(enabled): + import simplejson.decoder as dec + import simplejson.encoder as enc + import simplejson.scanner as scan + c_make_encoder = _import_c_make_encoder() + if enabled: + dec.scanstring = dec.c_scanstring or dec.py_scanstring + enc.c_make_encoder = c_make_encoder + enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or + enc.py_encode_basestring_ascii) + scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner + else: + dec.scanstring = dec.py_scanstring + enc.c_make_encoder = None + enc.encode_basestring_ascii = enc.py_encode_basestring_ascii + scan.make_scanner = scan.py_make_scanner + dec.make_scanner = scan.make_scanner + global _default_decoder + _default_decoder = JSONDecoder( + encoding=None, + object_hook=None, + object_pairs_hook=None, + ) + global _default_encoder + _default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + ) diff --git a/scrapy/xlib/simplejson/decoder.py b/scrapy/xlib/simplejson/decoder.py new file mode 100644 index 000000000..4cf4015f6 --- /dev/null +++ b/scrapy/xlib/simplejson/decoder.py @@ -0,0 +1,421 @@ +"""Implementation of JSONDecoder +""" +import re +import sys +import struct + +from simplejson.scanner import make_scanner +def _import_c_scanstring(): + try: + from simplejson._speedups import scanstring + return scanstring + except ImportError: + return None +c_scanstring = _import_c_scanstring() + +__all__ = ['JSONDecoder'] + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + # The struct module in Python 2.4 would get frexp() out of range here + # when an endian is specified in the format string. Fixed in Python 2.5+ + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + + +class JSONDecodeError(ValueError): + """Subclass of ValueError with the following additional properties: + + msg: The unformatted error message + doc: The JSON document being parsed + pos: The start index of doc where parsing failed + end: The end index of doc where parsing failed (may be None) + lineno: The line corresponding to pos + colno: The column corresponding to pos + endlineno: The line corresponding to end (may be None) + endcolno: The column corresponding to end (may be None) + + """ + def __init__(self, msg, doc, pos, end=None): + ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) + self.msg = msg + self.doc = doc + self.pos = pos + self.end = end + self.lineno, self.colno = linecol(doc, pos) + if end is not None: + self.endlineno, self.endcolno = linecol(doc, pos) + else: + self.endlineno, self.endcolno = None, None + + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + + +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups + lineno, colno = linecol(doc, pos) + if end is None: + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, +} + +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def py_scanstring(s, end, encoding=None, strict=True, + _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + end = chunk.end() + content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows + if terminator == '"': + break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise JSONDecodeError(msg, s, end) + else: + _append(terminator) + continue + try: + esc = s[end] + except IndexError: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + # If not a unicode escape sequence, must be in the lookup table + if esc != 'u': + try: + char = _b[esc] + except KeyError: + msg = "Invalid \\escape: " + repr(esc) + raise JSONDecodeError(msg, s, end) + end += 1 + else: + # Unicode escape sequence + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise JSONDecodeError(msg, s, end) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise JSONDecodeError(msg, s, end) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise JSONDecodeError(msg, s, end) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) + return u''.join(chunks), end + + +# Use speedup if available +scanstring = c_scanstring or py_scanstring + +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, + object_pairs_hook, memo=None, + _w=WHITESPACE.match, _ws=WHITESPACE_STR): + # Backwards compatibility + if memo is None: + memo = {} + memo_get = memo.setdefault + pairs = [] + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty + nextchar = s[end:end + 1] + # Normally we expect nextchar == '"' + if nextchar != '"': + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + 1 + elif nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end) + end += 1 + while True: + key, end = scanstring(s, end, encoding, strict) + key = memo_get(key, key) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". + if s[end:end + 1] != ':': + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise JSONDecodeError("Expecting : delimiter", s, end) + + end += 1 + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + pairs.append((key, value)) + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 + + if nextchar == '}': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end - 1) + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + + end += 1 + if nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end - 1) + + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = dict(pairs) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + values = [] + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array + if nextchar == ']': + return values, end + 1 + _append = values.append + while True: + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + _append(value) + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end) + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + return values, end + +class JSONDecoder(object): + """Simple JSON decoder + + Performs the following translations in decoding by default: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + + """ + + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True, + object_pairs_hook=None): + """ + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + *strict* controls the parser's behavior when it encounters an + invalid control character in a string. The default setting of + ``True`` means that unescaped control characters are parse errors, if + ``False`` then control characters will be allowed in strings. + + """ + self.encoding = encoding + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.memo = {} + self.scan_once = make_scanner(self) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise JSONDecodeError("Extra data", s, end, len(s)) + return obj + + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` + beginning with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + + """ + try: + obj, end = self.scan_once(s, idx) + except StopIteration: + raise JSONDecodeError("No JSON object could be decoded", s, idx) + return obj, end diff --git a/scrapy/xlib/simplejson/encoder.py b/scrapy/xlib/simplejson/encoder.py new file mode 100644 index 000000000..cab845653 --- /dev/null +++ b/scrapy/xlib/simplejson/encoder.py @@ -0,0 +1,501 @@ +"""Implementation of JSONEncoder +""" +import re +from decimal import Decimal + +def _import_speedups(): + try: + from simplejson import _speedups + return _speedups.encode_basestring_ascii, _speedups.make_encoder + except ImportError: + return None, None +c_encode_basestring_ascii, c_make_encoder = _import_speedups() + +from simplejson.decoder import PosInf + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +FLOAT_REPR = repr + +def encode_basestring(s): + """Return a JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + return ESCAPE_DCT[match.group(0)] + return u'"' + ESCAPE.sub(replace, s) + u'"' + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + #return '\\u{0:04x}'.format(n) + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii) + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8', default=None, + use_decimal=False): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + + If use_decimal is true (not the default), ``decimal.Decimal`` will + be supported directly by the encoder. For the inverse, decode JSON + with ``parse_float=decimal.Decimal``. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.use_decimal = use_decimal + if isinstance(indent, (int, long)): + indent = ' ' * indent + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + if default is not None: + self.default = default + self.encoding = encoding + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from simplejson import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + if self.ensure_ascii: + return ''.join(chunks) + else: + return u''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, + _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on + # the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + key_memo = {} + if (_one_shot and c_make_encoder is not None + and not self.indent and not self.sort_keys): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, key_memo, self.use_decimal) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, self.use_decimal) + try: + return _iterencode(o, 0) + finally: + key_memo.clear() + + +class JSONEncoderForHTML(JSONEncoder): + """An encoder that produces JSON safe to embed in HTML. + + To embed JSON content in, say, a script tag on a web page, the + characters &, < and > should be escaped. They cannot be escaped + with the usual entities (e.g. &) because they are not expanded + within