diff --git a/AUTHORS b/AUTHORS index a0fbe722f..1392aa71f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,28 +1,25 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and -improved by Insophia (insophia.com), with the sponsorship of By Design (the -company behind Mydeco). +improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to +bootstrap the project. -Here is the list of the primary authors & contributors, along with their user -name (in Scrapy trac/subversion). Emails are intentionally left out to avoid -spam. +Here is the list of the primary authors & contributors: - * Pablo Hoffman (pablo) - * Daniel Graña (daniel) - * Martin Olveyra (olveyra) - * Gabriel García (elpolilla) - * Michael Cetrulo (samus_) - * Artem Bogomyagkov (artem) - * Damian Canabal (calarval) - * Andres Moreira (andres) - * Ismael Carnales (ismael) - * Matías Aguirre (omab) - * German Hoffman (german) - * Anibal Pacheco (anibal) + * Pablo Hoffman + * Daniel Graña + * Martin Olveyra + * Gabriel García + * Michael Cetrulo + * Artem Bogomyagkov + * Damian Canabal + * Andres Moreira + * Ismael Carnales + * Matías Aguirre + * German Hoffmann + * Anibal Pacheco * Bruno Deferrari * Shane Evans - -And here is the list of people who have helped to put the Scrapy homepage live: - - * Ezequiel Rivero (ezequiel) + * Ezequiel Rivero + * Patrick Mezard + * Rolando Espinoza diff --git a/bin/scrapy.tac b/bin/scrapy.tac new file mode 100644 index 000000000..35e25c56f --- /dev/null +++ b/bin/scrapy.tac @@ -0,0 +1,5 @@ +from twisted.application.service import Application +from scrapy.service import ScrapyService + +application = Application("Scrapy") +ScrapyService().setServiceParent(application) diff --git a/docs/experimental/crawlspider-v2.rst b/docs/experimental/crawlspider-v2.rst new file mode 100644 index 000000000..a1ea09c48 --- /dev/null +++ b/docs/experimental/crawlspider-v2.rst @@ -0,0 +1,128 @@ +.. _topics-crawlspider-v2: + +============== +CrawlSpider v2 +============== + +Introduction +============ + +TODO: introduction + +Rules Matching +============== + +TODO: describe purpose of rules + +Request Extractors & Processors +=============================== + +TODO: describe purpose of extractors & processors + +Examples +======== + +TODO: plenty of examples + + +.. module:: scrapy.contrib_exp.crawlspider.spider + :synopsis: CrawlSpider + + +Reference +========= + +CrawlSpider +----------- + +TODO: describe crawlspider + +.. class:: CrawlSpider + + TODO: describe class + + +.. module:: scrapy.contrib_exp.crawlspider.rules + :synopsis: Rules + +Rules +----- + +TODO: describe spider rules + +.. class:: Rule + + TODO: describe Rules class + + +.. module:: scrapy.contrib_exp.crawlspider.reqext + :synopsis: Request Extractors + +Request Extractors +------------------ + +TODO: describe extractors purpose + +.. class:: BaseSgmlRequestExtractor + + TODO: describe base extractor + +.. class:: SgmlRequestExtractor + + TODO: describe sgml extractor + +.. class:: XPathRequestExtractor + + TODO: describe xpath request extractor + + +.. module:: scrapy.contrib_exp.crawlspider.reqproc + :synopsis: Request Processors + +Request Processors +------------------ + +TODO: describe request processors + +.. class:: Canonicalize + + TODO: describe proc + +.. class:: Unique + + TODO: describe unique + +.. class:: FilterDomain + + TODO: describe filter domain + +.. class:: FilterUrl + + TODO: describe filter url + + +.. module:: scrapy.contrib_exp.crawlspider.matchers + :synopsis: Matchers + +Request/Response Matchers +------------------------- + +TODO: describe matchers + +.. class:: BaseMatcher + + TODO: describe base matcher + +.. class:: UrlMatcher + + TODO: describe url matcher + +.. class:: UrlRegexMatcher + + TODO: describe UrlListMatcher + +.. class:: UrlListMatcher + + TODO: describe url list matcher + + diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 47f4ee4ac..f63ceddd9 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -21,3 +21,4 @@ it's properly merged) . Use at your own risk. djangoitems scheduler-middleware + crawlspider-v2 diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7eec036cf..5dd61100e 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -128,7 +128,8 @@ Finally, here's the spider code:: class MininovaSpider(CrawlSpider): - domain_name = 'mininova.org' + name = 'mininova.org' + allowed_domains = ['mininova.org'] start_urls = ['http://www.mininova.org/today'] rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index e8aad451b..8fb6c7db5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -102,8 +102,8 @@ to parse the contents of those pages to extract :ref:`items `. To create a Spider, you must subclass :class:`scrapy.spider.BaseSpider`, and define the three main, mandatory, attributes: -* :attr:`~scrapy.spider.BaseSpider.domain_name`: identifies the Spider. It must - be unique, that is, you can't set the same domain name for different Spiders. +* :attr:`~scrapy.spider.BaseSpider.name`: identifies the Spider. It must be + unique, that is, you can't set the same name for different Spiders. * :attr:`~scrapy.spider.BaseSpider.start_urls`: is a list of URLs where the Spider will begin to crawl from. So, the first pages downloaded will be those @@ -128,7 +128,8 @@ This is the code for our first Spider, save it in a file named from scrapy.spider import BaseSpider class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -354,7 +355,8 @@ Let's add this code to our spider:: from scrapy.selector import HtmlXPathSelector class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -398,7 +400,8 @@ scraped so far, the code for our Spider should be like this:: from dmoz.items import DmozItem class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -420,8 +423,8 @@ scraped so far, the code for our Spider should be like this:: Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: - [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'Text Processing in Python'], 'link': [u'http://gnosis.cx/TPiP/'], 'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n']}) in - [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'XML Processing with Python'], 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], 'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n']}) in + [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in + [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], link=[u'http://www.informit.com/store/product.aspx?isbn=0130211192'], title=[u'XML Processing with Python']) in Storing the data (using an Item Pipeline) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ec3679649..dfd30d3b2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -199,7 +199,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' - domain_name = 'intranet.example.com' + name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e8f6bef75..cbde79421 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -52,7 +52,7 @@ Exporter to export scraped items to different files, one per spider:: self.files = {} def spider_opened(self, spider): - file = open('%s_products.xml' % spider.domain_name, 'w+b') + file = open('%s_products.xml' % spider.name, 'w+b') self.files[spider] = file self.exporter = XmlItemExporter(file) self.exporter.start_exporting() diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index fc7367bac..972d6ea54 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -105,10 +105,10 @@ every time a domain/spider is opened and closed:: dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def spider_opened(self, spider): - log.msg("opened spider %s" % spider.domain_name) + log.msg("opened spider %s" % spider.name) def spider_closed(self, spider): - log.msg("closed spider %s" % spider.domain_name) + log.msg("closed spider %s" % spider.name) .. _topics-extensions-ref-manager: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 3e3cd94d0..649b2667e 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -79,7 +79,8 @@ This is how the spider would look so far:: from scrapy.contrib.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allowed_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index d4e185aa1..ba69c93b3 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -98,10 +98,10 @@ spider returns multiples items with the same id:: del self.duplicates[spider] def process_item(self, spider, item): - if item.id in self.duplicates[spider]: + if item['id'] in self.duplicates[spider]: raise DropItem("Duplicate item found: %s" % item) else: - self.duplicates[spider].add(item.id) + self.duplicates[spider].add(item['id']) return item Built-in Item Pipelines reference diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index f57128287..5e1a0e4bb 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -129,3 +129,14 @@ scrapy.log module Log level for debugging messages (recommended level for development) +Logging settings +================ + +These settings can be used to configure the logging: + +* :setting:`LOG_ENABLED` +* :setting:`LOG_ENCODING` +* :setting:`LOG_FILE` +* :setting:`LOG_LEVEL` +* :setting:`LOG_STDOUT` + diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index cffcfdee4..084108164 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -321,7 +321,7 @@ user name and password. You can use the :meth:`FormRequest.from_response` method for this job. Here's an example spider which uses it:: class LoginSpider(BaseSpider): - domain_name = 'example.com' + name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] def parse(self, response): @@ -466,12 +466,14 @@ TextResponse objects .. attribute:: TextResponse.encoding - A string with the encoding of this response. The encoding is resolved in the - following order: + A string with the encoding of this response. The encoding is resolved by + trying the following mechanisms, in order: 1. the encoding passed in the constructor `encoding` argument - 2. the encoding declared in the Content-Type HTTP header + 2. the encoding declared in the Content-Type HTTP header. If this + encoding is not valid (ie. unknown), it is ignored and the next + resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class doesn't provide any special functionality for this. However, the @@ -483,23 +485,11 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.headers_encoding() - - Returns a string with the encoding declared in the headers (ie. the - Content-Type HTTP header). - - .. method:: TextResponse.body_encoding() - - Returns a string with the encoding of the body, either declared or inferred - from its contents. The body encoding declaration is implemented in - :class:`TextResponse` subclasses such as: :class:`HtmlResponse` or - :class:`XmlResponse`. - .. method:: TextResponse.body_as_unicode() Returns the body of the response as unicode. This is equivalent to:: - response.body.encode(response.encoding) + response.body.decode(response.encoding) But **not** equivalent to:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b3dbc78f..e64ee329b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -340,16 +340,6 @@ Default: ``True`` Whether to collect depth stats. -.. setting:: DOMAIN_SCHEDULER - -SPIDER_SCHEDULER ----------------- - -Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` - -The Spider Scheduler to use. The spider scheduler returns the next spider to -scrape. - .. setting:: DOWNLOADER_DEBUG DOWNLOADER_DEBUG @@ -418,6 +408,15 @@ supported. Example:: DOWNLOAD_DELAY = 0.25 # 250 ms of delay +This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting (which is enabled by default). By default, Scrapy doesn't wait a fixed +amount of time between requests, but uses a random interval between 0.5 and 1.5 +* :setting:`DOWNLOAD_DELAY`. + +Another way to change the download delay (per spider, instead of globally) is +by using the ``download_delay`` spider attribute, which takes more precedence +than this setting. + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -439,6 +438,69 @@ The class used to detect and filter duplicate requests. The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint (using ``scrapy.utils.request.request_fingerprint``) and grouping per domain. +.. setting:: ENCODING_ALIASES + +ENCODING_ALIASES +---------------- + +Default: ``{}`` + +A mapping of custom encoding aliases for your project, where the keys are the +aliases (and must be lower case) and the values are the encodings they map to. + +This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which +contains some default mappings. + +.. setting:: ENCODING_ALIASES_BASE + +ENCODING_ALIASES_BASE +--------------------- + +Default:: + + { + # gb2312 is superseded by gb18030 + 'gb2312': 'gb18030', + 'chinese': 'gb18030', + 'csiso58gb231280': 'gb18030', + 'euc- cn': 'gb18030', + 'euccn': 'gb18030', + 'eucgb2312-cn': 'gb18030', + 'gb2312-1980': 'gb18030', + 'gb2312-80': 'gb18030', + 'iso- ir-58': 'gb18030', + # gbk is superseded by gb18030 + 'gbk': 'gb18030', + '936': 'gb18030', + 'cp936': 'gb18030', + 'ms936': 'gb18030', + # latin_1 is a subset of cp1252 + 'latin_1': 'cp1252', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'l1': 'cp1252', + # others + 'zh-cn': 'gb18030', + 'win-1251': 'cp1251', + 'macintosh' : 'mac_roman', + 'x-sjis': 'shift_jis', + } + +The default encoding aliases defined in Scrapy. Don't override this setting in +your project, override :setting:`ENCODING_ALIASES` instead. + +The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is +due to a well known browser hack. For more information see: `Character +encodings in HTML`_. + +.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 +.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 +.. _Character encodings in HTML: http://en.wikipedia.org/wiki/Character_encodings_in_HTML + .. setting:: EXTENSIONS EXTENSIONS @@ -517,7 +579,16 @@ LOG_ENABLED Default: ``True`` -Enable logging. +Whether to enable logging. + +.. setting:: LOG_ENCODING + +LOG_ENCODING +------------ + +Default: ``'utf-8'`` + +The encoding to use for logging. .. setting:: LOG_FILE @@ -677,6 +748,27 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' +.. setting:: RANDOMIZE_DOWNLOAD_DELAY + +RANDOMIZE_DOWNLOAD_DELAY +------------------------ + +Default: ``True`` + +If enabled, Scrapy will wait a random amount of time (between 0.5 and 1.5 +* :setting:`DOWNLOAD_DELAY`) while fetching requests from the same +spider. + +This randomization decreases the chance of the crawler being detected (and +subsequently blocked) by sites which analyze requests looking for statistically +significant similarities in the time between their times. + +The randomization policy is the same used by `wget`_ ``--random-wait`` option. + +If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. + +.. _wget: http://www.gnu.org/software/wget/manual/wget.html + .. setting:: REDIRECT_MAX_TIMES REDIRECT_MAX_TIMES @@ -773,7 +865,7 @@ The scheduler to use for crawling. SCHEDULER_ORDER --------------- -Default: ``'BFO'`` +Default: ``'DFO'`` Scope: ``scrapy.core.scheduler`` @@ -858,6 +950,16 @@ Example:: SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev'] +.. setting:: SPIDER_SCHEDULER + +SPIDER_SCHEDULER +---------------- + +Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` + +The Spider Scheduler to use. The spider scheduler returns the next spider to +scrape. + .. setting:: STATS_CLASS STATS_CLASS diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 6e3de9bc1..45d9cf0c3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -163,7 +163,7 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function. Here's an example of how you would call it from your spider:: class MySpider(BaseSpider): - domain_name = 'example.com' + ... def parse(self, response): if response.url == 'http://www.example.com/products.php': diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c0e57ed06..2889378cd 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -64,19 +64,18 @@ single Python class that defines one or more of the following methods: This method is called for each response that goes through the spider middleware and into the spider, for processing. - :meth:`process_spider_input` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item` - objects. + :meth:`process_spider_input` should return ``None`` or raise and + exception. - If returns ``None``, Scrapy will continue processing this response, + If it returns ``None``, Scrapy will continue processing this response, executing all other middlewares until, finally, the response is handled to the spider for processing. - If returns an iterable, Scrapy won't bother calling any other spider - middleware :meth:`process_spider_input` and will return the iterable - back in the other direction for the :meth:`process_spider_output` to - process it, or the :meth:`process_spider_exception` if it raised an - exception. + If it raises an exception, Scrapy won't bother calling any other spider + middleware :meth:`process_spider_input` and will call the request + errback. The output of the errback is chained back in the other + direction for :meth:`process_spider_output` to process it, or + :meth:`process_spider_exception` if it raised an exception. :param reponse: the response being processed :type response: :class:`~scrapy.http.Response` object @@ -89,7 +88,7 @@ single Python class that defines one or more of the following methods: This method is called with the results returned from the Spider, after it has processed the response. - + :meth:`process_spider_output` must return an iterable of :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item` objects. @@ -210,11 +209,8 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. - This middleware filters out every request whose host names don't match - :attr:`~scrapy.spider.BaseSpider.domain_name`, or the spider - :attr:`~scrapy.spider.BaseSpider.domain_name` prefixed by "www.". - Spider can add more domains to exclude using - :attr:`~scrapy.spider.BaseSpider.extra_domain_names` attribute. + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e60549098..b69f9a445 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -70,20 +70,22 @@ BaseSpider requests the given ``start_urls``/``start_requests``, and calls the spider's method ``parse`` for each of the resulting responses. - .. attribute:: domain_name + .. attribute:: name - A string which defines the domain name for this spider, which will also be - the unique identifier for this spider (which means you can't have two - spider with the same ``domain_name``). This is the most important spider - attribute and it's required, and it's the name by which Scrapy will known - the spider. + A string which defines the name for this spider. The spider name is how + the spider is located (and instantiated) by Scrapy, so it must be + unique. However, nothing prevents you from instantiating more than one + instance of the same spider. This is the most important spider attribute + and it's required. - .. attribute:: extra_domain_names + Is recommended to name your spiders after the domain that their crawl. - An optional list of strings containing additional domains that this - spider is allowed to crawl. Requests for URLs not belonging to the - domain name specified in :attr:`domain_name` or this list won't be - followed. + .. attribute:: allowed_domains + + An optional list of strings containing domains that this spider is + allowed to crawl. Requests for URLs not belonging to the domain names + specified in this list won't be followed if + :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls @@ -144,7 +146,7 @@ BaseSpider .. method:: log(message, [level, component]) Log a message using the :func:`scrapy.log.msg` function, automatically - populating the domain argument with the :attr:`domain_name` of this + populating the spider argument with the :attr:`name` of this spider. For more information see :ref:`topics-logging`. @@ -157,7 +159,8 @@ Let's see an example:: from scrapy.spider import BaseSpider class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -177,7 +180,8 @@ Another example returning multiples Requests and Items from a single callback:: from myproject.items import MyItem class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -254,7 +258,8 @@ Let's now take a look at an example CrawlSpider with rules:: from scrapy.item import Item class MySpider(CrawlSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com'] rules = ( @@ -378,7 +383,8 @@ These spiders are pretty easy to use, let's have at one example:: from myproject.items import TestItem class MySpider(XMLFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.xml'] iterator = 'iternodes' # This is actually unnecesary, since it's the default value itertag = 'item' @@ -435,7 +441,8 @@ Let's see an example similar to the previous one, but using a from myproject.items import TestItem class MySpider(CSVFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.csv'] delimiter = ';' headers = ['id', 'name', 'description'] diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 971a33364..34a3c8b7c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -204,15 +204,15 @@ MemoryStatsCollector A simple stats collector that keeps the stats of the last scraping run (for each spider) in memory, after they're closed. The stats can be accessed - through the :attr:`domain_stats` attribute, which is a dict keyed by spider + through the :attr:`spider_stats` attribute, which is a dict keyed by spider domain name. This is the default Stats Collector used in Scrapy. - .. attribute:: domain_stats + .. attribute:: spider_stats - A dict of dicts (keyed by spider domain name) containing the stats of - the last scraping run for each domain. + A dict of dicts (keyed by spider name) containing the stats of the last + scraping run for each spider. DummyStatsCollector ------------------- @@ -240,11 +240,11 @@ SimpledbStatsCollector In addition to the existing stats keys the following keys are added at persitance time: - * ``domain``: the spider domain (so you can use it later for querying stats - for that domain) + * ``spider``: the spider name (so you can use it later for querying stats + for that spider) * ``timestamp``: the timestamp when the stats were persisited - Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB + Both the ``spider`` and ``timestamp`` are used for generating the SimpleDB item name in order to avoid overwriting stats of previous scraping runs. As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and diff --git a/examples/experimental/googledir/googledir/__init__.py b/examples/experimental/googledir/googledir/__init__.py new file mode 100644 index 000000000..3104ef709 --- /dev/null +++ b/examples/experimental/googledir/googledir/__init__.py @@ -0,0 +1 @@ +# googledir project diff --git a/examples/experimental/googledir/googledir/items.py b/examples/experimental/googledir/googledir/items.py new file mode 100644 index 000000000..decc2c9ba --- /dev/null +++ b/examples/experimental/googledir/googledir/items.py @@ -0,0 +1,16 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class GoogledirItem(Item): + + name = Field(default='') + url = Field(default='') + description = Field(default='') + + def __str__(self): + return "Google Category: name=%s url=%s" \ + % (self['name'], self['url']) diff --git a/examples/experimental/googledir/googledir/pipelines.py b/examples/experimental/googledir/googledir/pipelines.py new file mode 100644 index 000000000..f775b254c --- /dev/null +++ b/examples/experimental/googledir/googledir/pipelines.py @@ -0,0 +1,22 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +from scrapy.core.exceptions import DropItem + +class FilterWordsPipeline(object): + """ + A pipeline for filtering out items which contain certain + words in their description + """ + + # put all words in lowercase + words_to_filter = ['politics', 'religion'] + + def process_item(self, spider, item): + for word in self.words_to_filter: + if word in unicode(item['description']).lower(): + raise DropItem("Contains forbidden word: %s" % word) + else: + return item diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py new file mode 100644 index 000000000..4e3c11163 --- /dev/null +++ b/examples/experimental/googledir/googledir/settings.py @@ -0,0 +1,21 @@ +# Scrapy settings for googledir project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'googledir' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['googledir.spiders'] +NEWSPIDER_MODULE = 'googledir.spiders' +DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + +ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline'] diff --git a/examples/experimental/googledir/googledir/spiders/__init__.py b/examples/experimental/googledir/googledir/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py new file mode 100644 index 000000000..2ed7c52a6 --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -0,0 +1,41 @@ +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule + +from googledir.items import GoogledirItem + +class GoogleDirectorySpider(CrawlSpider): + + name = 'google_directory' + allowed_domains = ['directory.google.com'] + start_urls = ['http://directory.google.com/'] + + rules = ( + # search for categories pattern and follow links + Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True), + ) + + def parse_category(self, response): + # The main selector we're using to extract data from the page + main_selector = HtmlXPathSelector(response) + + # The XPath to website links in the directory page + xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font' + + # Get a list of (sub) selectors to each website node pointed by the XPath + sub_selectors = main_selector.select(xpath) + + # Iterate over the sub-selectors to extract data for each website + for selector in sub_selectors: + item = GoogledirItem() + + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('name', 'a/text()') + l.add_xpath('url', 'a/@href') + l.add_xpath('description', 'font[2]/text()') + + # Here we populate the item and yield it + yield l.load_item() + +SPIDER = GoogleDirectorySpider() + diff --git a/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py new file mode 100644 index 000000000..552421ac3 --- /dev/null +++ b/examples/experimental/googledir/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings') + +from scrapy.command.cmdline import execute +execute() diff --git a/examples/experimental/imdb/imdb/__init__.py b/examples/experimental/imdb/imdb/__init__.py new file mode 100644 index 000000000..5bb534f79 --- /dev/null +++ b/examples/experimental/imdb/imdb/__init__.py @@ -0,0 +1 @@ +# package diff --git a/examples/experimental/imdb/imdb/items.py b/examples/experimental/imdb/imdb/items.py new file mode 100644 index 000000000..03bb5c2c3 --- /dev/null +++ b/examples/experimental/imdb/imdb/items.py @@ -0,0 +1,12 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class ImdbItem(Item): + # define the fields for your item here like: + # name = Field() + title = Field() + url = Field() diff --git a/examples/experimental/imdb/imdb/pipelines.py b/examples/experimental/imdb/imdb/pipelines.py new file mode 100644 index 000000000..e60714159 --- /dev/null +++ b/examples/experimental/imdb/imdb/pipelines.py @@ -0,0 +1,8 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +class ImdbPipeline(object): + def process_item(self, spider, item): + return item diff --git a/examples/experimental/imdb/imdb/settings.py b/examples/experimental/imdb/imdb/settings.py new file mode 100644 index 000000000..de026dc14 --- /dev/null +++ b/examples/experimental/imdb/imdb/settings.py @@ -0,0 +1,20 @@ +# Scrapy settings for imdb project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'imdb' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['imdb.spiders'] +NEWSPIDER_MODULE = 'imdb.spiders' +DEFAULT_ITEM_CLASS = 'imdb.items.ImdbItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + diff --git a/examples/experimental/imdb/imdb/spiders/__init__.py b/examples/experimental/imdb/imdb/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/imdb/imdb/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py new file mode 100644 index 000000000..8c2ebcd01 --- /dev/null +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -0,0 +1,141 @@ +from scrapy.http import Request +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize, \ + FilterDupes, FilterUrl +from scrapy.utils.url import urljoin_rfc + +from imdb.items import ImdbItem, Field + +from itertools import chain, imap, izip + +class UsaOpeningWeekMovie(ImdbItem): + pass + +class UsaTopWeekMovie(ImdbItem): + pass + +class Top250Movie(ImdbItem): + rank = Field() + rating = Field() + year = Field() + votes = Field() + +class MovieItem(ImdbItem): + release_date = Field() + tagline = Field() + + +class ImdbSiteSpider(CrawlSpider): + name = 'imdb.com' + allowed_domains = ['imdb.com'] + start_urls = ['http://www.imdb.com/'] + + # extract requests using this classes from urls matching 'follow' flag + request_extractors = [ + SgmlRequestExtractor(tags=['a'], attrs=['href']), + ] + + # process requests using this classes from urls matching 'follow' flag + request_processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(deny=r'/tt\d+/$'), # deny movie url as we will dispatch + # manually the movie requests + ] + + # include domain bit for demo purposes + rules = ( + # these two rules expects requests from start url + Rule(r'imdb.com/nowplaying/$', 'parse_now_playing'), + Rule(r'imdb.com/chart/top$', 'parse_top_250'), + # this rule will parse requests manually dispatched + Rule(r'imdb.com/title/tt\d+/$', 'parse_movie_info'), + ) + + def parse_now_playing(self, response): + """Scrapes USA openings this week and top 10 in week""" + self.log("Parsing USA Top Week") + hxs = HtmlXPathSelector(response) + + _urljoin = lambda url: self._urljoin(response, url) + + # + # openings this week + # + openings = hxs.select('//table[@class="movies"]//a[@class="title"]') + boxoffice = hxs.select('//table[@class="boxoffice movies"]//a[@class="title"]') + + opening_titles = openings.select('text()').extract() + opening_urls = imap(_urljoin, openings.select('@href').extract()) + + box_titles = boxoffice.select('text()').extract() + box_urls = imap(_urljoin, boxoffice.select('@href').extract()) + + # items + opening_items = (UsaOpeningWeekMovie(title=title, url=url) + for (title, url) + in izip(opening_titles, opening_urls)) + + box_items = (UsaTopWeekMovie(title=title, url=url) + for (title, url) + in izip(box_titles, box_urls)) + + # movie requests + requests = imap(self.make_requests_from_url, + chain(opening_urls, box_urls)) + + return chain(opening_items, box_items, requests) + + def parse_top_250(self, response): + """Scrapes movies from top 250 list""" + self.log("Parsing Top 250") + hxs = HtmlXPathSelector(response) + + # scrap each row in the table + rows = hxs.select('//div[@id="main"]/table/tr//a/ancestor::tr') + for row in rows: + fields = row.select('td//text()').extract() + url, = row.select('td//a/@href').extract() + url = self._urljoin(response, url) + + item = Top250Movie() + item['title'] = fields[2] + item['url'] = url + item['rank'] = fields[0] + item['rating'] = fields[1] + item['year'] = fields[3] + item['votes'] = fields[4] + + # scrapped top250 item + yield item + # fetch movie + yield self.make_requests_from_url(url) + + def parse_movie_info(self, response): + """Scrapes movie information""" + self.log("Parsing Movie Info") + hxs = HtmlXPathSelector(response) + selector = hxs.select('//div[@class="maindetails"]') + + item = MovieItem() + # set url + item['url'] = response.url + + # use item loader for other attributes + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('title', './/h1/text()') + l.add_xpath('release_date', './/h5[text()="Release Date:"]' + '/following-sibling::div/text()') + l.add_xpath('tagline', './/h5[text()="Tagline:"]' + '/following-sibling::div/text()') + + yield l.load_item() + + def _urljoin(self, response, url): + """Helper to convert relative urls to absolute""" + return urljoin_rfc(response.url, url, response.encoding) + +SPIDER = ImdbSiteSpider() diff --git a/examples/experimental/imdb/scrapy-ctl.py b/examples/experimental/imdb/scrapy-ctl.py new file mode 100644 index 000000000..df57621b3 --- /dev/null +++ b/examples/experimental/imdb/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'imdb.settings') + +from scrapy.command.cmdline import execute +execute() diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index 054cef022..2af52bf1f 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -6,7 +6,8 @@ from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allow_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/examples/scripts/count_and_follow_links.py b/examples/scripts/count_and_follow_links.py deleted file mode 100644 index 4ead870fc..000000000 --- a/examples/scripts/count_and_follow_links.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Simple script to follow links from a start url. The links are followed in no -particular order. - -Usage: -count_and_follow_links.py - -Example: -count_and_follow_links.py http://scrapy.org/ 20 - -For each page visisted, this script will print the page body size and the -number of links found. -""" - -import sys -from urlparse import urljoin - -from scrapy.crawler import Crawler -from scrapy.selector import HtmlXPathSelector -from scrapy.http import Request, HtmlResponse - -links_followed = 0 - -def parse(response): - global links_followed - links_followed += 1 - if links_followed >= links_to_follow: - crawler.stop() - - # ignore non-HTML responses - if not isinstance(response, HtmlResponse): - return - - links = HtmlXPathSelector(response).select('//a/@href').extract() - abslinks = [urljoin(response.url, l) for l in links] - - print "page %2d/%d: %s" % (links_followed, links_to_follow, response.url) - print " size : %d bytes" % len(response.body) - print " links: %d" % len(links) - print - - return [Request(l, callback=parse) for l in abslinks] - -if len(sys.argv) != 3: - print __doc__ - sys.exit(2) - -start_url, links_to_follow = sys.argv[1], int(sys.argv[2]) -request = Request(start_url, callback=parse) -crawler = Crawler() -crawler.crawl(request) diff --git a/extras/sql/scraping.sql b/extras/sql/scraping.sql deleted file mode 100644 index c3314f5a1..000000000 --- a/extras/sql/scraping.sql +++ /dev/null @@ -1,72 +0,0 @@ -DROP TABLE IF EXISTS `url_history`; -DROP TABLE IF EXISTS `version`; -DROP TABLE IF EXISTS `url_status`; -DROP TABLE IF EXISTS `ticket`; -DROP TABLE IF EXISTS `domain_stats`; -DROP TABLE IF EXISTS `domain_stats_history`; -DROP TABLE IF EXISTS `domain_data_history`; - -CREATE TABLE `ticket` ( - `guid` char(40) NOT NULL, - `domain` varchar(255) default NULL, - `url` varchar(2048) default NULL, - `url_hash` char(40) default NULL, -- so we can join to url_status - PRIMARY KEY (`guid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `version` ( - `id` bigint(20) NOT NULL auto_increment, - `guid` char(40) NOT NULL, - `version` char(40) NOT NULL, - `seen` datetime NOT NULL, - PRIMARY KEY (`id`), - FOREIGN KEY (`guid`) REFERENCES ticket(guid) ON UPDATE CASCADE ON DELETE CASCADE, - UNIQUE KEY (`version`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_status` ( - -- see http://support.microsoft.com/kb/q208427/ for explanation of 2048 - `url_hash` char(40) NOT NULL, -- for faster searches - `url` varchar(2048) NOT NULL, - `parent_hash` char(40) default NULL, -- the url that was followed to this one - for reporting - `last_version` char(40) default NULL, -- can be null if it generated an error the last time is was checked - `last_checked` datetime NOT NULL, - PRIMARY KEY (`url_hash`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_history` ( - `url_hash` char(40) NOT NULL, - `version` char(40) NOT NULL, - `postdata_hash` char(40) default NULL, - `created` datetime NOT NULL, - PRIMARY KEY (`version`), - FOREIGN KEY (`url_hash`) REFERENCES url_status(url_hash) ON UPDATE CASCADE ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats` ( - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` text, - PRIMARY KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats_history` ( - `id` bigint(20) NOT NULL auto_increment, - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` varchar(2048) NOT NULL, - `stored` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`), - KEY `stored` (`stored`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_data_history` ( - `domain` varchar(255) NOT NULL, - `stored` datetime NOT NULL, - `data` text, - KEY `domain_stored` (`domain`, `stored`), - KEY `domain` (`domain`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 2ecc7749c..184b51cb1 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 8, 0, '', 0) -__version__ = "0.8" +version_info = (0, 9, 0, 'dev') +__version__ = "0.9-dev" import sys, os, warnings @@ -17,11 +17,6 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches -# add some common encoding aliases not included by default in Python -from scrapy.utils.encoding import add_encoding_alias -add_encoding_alias('gb2312', 'zh-cn') -add_encoding_alias('cp1251', 'win-1251') - # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index 25f531cea..1a190549d 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -7,20 +7,14 @@ import cProfile import scrapy from scrapy import log -from scrapy.spider import spiders from scrapy.xlib import lsprofcalltree from scrapy.conf import settings from scrapy.command.models import ScrapyCommand +from scrapy.utils.signal import send_catch_log -# This dict holds information about the executed command for later use -command_executed = {} - -def _save_command_executed(cmdname, cmd, args, opts): - """Save command executed info for later reference""" - command_executed['name'] = cmdname - command_executed['class'] = cmd - command_executed['args'] = args[:] - command_executed['opts'] = opts.__dict__.copy() +# Signal that carries information about the command which was executed +# args: cmdname, cmdobj, args, opts +command_executed = object() def _find_commands(dir): try: @@ -127,7 +121,8 @@ def execute(argv=None): sys.exit(2) del args[0] # remove command name from args - _save_command_executed(cmdname, cmd, args, opts) + send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \ + args=args, opts=opts) from scrapy.core.manager import scrapymanager scrapymanager.configure(control_reactor=True) ret = _run_command(cmd, args, opts) @@ -136,23 +131,25 @@ def execute(argv=None): def _run_command(cmd, args, opts): if opts.profile or opts.lsprof: - if opts.profile: - log.msg("writing cProfile stats to %r" % opts.profile) - if opts.lsprof: - log.msg("writing lsprof stats to %r" % opts.lsprof) - loc = locals() - p = cProfile.Profile() - p.runctx('ret = cmd.run(args, opts)', globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) - k = lsprofcalltree.KCacheGrind(p) - if opts.lsprof: - with open(opts.lsprof, 'w') as f: - k.output(f) - ret = loc['ret'] + return _run_command_profiled(cmd, args, opts) else: - ret = cmd.run(args, opts) - return ret + return cmd.run(args, opts) + +def _run_command_profiled(cmd, args, opts): + if opts.profile: + log.msg("writing cProfile stats to %r" % opts.profile) + if opts.lsprof: + log.msg("writing lsprof stats to %r" % opts.lsprof) + loc = locals() + p = cProfile.Profile() + p.runctx('ret = cmd.run(args, opts)', globals(), loc) + if opts.profile: + p.dump_stats(opts.profile) + k = lsprofcalltree.KCacheGrind(p) + if opts.lsprof: + with open(opts.lsprof, 'w') as f: + k.output(f) + return loc['ret'] if __name__ == '__main__': execute() diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index 3a04423fc..9c02b08e3 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -1,20 +1,27 @@ +from scrapy import log from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.conf import settings +from scrapy.http import Request +from scrapy.spider import spiders +from scrapy.utils.url import is_url +from collections import defaultdict class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] ..." + return "[options] ..." def short_desc(self): - return "Start crawling a domain or URL" + return "Start crawling from a spider or URL" def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider when arguments are urls") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \ help="don't follow links (for use with URLs only)") @@ -24,4 +31,45 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - scrapymanager.runonce(*args) + urls, names = self._split_urls_and_names(args) + for name in names: + scrapymanager.crawl_spider_name(name) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + for url in urls: + scrapymanager.crawl_url(url, spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + else: + for name, urls in self._group_urls_by_spider(urls): + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) + + scrapymanager.start() + + def _group_urls_by_spider(self, urls): + spider_urls = defaultdict(list) + for url in urls: + spider_names = spiders.find_by_request(Request(url)) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + return spider_urls.items() + + def _split_urls_and_names(self, args): + urls = [] + names = [] + for arg in args: + if is_url(arg): + urls.append(arg) + else: + names.append(arg) + return urls, names diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index bbc2efa9e..bb2df1400 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -1,7 +1,11 @@ import pprint +from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager +from scrapy.http import Request +from scrapy.spider import BaseSpider, spiders +from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -19,17 +23,33 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", + help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") def run(self, args, opts): - if len(args) != 1: - print "One URL is required" - return + if len(args) != 1 or not is_url(args[0]): + return False + responses = [] # to collect downloaded responses + request = Request(args[0], callback=responses.append, dont_filter=True) - responses = fetch(args) + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg("Could not find spider: %s" % opts.spider, log.ERROR) + else: + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default')) + + scrapymanager.crawl_request(request, spider) + scrapymanager.start() + + # display response if responses: if opts.headers: pprint.pprint(responses[0].headers) else: print responses[0].body + diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index f39d52d14..2289fee1a 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -15,10 +15,11 @@ SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders') def sanitize_module_name(module_name): - """Sanitize the given module name, by replacing dashes with underscores and - prefixing it with a letter if it doesn't start with one + """Sanitize the given module name, by replacing dashes and points + with underscores and prefixing it with a letter if it doesn't start + with one """ - module_name = module_name.replace('-', '_') + module_name = module_name.replace('-', '_').replace('.', '_') if module_name[0] not in string.ascii_letters: module_name = "a" + module_name return module_name @@ -28,7 +29,7 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] " + return "[options] " def short_desc(self): return "Generate new spider based on template passed with -t or --template" @@ -54,28 +55,37 @@ class Command(ScrapyCommand): print template.read() return - if len(args) < 2: + if len(args) != 2: return False - module = sanitize_module_name(args[0]) + name = args[0] domain = args[1] - spider = spiders.fromdomain(domain) - if spider and not opts.force: - print "Spider '%s' already exists in module:" % domain - print " %s" % spider.__module__ - sys.exit(1) + + module = sanitize_module_name(name) + + # if spider already exists and not force option then halt + try: + spider = spiders.create(name) + except KeyError: + pass + else: + if not opts.force: + print "Spider '%s' already exists in module:" % name + print " %s" % spider.__module__ + sys.exit(1) template_file = self._find_template(opts.template) if template_file: - self._genspider(module, domain, opts.template, template_file) + self._genspider(module, name, domain, opts.template, template_file) - def _genspider(self, module, domain, template_name, template_file): + def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" tvars = { 'project_name': settings.get('BOT_NAME'), 'ProjectName': string_camelcase(settings.get('BOT_NAME')), 'module': module, - 'site': domain, + 'name': name, + 'domain': domain, 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } @@ -86,7 +96,7 @@ class Command(ScrapyCommand): shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print "Created spider %r using template %r in module:" % (domain, \ + print "Created spider %r using template %r in module:" % (name, \ template_name) print " %s.%s" % (spiders_module.__name__, module) diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 5d5143f08..9aabe94ff 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -1,11 +1,15 @@ from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager from scrapy.http import Request from scrapy.item import BaseItem from scrapy.spider import spiders from scrapy.utils import display +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.url import is_url from scrapy import log +from collections import defaultdict + class Command(ScrapyCommand): requires_project = True @@ -18,6 +22,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider") parser.add_option("--nolinks", dest="nolinks", action="store_true", \ help="don't show extracted links") parser.add_option("--noitems", dest="noitems", action="store_true", \ @@ -37,18 +43,13 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for url: %s' % response.url, level=log.ERROR) - return (), () - if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) if not callback_fcn: - log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name)) + log.msg('Cannot find callback %s in %s spider' % (callback, spider.name)) return (), () - result = callback_fcn(response) + result = iterate_spider_output(callback_fcn(response)) links = [i for i in result if isinstance(i, Request)] items = [self.pipeline_process(i, spider, opts) for i in result if \ isinstance(i, BaseItem)] @@ -71,36 +72,68 @@ class Command(ScrapyCommand): display.pprint(list(links)) def run(self, args, opts): - if not args: - print "An URL is required" + if not len(args) == 1 or not is_url(args[0]): + return False + + request = Request(args[0]) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + return + else: + spider = scrapymanager._create_spider_for_request(request, \ + log_none=True, log_multiple=True) + + if not spider: return - for response in fetch(args): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for "%s"' % response.url) - continue + responses = [] # to collect downloaded responses + request = request.replace(callback=responses.append) - if self.callbacks: - for callback in self.callbacks: - items, links = self.run_callback(spider, response, callback, args, opts) - self.print_results(items, links, callback, opts) + scrapymanager.crawl_request(request, spider) + scrapymanager.start() - elif opts.rules: - rules = getattr(spider, 'rules', None) - if rules: - items, links = [], [] - for rule in rules: - if rule.callback and rule.link_extractor.matches(response.url): - items, links = self.run_callback(spider, response, rule.callback, args, opts) - self.print_results(items, links, rule.callback, opts) - break - else: - log.msg('No rules found for spider "%s", please specify a callback for parsing' \ - % spider.domain_name) - continue + if not responses: + log.msg('No response returned', log.ERROR, spider=spider) + return + # now process response + # - if callbacks defined then call each one print results + # - if --rules option given search for matching spider's rule + # - default print result using default 'parse' spider's callback + response = responses[0] + + if self.callbacks: + # apply each callback + for callback in self.callbacks: + items, links = self.run_callback(spider, response, + callback, args, opts) + self.print_results(items, links, callback, opts) + elif opts.rules: + # search for matching spider's rule + if hasattr(spider, 'rules') and spider.rules: + items, links = [], [] + for rule in spider.rules: + if rule.link_extractor.matches(response.url) \ + and rule.callback: + + items, links = self.run_callback(spider, + response, rule.callback, + args, opts) + self.print_results(items, links, + rule.callback, opts) + # first-match rule breaks rules loop + break else: - items, links = self.run_callback(spider, response, 'parse', args, opts) - self.print_results(items, links, 'parse', opts) + log.msg('No rules found for spider "%s", ' \ + 'please specify a callback for parsing' \ + % spider.name, log.ERROR) + else: + # default callback 'parse' + items, links = self.run_callback(spider, response, + 'parse', args, opts) + self.print_results(items, links, 'parse', opts) diff --git a/scrapy/command/commands/runspider.py b/scrapy/command/commands/runspider.py index e18f13cdd..2bfcd87f9 100644 --- a/scrapy/command/commands/runspider.py +++ b/scrapy/command/commands/runspider.py @@ -52,6 +52,10 @@ class Command(ScrapyCommand): dispatcher.connect(exporter.export_item, signal=signals.item_passed) exporter.start_exporting() module = _import_file(args[0]) - scrapymanager.runonce(module.SPIDER) + + # schedule spider and start engine + scrapymanager.crawl_spider(module.SPIDER) + scrapymanager.start() + if opts.output: exporter.finish_exporting() diff --git a/scrapy/command/commands/start.py b/scrapy/command/commands/start.py index 032f12bcc..2c7304787 100644 --- a/scrapy/command/commands/start.py +++ b/scrapy/command/commands/start.py @@ -9,4 +9,4 @@ class Command(ScrapyCommand): return "Start the Scrapy manager but don't run any spider (idle mode)" def run(self, args, opts): - scrapymanager.start(*args) + scrapymanager.start(keep_alive=True) diff --git a/scrapy/command/commands/startproject.py b/scrapy/command/commands/startproject.py index eed8ae260..ee44026ce 100644 --- a/scrapy/command/commands/startproject.py +++ b/scrapy/command/commands/startproject.py @@ -7,7 +7,7 @@ from os.path import join, exists import scrapy from scrapy.command import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase -from scrapy.utils.python import ignore_patterns, copytree +from scrapy.utils.py26 import ignore_patterns, copytree TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project') diff --git a/scrapy/command/models.py b/scrapy/command/models.py index b019c3e52..049365ebb 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -57,8 +57,6 @@ class ScrapyCommand(object): help="log level (default: %s)" % settings['LOGLEVEL']) group.add_option("--nolog", action="store_true", dest="nolog", \ help="disable logging completely") - group.add_option("--spider", dest="spider", default=None, \ - help="always use this spider when arguments are urls") group.add_option("--profile", dest="profile", metavar="FILE", default=None, \ help="write python cProfile stats to FILE") group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \ @@ -99,10 +97,6 @@ class ScrapyCommand(object): if opts.nolog: settings.overrides['LOG_ENABLED'] = False - if opts.spider: - from scrapy.spider import spiders - spiders.force_domain = opts.spider - if opts.pidfile: with open(opts.pidfile, "w") as f: f.write(str(os.getpid())) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 8892e41b4..b80e89385 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -71,6 +71,40 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' +ENCODING_ALIASES = {} + +ENCODING_ALIASES_BASE = { + # gb2312 is superseded by gb18030 + 'gb2312': 'gb18030', + 'chinese': 'gb18030', + 'csiso58gb231280': 'gb18030', + 'euc- cn': 'gb18030', + 'euccn': 'gb18030', + 'eucgb2312-cn': 'gb18030', + 'gb2312-1980': 'gb18030', + 'gb2312-80': 'gb18030', + 'iso- ir-58': 'gb18030', + # gbk is superseded by gb18030 + 'gbk': 'gb18030', + '936': 'gb18030', + 'cp936': 'gb18030', + 'ms936': 'gb18030', + # latin_1 is a subset of cp1252 + 'latin_1': 'cp1252', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'l1': 'cp1252', + # others + 'zh-cn': 'gb18030', + 'win-1251': 'cp1251', + 'macintosh' : 'mac_roman', + 'x-sjis': 'shift_jis', +} + EXTENSIONS = {} EXTENSIONS_BASE = { @@ -101,6 +135,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' ITEM_PIPELINES = [] LOG_ENABLED = True +LOG_ENCODING = 'utf-8' LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' @@ -122,6 +157,8 @@ MYSQL_CONNECTION_SETTINGS = {} NEWSPIDER_MODULE = '' +RANDOMIZE_DOWNLOAD_DELAY = True + REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 @@ -150,7 +187,7 @@ SCHEDULER_MIDDLEWARES_BASE = { 'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware': 500, } -SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO +SCHEDULER_ORDER = 'DFO' SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager' diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 1f54b7634..b62c9e6b3 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -5,13 +5,13 @@ because Amazon Web Service use timestamps for authentication. """ import os -import time - -from scrapy.utils.httpobj import urlparse_cached +from time import strftime, gmtime from scrapy.utils.aws import sign_request from scrapy.conf import settings + class AWSMiddleware(object): + def __init__(self): self.access_key = settings['AWS_ACCESS_KEY_ID'] or \ os.environ.get('AWS_ACCESS_KEY_ID') @@ -19,9 +19,6 @@ class AWSMiddleware(object): os.environ.get('AWS_SECRET_ACCESS_KEY') def process_request(self, request, spider): - hostname = urlparse_cached(request).hostname - if spider.domain_name == 's3.amazonaws.com' \ - or (hostname and hostname.endswith('s3.amazonaws.com')): - request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \ - time.gmtime()) + if request.meta.get('sign_s3_request'): + request.headers['Date'] = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime()) sign_request(request, self.access_key, self.secret_key) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 239c05bc7..341e9b5a1 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -108,7 +108,7 @@ class FilesystemCacheStorage(object): def _get_request_path(self, spider, request): key = request_fingerprint(request) - return join(self.cachedir, spider.domain_name, key[0:2], key) + return join(self.cachedir, spider.name, key[0:2], key) def _read_meta(self, spider, request): rpath = self._get_request_path(spider, request) diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 249862824..1c6297b49 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,5 @@ from scrapy import log +from scrapy.http import HtmlResponse from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh from scrapy.core.exceptions import IgnoreRequest @@ -24,10 +25,11 @@ class RedirectMiddleware(object): redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) - interval, url = get_meta_refresh(response) - if url and interval < self.max_metarefresh_delay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + if isinstance(response, HtmlResponse): + interval, url = get_meta_refresh(response) + if url and interval < self.max_metarefresh_delay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index 4bd1b46b3..d26e9607b 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -1,9 +1,5 @@ from scrapy.contrib.exporter import BaseItemExporter - -try: - import json -except ImportError: - import simplejson as json +from scrapy.utils.py26 import json class JsonLinesItemExporter(BaseItemExporter): diff --git a/scrapy/contrib/groupsettings.py b/scrapy/contrib/groupsettings.py deleted file mode 100644 index 4bad4100b..000000000 --- a/scrapy/contrib/groupsettings.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Extensions to override scrapy settings with per-group settings according to the -group the spider belongs to. It only overrides the settings when running the -crawl command with *only one domain as argument*. -""" - -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured -from scrapy.command.cmdline import command_executed - -class GroupSettings(object): - - def __init__(self): - if not settings.getbool("GROUPSETTINGS_ENABLED"): - raise NotConfigured - - if command_executed and command_executed['name'] == 'crawl': - mod = __import__(settings['GROUPSETTINGS_MODULE'], {}, {}, ['']) - args = command_executed['args'] - if len(args) == 1 and not args[0].startswith('http://'): - domain = args[0] - settings.overrides.update(mod.default_settings) - for group, domains in mod.group_spiders.iteritems(): - if domain in domains: - settings.overrides.update(mod.group_settings.get(group, {})) - diff --git a/scrapy/contrib/ibl/__init__.py b/scrapy/contrib/ibl/__init__.py new file mode 100644 index 000000000..52c6e08b2 --- /dev/null +++ b/scrapy/contrib/ibl/__init__.py @@ -0,0 +1,16 @@ +""" +This contrib implements an automatic extraction library based on an Instance +Based Learning (IBL) algorithm, as described in the following papers: + + A hierarchical approach to wrapper induction + http://portal.acm.org/citation.cfm?id=301191 + + Extracting web data using instance based learning + http://portal.acm.org/citation.cfm?id=1265174 + +This code has some additional dependencies too: + +* numpy +* nltk + +""" diff --git a/scrapy/contrib/ibl/descriptor.py b/scrapy/contrib/ibl/descriptor.py new file mode 100644 index 000000000..9780ee0cf --- /dev/null +++ b/scrapy/contrib/ibl/descriptor.py @@ -0,0 +1,51 @@ +""" +Extended types for IBL extraction +""" +from itertools import chain + +from scrapy.contrib.ibl.extractors import text + +class FieldDescriptor(object): + """description of a scraped attribute""" + __slots__ = ('name', 'description', 'extractor', 'required', 'allow_markup') + + def __init__(self, name, description, extractor=text, required=False, + allow_markup=False): + self.name = name + self.description = description + self.extractor = extractor + self.required = required + self.allow_markup = allow_markup + + def __str__(self): + return "FieldDescriptor(%s)" % self.name + +class ItemDescriptor(object): + """Simple auto scraping item descriptor. + + This used to describe type-specific operations and may be overridden where + necessary. + """ + + def __init__(self, name, description, attribute_descriptors): + self.name = name + self.attribute_map = dict((d.name, d) for d in attribute_descriptors) + self._required_attributes = [d.name for d in attribute_descriptors \ + if d.required] + + def validated(self, data): + """Only return the items in the data that are valid""" + return [d for d in data if self._item_validates(d)] + + def _item_validates(self, item): + """simply checks that all mandatory attributes are present""" + variant_attrs = set(chain(* + [v.keys() for v in item.get('variants', [])])) + return all([(name in item or name in variant_attrs) \ + for name in self._required_attributes]) + + def get_required_attributes(self): + return self._required_attributes + + def __str__(self): + return "ItemDescriptor(%s)" % self.name diff --git a/scrapy/contrib/ibl/extraction/__init__.py b/scrapy/contrib/ibl/extraction/__init__.py new file mode 100644 index 000000000..4d5cf6559 --- /dev/null +++ b/scrapy/contrib/ibl/extraction/__init__.py @@ -0,0 +1,126 @@ +""" +IBL module + +This contains an extraction algorithm based on the paper Extracting Web Data +Using Instance-Based Learning by Yanhong Zhai and Bing Liu. + +It defines the InstanceBasedLearningExtractor class, which implements this +extraction algorithm. + +Main departures from the original algorithm: + * there is no limit in prefix or suffix size + * we have "attribute adaptors" that allow generic post processing and may + affect the extraction process. For example, a price field may require a + numeric value to be present. + * tags can be inserted to extract regions not wrapped by html tags. These + regions are then identified using the longest unique character prefix and + suffix. +""" +from operator import itemgetter +from .regionextract import build_extraction_tree +from .pageparsing import parse_template, parse_extraction_page +from .pageobjects import TokenDict, AnnotationText +from .similarity import common_prefix + +class InstanceBasedLearningExtractor(object): + """Implementation of the instance based learning algorithm to + extract data from web pages. + """ + + def __init__(self, templates, type_descriptor=None, trace=False): + """Initialise this extractor + + templates should contain a sequence of strings, each containing + annotated html that will be used as templates for extraction. + + Tags surrounding areas to be extracted must contain a + 'data-scrapy-annotate' attribute and the value must be the name + of the attribute. If the tag was inserted and was not present in the + original page, the data-scrapy-generated attribute must be present. + + type_descriptor may contain a type descriptor describing the item + to be extracted. + + if trace is true, the returned extracted data will have a 'trace' + property that contains a trace of the extraction execution. + """ + self.token_dict = TokenDict() + parsed_plus_templates = [(parse_template(self.token_dict, t), t) for t in templates] + parsed_plus_epages = [(p, parse_extraction_page(self.token_dict, t)) for p, t \ + in parsed_plus_templates if _annotation_count(p)] + parsed_templates = map(itemgetter(0), parsed_plus_epages) + + # calculate common text prefixes for annotations of same field across all templates + extracted_text = {} + extraction_pages = map(itemgetter(1), parsed_plus_epages) + for i, parsed in enumerate(parsed_templates): + for annot in parsed.annotations: + if annot.match_common_prefix: + field = annot.surrounds_attribute + if field is not None: + descriptor = type_descriptor.attribute_map.get(field) if type_descriptor else None + allow_markup = descriptor.allow_markup if descriptor else False + start, end = annot.start_index, annot.end_index + if allow_markup: + text = extraction_pages[i].html_between_tokens(start, end) + else: + text = extraction_pages[i].text_between_tokens(start, end) + extracted_text.setdefault(field, []).append(text) + common_prefixes = {} + for field, data in extracted_text.iteritems(): + if len(data) > 1: + cprefix = common_prefix(*data) + if cprefix: + common_prefixes[field] = "".join(cprefix).strip() + # now apply common prefixes to annotations + for i, parsed in enumerate(parsed_templates): + for annot in parsed.annotations: + for field, prefix in common_prefixes.iteritems(): + if annot.surrounds_attribute == field and not annot.annotation_text: + annot.annotation_text = AnnotationText(prefix) + + # templates with more attributes are considered first + sorted_templates = sorted(parsed_templates, key=_annotation_count, reverse=True) + self.extraction_trees = [build_extraction_tree(t, type_descriptor, + trace) for t in sorted_templates] + self.validated = type_descriptor.validated if type_descriptor else \ + self._filter_not_none + + def extract(self, html, pref_template_id=None, useone=False): + """extract data from an html page + + If pref_template_url is specified, the template with that url will be + used first. + if useone is True and no data was extracted, no additional template will + be tried. If False and no data was extracted, try with rest of item templates + """ + extraction_page = parse_extraction_page(self.token_dict, html) + if pref_template_id is not None: + if useone: + extraction_trees = [x for x in self.extraction_trees if x.template.id == pref_template_id] + else: + extraction_trees = sorted(self.extraction_trees, + key=lambda x: x.template.id != pref_template_id) + else: + extraction_trees = self.extraction_trees + + for extraction_tree in extraction_trees: + extracted = extraction_tree.extract(extraction_page) + correctly_extracted = self.validated(extracted) + extra_required = extraction_tree.template.extra_required_attrs + correctly_extracted = [c for c in correctly_extracted if \ + extra_required.intersection(c.keys()) == extra_required ] + if len(correctly_extracted) > 0: + return correctly_extracted, extraction_tree.template.id + return None, None + + def __str__(self): + return "InstanceBasedLearningExtractor[\n%s\n]" % \ + (',\n'.join(map(str, self.extraction_trees))) + + @staticmethod + def _filter_not_none(items): + return [d for d in items if d is not None] + +def _annotation_count(template): + return len(template.annotations) diff --git a/scrapy/contrib/ibl/extraction/pageobjects.py b/scrapy/contrib/ibl/extraction/pageobjects.py new file mode 100644 index 000000000..c57cdfa75 --- /dev/null +++ b/scrapy/contrib/ibl/extraction/pageobjects.py @@ -0,0 +1,230 @@ +""" +Page objects + +This module contains objects representing pages and parts of pages (e.g. tokens +and annotations) used in the instance based learning algorithm. +""" +from numpy import array, ndarray + +from scrapy.contrib.ibl.htmlpage import HtmlTagType + +class TokenType(object): + """constants for token types""" + WORD = 0 + OPEN_TAG = HtmlTagType.OPEN_TAG + CLOSE_TAG = HtmlTagType.CLOSE_TAG + NON_PAIRED_TAG = HtmlTagType.UNPAIRED_TAG + +class TokenDict(object): + """Mapping from parse tokens to integers + + >>> d = TokenDict() + >>> d.tokenid('i') + 0 + >>> d.tokenid('b') + 1 + >>> d.tokenid('i') + 0 + + Tokens can be searched for by id + >>> d.find_token(1) + 'b' + + The lower 24 bits store the token reference and the higher bits the type. + """ + + def __init__(self): + self.token_ids = {} + + def tokenid(self, token, token_type=TokenType.WORD): + """create an integer id from the token and token type passed""" + tid = self.token_ids.setdefault(token, len(self.token_ids)) + return tid | (token_type << 24) + + @staticmethod + def token_type(token): + """extract the token type from the token id passed""" + return token >> 24 + + def find_token(self, tid): + """Search for a tag with the given ID + + This is O(N) and is only intended for debugging + """ + tid &= 0xFFFFFF + if tid >= len(self.token_ids) or tid < 0: + raise ValueError("tag id %s out of range" % tid) + + for (token, token_id) in self.token_ids.items(): + if token_id == tid: + return token + assert False, "token dictionary is corrupt" + + def token_string(self, tid): + """create a string representation of a token + + This is O(N). + """ + templates = ["%s", "<%s>", "", "<%s/>"] + return templates[tid >> 24] % self.find_token(tid) + +class Page(object): + """Basic representation of a page. This consists of a reference to a + dictionary of tokens and an array of raw token ids + """ + + __slots__ = ('token_dict', 'page_tokens') + + def __init__(self, token_dict, page_tokens): + self.token_dict = token_dict + # use a numpy array becuase we can index/slice easily and efficiently + if not isinstance(page_tokens, ndarray): + page_tokens = array(page_tokens) + self.page_tokens = page_tokens + +class TemplatePage(Page): + __slots__ = ('annotations', 'id', 'ignored_regions', 'extra_required_attrs') + + def __init__(self, token_dict, page_tokens, annotations, template_id=None, \ + ignored_regions=None, extra_required=None): + Page.__init__(self, token_dict, page_tokens) + # ensure order is the same as start tag order in the original page + annotations = sorted(annotations, key=lambda x: x.end_index, reverse=True) + self.annotations = sorted(annotations, key=lambda x: x.start_index) + self.id = template_id + self.ignored_regions = ignored_regions or [] + self.extra_required_attrs = set(extra_required or []) + + def __str__(self): + summary = [] + for index, token in enumerate(self.page_tokens): + text = "%s: %s" % (index, self.token_dict.find_token(token)) + summary.append(text) + return "TemplatePage\n============\nTokens: (index, token)\n%s\nAnnotations: %s\n" % \ + ('\n'.join(summary), '\n'.join(map(str, self.annotations))) + +class ExtractionPage(Page): + """Parsed data belonging to a web page upon which we wish to perform + extraction. + """ + __slots__ = ('text', + 'token_start_indexes', # index in text of the start of a token + 'token_follow_indexes', # index in text of data following token + 'tag_attributes' # a map from token index to tag attributes + ) + + def __init__(self, text, token_dict, page_tokens, token_start_indexes, + token_follow_indexes, tag_attributes): + Page.__init__(self, token_dict, page_tokens) + self.text = text + self.token_start_indexes = token_start_indexes + self.token_follow_indexes = token_follow_indexes + self.tag_attributes = tag_attributes + + def token_html(self, token_index): + """The raw html for a page token at the given index in the page_tokens + list + """ + text_start = self.token_start_indexes[token_index] + text_end = self.token_follow_indexes[token_index] + return self.text[text_start:text_end] + + def html_between_tokens(self, start_token_index, end_token_index): + """The raw html between the tokens at the specified indexes in the + page_tokens list + + This assumes start_token_index <= end_token_index + """ + text_start = self.token_follow_indexes[start_token_index] + text_end = self.token_start_indexes[end_token_index] + return self.text[text_start:text_end] + + def text_between_tokens(self, start_token_index, end_token_index, + tag_replacement=u' '): + """The text between the the tokens at the specified indexes in the + page_tokens list. Tags are replaced by tag_replacement (default one space + character) + """ + return tag_replacement.join([self.text[ + self.token_follow_indexes[i]:self.token_start_indexes[i+1]] \ + for i in xrange(start_token_index, end_token_index)]) + + def tag_attribute(self, token_index, attribute): + """The value of a tag attribute. The tag is identified by its + corresponding token index. + + If the tag or attribute is not present, None is returned + """ + return self.tag_attributes.get(token_index, {}).get(attribute) + + def __str__(self): + summary = [] + for (token, start, follow) in zip(self.page_tokens, self.token_start_indexes, + self.token_follow_indexes): + text = "%s %s-%s (%s)" % (self.token_dict.find_token(token), start, follow, + self.text[start:follow]) + summary.append(text) + return "ExtractionPage\n==============\nTokens: %s\n\nRaw text: %s\n\n" \ + "Tag attributes: %s\n" % ('\n'.join(summary), self.text, + self.tag_attributes) + +class AnnotationText(object): + __slots__ = ('start_text', 'follow_text') + + def __init__(self, start_text=None, follow_text=None): + self.start_text = start_text + self.follow_text = follow_text + + def __str__(self): + return "AnnotationText(%s..%s)" % \ + (repr(self.start_text), repr(self.follow_text)) + +class AnnotationTag(object): + """A tag that annotates part of the document + + It has the following properties: + start_index - index of the token for the opening tag + end_index - index of the token for the closing tag + surrounds_attribute - the attribute name surrounded by this tag + tag_attributes - list of (tag attribute, extracted attribute) tuples + for each item to be extracted from a tag attribute + annotation_text - text prefix and suffix for the attribute to be extracted + match_common_prefix - use this annotation for calculating across-template prefixes + """ + __slots__ = ('surrounds_attribute', 'start_index', 'end_index', + 'tag_attributes', 'annotation_text', 'variant_id', + 'surrounds_variant','match_common_prefix') + + def __init__(self, start_index, end_index, surrounds_attribute=None, + annotation_text=None, tag_attributes=None, variant_id=None, + surrounds_variant=None, match_common_prefix=False): + self.start_index = start_index + self.end_index = end_index + self.surrounds_attribute = surrounds_attribute + self.annotation_text = annotation_text + self.tag_attributes = tag_attributes or [] + self.variant_id = variant_id + self.surrounds_variant = surrounds_variant + self.match_common_prefix = match_common_prefix + + def __str__(self): + return "AnnotationTag(%s)" % ", ".join( + ["%s=%s" % (s, getattr(self, s)) \ + for s in self.__slots__ if getattr(self, s)]) + + def __repr__(self): + return str(self) + +class LabelledRegion(object): + __slots__ = ('start_index', 'end_index') + + def __init__(self, start, end): + self.start_index = start + self.end_index = end + + def __str__(self): + return "LabelledRegion (%s, %s)" % (self.start_index, self.end_index) + + def __repr__(self): + return str(self) + diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py new file mode 100644 index 000000000..486862ae5 --- /dev/null +++ b/scrapy/contrib/ibl/extraction/pageparsing.py @@ -0,0 +1,318 @@ +""" +Page parsing + +Parsing of web pages for extraction task. +""" +from collections import defaultdict +from numpy import array + +from scrapy.utils.py26 import json + +from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage +from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, + TemplatePage, ExtractionPage, AnnotationText, TokenDict) + +def parse_strings(template_html, extraction_html): + """Create a template and extraction page from raw strings + + this is useful for testing purposes + """ + t = TokenDict() + template_page = HtmlPage(body=template_html) + extraction_page = HtmlPage(body=extraction_html) + return (parse_template(t, template_page), + parse_extraction_page(t, extraction_page)) + +def parse_template(token_dict, template_html): + """Create an TemplatePage object by parsing the annotated html""" + parser = TemplatePageParser(token_dict) + parser.feed(template_html) + return parser.to_template() + +def parse_extraction_page(token_dict, page_html): + """Create an ExtractionPage object by parsing the html""" + parser = ExtractionPageParser(token_dict) + parser.feed(page_html) + return parser.to_extraction_page() + +class InstanceLearningParser(object): + """Base parser for instance based learning algorithm + + This does not require correct HTML and the parsing method should not alter + the original tag order. It is important that parsing results do not vary. + """ + def __init__(self, token_dict): + self.token_dict = token_dict + self.token_list = [] + + def _add_token(self, token, token_type, start, end): + tid = self.token_dict.tokenid(token, token_type) + self.token_list.append(tid) + + def feed(self, html_page): + self.html_page = html_page + self.previous_element_class = None + for data in html_page.parsed_body: + if isinstance(data, HtmlTag): + self._add_token(data.tag, data.tag_type, data.start, data.end) + self.handle_tag(data) + else: + self.handle_data(data) + self.previous_element_class = data.__class__ + + def handle_data(self, html_data_fragment): + pass + + def handle_tag(self, html_tag): + pass + +_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"] + +class TemplatePageParser(InstanceLearningParser): + """Template parsing for instance based learning algorithm""" + + def __init__(self, token_dict): + InstanceLearningParser.__init__(self, token_dict) + self.annotations = [] + self.ignored_regions = [] + self.extra_required_attrs = [] + self.ignored_tag_stacks = defaultdict(list) + # tag names that have not been completed + self.labelled_tag_stacks = defaultdict(list) + self.replacement_stacks = defaultdict(list) + self.unpairedtag_stack = [] + self.variant_stack = [] + self.prev_data = None + self.last_text_region = None + self.next_tag_index = 0 + + def handle_tag(self, html_tag): + if self.last_text_region: + self._process_text('') + + if html_tag.tag_type == HtmlTagType.OPEN_TAG: + self._handle_open_tag(html_tag) + elif html_tag.tag_type == HtmlTagType.CLOSE_TAG: + self._handle_close_tag(html_tag) + else: + # the tag is not paired, it can contain only attribute annotations + self._handle_unpaired_tag(html_tag) + + @staticmethod + def _read_template_annotation(html_tag): + template_attr = html_tag.attributes.get('data-scrapy-annotate') + if template_attr is None: + return None + unescaped = template_attr.replace('"', '"') + return json.loads(unescaped) + + @staticmethod + def _read_bool_template_attribute(html_tag, attribute): + return html_tag.attributes.get("data-scrapy-" + attribute) == "true" + + def _close_unpaired_tag(self): + self.unpairedtag_stack[0].end_index = self.next_tag_index + self.unpairedtag_stack = [] + + def _handle_unpaired_tag(self, html_tag): + if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img": + self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1)) + elif self._read_bool_template_attribute(html_tag, "ignore-beneath") and html_tag.tag == "img": + self.ignored_regions.append((self.next_tag_index, None)) + jannotation = self._read_template_annotation(html_tag) + if jannotation: + if self.unpairedtag_stack: + self._close_unpaired_tag() + + annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1) + attribute_annotations = jannotation.get('annotations', {}).items() + for extract_attribute, tag_value in attribute_annotations: + if extract_attribute == 'content': + annotation.surrounds_attribute = tag_value + self.unpairedtag_stack.append(annotation) + else: + annotation.tag_attributes.append((extract_attribute, tag_value)) + self.annotations.append(annotation) + if jannotation.get('common_prefix', False): + annotation.match_common_prefix = True + + self.extra_required_attrs.extend(jannotation.get('required', [])) + + self.next_tag_index += 1 + + def _handle_open_tag(self, html_tag): + if self._read_bool_template_attribute(html_tag, "ignore"): + if html_tag.tag == "img": + self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1)) + else: + self.ignored_regions.append((self.next_tag_index, None)) + self.ignored_tag_stacks[html_tag.tag].append(html_tag) + + elif self.ignored_tag_stacks.get(html_tag.tag): + self.ignored_tag_stacks[html_tag.tag].append(None) + if self._read_bool_template_attribute(html_tag, "ignore-beneath"): + self.ignored_regions.append((self.next_tag_index, None)) + + replacement = html_tag.attributes.pop("data-scrapy-replacement", None) + if replacement: + self.token_list.pop() + self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end) + self.replacement_stacks[html_tag.tag].append(replacement) + elif html_tag.tag in self.replacement_stacks: + self.replacement_stacks[html_tag.tag].append(None) + + if self.unpairedtag_stack: + if html_tag.tag in _END_UNPAIREDTAG_TAGS: + self._close_unpaired_tag() + else: + self.unpairedtag_stack.append(html_tag.tag) + + # can't be a p inside another p. Also, an open p element closes + # a previous open p element. + if html_tag.tag == "p" and html_tag.tag in self.labelled_tag_stacks: + annotation = self.labelled_tag_stacks.pop(html_tag.tag)[0] + annotation.end_index = self.next_tag_index + self.annotations.append(annotation) + + jannotation = self._read_template_annotation(html_tag) + if not jannotation: + if html_tag.tag in self.labelled_tag_stacks: + # add this tag to the stack to match correct end tag + self.labelled_tag_stacks[html_tag.tag].append(None) + self.next_tag_index += 1 + return + + annotation = AnnotationTag(self.next_tag_index, None) + if jannotation.get('generated', False): + self.token_list.pop() + annotation.start_index -= 1 + if self.previous_element_class == HtmlTag: + annotation.annotation_text = AnnotationText('') + else: + annotation.annotation_text = AnnotationText(self.prev_data) + if self._read_bool_template_attribute(html_tag, "ignore") \ + or self._read_bool_template_attribute(html_tag, "ignore-beneath"): + ignored = self.ignored_regions.pop() + self.ignored_regions.append((ignored[0]-1, ignored[1])) + + if jannotation.get('common_prefix', False): + annotation.match_common_prefix = True + + self.extra_required_attrs.extend(jannotation.get('required', [])) + + variant_id = jannotation.get('variant', 0) + if variant_id > 0: + self.variant_stack.append(variant_id) + annotation.surrounds_variant = variant_id + attribute_annotations = jannotation.get('annotations', {}).items() + for extract_attribute, tag_value in attribute_annotations: + if extract_attribute == 'content': + annotation.surrounds_attribute = tag_value + else: + annotation.tag_attributes.append((extract_attribute, tag_value)) + + if annotation.annotation_text is None: + self.next_tag_index += 1 + if self.variant_stack: + variant_id = self.variant_stack[-1] + if variant_id == '0': + variant_id = None + annotation.variant_id = variant_id + + # look for a closing tag if the content is important + if annotation.surrounds_attribute or annotation.surrounds_variant: + self.labelled_tag_stacks[html_tag.tag].append(annotation) + else: + annotation.end_index = annotation.start_index + 1 + self.annotations.append(annotation) + + def _handle_close_tag(self, html_tag): + + if self.unpairedtag_stack: + if html_tag.tag == self.unpairedtag_stack[-1]: + self.unpairedtag_stack.pop() + else: + self._close_unpaired_tag() + ignored_tags = self.ignored_tag_stacks.get(html_tag.tag) + if ignored_tags is not None: + tag = ignored_tags.pop() + if isinstance(tag, HtmlTag): + for i in range(-1, -len(self.ignored_regions) - 1, -1): + if self.ignored_regions[i][1] is None: + self.ignored_regions[i] = (self.ignored_regions[i][0], self.next_tag_index) + break + if len(ignored_tags) == 0: + del self.ignored_tag_stacks[html_tag.tag] + + if html_tag.tag in self.replacement_stacks: + replacement = self.replacement_stacks[html_tag.tag].pop() + if replacement: + self.token_list.pop() + self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end) + if len(self.replacement_stacks[html_tag.tag]) == 0: + del self.replacement_stacks[html_tag.tag] + + labelled_tags = self.labelled_tag_stacks.get(html_tag.tag) + if labelled_tags is None: + self.next_tag_index += 1 + return + annotation = labelled_tags.pop() + if annotation is None: + self.next_tag_index += 1 + else: + annotation.end_index = self.next_tag_index + self.annotations.append(annotation) + if annotation.annotation_text is not None: + self.token_list.pop() + self.last_text_region = annotation + else: + self.next_tag_index += 1 + if len(labelled_tags) == 0: + del self.labelled_tag_stacks[html_tag.tag] + if annotation.surrounds_variant and self.variant_stack: + prev = self.variant_stack.pop() + if prev != annotation.surrounds_variant: + raise ValueError("unbalanced variant annotation tags") + + def handle_data(self, html_data_fragment): + fragment_text = self.html_page.fragment_data(html_data_fragment) + self._process_text(fragment_text) + + def _process_text(self, text): + if self.last_text_region is not None: + self.last_text_region.annotation_text.follow_text = text + self.last_text_region = None + self.prev_data = text + + def to_template(self): + """create a TemplatePage from the data fed to this parser""" + return TemplatePage(self.token_dict, self.token_list, self.annotations, + self.html_page.page_id, self.ignored_regions, self.extra_required_attrs) + +class ExtractionPageParser(InstanceLearningParser): + """Parse an HTML page for extraction using the instance based learning + algorithm + + This needs to extract the tokens in a similar way to LabelledPageParser, + it needs to also maintain a mapping from token index to the original content + so that once regions are identified, the original content can be extracted. + """ + def __init__(self, token_dict): + InstanceLearningParser.__init__(self, token_dict) + self.page_data = [] + self.token_start_index = [] + self.token_follow_index = [] + self.tag_attrs = {} + + def _add_token(self, token, token_type, start, end): + InstanceLearningParser._add_token(self, token, token_type, start, end) + self.token_start_index.append(start) + self.token_follow_index.append(end) + + def handle_tag(self, html_tag): + if html_tag.attributes: + self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes + + def to_extraction_page(self): + return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list), + self.token_start_index, self.token_follow_index, self.tag_attrs) diff --git a/scrapy/contrib/ibl/extraction/regionextract.py b/scrapy/contrib/ibl/extraction/regionextract.py new file mode 100644 index 000000000..9b5aab166 --- /dev/null +++ b/scrapy/contrib/ibl/extraction/regionextract.py @@ -0,0 +1,651 @@ +""" +Region Extract + +Custom extraction for regions in a document +""" +import operator +import copy +import pprint +import cStringIO +from itertools import groupby + +import nltk +from numpy import array + +from scrapy.contrib.ibl.descriptor import FieldDescriptor +from scrapy.contrib.ibl.extraction.similarity import (similar_region, + longest_unique_subsequence, common_prefix) +from scrapy.contrib.ibl.extraction.pageobjects import AnnotationTag, LabelledRegion + +def build_extraction_tree(template, type_descriptor, trace=True): + """Build a tree of region extractors corresponding to the + template + """ + attribute_map = type_descriptor.attribute_map if type_descriptor else None + extractors = BasicTypeExtractor.create(template.annotations, attribute_map) + if trace: + extractors = TraceExtractor.apply(template, extractors) + for cls in (RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor, + RecordExtractor): + extractors = cls.apply(template, extractors) + if trace: + extractors = TraceExtractor.apply(template, extractors) + + return TemplatePageExtractor(template, extractors) + +_ID = lambda x: x +_DEFAULT_DESCRIPTOR = FieldDescriptor('none', None) + +def _labelled(obj): + """ + Returns labelled element of the object (extractor or labelled region) + """ + if hasattr(obj, "annotation"): + return obj.annotation + return obj + +def _compose(f, g): + """given unary functions f and g, return a function that computes f(g(x)) + """ + def _exec(x): + ret = g(x) + return f(ret) if ret is not None else None + return _exec + +class BasicTypeExtractor(object): + """The BasicTypeExtractor extracts single attributes corresponding to + annotations. + + For example: + >>> from scrapy.contrib.ibl.extraction.pageparsing import parse_strings + >>> template, page = parse_strings( \ + u'

x

', u'

a name

') + >>> ex = BasicTypeExtractor(template.annotations[0]) + >>> ex.extract(page, 0, 1, None) + [(u'name', u' a name')] + + It supports attribute descriptors + >>> descriptor = FieldDescriptor('name', None, lambda x: x.strip()) + >>> ex = BasicTypeExtractor(template.annotations[0], {'name': descriptor}) + >>> ex.extract(page, 0, 1, None) + [(u'name', u'a name')] + + It supports ignoring regions + >>> template, page = parse_strings(\ + u'
x xx
',\ + u'
a name id-9
') + >>> ex = BasicTypeExtractor(template.annotations[0]) + >>> ex.extract(page, 0, 3, [LabelledRegion(*(1,2))]) + [(u'name', u'a name')] + """ + + def __init__(self, annotation, attribute_descriptors=None): + self.annotation = annotation + if attribute_descriptors is None: + attribute_descriptors = {} + + if annotation.surrounds_attribute: + descriptor = attribute_descriptors.get(annotation.surrounds_attribute) + if descriptor: + self.content_validate = descriptor.extractor + self.allow_markup = descriptor.allow_markup + else: + self.content_validate = _ID + self.allow_markup = False + self.extract = self._extract_content + + if annotation.tag_attributes: + self.tag_data = [] + for (tag_attr, extraction_attr) in annotation.tag_attributes: + descriptor = attribute_descriptors.get(extraction_attr) + extractf = descriptor.extractor if descriptor else _ID + self.tag_data.append((extractf, tag_attr, extraction_attr)) + + self.extract = self._extract_both if \ + annotation.surrounds_attribute else self._extract_attribute + + def _extract_both(self, page, start_index, end_index, ignored_regions=None): + return self._extract_content(page, start_index, end_index, ignored_regions) + \ + self._extract_attribute(page, start_index, end_index, ignored_regions) + + def _extract_content(self, extraction_page, start_index, end_index, ignored_regions=None): + # we might want to add opening/closing ul/ol/table if we have the + # middle of a region. This would require support in the scrapy + # cleansing. + complete_data = "" + start = start_index + end = ignored_regions[0].start_index if ignored_regions else end_index + while start is not None: + if self.allow_markup: + data = extraction_page.html_between_tokens(start, end) + else: + data = extraction_page.text_between_tokens(start, end) + complete_data += data + if ignored_regions: + start = ignored_regions[0].end_index + ignored_regions.pop(0) + end = ignored_regions[0].start_index if ignored_regions else end_index + else: + start = None + complete_data = self.content_validate(complete_data) + return [(self.annotation.surrounds_attribute, complete_data)] if complete_data else [] + + def _extract_attribute(self, extraction_page, start_index, end_index, ignored_regions=None): + data = [] + for (f, ta, ea) in self.tag_data: + tag_value = extraction_page.tag_attribute(start_index, ta) + if tag_value: + extracted = f(tag_value) + if extracted is not None: + data.append((ea, extracted)) + return data + + @classmethod + def create(cls, annotations, attribute_descriptors=None): + """Create a list of basic extractors from the given annotations + and attribute descriptors + """ + if attribute_descriptors is None: + attribute_descriptors = {} + return [cls._create_basic_extractor(annotation, attribute_descriptors) \ + for annotation in annotations \ + if annotation.surrounds_attribute or annotation.tag_attributes] + + @staticmethod + def _create_basic_extractor(annotation, attribute_descriptors): + """Create a basic type extractor for the annotation""" + text_region = annotation.annotation_text + if text_region is not None: + if annotation.match_common_prefix: + region_extract = TextPrefixRegionDataExtractor(text_region.start_text).extract + else: + region_extract = TextRegionDataExtractor(text_region.start_text, + text_region.follow_text).extract + # copy attribute_descriptors and add the text extractor + descriptor_copy = dict(attribute_descriptors) + attr_descr = descriptor_copy.get(annotation.surrounds_attribute, + _DEFAULT_DESCRIPTOR) + attr_descr = copy.copy(attr_descr) + attr_descr.extractor = _compose(attr_descr.extractor, region_extract) + descriptor_copy[annotation.surrounds_attribute] = attr_descr + attribute_descriptors = descriptor_copy + return BasicTypeExtractor(annotation, attribute_descriptors) + + def extracted_item(self): + """key used to identify the item extracted""" + return (self.annotation.surrounds_attribute, self.annotation.tag_attributes) + + def __repr__(self): + return str(self) + + def __str__(self): + messages = ['BasicTypeExtractor('] + if self.annotation.surrounds_attribute: + messages += [self.annotation.surrounds_attribute, ': ', + 'html content' if self.allow_markup else 'text content', + ] + if self.content_validate != _ID: + messages += [', extracted with \'', + self.content_validate.__name__, '\''] + + if self.annotation.tag_attributes: + if self.annotation.surrounds_attribute: + messages.append(';') + for (f, ta, ea) in self.tag_data: + messages += [ea, ': tag attribute "', ta, '"'] + if f != _ID: + messages += [', validated by ', str(f)] + messages.append(", template[%s:%s])" % \ + (self.annotation.start_index, self.annotation.end_index)) + return ''.join(messages) + +class RepeatedDataExtractor(object): + """Data extractor for handling repeated data""" + + def __init__(self, prefix, suffix, extractors): + self.prefix = array(prefix) + self.suffix = array(suffix) + self.extractor = copy.copy(extractors[0]) + self.annotation = copy.copy(self.extractor.annotation) + self.annotation.end_index = extractors[-1].annotation.end_index + + def extract(self, page, start_index, end_index, ignored_regions): + """repeatedly find regions bounded by the repeated + prefix and suffix and extract them + """ + prefixlen = len(self.prefix) + suffixlen = len(self.suffix) + index = max(0, start_index - prefixlen) + max_index = min(len(page.page_tokens) - suffixlen, end_index + len(self.suffix)) + max_start_index = max_index - prefixlen + extracted = [] + while index <= max_start_index: + prefix_end = index + prefixlen + if (page.page_tokens[index:prefix_end] == self.prefix).all(): + for peek in xrange(prefix_end, max_index): + if (page.page_tokens[peek:peek + suffixlen] \ + == self.suffix).all(): + extracted += self.extractor.extract(page, + prefix_end - 1, peek, ignored_regions) + index = max(peek, index + 1) + break + else: + break + else: + index += 1 + return extracted + + @staticmethod + def apply(template, extractors): + tokens = template.page_tokens + output_extractors = [] + group_key = lambda x: x.extracted_item() + for extr_key, extraction_group in groupby(extractors, group_key): + extraction_group = list(extraction_group) + if extr_key is None or len(extraction_group) == 1: + output_extractors += extraction_group + continue + + separating_tokens = [ \ + tokens[x.annotation.end_index:y.annotation.start_index+1] \ + for (x, y) in zip(extraction_group[:-1], extraction_group[1:])] + + # calculate the common prefix + group_start = extraction_group[0].annotation.start_index + prefix_start = max(0, group_start - len(separating_tokens[0])) + first_prefix = tokens[prefix_start:group_start+1] + prefixes = [first_prefix] + separating_tokens + prefix_pattern = list(reversed( + common_prefix(*map(reversed, prefixes)))) + + # calculate the common suffix + group_end = extraction_group[-1].annotation.end_index + last_suffix = tokens[group_end:group_end + \ + len(separating_tokens[-1])] + suffixes = separating_tokens + [last_suffix] + suffix_pattern = common_prefix(*suffixes) + + # create a repeated data extractor, if there is a suitable + # prefix and suffix. (TODO: tune this heuristic) + matchlen = len(prefix_pattern) + len(suffix_pattern) + if matchlen >= len(separating_tokens): + group_extractor = RepeatedDataExtractor(prefix_pattern, + suffix_pattern, extraction_group) + output_extractors.append(group_extractor) + else: + output_extractors += extraction_group + return output_extractors + + def extracted_item(self): + """key used to identify the item extracted""" + return self.extractor.extracted_item() + + def __repr__(self): + return "Repeat(%r)" % self.extractor + + def __str__(self): + return "Repeat(%s)" % self.extractor + +class TransposedDataExtractor(object): + """ """ + pass + +_namef = operator.itemgetter(0) +_valuef = operator.itemgetter(1) +def _attrs2dict(attributes): + """convert a list of attributes (name, value) tuples + into a dict of lists. + + For example: + >>> l = [('name', 'sofa'), ('colour', 'red'), ('colour', 'green')] + >>> _attrs2dict(l) == {'name': ['sofa'], 'colour': ['red', 'green']} + True + """ + grouped_data = groupby(sorted(attributes, key=_namef), _namef) + return dict((name, map(_valuef, data)) for (name, data) in grouped_data) + +class RecordExtractor(object): + """The RecordExtractor will extract records given annotations. + + It looks for a similar region in the target document, using the ibl + similarity algorithm. The annotations are partitioned by the first similar + region found and searched recursively. + + Records are represented as dicts mapping attribute names to lists + containing their values. + + For example: + >>> from scrapy.contrib.ibl.extraction.pageparsing import parse_strings + >>> template, page = parse_strings( \ + u'

x

' + \ + u'

y

', \ + u'

name

description

') + >>> basic_extractors = map(BasicTypeExtractor, template.annotations) + >>> ex = RecordExtractor.apply(template, basic_extractors)[0] + >>> ex.extract(page) + [{u'description': [u'description'], u'name': [u'name']}] + """ + + def __init__(self, extractors, template_tokens): + """Construct a RecordExtractor for the given annotations and their + corresponding region extractors + """ + self.extractors = extractors + self.template_tokens = template_tokens + self.template_ignored_regions = [] + start_index = min(e.annotation.start_index for e in extractors) + end_index = max(e.annotation.end_index for e in extractors) + self.annotation = AnnotationTag(start_index, end_index) + + def extract(self, page, start_index=0, end_index=None, ignored_regions=None): + """extract data from an extraction page + + The region in the page to be extracted from may be specified using + start_index and end_index + """ + ignored_regions = [LabelledRegion(*i) for i in (ignored_regions or [])] + region_elements = sorted(self.extractors + ignored_regions, key=lambda x: _labelled(x).start_index) + _, _, attributes = self._doextract(page, region_elements, start_index, + end_index) + # collect variant data, maintaining the order of variants + variant_ids = []; variants = {}; items = [] + for k, v in attributes: + if isinstance(k, int): + if k in variants: + variants[k] += v + else: + variant_ids.append(k) + variants[k] = v + else: + items.append((k, v)) + + variant_records = [('variants', _attrs2dict(variants[vid])) \ + for vid in variant_ids] + items += variant_records + return [_attrs2dict(items)] + + def _doextract(self, page, region_elements, start_index, end_index, nested_regions=None, ignored_regions=None): + """Carry out extraction of records using the given annotations + in the page tokens bounded by start_index and end_index + """ + # reorder extractors leaving nested ones for the end and separating + # ignore regions + nested_regions = nested_regions or [] + ignored_regions = ignored_regions or [] + first_region, following_regions = region_elements[0], region_elements[1:] + while following_regions and _labelled(following_regions[0]).start_index \ + < _labelled(first_region).end_index: + region = following_regions.pop(0) + labelled = _labelled(region) + if isinstance(labelled, AnnotationTag) or (nested_regions and \ + _labelled(nested_regions[-1]).start_index < labelled.start_index \ + < _labelled(nested_regions[-1]).end_index): + nested_regions.append(region) + else: + ignored_regions.append(region) + extracted_data = [] + # end_index is inclusive, but similar_region treats it as exclusive + end_region = None if end_index is None else end_index + 1 + labelled = _labelled(first_region) + score, pindex, sindex = \ + similar_region(page.page_tokens, self.template_tokens, + labelled, start_index, end_region) + if score > 0: + if isinstance(labelled, AnnotationTag): + similar_ignored_regions = [] + start = pindex + for i in ignored_regions: + s, p, e = similar_region(page.page_tokens, self.template_tokens, \ + i, start, sindex) + if s > 0: + similar_ignored_regions.append(LabelledRegion(*(p, e))) + start = e or start + extracted_data = first_region.extract(page, pindex, sindex, similar_ignored_regions) + if extracted_data: + if first_region.annotation.variant_id: + extracted_data = [(first_region.annotation.variant_id, extracted_data)] + + if nested_regions: + _, _, nested_data = self._doextract(page, nested_regions, pindex, sindex) + extracted_data += nested_data + if following_regions: + _, _, following_data = self._doextract(page, following_regions, sindex or start_index, end_region) + extracted_data += following_data + + elif following_regions: + end_index, _, following_data = self._doextract(page, following_regions, start_index, end_region) + if end_index is not None: + pindex, sindex, extracted_data = self._doextract(page, [first_region], start_index, end_index - 1, nested_regions, ignored_regions) + extracted_data += following_data + elif nested_regions: + _, _, nested_data = self._doextract(page, nested_regions, start_index, end_region) + extracted_data += nested_data + return pindex, sindex, extracted_data + + @classmethod + def apply(cls, template, extractors): + return [cls(extractors, template.page_tokens)] + + def extracted_item(self): + return [self.__class__.__name__] + \ + sorted(e.extracted_item() for e in self.extractors) + + def __repr__(self): + return str(self) + + def __str__(self): + stream = cStringIO.StringIO() + pprint.pprint(self.extractors, stream) + stream.seek(0) + template_data = stream.read() + if template_data: + return "%s[\n%s\n]" % (self.__class__.__name__, template_data) + return "%s[none]" % (self.__class__.__name__) + +class AdjacentVariantExtractor(RecordExtractor): + """Extractor for variants + + This simply extends the RecordExtractor to output data in a "variants" + attribute. + + The "apply" method will only apply to variants whose items are all adjacent and + it will appear as one record so that it can be handled by the RepeatedDataExtractor. + """ + + def extract(self, page, start_index=0, end_index=None, ignored_regions=None): + records = RecordExtractor.extract(self, page, start_index, end_index, ignored_regions) + return [('variants', r['variants'][0]) for r in records if r] + + @classmethod + def apply(cls, template, extractors): + adjacent_variants = set([]) + variantf = lambda x: x.annotation.variant_id + for vid, egroup in groupby(extractors, variantf): + if not vid: + continue + if vid in adjacent_variants: + adjacent_variants.remove(vid) + elif len(list(egroup)) > 1: + adjacent_variants.add(vid) + + new_extractors = [] + for variant, group_seq in groupby(extractors, variantf): + group_seq = list(group_seq) + if variant in adjacent_variants: + record_extractor = AdjacentVariantExtractor(group_seq, template.page_tokens) + new_extractors.append(record_extractor) + else: + new_extractors += group_seq + return new_extractors + + def __repr__(self): + return str(self) + +class TraceExtractor(object): + """Extractor that wraps other extractors and prints an execution + trace of the extraction process to aid debugging + """ + + def __init__(self, traced, template): + self.traced = traced + self.annotation = traced.annotation + tstart = traced.annotation.start_index + tend = traced.annotation.end_index + self.tprefix = " ".join([template.token_dict.token_string(t) + for t in template.page_tokens[tstart-4:tstart+1]]) + self.tsuffix = " ".join([template.token_dict.token_string(t) + for t in template.page_tokens[tend:tend+5]]) + + def summarize_trace(self, page, start, end, ret): + text_start = page.token_follow_indexes[start] + text_end = page.token_start_indexes[end or -1] + page_snippet = "(...%s)%s(%s...)" % ( + page.text[text_start-50:text_start].replace('\n', ' '), + page.text[text_start:text_end], + page.text[text_end:text_end+50].replace('\n', ' ')) + pre_summary = "\nstart %s page[%s:%s]\n" % (self.traced.__class__.__name__, start, end) + post_summary = """ +%s page[%s:%s] + +html +%s + +annotation +...%s +%s +%s... + +extracted +%s + """ % (self.traced.__class__.__name__, start, end, page_snippet, + self.tprefix, self.annotation, self.tsuffix, [r for r in ret if 'trace' not in r]) + return pre_summary, post_summary + + def extract(self, page, start, end, ignored_regions): + ret = self.traced.extract(page, start, end, ignored_regions) + if not ret: + return [] + + # handle records by inserting a trace and combining with variant traces + if len(ret) == 1 and isinstance(ret[0], dict): + item = ret[0] + trace = item.pop('trace', []) + variants = item.get('variants', ()) + for variant in variants: + trace += variant.pop('trace', []) + pre_summary, post_summary = self.summarize_trace(page, start, end, ret) + item['trace'] = [pre_summary] + trace + [post_summary] + return ret + + pre_summary, post_summary = self.summarize_trace(page, start, end, ret) + return [('trace', pre_summary)] + ret + [('trace', post_summary)] + + @staticmethod + def apply(template, extractors): + output = [] + for extractor in extractors: + if not isinstance(extractor, TraceExtractor): + extractor = TraceExtractor(extractor, template) + output.append(extractor) + return output + + def extracted_item(self): + return self.traced.extracted_item() + + def __repr__(self): + return "Trace(%s)" % repr(self.traced) + +class TemplatePageExtractor(object): + """Top level extractor for a template page""" + + def __init__(self, template, extractors): + # fixme: handle multiple items per page + self.extractor = extractors[0] + self.template = template + + def extract(self, page, start_index=0, end_index=None): + return self.extractor.extract(page, start_index, end_index, self.template.ignored_regions) + + def __repr__(self): + return repr(self.extractor) + + def __str__(self): + return str(self.extractor) + +_tokenize = nltk.tokenize.WordPunctTokenizer().tokenize + +class TextRegionDataExtractor(object): + """Data Extractor for extracting text fragments from within a larger + body of text. It extracts based on the longest unique prefix and suffix. + + for example: + >>> extractor = TextRegionDataExtractor('designed by ', '.') + >>> extractor.extract("by Marc Newson.") + 'Marc Newson' + + Both prefix and suffix are optional: + >>> extractor = TextRegionDataExtractor('designed by ') + >>> extractor.extract("by Marc Newson.") + 'Marc Newson.' + >>> extractor = TextRegionDataExtractor(suffix='.') + >>> extractor.extract("by Marc Newson.") + 'by Marc Newson' + + It requires a minimum match of at least one word or punctuation character: + >>> extractor = TextRegionDataExtractor('designed by') + >>> extractor.extract("y Marc Newson.") is None + True + """ + def __init__(self, prefix=None, suffix=None): + self.prefix = (prefix or '')[::-1] + self.suffix = suffix or '' + self.minprefix = self.minmatch(self.prefix) + self.minsuffix = self.minmatch(self.suffix) + + @staticmethod + def minmatch(matchstring): + """the minimum number of characters that should match in order + to consider it a match for that string. + + This uses the last word of punctuation character + """ + tokens = _tokenize(matchstring or '') + return len(tokens[0]) if tokens else 0 + + def extract(self, text): + """attempt to extract a substring from the text""" + pref_index = 0 + if self.minprefix > 0: + rev_idx, plen = longest_unique_subsequence(text[::-1], self.prefix) + if plen < self.minprefix: + return None + pref_index = -rev_idx + if self.minsuffix == 0: + return text[pref_index:] + sidx, slen = longest_unique_subsequence(text[pref_index:], self.suffix) + if slen < self.minsuffix: + return None + return text[pref_index:pref_index + sidx] + +class TextPrefixRegionDataExtractor(object): + """ + Data extractor for extracting text fragment from within a + larger body of text, based on a fixed prefix. + >>> extractor = TextPrefixRegionDataExtractor("£s;") + >>> extractor.extract("£s; 17.00") + ' 17.00' + >>> extractor.extract("€ 17.00") is None + True + >>> extractor.extract("$ 17.00") is None + True + >>> extractor.extract(" £s; 17.00 ") + ' 17.00 ' + """ + def __init__(self, prefix): + self.prefix = prefix + def extract(self, text): + text = text.lstrip() + # attempt to extract a substring from the text + if text.startswith(self.prefix): + return text.replace(self.prefix, '') + diff --git a/scrapy/contrib/ibl/extraction/similarity.py b/scrapy/contrib/ibl/extraction/similarity.py new file mode 100644 index 000000000..9bb8e24c3 --- /dev/null +++ b/scrapy/contrib/ibl/extraction/similarity.py @@ -0,0 +1,136 @@ +""" +Similarity calculation for Instance based extraction algorithm. +""" +from itertools import izip, count +from operator import itemgetter +from heapq import nlargest + +def common_prefix_length(a, b): + """Calculate the length of the common prefix in both sequences passed. + + For example, the common prefix in this example is [1, 3] + >>> common_prefix_length([1, 3, 4], [1, 3, 5, 1]) + 2 + + If there is no common prefix, 0 is returned + >>> common_prefix_length([1], []) + 0 + """ + i = -1 + for i, x, y in izip(count(), a, b): + if x != y: + return i + return i + 1 + +def common_prefix(*sequences): + """determine the common prefix of all sequences passed + + For example: + >>> common_prefix('abcdef', 'abc', 'abac') + ['a', 'b'] + """ + prefix = [] + for sample in izip(*sequences): + first = sample[0] + if all(x == first for x in sample[1:]): + prefix.append(first) + else: + break + return prefix + +def longest_unique_subsequence(to_search, subsequence, range_start=0, + range_end=None): + """Find the longest unique subsequence of items in a list or array. This + searches the to_search list or array looking for the longest overlapping + match with subsequence. If the largest match is unique (there is no other + match of equivalent length), the index and length of match is returned. If + there is no match, (None, None) is returned. + + Please see section 3.2 of Extracting Web Data Using Instance-Based + Learning by Yanhong Zhai and Bing Liu + + For example, the longest match occurs at index 2 and has length 3 + >>> to_search = [6, 3, 2, 4, 3, 2, 5] + >>> longest_unique_subsequence(to_search, [2, 4, 3]) + (2, 3) + + When there are two equally long subsequences, it does not generate a match + >>> longest_unique_subsequence(to_search, [3, 2]) + (None, None) + + range_start and range_end specify a range in which the match must begin + >>> longest_unique_subsequence(to_search, [3, 2], 3) + (4, 2) + >>> longest_unique_subsequence(to_search, [3, 2], 0, 2) + (1, 2) + """ + startval = subsequence[0] + if range_end is None: + range_end = len(to_search) + + # the comparison to startval ensures only matches of length >= 1 and + # reduces the number of calls to the common_length function + matches = ((i, common_prefix_length(to_search[i:], subsequence)) \ + for i in xrange(range_start, range_end) if startval == to_search[i]) + best2 = nlargest(2, matches, key=itemgetter(1)) + # if there is a single unique best match, return that + if len(best2) == 1 or len(best2) == 2 and best2[0][1] != best2[1][1]: + return best2[0] + return None, None + +def similar_region(extracted_tokens, template_tokens, labelled_region, + range_start=0, range_end=None): + """Given a labelled section in a template, identify a similar region + in the extracted tokens. + + The start and end index of the similar region in the extracted tokens + is returned. + + This will return a tuple containing: + (match score, start index, end index) + where match score is the sum of the length of the matching prefix and + suffix. If there is no unique match, (0, None, None) will be returned. + + start_index and end_index specify a range in which the match must begin + """ + data_length = len(extracted_tokens) + if range_end is None: + range_end = data_length + # calculate the prefix score by finding a longest subsequence in + # reverse order + reverse_prefix = template_tokens[labelled_region.start_index::-1] + reverse_tokens = extracted_tokens[::-1] + (rpi, pscore) = longest_unique_subsequence(reverse_tokens, reverse_prefix, + data_length - range_end, data_length - range_start) + + # None means nothing exracted. Index 0 means there cannot be a suffix. + if not rpi: + return 0, None, None + + # convert to an index from the start instead of in reverse + prefix_index = len(extracted_tokens) - rpi - 1 + + if labelled_region.end_index is None: + return pscore, prefix_index, None + + suffix = template_tokens[labelled_region.end_index:] + + # if it's not a paired tag, use the best match between prefix & suffix + if labelled_region.start_index == labelled_region.end_index: + (match_index, sscore) = longest_unique_subsequence(extracted_tokens, + suffix, prefix_index, range_end) + if match_index == prefix_index: + return (pscore + sscore, prefix_index, match_index) + elif pscore > sscore: + return pscore, prefix_index, prefix_index + elif sscore > pscore: + return sscore, match_index, match_index + return 0, None, None + + # calculate the suffix match on the tokens following the prefix. We could + # consider the whole page and require a good match. + (match_index, sscore) = longest_unique_subsequence(extracted_tokens, + suffix, prefix_index + 1, range_end) + if match_index is None: + return 0, None, None + return (pscore + sscore, prefix_index, match_index) diff --git a/scrapy/contrib/ibl/extractors.py b/scrapy/contrib/ibl/extractors.py new file mode 100644 index 000000000..9c460df10 --- /dev/null +++ b/scrapy/contrib/ibl/extractors.py @@ -0,0 +1,156 @@ +""" +Extractors for attributes +""" +import re +import urlparse + +from scrapy.utils.markup import remove_entities +from scrapy.utils.url import safe_url_string + +#FIXME: the use of "." needs to be localized +_NUMERIC_ENTITIES = re.compile("&#([0-9]+)(?:;|\s)", re.U) +_PRICE_NUMBER_RE = re.compile('(?:^|[^a-zA-Z0-9])(\d+(?:\.\d+)?)(?:$|[^a-zA-Z0-9])') +_NUMBER_RE = re.compile('(\d+(?:\.\d+)?)') + +_IMAGES = ( + 'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif', + 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', +) + +_IMAGES_TYPES = '|'.join(_IMAGES) +_CSS_IMAGERE = re.compile("background(?:-image)?\s*:\s*url\((.*?)\)", re.I) +_BASE_PATH_RE = "/?(?:[^/]+/)*(?:.+%s)" +_IMAGE_PATH_RE = re.compile(_BASE_PATH_RE % '\.(?:%s)' % _IMAGES_TYPES, re.I) +_GENERIC_PATH_RE = re.compile(_BASE_PATH_RE % '', re.I) + +def text(txt): + stripped = txt.strip() if txt else None + if stripped: + return stripped + +def contains_any_numbers(txt): + """text that must contain at least one number + >>> contains_any_numbers('foo') + >>> contains_any_numbers('$67 at 15% discount') + '$67 at 15% discount' + """ + if _NUMBER_RE.search(txt) is not None: + return txt + +def contains_prices(txt): + """text must contain a number that is not joined to text""" + if _PRICE_NUMBER_RE.findall(txt) is not None: + return txt + +def contains_numbers(txt, count=1): + """Must contain a certain amount of numbers + + >>> contains_numbers('foo', 2) + >>> contains_numbers('this 1 has 2 numbers', 2) + 'this 1 has 2 numbers' + """ + numbers = _NUMBER_RE.findall(txt) + if len(numbers) == count: + return txt + +def extract_number(txt): + """Extract a numeric value. + + This will fail if more than one numeric value is present. + + >>> extract_number(' 45.3') + '45.3' + >>> extract_number(' 45.3, 7') + + It will handle unescaped entities: + >>> extract_number(u'£129.99') + u'129.99' + """ + txt = _NUMERIC_ENTITIES.sub(lambda m: unichr(int(m.groups()[0])), txt) + numbers = _NUMBER_RE.findall(txt) + if len(numbers) == 1: + return numbers[0] + +def url(txt): + """convert text to a url + + this is quite conservative, since relative urls are supported + """ + txt = txt.strip("\t\r\n '\"") + if txt: + return txt + +def image_url(txt): + """convert text to a url + + this is quite conservative, since relative urls are supported + Example: + + >>> image_url('') + + >>> image_url(' ') + + >>> image_url(' \\n\\n ') + + >>> image_url('foo-bar.jpg') + ['foo-bar.jpg'] + >>> image_url('/images/main_logo12.gif') + ['/images/main_logo12.gif'] + >>> image_url("http://www.image.com/image.jpg") + ['http://www.image.com/image.jpg'] + >>> image_url("http://www.domain.com/path1/path2/path3/image.jpg") + ['http://www.domain.com/path1/path2/path3/image.jpg'] + >>> image_url("/path1/path2/path3/image.jpg") + ['/path1/path2/path3/image.jpg'] + >>> image_url("path1/path2/image.jpg") + ['path1/path2/image.jpg'] + >>> image_url("background-image : url(http://www.site.com/path1/path2/image.jpg)") + ['http://www.site.com/path1/path2/image.jpg'] + >>> image_url("background-image : url('http://www.site.com/path1/path2/image.jpg')") + ['http://www.site.com/path1/path2/image.jpg'] + >>> image_url('background-image : url("http://www.site.com/path1/path2/image.jpg")') + ['http://www.site.com/path1/path2/image.jpg'] + >>> image_url("background : url(http://www.site.com/path1/path2/image.jpg)") + ['http://www.site.com/path1/path2/image.jpg'] + >>> image_url("background : url('http://www.site.com/path1/path2/image.jpg')") + ['http://www.site.com/path1/path2/image.jpg'] + >>> image_url('background : url("http://www.site.com/path1/path2/image.jpg")') + ['http://www.site.com/path1/path2/image.jpg'] + >>> image_url('/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350') + ['/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350'] + >>> image_url('http://www.site.com/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350') + ['http://www.site.com/getimage.php?image=totalgardens/outbbq2_400.jpg&type=prod&resizeto=350'] + >>> image_url('http://s7d4.scene7.com/is/image/Kohler/jaa03267?hei=425&wid=457&op_usm=2,1,2,1&qlt=80') + ['http://s7d4.scene7.com/is/image/Kohler/jaa03267?hei=425&wid=457&op_usm=2,1,2,1&qlt=80'] + >>> image_url('../image.aspx?thumb=true&boxSize=175&img=Unknoportrait[1].jpg') + ['../image.aspx?thumb=true&boxSize=175&img=Unknoportrait%5B1%5D.jpg'] + >>> image_url('http://www.sundancecatalog.com/mgen/catalog/test.ms?args=%2245932|MERIDIAN+PENDANT|.jpg%22&is=336,336,0xffffff') + ['http://www.sundancecatalog.com/mgen/catalog/test.ms?args=%2245932|MERIDIAN+PENDANT|.jpg%22&is=336,336,0xffffff'] + >>> image_url('http://www.site.com/image.php') + ['http://www.site.com/image.php'] + >>> image_url('background-image:URL(http://s7d5.scene7.com/is/image/wasserstrom/165133?wid=227&hei=227&defaultImage=noimage_wasserstrom)') + ['http://s7d5.scene7.com/is/image/wasserstrom/165133?wid=227&hei=227&defaultImage=noimage_wasserstrom'] + + """ + txt = url(txt) + imgurl = None + if txt: + # check if the text is style content + m = _CSS_IMAGERE.search(txt) + txt = m.groups()[0] if m else txt + parsed = urlparse.urlparse(txt) + path = None + m = _IMAGE_PATH_RE.search(parsed.path) + if m: + path = m.group() + elif parsed.query: + m = _GENERIC_PATH_RE.search(parsed.path) + if m: + path = m.group() + if path is not None: + parsed = list(parsed) + parsed[2] = path + imgurl = urlparse.urlunparse(parsed) + if not imgurl: + imgurl = txt + return [safe_url_string(remove_entities(url(imgurl)))] if imgurl else None diff --git a/scrapy/contrib/ibl/htmlpage.py b/scrapy/contrib/ibl/htmlpage.py new file mode 100644 index 000000000..daeef4053 --- /dev/null +++ b/scrapy/contrib/ibl/htmlpage.py @@ -0,0 +1,212 @@ +""" +htmlpage + +Container object for representing html pages in the IBL system. This +encapsulates page related information and prevents parsing multiple times. +""" +import re +import hashlib + +from scrapy.utils.python import str_to_unicode + +def create_page_from_jsonpage(jsonpage, body_key): + """Create an HtmlPage object from a dict object conforming to the schema + for a page + + `body_key` is the key where the body is stored and can be either 'body' + (original page with annotations - if any) or 'original_body' (original + page, always). Classification typically uses 'original_body' to avoid + confusing the classifier with annotated pages, while extraction uses 'body' + to pass the annotated pages. + """ + url = jsonpage['url'] + headers = jsonpage.get('headers') + body = str_to_unicode(jsonpage[body_key]) + page_id = jsonpage.get('page_id') + return HtmlPage(url, headers, body, page_id) + +class HtmlPage(object): + def __init__(self, url=None, headers=None, body=None, page_id=None): + assert isinstance(body, unicode), "unicode expected, got: %s" % type(body).__name__ + self.headers = headers or {} + self.body = body + self.url = url or u'' + if page_id is None and url: + self.page_id = hashlib.sha1(url).hexdigest() + else: + self.page_id = page_id + + + def _set_body(self, body): + self._body = body + self.parsed_body = list(parse_html(body)) + + body = property(lambda x: x._body, _set_body) + + def fragment_data(self, data_fragment): + return self.body[data_fragment.start:data_fragment.end] + +class HtmlTagType(object): + OPEN_TAG = 1 + CLOSE_TAG = 2 + UNPAIRED_TAG = 3 + +class HtmlDataFragment(object): + __slots__ = ('start', 'end') + + def __init__(self, start, end): + self.start = start + self.end = end + + def __str__(self): + return "" % (self.start, self.end) + + def __repr__(self): + return str(self) + +class HtmlTag(HtmlDataFragment): + __slots__ = ('tag_type', 'tag', 'attributes') + + def __init__(self, tag_type, tag, attributes, start, end): + HtmlDataFragment.__init__(self, start, end) + self.tag_type = tag_type + self.tag = tag + self.attributes = attributes + + def __str__(self): + return "" % (self.tag, ', '.join(sorted\ + (["%s: %s" % (k, repr(v)) for k, v in self.attributes.items()])), self.start, self.end) + + def __repr__(self): + return str(self) + +_ATTR = "((?:[^=/>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?" +_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>" +_DOCTYPE = r"" + +_ATTR_REGEXP = re.compile(_ATTR, re.I | re.DOTALL) +_HTML_REGEXP = re.compile(_TAG, re.I | re.DOTALL) +_DOCTYPE_REGEXP = re.compile("(?:%s)" % _DOCTYPE) +_COMMENT_RE = re.compile("()", re.DOTALL) +_SCRIPT_RE = re.compile("().*?()", re.DOTALL | re.I) + +def parse_html(text): + """Higher level html parser. Calls lower level parsers and joins sucesive + HtmlDataFragment elements in a single one. + """ + script_layer = lambda x: _parse_clean_html(x, _SCRIPT_RE, HtmlTag, _simple_parse_html) + comment_layer = lambda x: _parse_clean_html(x, _COMMENT_RE, HtmlDataFragment, script_layer) + delayed_element = None + for element in comment_layer(text): + if isinstance(element, HtmlTag): + if delayed_element is not None: + yield delayed_element + delayed_element = None + yield element + else:# element is HtmlDataFragment + if delayed_element is not None: + delayed_element.start = min(element.start, delayed_element.start) + delayed_element.end = max(element.end, delayed_element.end) + else: + delayed_element = element + if delayed_element is not None: + yield delayed_element + +def _parse_clean_html(text, regex, htype, func): + """ + Removes regions from text, passes the cleaned text to the lower parse layer, + and reinserts removed regions. + regex - regular expression that defines regions to be removed/re inserted + htype - the html parser type of the removed elements + func - function that performs the lower parse layer + """ + removed = [[m.start(), m.end(), m.groups()] for m in regex.finditer(text)] + + cleaned = regex.sub("", text) + shift = 0 + for element in func(cleaned): + element.start += shift + element.end += shift + while removed: + if element.end <= removed[0][0]: + yield element + break + else: + start, end, groups = removed.pop(0) + add = end - start + element.end += add + shift += add + if element.start >= start: + element.start += add + elif isinstance(element, HtmlTag): + yield element + break + + if element.start < start: + yield HtmlDataFragment(element.start, start) + element.start = end + + if htype == HtmlTag: + begintag = _parse_tag(_HTML_REGEXP.match(groups[0])) + endtag = _parse_tag(_HTML_REGEXP.match(groups[1])) + begintag.start = start + begintag.end += start + + endtag.start = end - endtag.end + endtag.end = end + content = None + if begintag.end < endtag.start: + content = HtmlDataFragment(begintag.end, endtag.start) + yield begintag + if content is not None: + yield content + yield endtag + else: + yield htype(start, end) + else: + yield element + +def _simple_parse_html(text): + """Simple html parse. It returns a sequence of HtmlTag and HtmlDataFragment + objects. Does not ignore any region. + """ + # If have doctype remove it. + start_pos = 0 + match = _DOCTYPE_REGEXP.match(text) + if match: + start_pos = match.end() + prev_end = start_pos + for match in _HTML_REGEXP.finditer(text, start_pos): + start = match.start() + end = match.end() + + if start > prev_end: + yield HtmlDataFragment(prev_end, start) + + yield _parse_tag(match) + prev_end = end + textlen = len(text) + if prev_end < textlen: + yield HtmlDataFragment(prev_end, textlen) + +def _parse_tag(match): + """ + parse a tag matched by _HTML_REGEXP + """ + data = match.groups() + closing, tag, attr_text = data[:3] + # if tag is None then the match is a comment + if tag is not None: + unpaired = data[-1] + if closing: + tag_type = HtmlTagType.CLOSE_TAG + elif unpaired: + tag_type = HtmlTagType.UNPAIRED_TAG + else: + tag_type = HtmlTagType.OPEN_TAG + attributes = [] + for attr_match in _ATTR_REGEXP.findall(attr_text): + name = attr_match[0].lower() + values = [v for v in attr_match[1:] if v] + attributes.append((name, values[0] if values else None)) + return HtmlTag(tag_type, tag.lower(), dict(attributes), match.start(), match.end()) diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index 6021bbf5a..9f8666467 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -1,6 +1,6 @@ """ This module provides a mechanism for collecting one (or more) sample items per -domain. +spider. The items are collected in a dict of guid->item and persisted by pickling that dict into a file. @@ -8,7 +8,7 @@ dict into a file. This can be useful for testing changes made to the framework or other common code that affects several spiders. -It uses the scrapy stats service to keep track of which domains are already +It uses the scrapy stats service to keep track of which spiders are already sampled. Settings that affect this module: @@ -48,7 +48,7 @@ class ItemSamplerPipeline(object): raise NotConfigured self.items = {} self.spiders_count = 0 - self.empty_domains = set() + self.empty_spiders = set() dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -66,21 +66,21 @@ class ItemSamplerPipeline(object): def engine_stopped(self): with open(self.filename, 'w') as f: pickle.dump(self.items, f) - if self.empty_domains: - log.msg("No products sampled for: %s" % " ".join(self.empty_domains), \ + if self.empty_spiders: + log.msg("No products sampled for: %s" % " ".join(self.empty_spiders), \ level=log.WARNING) def spider_closed(self, spider, reason): if reason == 'finished' and not stats.get_value("items_sampled", spider=spider): - self.empty_domains.add(spider.domain_name) + self.empty_spiders.add(spider.name) self.spiders_count += 1 - log.msg("Sampled %d domains so far (%d empty)" % (self.spiders_count, \ - len(self.empty_domains)), level=log.INFO) + log.msg("Sampled %d spiders so far (%d empty)" % (self.spiders_count, \ + len(self.empty_spiders)), level=log.INFO) class ItemSamplerMiddleware(object): - """This middleware drops items and requests (when domain sampling has been - completed) to accelerate the processing of remaining domains""" + """This middleware drops items and requests (when spider sampling has been + completed) to accelerate the processing of remaining spiders""" def __init__(self): if not settings['ITEMSAMPLER_FILE']: diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index 2714fb562..fb3fd661b 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -26,7 +26,7 @@ class HtmlParserLinkExtractor(HTMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 711623b81..88dd78577 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -3,7 +3,6 @@ This module implements the HtmlImageLinkExtractor for extracting image links only. """ -import urlparse from scrapy.link import Link from scrapy.utils.url import canonicalize_url, urljoin_rfc @@ -25,13 +24,13 @@ class HTMLImageLinkExtractor(object): self.unique = unique self.canonicalize = canonicalize - def extract_from_selector(self, selector, parent=None): + def extract_from_selector(self, selector, encoding, parent=None): ret = [] def _add_link(url_sel, alt_sel=None): url = flatten([url_sel.extract()]) alt = flatten([alt_sel.extract()]) if alt_sel else (u'', ) if url: - ret.append(Link(unicode_to_str(url[0]), alt[0])) + ret.append(Link(unicode_to_str(url[0], encoding), alt[0])) if selector.xmlNode.type == 'element': if selector.xmlNode.name == 'img': @@ -41,7 +40,7 @@ class HTMLImageLinkExtractor(object): children = selector.select('child::*') if len(children): for child in children: - ret.extend(self.extract_from_selector(child, parent=selector)) + ret.extend(self.extract_from_selector(child, encoding, parent=selector)) elif selector.xmlNode.name == 'a' and not parent: _add_link(selector.select('@href'), selector.select('@title')) else: @@ -52,7 +51,7 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) base_url = xs.select('//base/@href').extract() - base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) + base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url links = [] for location in self.locations: @@ -64,7 +63,7 @@ class HTMLImageLinkExtractor(object): continue for selector in selectors: - links.extend(self.extract_from_selector(selector)) + links.extend(self.extract_from_selector(selector, response.encoding)) seen, ret = set(), [] for link in links: diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py index 390e3a304..27cd0697a 100644 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ b/scrapy/contrib/linkextractors/lxmlparser.py @@ -29,7 +29,7 @@ class LxmlLinkExtractor(object): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 08e1e4526..1de044df3 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -16,8 +16,9 @@ def clean_link(link_text): class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" + def _extract_links(self, response_text, response_url, response_encoding): - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip() diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index d548626ab..17877031d 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -21,15 +21,14 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): self.unique = unique def _extract_links(self, response_text, response_url, response_encoding): + """ Do the real extraction work """ self.reset() self.feed(response_text) self.close() - links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links - ret = [] - base_url = self.base_url if self.base_url else response_url - for link in links: + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url + for link in self.links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) link.text = str_to_unicode(link.text, response_encoding) @@ -37,9 +36,19 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): return ret + def _process_links(self, links): + """ Normalize and filter extracted links + + The subclass should override it if neccessary + """ + links = unique_list(links, key=lambda link: link.url) if self.unique else links + return links + def extract_links(self, response): # wrapper needed to allow to work directly with text - return self._extract_links(response.body, response.url, response.encoding) + links = self._extract_links(response.body, response.url, response.encoding) + links = self._process_links(links) + return links def reset(self): FixedSGMLParser.reset(self) @@ -93,12 +102,16 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): def extract_links(self, response): if self.restrict_xpaths: hxs = HtmlXPathSelector(response) - html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ + html = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ for xpath_expr in self.restrict_xpaths) - links = self._extract_links(html_slice, response.url, response.encoding) else: - links = BaseSgmlLinkExtractor.extract_links(self, response) + html = response.body + links = self._extract_links(html, response.url, response.encoding) + links = self._process_links(links) + return links + + def _process_links(self, links): links = [link for link in links if _is_valid_url(link.url)] if self.allow_res: @@ -114,6 +127,7 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): for link in links: link.url = canonicalize_url(link.url) + links = BaseSgmlLinkExtractor._process_links(self, links) return links def matches(self, url): diff --git a/scrapy/contrib/pipeline/fileexport.py b/scrapy/contrib/pipeline/fileexport.py index d20b83c2a..6e2109d01 100644 --- a/scrapy/contrib/pipeline/fileexport.py +++ b/scrapy/contrib/pipeline/fileexport.py @@ -8,6 +8,7 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.exceptions import NotConfigured from scrapy.contrib import exporter +from scrapy.contrib.exporter import jsonlines from scrapy.conf import settings class FileExportPipeline(object): @@ -48,7 +49,6 @@ class FileExportPipeline(object): elif format == 'pickle': exp = exporter.PickleItemExporter(file, **exp_kwargs) elif format == 'json': - from scrapy.contrib.exporter import jsonlines exp = jsonlines.JsonLinesItemExporter(file, **exp_kwargs) else: raise NotConfigured("Unsupported export format: %s" % format) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 3a93553dd..46eef385c 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -47,7 +47,7 @@ class FSImagesStore(object): dispatcher.connect(self.spider_closed, signals.spider_closed) def spider_closed(self, spider): - self.created_directories.pop(spider.domain_name, None) + self.created_directories.pop(spider.name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) @@ -92,7 +92,7 @@ class _S3AmazonAWSSpider(BaseSpider): It means that a spider that uses download_delay or alike is not going to be delayed even more because it is uploading images to s3. """ - domain_name = "s3.amazonaws.com" + name = "s3.amazonaws.com" start_urls = ['http://s3.amazonaws.com/'] max_concurrent_requests = 100 @@ -143,7 +143,7 @@ class S3ImagesStore(object): def _build_request(self, key, method, body=None, headers=None): url = 'http://%s.s3.amazonaws.com/%s%s' % (self.bucket, self.prefix, key) return Request(url, method=method, body=body, headers=headers, \ - priority=self.request_priority) + meta={'sign_s3_request': True}, priority=self.request_priority) def _download_request(self, request, info): """This method is used for HEAD and PUT requests sent to amazon S3 @@ -206,8 +206,8 @@ class ImagesPipeline(MediaPipeline): referer = request.headers.get('Referer') if response.status != 200: - log.msg('Image (http-error): Error downloading image from %s referred in <%s>' \ - % (request, referer), level=log.WARNING, spider=info.spider) + log.msg('Image (code: %s): Error downloading image from %s referred in <%s>' \ + % (response.status, request, referer), level=log.WARNING, spider=info.spider) raise ImageException if not response.body: diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 229301c2c..1c41b625d 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -4,7 +4,6 @@ spiders """ import sys -import urlparse from twisted.plugin import getCache from twisted.python.rebuild import rebuild @@ -19,42 +18,38 @@ class TwistedPluginSpiderManager(object): def __init__(self): self.loaded = False - self.force_domain = None - self._invaliddict = {} self._spiders = {} - def fromdomain(self, domain): - return self._spiders.get(domain) + def create(self, spider_name, **spider_kwargs): + """Returns a Spider instance for the given spider name, using the given + spider arguments. If the sipder name is not found, it raises a + KeyError. + """ + spider = self._spiders[spider_name] + spider.__dict__.update(spider_kwargs) + return spider - def fromurl(self, url): - if self.force_domain: - return self._spiders.get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self._spiders: # try first locating by domain - return self._spiders[domain] - else: # else search spider by spider - plist = self._spiders.values() - for p in plist: - if url_is_from_spider(url, p): - return p + def find_by_request(self, request): + """Returns list of spiders names that match the given Request""" + return [name for name, spider in self._spiders.iteritems() + if url_is_from_spider(request.url, spider)] def list(self): + """Returns list of spiders available.""" return self._spiders.keys() def load(self, spider_modules=None): + """Load spiders from module directory.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._invaliddict = {} self._spiders = {} modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] for module in modules: for spider in self._getspiders(ISpider, module): ISpider.validateInvariants(spider) - self._spiders[spider.domain_name] = spider + self._spiders[spider.name] = spider self.loaded = True def _getspiders(self, interface, package): @@ -77,14 +72,14 @@ class TwistedPluginSpiderManager(object): """Reload spider module to release any resources held on to by the spider """ - domain = spider.domain_name - if domain not in self._spiders: + name = spider.name + if name not in self._spiders: return - spider = self._spiders[domain] + spider = self._spiders[name] module_name = spider.__module__ module = sys.modules[module_name] if hasattr(module, 'SPIDER'): log.msg("Reloading module %s" % module_name, spider=spider, \ level=log.DEBUG) new_module = rebuild(module, doLog=0) - self._spiders[domain] = new_module.SPIDER + self._spiders[name] = new_module.SPIDER diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py index 78cecfdf7..6da26aa2d 100644 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -3,6 +3,16 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ +from scrapy.core.exceptions import IgnoreRequest + + +class HttpErrorException(IgnoreRequest): + """A non-200 response was filtered""" + + def __init__(self, response, *args, **kwargs): + self.response = response + super(HttpErrorException, self).__init__(*args, **kwargs) + class HttpErrorMiddleware(object): @@ -15,4 +25,5 @@ class HttpErrorMiddleware(object): allowed_statuses = getattr(spider, 'handle_httpstatus_list', ()) if response.status in allowed_statuses: return - return [] + raise HttpErrorException(response, 'Ignoring non-200 response') + diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 5e40d9915..f28b0d53d 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -47,8 +47,7 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - domains = [spider.domain_name] + spider.extra_domain_names - self.host_regexes[spider] = self.get_host_regex(domains) + self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains) self.domains_seen[spider] = set() def spider_closed(self, spider): diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 648765b06..2e74dab4e 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -59,9 +59,9 @@ class CrawlSpider(InitSpider): """ rules = () - def __init__(self): + def __init__(self, *a, **kw): """Constructor takes care of compiling rules""" - super(CrawlSpider, self).__init__() + super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() def parse(self, response): diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py index b37591ca5..f759fd81f 100644 --- a/scrapy/contrib/spiders/init.py +++ b/scrapy/contrib/spiders/init.py @@ -3,8 +3,8 @@ from scrapy.spider import BaseSpider class InitSpider(BaseSpider): """Base Spider with initialization facilities""" - def __init__(self): - super(InitSpider, self).__init__() + def __init__(self, *a, **kw): + super(InitSpider, self).__init__(*a, **kw) self._postinit_reqs = [] self._init_complete = False self._init_started = False diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index c2185e0a7..fc76ccfbf 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -23,6 +23,6 @@ class StatsMailer(object): mail = MailSender() body = "Global stats\n\n" body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.domain_name + body += "\n\n%s stats\n\n" % spider.name body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - mail.send(self.recipients, "Scrapy stats for: %s" % spider.domain_name, body) + mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 22afc20c1..2e61e8bc3 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -60,7 +60,7 @@ class LiveStats(object): runtime = datetime.now() - stats.started s += '%s%d%d%d%d%d%d%s%s\n' % \ - (spider.domain_name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) + (spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) totdomains += 1 totscraped += stats.scraped diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 9719d12f0..ce3fdffd6 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -25,18 +25,18 @@ class Spiderctl(object): dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) def spider_opened(self, spider): - self.running[spider.domain_name] = spider + self.running[spider.name] = spider def spider_closed(self, spider): - del self.running[spider.domain_name] - self.finished.add(spider.domain_name) + del self.running[spider.name] + self.finished.add(spider.name) def webconsole_render(self, wc_request): if wc_request.args: changes = self.webconsole_control(wc_request) - self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders] - self.idle = [d for d in self.enabled_domains if d not in self.scheduled + self.scheduled = [s.name for s in scrapyengine.spider_scheduler._pending_spiders] + self.idle = [d for d in self.enabled_spiders if d not in self.scheduled and d not in self.running and d not in self.finished] @@ -53,9 +53,9 @@ class Spiderctl(object): # idle s += "\n" s += '
\n' - s += '\n' + for name in sorted(self.idle): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -65,9 +65,9 @@ class Spiderctl(object): # scheduled s += "\n" s += '\n' - s += '\n' + for name in self.scheduled: + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -78,9 +78,9 @@ class Spiderctl(object): # running s += "\n" s += '\n' - s += '\n' + for name in sorted(self.running): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -90,9 +90,9 @@ class Spiderctl(object): # finished s += "\n" s += '\n' - s += '\n' + for name in sorted(self.finished): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -114,42 +114,42 @@ class Spiderctl(object): args = wc_request.args s = "
\n" - if "stop_running_domains" in args: + if "stop_running_spiders" in args: s += "

" - stopped_domains = [] - for domain in args["stop_running_domains"]: - if domain in self.running: - scrapyengine.close_spider(self.running[domain]) - stopped_domains.append(domain) - s += "Stopped spiders:

  • %s
" % "
  • ".join(stopped_domains) + stopped_spiders = [] + for name in args["stop_running_spiders"]: + if name in self.running: + scrapyengine.close_spider(self.running[name]) + stopped_spiders.append(name) + s += "Stopped spiders:
    • %s
    " % "
  • ".join(stopped_spiders) s += "

    " - if "remove_pending_domains" in args: + if "remove_pending_spiders" in args: removed = [] - for domain in args["remove_pending_domains"]: - if scrapyengine.spider_scheduler.remove_pending_domain(domain): - removed.append(domain) + for name in args["remove_pending_spiders"]: + if scrapyengine.spider_scheduler.remove_pending_spider(name): + removed.append(name) if removed: s += "

    " - s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_domains"]) + s += "Removed scheduled spiders:
    • %s
    " % "
  • ".join(args["remove_pending_spiders"]) s += "

    " - if "add_pending_domains" in args: - for domain in args["add_pending_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) + if "add_pending_spiders" in args: + for name in args["add_pending_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) s += "

    " - s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_domains"]) + s += "Scheduled spiders:
    • %s
    " % "
  • ".join(args["add_pending_spiders"]) s += "

    " - if "rerun_finished_domains" in args: - for domain in args["rerun_finished_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) - self.finished.remove(domain) + if "rerun_finished_spiders" in args: + for name in args["rerun_finished_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) + self.finished.remove(name) s += "

    " - s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) + s += "Re-scheduled finished spiders:
    • %s
    " % "
  • ".join(args["rerun_finished_spiders"]) s += "

    " return s def webconsole_discover_module(self): - self.enabled_domains = spiders.list() + self.enabled_spiders = spiders.list() return self diff --git a/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py index 837cbcf81..e8c707f64 100644 --- a/scrapy/contrib/webconsole/stats.py +++ b/scrapy/contrib/webconsole/stats.py @@ -23,7 +23,7 @@ class StatsDump(object): s += "

    Global stats

    \n" s += stats_html_table(stats.get_stats()) for spider, spider_stats in stats.iter_spider_stats(): - s += "

    %s

    \n" % spider.domain_name + s += "

    %s

    \n" % spider.name s += stats_html_table(spider_stats) s += "\n" s += "\n" diff --git a/scrapy/contrib_exp/crawlspider/__init__.py b/scrapy/contrib_exp/crawlspider/__init__.py new file mode 100644 index 000000000..03173eb38 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/__init__.py @@ -0,0 +1,4 @@ +"""CrawlSpider v2""" + +from .rules import Rule +from .spider import CrawlSpider diff --git a/scrapy/contrib_exp/crawlspider/matchers.py b/scrapy/contrib_exp/crawlspider/matchers.py new file mode 100644 index 000000000..3ef259c67 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/matchers.py @@ -0,0 +1,61 @@ +""" +Request/Response Matchers + +Perform evaluation to Request or Response attributes +""" + +import re + +class BaseMatcher(object): + """Base matcher. Returns True by default.""" + + def matches_request(self, request): + """Performs Request Matching""" + return True + + def matches_response(self, response): + """Performs Response Matching""" + return True + + +class UrlMatcher(BaseMatcher): + """Matches URL attribute""" + + def __init__(self, url): + """Initialize url attribute""" + self._url = url + + def matches_url(self, url): + """Returns True if given url is equal to matcher's url""" + return self._url == url + + def matches_request(self, request): + """Returns True if Request's url matches initial url""" + return self.matches_url(request.url) + + def matches_response(self, response): + """Returns True if Response's url matches initial url""" + return self.matches_url(response.url) + + +class UrlRegexMatcher(UrlMatcher): + """Matches URL using regular expression""" + + def __init__(self, regex, flags=0): + """Initialize regular expression""" + self._regex = re.compile(regex, flags) + + def matches_url(self, url): + """Returns True if url matches regular expression""" + return self._regex.search(url) is not None + + +class UrlListMatcher(UrlMatcher): + """Matches if URL is in List""" + + def __init__(self, urls): + self._urls = urls + + def matches_url(self, url): + """Returns True if url is in urls list""" + return url in self._urls diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py new file mode 100644 index 000000000..e23e78082 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqext.py @@ -0,0 +1,117 @@ +"""Request Extractors""" +from scrapy.http import Request +from scrapy.selector import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import FixedSGMLParser, str_to_unicode +from scrapy.utils.url import safe_url_string, urljoin_rfc + +from itertools import ifilter + + +class BaseSgmlRequestExtractor(FixedSGMLParser): + """Base SGML Request Extractor""" + + def __init__(self, tag='a', attr='href'): + """Initialize attributes""" + FixedSGMLParser.__init__(self) + + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.current_request = None + + def extract_requests(self, response): + """Returns list of requests extracted from response""" + return self._extract_requests(response.body, response.url, + response.encoding) + + def _extract_requests(self, response_text, response_url, response_encoding): + """Extract requests with absolute urls""" + self.reset() + self.feed(response_text) + self.close() + + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url + self._make_absolute_urls(base_url, response_encoding) + self._fix_link_text_encoding(response_encoding) + + return self.requests + + def _make_absolute_urls(self, base_url, encoding): + """Makes all request's urls absolute""" + for req in self.requests: + url = req.url + # make absolute url + url = urljoin_rfc(base_url, url, encoding) + url = safe_url_string(url, encoding) + # replace in-place request's url + req.url = url + + def _fix_link_text_encoding(self, encoding): + """Convert link_text to unicode for each request""" + for req in self.requests: + req.meta.setdefault('link_text', '') + req.meta['link_text'] = str_to_unicode(req.meta['link_text'], + encoding) + + def reset(self): + """Reset state""" + FixedSGMLParser.reset(self) + self.requests = [] + self.base_url = None + + def unknown_starttag(self, tag, attrs): + """Process unknown start tag""" + if 'base' == tag: + self.base_url = dict(attrs).get('href') + + _matches = lambda (attr, value): self.scan_attr(attr) \ + and value is not None + if self.scan_tag(tag): + for attr, value in ifilter(_matches, attrs): + req = Request(url=value) + self.requests.append(req) + self.current_request = req + + def unknown_endtag(self, tag): + """Process unknown end tag""" + self.current_request = None + + def handle_data(self, data): + """Process data""" + current = self.current_request + if current and not 'link_text' in current.meta: + current.meta['link_text'] = data.strip() + + +class SgmlRequestExtractor(BaseSgmlRequestExtractor): + """SGML Request Extractor""" + + def __init__(self, tags=None, attrs=None): + """Initialize with custom tag & attribute function checkers""" + # defaults + tags = tuple(tags) if tags else ('a', 'area') + attrs = tuple(attrs) if attrs else ('href', ) + + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) + +# TODO: move to own file +class XPathRequestExtractor(SgmlRequestExtractor): + """SGML Request Extractor with XPath restriction""" + + def __init__(self, restrict_xpaths, tags=None, attrs=None): + """Initialize XPath restrictions""" + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + SgmlRequestExtractor.__init__(self, tags, attrs) + + def extract_requests(self, response): + """Restrict to XPath regions""" + hxs = HtmlXPathSelector(response) + fragments = (''.join( + html_frag for html_frag in hxs.select(xpath).extract() + ) for xpath in self.restrict_xpaths) + html_slice = ''.join(html_frag for html_frag in fragments) + return self._extract_requests(html_slice, response.url, + response.encoding) + diff --git a/scrapy/contrib_exp/crawlspider/reqgen.py b/scrapy/contrib_exp/crawlspider/reqgen.py new file mode 100644 index 000000000..3858fbcf7 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqgen.py @@ -0,0 +1,27 @@ +"""Request Generator""" +from itertools import imap + +class RequestGenerator(object): + """Extracto and process requests from response""" + + def __init__(self, req_extractors, req_processors, callback, spider=None): + """Initialize attributes""" + self._request_extractors = req_extractors + self._request_processors = req_processors + #TODO: resolve callback? + self._callback = callback + + def generate_requests(self, response): + """Extract and process new requests from response. + Attach callback to each request as default callback.""" + requests = [] + for ext in self._request_extractors: + requests.extend(ext.extract_requests(response)) + + for proc in self._request_processors: + requests = proc(requests) + + # return iterator + # @@@ creates new Request object with callback + return imap(lambda r: r.replace(callback=self._callback), requests) + diff --git a/scrapy/contrib_exp/crawlspider/reqproc.py b/scrapy/contrib_exp/crawlspider/reqproc.py new file mode 100644 index 000000000..d39399b20 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqproc.py @@ -0,0 +1,111 @@ +"""Request Processors""" +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.url import canonicalize_url, url_is_from_any_domain + +from itertools import ifilter, imap + +import re + +class Canonicalize(object): + """Canonicalize Request Processor""" + def _replace_url(self, req): + # replace in-place + req.url = canonicalize_url(req.url) + return req + + def __call__(self, requests): + """Canonicalize all requests' urls""" + return imap(self._replace_url, requests) + + +class FilterDupes(object): + """Filter duplicate Requests""" + + def __init__(self, *attributes): + """Initialize comparison attributes""" + self._attributes = tuple(attributes) if attributes \ + else tuple(['url']) + + def _equal_attr(self, obj1, obj2, attr): + return getattr(obj1, attr) == getattr(obj2, attr) + + def _requests_equal(self, req1, req2): + """Attribute comparison helper""" + # look for not equal attribute + _not_equal = lambda attr: not self._equal_attr(req1, req2, attr) + for attr in ifilter(_not_equal, self._attributes): + return False + # all attributes equal + return True + + def _request_in(self, request, requests_seen): + """Check if request is in given requests seen list""" + _req_seen = lambda r: self._requests_equal(r, request) + for seen in ifilter(_req_seen, requests_seen): + return True + # request not seen + return False + + def __call__(self, requests): + """Filter seen requests""" + # per-call duplicates filter + self.requests_seen = set() + _not_seen = lambda r: not self._request_in(r, self.requests_seen) + for req in ifilter(_not_seen, requests): + yield req + # registry seen request + self.requests_seen.add(req) + + +class FilterDomain(object): + """Filter request's domain""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + self.allow = tuple(arg_to_iter(allow)) + self.deny = tuple(arg_to_iter(deny)) + + def __call__(self, requests): + """Filter domains""" + processed = (req for req in requests) + + if self.allow: + processed = (req for req in requests + if url_is_from_any_domain(req.url, self.allow)) + if self.deny: + processed = (req for req in requests + if not url_is_from_any_domain(req.url, self.deny)) + + return processed + + +class FilterUrl(object): + """Filter request's url""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + _re_type = type(re.compile('', 0)) + + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + def __call__(self, requests): + """Filter request's url based on allow/deny rules""" + #TODO: filter valid urls here? + processed = (req for req in requests) + + if self.allow_res: + processed = (req for req in requests + if self._matches(req.url, self.allow_res)) + if self.deny_res: + processed = (req for req in requests + if not self._matches(req.url, self.deny_res)) + + return processed + + def _matches(self, url, regexs): + """Returns True if url matches any regex in given list""" + return any(r.search(url) for r in regexs) + diff --git a/scrapy/contrib_exp/crawlspider/rules.py b/scrapy/contrib_exp/crawlspider/rules.py new file mode 100644 index 000000000..ff1691ad0 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/rules.py @@ -0,0 +1,100 @@ +"""Crawler Rules""" +from scrapy.http import Request +from scrapy.http import Response + +from functools import partial +from itertools import ifilter + +from .matchers import BaseMatcher +# default strint-to-matcher class +from .matchers import UrlRegexMatcher + +class CompiledRule(object): + """Compiled version of Rule""" + def __init__(self, matcher, callback=None, follow=False): + """Initialize attributes checking type""" + assert isinstance(matcher, BaseMatcher) + assert callback is None or callable(callback) + assert isinstance(follow, bool) + + self.matcher = matcher + self.callback = callback + self.follow = follow + + +class Rule(object): + """Crawler Rule""" + def __init__(self, matcher=None, callback=None, follow=False, **kwargs): + """Store attributes""" + self.matcher = matcher + self.callback = callback + self.cb_kwargs = kwargs if kwargs else {} + self.follow = True if follow else False + + if self.callback is None and self.follow is False: + raise ValueError("Rule must either have a callback or " + "follow=True: %r" % self) + + def __repr__(self): + return "Rule(matcher=%r, callback=%r, follow=%r, **%r)" \ + % (self.matcher, self.callback, self.follow, self.cb_kwargs) + + +class RulesManager(object): + """Rules Manager""" + def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): + """Initialize rules using spider and default matcher""" + self._rules = tuple() + + # compile absolute/relative-to-spider callbacks""" + for rule in rules: + # prepare matcher + if rule.matcher is None: + # instance BaseMatcher by default + matcher = BaseMatcher() + elif isinstance(rule.matcher, BaseMatcher): + matcher = rule.matcher + else: + # matcher not BaseMatcher, check for string + if isinstance(rule.matcher, basestring): + # instance default matcher + matcher = default_matcher(rule.matcher) + else: + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) + + # prepare callback + if callable(rule.callback): + callback = rule.callback + elif not rule.callback is None: + # callback from spider + callback = getattr(spider, rule.callback) + + if not callable(callback): + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) + else: + callback = None + + if rule.cb_kwargs: + # build partial callback + callback = partial(callback, **rule.cb_kwargs) + + # append compiled rule to rules list + crule = CompiledRule(matcher, callback, follow=rule.follow) + self._rules += (crule, ) + + def get_rule_from_request(self, request): + """Returns first rule that matches given Request""" + _matches = lambda r: r.matcher.matches_request(request) + for rule in ifilter(_matches, self._rules): + # return first match of iterator + return rule + + def get_rule_from_response(self, response): + """Returns first rule that matches given Response""" + _matches = lambda r: r.matcher.matches_response(response) + for rule in ifilter(_matches, self._rules): + # return first match of iterator + return rule + diff --git a/scrapy/contrib_exp/crawlspider/spider.py b/scrapy/contrib_exp/crawlspider/spider.py new file mode 100644 index 000000000..e2bf1a34b --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/spider.py @@ -0,0 +1,69 @@ +"""CrawlSpider v2""" +from scrapy.spider import BaseSpider +from scrapy.utils.spider import iterate_spider_output + +from .matchers import UrlListMatcher +from .rules import Rule, RulesManager +from .reqext import SgmlRequestExtractor +from .reqgen import RequestGenerator +from .reqproc import Canonicalize, FilterDupes + +class CrawlSpider(BaseSpider): + """CrawlSpider v2""" + + request_extractors = None + request_processors = None + rules = [] + + def __init__(self, *a, **kw): + """Initialize dispatcher""" + super(CrawlSpider, self).__init__(*a, **kw) + + # auto follow start urls + if self.start_urls: + _matcher = UrlListMatcher(self.start_urls) + # append new rule using type from current self.rules + rules = self.rules + type(self.rules)([ + Rule(_matcher, follow=True) + ]) + else: + rules = self.rules + + # set defaults if not set + if self.request_extractors is None: + # default link extractor. Extracts all links from response + self.request_extractors = [ SgmlRequestExtractor() ] + + if self.request_processors is None: + # default proccessor. Filter duplicates requests + self.request_processors = [ FilterDupes() ] + + + # wrap rules + self._rulesman = RulesManager(rules, spider=self) + # generates new requests with given callback + self._reqgen = RequestGenerator(self.request_extractors, + self.request_processors, + callback=self.parse) + + def parse(self, response): + """Dispatch callback and generate requests""" + # get rule for response + rule = self._rulesman.get_rule_from_response(response) + + if rule: + # dispatch callback if set + if rule.callback: + output = iterate_spider_output(rule.callback(response)) + for req_or_item in output: + yield req_or_item + + if rule.follow: + for req in self._reqgen.generate_requests(response): + # only dispatch request if has matching rule + if self._rulesman.get_rule_from_request(req): + yield req + else: + self.log("No rule for response %s" % response, level=log.WARNING) + + diff --git a/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py deleted file mode 100644 index 869c62c58..000000000 --- a/scrapy/contrib_exp/pipeline/shoveitem.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -A pipeline to persist objects using shove. - -Shove is a "new generation" shelve. For more information see: -http://pypi.python.org/pypi/shove -""" - -from string import Template - -from shove import Shove -from scrapy.xlib.pydispatch import dispatcher - -from scrapy import log -from scrapy.core import signals -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured - -class ShoveItemPipeline(object): - - def __init__(self): - self.uritpl = settings['SHOVEITEM_STORE_URI'] - if not self.uritpl: - raise NotConfigured - self.opts = settings['SHOVEITEM_STORE_OPT'] or {} - self.stores = {} - - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_item(self, spider, item): - guid = str(item.guid) - - if guid in self.stores[spider]: - if self.stores[spider][guid] == item: - status = 'old' - else: - status = 'upd' - else: - status = 'new' - - if not status == 'old': - self.stores[spider][guid] = item - self.log(spider, item, status) - return item - - def spider_opened(self, spider): - uri = Template(self.uritpl).substitute(domain=spider.domain_name) - self.stores[spider] = Shove(uri, **self.opts) - - def spider_closed(self, spider): - self.stores[spider].sync() - - def log(self, spider, item, status): - log.msg("Shove (%s): Item guid=%s" % (status, item.guid), level=log.DEBUG, \ - spider=spider) diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index b53db5811..aec17b05e 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -2,6 +2,7 @@ Download web pages using asynchronous IO """ +import random from time import time from twisted.internet import reactor, defer @@ -20,15 +21,21 @@ class SpiderInfo(object): def __init__(self, download_delay=None, max_concurrent_requests=None): if download_delay is None: - self.download_delay = settings.getfloat('DOWNLOAD_DELAY') + self._download_delay = settings.getfloat('DOWNLOAD_DELAY') else: - self.download_delay = download_delay - if self.download_delay: + self._download_delay = float(download_delay) + if self._download_delay: self.max_concurrent_requests = 1 elif max_concurrent_requests is None: self.max_concurrent_requests = settings.getint('CONCURRENT_REQUESTS_PER_SPIDER') else: self.max_concurrent_requests = max_concurrent_requests + if self._download_delay and settings.getbool('RANDOMIZE_DOWNLOAD_DELAY'): + # same policy as wget --random-wait + self.random_delay_interval = (0.5*self._download_delay, \ + 1.5*self._download_delay) + else: + self.random_delay_interval = None self.active = set() self.queue = [] @@ -44,6 +51,12 @@ class SpiderInfo(object): # use self.active to include requests in the downloader middleware return len(self.active) > 2 * self.max_concurrent_requests + def download_delay(self): + if self.random_delay_interval: + return random.uniform(*self.random_delay_interval) + else: + return self._download_delay + def cancel_request_calls(self): for call in self.next_request_calls: call.cancel() @@ -99,8 +112,9 @@ class Downloader(object): # Delay queue processing if a download_delay is configured now = time() - if site.download_delay: - penalty = site.download_delay - now + site.lastseen + delay = site.download_delay() + if delay: + penalty = delay - now + site.lastseen if penalty > 0: d = defer.Deferred() d.addCallback(self._process_queue) diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index 2078aaa95..b1dc3754f 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -1,5 +1,4 @@ import signal -from collections import defaultdict from twisted.internet import reactor @@ -7,54 +6,13 @@ from scrapy.extension import extensions from scrapy import log from scrapy.http import Request from scrapy.core.engine import scrapyengine -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import spiders from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import is_url from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -def _get_spider_requests(*args): - """Collect requests and spiders from the given arguments. Returns a dict of - spider -> list of requests - """ - spider_requests = defaultdict(list) - for arg in args: - if isinstance(arg, tuple): - request, spider = arg - spider_requests[spider] = request - elif isinstance(arg, Request): - spider = spiders.fromurl(arg.url) or BaseSpider('default') - if spider: - spider_requests[spider] += [arg] - else: - log.msg('Could not find spider for request: %s' % arg, log.ERROR) - elif isinstance(arg, BaseSpider): - spider_requests[arg] += arg.start_requests() - elif is_url(arg): - spider = spiders.fromurl(arg) or BaseSpider('default') - if spider: - for req in arg_to_iter(spider.make_requests_from_url(arg)): - spider_requests[spider] += [req] - else: - log.msg('Could not find spider for url: %s' % arg, log.ERROR) - elif isinstance(arg, basestring): - spider = spiders.fromdomain(arg) - if spider: - spider_requests[spider] += spider.start_requests() - else: - log.msg('Could not find spider for domain: %s' % arg, log.ERROR) - else: - raise TypeError("Unsupported argument: %r" % arg) - return spider_requests - class ExecutionManager(object): - """Process a list of sites or urls. - This class should be used in a main for process a list of sites/urls. - - It extracts products and could be used to store results in a database or - just for testing spiders. - """ def __init__(self): self.interrupted = False self.configured = False @@ -78,24 +36,46 @@ class ExecutionManager(object): scrapyengine.configure() self.configured = True - def crawl(self, *args): - """Schedule the given args for crawling. args is a list of urls or domains""" + def crawl_url(self, url, spider=None): + """Schedule given url for crawling.""" + if spider is None: + spider = self._create_spider_for_request(Request(url), log_none=True, \ + log_multiple=True) + if spider: + requests = arg_to_iter(spider.make_requests_from_url(url)) + self._crawl_requests(requests, spider) + + def crawl_request(self, request, spider=None): + """Schedule request for crawling.""" assert self.configured, "Scrapy Manager not yet configured" - spider_requests = _get_spider_requests(*args) - for spider, requests in spider_requests.iteritems(): - for request in requests: - scrapyengine.crawl(request, spider) + if spider is None: + spider = self._create_spider_for_request(request, log_none=True, \ + log_multiple=True) + if spider: + scrapyengine.crawl(request, spider) - def runonce(self, *args): - """Run the engine until it finishes scraping all domains and then exit""" - self.crawl(*args) - scrapyengine.start() - if self.control_reactor: - reactor.run(installSignalHandlers=False) + def crawl_spider_name(self, name): + """Schedule given spider by name for crawling.""" + try: + spider = spiders.create(name) + except KeyError: + log.msg('Could not find spider: %s' % name, log.ERROR) + else: + self.crawl_spider(spider) - def start(self): + def crawl_spider(self, spider): + """Schedule spider for crawling.""" + requests = spider.start_requests() + self._crawl_requests(requests, spider) + + def _crawl_requests(self, requests, spider): + """Shortcut to schedule a list of requests""" + for req in requests: + self.crawl_request(req, spider) + + def start(self, keep_alive=False): """Start the scrapy server, without scheduling any domains""" - scrapyengine.keep_alive = True + scrapyengine.keep_alive = keep_alive scrapyengine.start() if self.control_reactor: reactor.run(installSignalHandlers=False) @@ -105,6 +85,17 @@ class ExecutionManager(object): self.interrupted = True scrapyengine.stop() + def _create_spider_for_request(self, request, default=None, log_none=False, \ + log_multiple=False): + spider_names = spiders.find_by_request(request) + if len(spider_names) == 1: + return spiders.create(spider_names[0]) + if len(spider_names) > 1 and log_multiple: + log.msg('More than one spider found for: %s' % request, log.ERROR) + if len(spider_names) == 0 and log_none: + log.msg('Could not find spider for: %s' % request, log.ERROR) + return default + def _signal_shutdown(self, signum, _): signame = signal_names[signum] log.msg("Received %s, shutting down gracefully. Send again to force " \ diff --git a/scrapy/crawler.py b/scrapy/crawler.py deleted file mode 100644 index e793d9125..000000000 --- a/scrapy/crawler.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Crawler class - -The Crawler class can be used to crawl pages using the Scrapy crawler from -outside a Scrapy project, for example, from a standalone script. - -To use it, instantiate it and call the "crawl" method with one (or more) -requests. For example: - - >>> from scrapy.crawler import Crawler - >>> from scrapy.http import Request - >>> def parse_response(response): - ... print "Visited: %s" % response.url - ... - >>> request = Request('http://scrapy.org', callback=parse_response) - >>> crawler = Crawler() - >>> crawler.crawl(request) - Visited: http://scrapy.org - >>> - -Request callbacks follow the same API of spiders callback, which means that all -requests returned from the callbacks will be followed. - -See examples/scripts/count_and_follow_links.py for a more detailed example. - -WARNING: The Crawler class currently has a big limitation - it cannot be used -more than once in the same Python process. This is due to the fact that Twisted -reactors cannot be restarted. Hopefully, this limitation will be removed in the -future. -""" - -from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.conf import settings as scrapy_settings -from scrapy import log - -class Crawler(object): - - def __init__(self, enable_log=False, stop_on_error=False, silence_errors=False, \ - settings=None): - self.stop_on_error = stop_on_error - self.silence_errors = silence_errors - # disable offsite middleware (by default) because it prevents free crawling - if settings is not None: - settings.overrides.update(settings) - scrapy_settings.overrides['SPIDER_MIDDLEWARES'] = { - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None} - scrapy_settings.overrides['LOG_ENABLED'] = enable_log - scrapymanager.configure() - dispatcher.connect(self._logmessage_received, signal=log.logmessage_received) - - def crawl(self, *args): - scrapymanager.runonce(*args) - - def stop(self): - scrapyengine.stop() - log.log_level = log.SILENT - scrapyengine.kill() - - def _logmessage_received(self, message, level): - if level <= log.ERROR: - if not self.silence_errors: - print "Crawler error: %s" % message - if self.stop_on_error: - self.stop() diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 187238b1c..4dce92880 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -96,20 +96,12 @@ class Request(object_ref): """Return a copy of this Request""" return self.replace() - def replace(self, url=None, callback=None, method=None, headers=None, body=None, \ - cookies=None, meta=None, encoding=None, priority=None, \ - dont_filter=None, errback=None): + def replace(self, *args, **kwargs): """Create a new Request with the same attributes except for those given new values. """ - return self.__class__(url=self.url if url is None else url, - callback=callback, - method=self.method if method is None else method, - headers=copy.deepcopy(self.headers) if headers is None else headers, - body=self.body if body is None else body, - cookies=self.cookies if cookies is None else cookies, - meta=self.meta if meta is None else meta, - encoding=self.encoding if encoding is None else encoding, - priority=self.priority if priority is None else priority, - dont_filter=self.dont_filter if dont_filter is None else dont_filter, - errback=errback) + for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', \ + 'encoding', 'priority', 'dont_filter']: + kwargs.setdefault(x, getattr(self, x)) + cls = kwargs.pop('cls', self.__class__) + return cls(*args, **kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f632ea117..251e77ad7 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -71,18 +71,11 @@ class Response(object_ref): """Return a copy of this Response""" return self.replace() - def replace(self, url=None, status=None, headers=None, body=None, meta=None, \ - flags=None, cls=None, **kwargs): + def replace(self, *args, **kwargs): """Create a new Response with the same attributes except for those given new values. """ - if cls is None: - cls = self.__class__ - new = cls(url=self.url if url is None else url, - status=self.status if status is None else status, - headers=copy.deepcopy(self.headers) if headers is None else headers, - body=self.body if body is None else body, - meta=self.meta if meta is None else meta, - flags=self.flags if flags is None else flags, - **kwargs) - return new + for x in ['url', 'status', 'headers', 'body', 'meta', 'flags']: + kwargs.setdefault(x, getattr(self, x)) + cls = kwargs.pop('cls', self.__class__) + return cls(*args, **kwargs) diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index f1557e6f7..dc812ac0e 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -23,9 +23,6 @@ class HtmlResponse(TextResponse): METATAG_RE = re.compile(r'[\w-]+)') XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) - def body_encoding(self): - return self._body_declared_encoding() or super(XmlResponse, self).body_encoding() - @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] diff --git a/scrapy/log.py b/scrapy/log.py index 71897ffee..c8e969928 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -29,8 +29,9 @@ BOT_NAME = settings['BOT_NAME'] # args: message, level, spider logmessage_received = object() -# default logging level +# default values log_level = DEBUG +log_encoding = 'utf-8' started = False @@ -47,11 +48,12 @@ def _get_log_level(level_name_or_id=None): def start(logfile=None, loglevel=None, logstdout=None): """Initialize and start logging facility""" - global log_level, started + global log_level, log_encoding, started if started or not settings.getbool('LOG_ENABLED'): return log_level = _get_log_level(loglevel) + log_encoding = settings['LOG_ENCODING'] started = True # set log observer @@ -73,8 +75,8 @@ def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): "use 'spider' argument instead", DeprecationWarning, stacklevel=2) dispatcher.send(signal=logmessage_received, message=message, level=level, \ spider=spider) - system = domain or (spider.domain_name if spider else component) - msg_txt = unicode_to_str("%s: %s" % (level_names[level], message)) + system = domain or (spider.name if spider else component) + msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding) log.msg(msg_txt, system=system) def exc(message, level=ERROR, component=BOT_NAME, domain=None, spider=None): @@ -91,7 +93,7 @@ def err(_stuff=None, _why=None, **kwargs): import warnings warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \ "use 'spider' argument instead", DeprecationWarning, stacklevel=2) - kwargs['system'] = domain or (spider.domain_name if spider else component) + kwargs['system'] = domain or (spider.name if spider else component) if _why: - _why = unicode_to_str("ERROR: %s" % _why) + _why = unicode_to_str("ERROR: %s" % _why, log_encoding) log.err(_stuff, _why, **kwargs) diff --git a/scrapy/mail.py b/scrapy/mail.py index 8275e75cb..87825ed9c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -47,34 +47,26 @@ class MailSender(object): part = MIMEBase(*mimetype.split('/')) part.set_payload(f.read()) Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="%s"' % attach_name) + part.add_header('Content-Disposition', 'attachment; filename="%s"' \ + % attach_name) msg.attach(part) else: msg.set_payload(body) - # FIXME --------------------------------------------------------------------- - # There seems to be a problem with sending emails using deferreds when - # the last thing left to do is sending the mail, cause the engine stops - # the reactor and the email don't get send. we need to fix this. until - # then, we'll revert to use Python standard (IO-blocking) smtplib. - - #dfd = self._sendmail(self.smtphost, self.mailfrom, rcpts, msg.as_string()) - #dfd.addCallbacks(self._sent_ok, self._sent_failed, - # callbackArgs=[to, cc, subject, len(attachs)], - # errbackArgs=[to, cc, subject, len(attachs)]) - import smtplib - smtp = smtplib.SMTP(self.smtphost) - smtp.sendmail(self.mailfrom, rcpts, msg.as_string()) - log.msg('Mail sent: To=%s Cc=%s Subject="%s"' % (to, cc, subject)) - smtp.close() - # --------------------------------------------------------------------------- + dfd = self._sendmail(self.smtphost, self.mailfrom, rcpts, msg.as_string()) + dfd.addCallbacks(self._sent_ok, self._sent_failed, + callbackArgs=[to, cc, subject, len(attachs)], + errbackArgs=[to, cc, subject, len(attachs)]) + reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) def _sent_ok(self, result, to, cc, subject, nattachs): - log.msg('Mail sent OK: To=%s Cc=%s Subject="%s" Attachs=%d' % (to, cc, subject, nattachs)) + log.msg('Mail sent OK: To=%s Cc=%s Subject="%s" Attachs=%d' % \ + (to, cc, subject, nattachs)) def _sent_failed(self, failure, to, cc, subject, nattachs): errstr = str(failure.value) - log.msg('Unable to send mail: To=%s Cc=%s Subject="%s" Attachs=%d - %s' % (to, cc, subject, nattachs, errstr), level=log.ERROR) + log.msg('Unable to send mail: To=%s Cc=%s Subject="%s" Attachs=%d - %s' % \ + (to, cc, subject, nattachs, errstr), level=log.ERROR) def _sendmail(self, smtphost, from_addr, to_addrs, msg, port=25): """ This is based on twisted.mail.smtp.sendmail except that it diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 37f86d431..bf2dc52c7 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -29,8 +29,8 @@ class XPathSelector(object_ref): self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc elif text: - response = TextResponse(url='about:blank', body=unicode_to_str(text), \ - encoding='utf-8') + response = TextResponse(url='about:blank', \ + body=unicode_to_str(text, 'utf-8'), encoding='utf-8') self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc self.expr = expr diff --git a/scrapy/service.py b/scrapy/service.py new file mode 100644 index 000000000..7247fd0ca --- /dev/null +++ b/scrapy/service.py @@ -0,0 +1,60 @@ +import sys, os + +from twisted.python import log +from twisted.internet import reactor, protocol, error +from twisted.application.service import Service + +from scrapy.utils.py26 import cpu_count +from scrapy.conf import settings + + +class ScrapyService(Service): + + def startService(self): + reactor.callWhenRunning(self.start_processes) + + def start_processes(self): + for i in range(cpu_count()): + self.start_process(i+1) + + def start_process(self, id): + args = [sys.executable, '-m', 'scrapy.service'] + env = os.environ.copy() + self.set_log_file(env, id) + pp = ScrapyProcessProtocol(self, id, env.get('SCRAPY_LOG_FILE')) + reactor.spawnProcess(pp, sys.executable, args=args, env=env) + + def set_log_file(self, env, suffix): + logfile = settings['LOG_FILE'] + if logfile: + file, ext = os.path.splitext(logfile) + env['SCRAPY_LOG_FILE'] = "%s-%s%s" % (file, suffix, ext) + + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + + def __init__(self, service, id, logfile): + self.service = service + self.id = id + self.logfile = logfile + self.pid = None + + def connectionMade(self): + self.pid = self.transport.pid + log.msg("Process %r started: pid=%r logfile=%r" % (self.id, self.pid, \ + self.logfile)) + + def processEnded(self, status): + if isinstance(status.value, error.ProcessDone): + log.msg("Process %r finished: pid=%r logfile=%r" % (self.id, \ + self.pid, self.logfile)) + else: + log.msg("Process %r died: exitstatus=%r pid=%r logfile=%r" % \ + (self.id, status.value.exitCode, self.pid, self.logfile)) + reactor.callLater(5, self.service.start_process, self.id) + + +if __name__ == '__main__': + from scrapy.core.manager import scrapymanager + scrapymanager.configure() + scrapymanager.start(keep_alive=True) diff --git a/scrapy/shell.py b/scrapy/shell.py index 8af6af006..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -35,6 +35,7 @@ def parse_url(url): u = urlparse.urlparse(url) return url + class Shell(object): requires_project = False @@ -52,18 +53,21 @@ class Shell(object): else: url = parse_url(request_or_url) request = Request(url) - spider = spiders.fromurl(url) or BaseSpider('default') + + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default'), log_multiple=True) + print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ request, spider) if response: - self.populate_vars(url, response, request) + self.populate_vars(url, response, request, spider) if print_help: self.print_help() else: print "Done - use shelp() to see available objects" - def populate_vars(self, url=None, response=None, request=None): + def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class() self.vars['item'] = item if url: @@ -73,7 +77,7 @@ class Shell(object): self.vars['url'] = url self.vars['response'] = response self.vars['request'] = request - self.vars['spider'] = spiders.fromurl(url) + self.vars['spider'] = spider if not self.nofetch: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser @@ -104,7 +108,7 @@ class Shell(object): signal.signal(signal.SIGINT, signal.SIG_IGN) reactor.callInThread(self._console_thread, url) - scrapymanager.start() + scrapymanager.start(keep_alive=True) def inspect_response(self, response): print diff --git a/scrapy/spider/middleware.py b/scrapy/spider/middleware.py index 416b3cc9b..4a8342c0a 100644 --- a/scrapy/spider/middleware.py +++ b/scrapy/spider/middleware.py @@ -7,11 +7,11 @@ docs/topics/spider-middleware.rst """ from scrapy import log +from twisted.python.failure import Failure from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred -from scrapy.http import Request from scrapy.conf import settings def _isiterable(possible_iterator): @@ -60,12 +60,14 @@ class SpiderMiddlewareManager(object): def process_spider_input(response): for method in self.spider_middleware: - result = method(response=response, spider=spider) - assert result is None or _isiterable(result), \ - 'Middleware %s must returns None or an iterable object, got %s ' % \ - (fname(method), type(result)) - if result is not None: - return result + try: + result = method(response=response, spider=spider) + assert result is None, \ + 'Middleware %s must returns None or ' \ + 'raise an exception, got %s ' \ + % (fname(method), type(result)) + except: + return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) def process_spider_exception(_failure): diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index a9b64995e..c1ac2c1d4 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -3,6 +3,9 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + +import warnings + from zope.interface import Interface, Attribute, invariant, implements from twisted.plugin import IPlugin @@ -11,17 +14,9 @@ from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref -def _valid_domain_name(obj): - """Check the domain name specified is valid""" - if not obj.domain_name: - raise ValueError("Spider 'domain_name' attribute is required") - class ISpider(Interface, IPlugin) : - """Interface to be implemented by site-specific web spiders""" - - domain_name = Attribute("The domain name of the site to be scraped.") - - invariant(_valid_domain_name) + """Interface used by TwistedPluginSpiderManager to discover spiders""" + pass class BaseSpider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -31,19 +26,37 @@ class BaseSpider(object_ref): implements(ISpider) # XXX: class attributes kept for backwards compatibility - domain_name = None + name = None start_urls = [] - extra_domain_names = [] + allowed_domains = [] - def __init__(self, domain_name=None): - if domain_name is not None: - self.domain_name = domain_name + def __init__(self, name=None, **kwargs): + self.__dict__.update(kwargs) + # XXX: SEP-12 backward compatibility (remove for 0.10) + if hasattr(self, 'domain_name'): + warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead and Spider.allowed_domains", \ + DeprecationWarning, stacklevel=4) + self.name = self.domain_name + self.allowed_domains = [self.name] + if hasattr(self, 'extra_domain_names'): + warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \ + DeprecationWarning, stacklevel=4) + self.allowed_domains += list(self.extra_domain_names) + + if name is not None: + self.name = name # XXX: create instance attributes (class attributes were kept for # backwards compatibility) if not self.start_urls: self.start_urls = [] - if not self.extra_domain_names: - self.extra_domain_names = [] + if not self.allowed_domains: + self.allowed_domains = [] + if not self.name: + raise ValueError("%s must have a name" % type(self).__name__) + + # XXX: SEP-12 forward compatibility (remove for 0.10) + self.domain_name = self.name + self.extra_domain_names = self.allowed_domains def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this @@ -67,6 +80,6 @@ class BaseSpider(object_ref): pass def __str__(self): - return "<%s %r>" % (type(self).__name__, self.domain_name) + return "<%s %r>" % (type(self).__name__, self.name) __repr__ = __str__ diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index f41916818..0acf36073 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -76,11 +76,11 @@ class MemoryStatsCollector(StatsCollector): def __init__(self): super(MemoryStatsCollector, self).__init__() - self.domain_stats = {} + self.spider_stats = {} def _persist_stats(self, stats, spider=None): if spider is not None: - self.domain_stats[spider.domain_name] = stats + self.spider_stats[spider.name] = stats class DummyStatsCollector(StatsCollector): diff --git a/scrapy/stats/collector/mysql.py b/scrapy/stats/collector/mysql.py deleted file mode 100644 index 2cf145347..000000000 --- a/scrapy/stats/collector/mysql.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -A Stats collector for persisting stats (pickled) to a MySQL db -""" - -import cPickle as pickle -from datetime import datetime - -from scrapy.stats.collector import StatsCollector -from scrapy.utils.mysql import mysql_connect -from scrapy.conf import settings - -class MysqlStatsCollector(StatsCollector): - - def __init__(self): - super(MysqlStatsCollector, self).__init__() - mysqluri = settings['STATS_MYSQL_URI'] - self._mysql_conn = mysql_connect(mysqluri, use_unicode=False) if mysqluri else None - - def _persist_stats(self, stats, spider=None): - if spider is None: # only store spider-specific stats - return - if self._mysql_conn is None: - return - stored = datetime.utcnow() - datas = pickle.dumps(stats) - table = 'domain_data_history' - - c = self._mysql_conn.cursor() - c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % table, \ - (spider.domain_name, stored, datas)) - self._mysql_conn.commit() diff --git a/scrapy/stats/collector/simpledb.py b/scrapy/stats/collector/simpledb.py index 850558442..d521c5e40 100644 --- a/scrapy/stats/collector/simpledb.py +++ b/scrapy/stats/collector/simpledb.py @@ -36,9 +36,9 @@ class SimpledbStatsCollector(StatsCollector): def _persist_to_sdb(self, spider, stats): ts = self._get_timestamp(spider).isoformat() - sdb_item_id = "%s_%s" % (spider.domain_name, ts) + sdb_item_id = "%s_%s" % (spider.name, ts) sdb_item = dict((k, self._to_sdb_value(v, k)) for k, v in stats.iteritems()) - sdb_item['domain'] = spider.domain_name + sdb_item['spider'] = spider.name sdb_item['timestamp'] = self._to_sdb_value(ts) connect_sdb().put_attributes(self._sdbdomain, sdb_item_id, sdb_item) diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index fa6f5ea6f..e3f89342d 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -4,5 +4,5 @@ # See: http://doc.scrapy.org/topics/item-pipeline.html class ${ProjectName}Pipeline(object): - def process_item(self, domain, item): + def process_item(self, spider, item): return item diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 2e3baf992..246015466 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,9 +1,10 @@ from scrapy.spider import BaseSpider class $classname(BaseSpider): - domain_name = "$site" + name = "$name" + allowed_domains = ["$domain"] start_urls = ( - 'http://www.$site/', + 'http://www.$domain/', ) def parse(self, response): diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 2d4f7fce6..578779c06 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -6,19 +6,20 @@ from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item class $classname(CrawlSpider): - domain_name = '$site' - start_urls = ['http://www.$site/'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/'] rules = ( - Rule(SgmlLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), + Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), ) def parse_item(self, response): - xs = HtmlXPathSelector(response) + hxs = HtmlXPathSelector(response) i = ${ProjectName}Item() - #i['site_id'] = xs.select('//input[@id="sid"]/@value').extract() - #i['name'] = xs.select('//div[@id="name"]').extract() - #i['description'] = xs.select('//div[@id="description"]').extract() + #i['domain_id'] = hxs.select('//input[@id="sid"]/@value').extract() + #i['name'] = hxs.select('//div[@id="name"]').extract() + #i['description'] = hxs.select('//div[@id="description"]').extract() return i SPIDER = $classname() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 794288570..c9a723000 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item class $classname(CSVFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.csv'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.csv'] # headers = ['id', 'name', 'description', 'image_link'] # delimiter = '\t' diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index f249b4537..f5ecbd707 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item class $classname(XMLFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.xml'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.xml'] def parse_item(self, response, selector): i = ${ProjectName}Item() diff --git a/scrapy/tests/__init__.py b/scrapy/tests/__init__.py index c77772af5..afd463740 100644 --- a/scrapy/tests/__init__.py +++ b/scrapy/tests/__init__.py @@ -6,9 +6,6 @@ To run all Scrapy unittests go to Scrapy main dir and type: bin/runtests.sh If you're in windows use runtests.bat instead. - -Keep in mind that some tests may be skipped if you don't have some (optional) -modules available like MySQLdb or simplejson, but that's not a problem. """ import os diff --git a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html index 602ea04cd..fecd86563 100644 --- a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -9,7 +9,7 @@ sample 2 -sample 3 text +sample 3 text sample 3 repetition diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index cb4e5ced1..6eed6d22a 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -59,10 +59,18 @@ class CommandTest(ProjectTest): class GenspiderCommandTest(CommandTest): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + self.assertEqual(0, self.call('genspider', 'test_name')) + assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + # pass two arguments . spider script should be created + self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) + assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + def test_template_default(self, *args): - self.assertEqual(0, self.call('genspider', 'testspider', 'test.com', *args)) - assert exists(join(self.proj_mod_path, 'spiders', 'testspider.py')) - self.assertEqual(1, self.call('genspider', 'otherspider', 'test.com')) + self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args)) + assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com')) def test_template_basic(self): self.test_template_default('--template=basic') diff --git a/scrapy/tests/test_contrib_exp_crawlspider_matchers.py b/scrapy/tests/test_contrib_exp_crawlspider_matchers.py new file mode 100644 index 000000000..4cb832aa1 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_matchers.py @@ -0,0 +1,94 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import Response + +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlListMatcher + +import re + +class MatchersTest(unittest.TestCase): + + def setUp(self): + pass + + def test_base_matcher(self): + matcher = BaseMatcher() + + request = Request('http://example.com') + response = Response('http://example.com') + + self.assertTrue(matcher.matches_request(request)) + self.assertTrue(matcher.matches_response(response)) + + def test_url_matcher(self): + matcher = UrlMatcher('http://example.com') + + request = Request('http://example.com') + response = Response('http://example.com') + + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_request(response)) + + request = Request('http://example2.com') + response = Response('http://example2.com') + + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_request(response)) + + def test_url_regex_matcher(self): + matcher = UrlRegexMatcher(r'sample') + urls = ( + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample4.html', + ) + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + matcher = UrlRegexMatcher(r'sample_fail') + for url in urls: + request, response = Request(url), Response(url) + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_response(response)) + + matcher = UrlRegexMatcher(r'SAMPLE\d+', re.IGNORECASE) + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + def test_url_list_matcher(self): + urls = ( + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample4.html', + ) + urls2 = ( + 'http://example.com/sample5.html', + 'http://example.com/sample6.html', + 'http://example.com/sample7.html', + 'http://example.com/sample8.html', + 'http://example.com/', + ) + matcher = UrlListMatcher(urls) + + # match urls + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + # non-match urls + for url in urls2: + request, response = Request(url), Response(url) + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_response(response)) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py new file mode 100644 index 000000000..0259b30fb --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py @@ -0,0 +1,156 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.tests import get_testdata + +from scrapy.contrib_exp.crawlspider.reqext import BaseSgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqext import XPathRequestExtractor + +class AbstractRequestExtractorTest(unittest.TestCase): + + def _requests_equals(self, list1, list2): + """Compares request's urls and link_text""" + for (r1, r2) in zip(list1, list2): + if r1.url != r2.url: + return False + if r1.meta['link_text'] != r2.meta['link_text']: + return False + # all equal + return True + + +class RequestExtractorTest(AbstractRequestExtractorTest): + + def test_basic(self): + base_url = 'http://example.org/somepage/index.html' + html = """Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + requests = [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/about.html', + meta={'link_text': 'About us'}), + Request('http://example.org/othercat.html', + meta={'link_text': 'Other category'}), + Request('http://example.org/', + meta={'link_text': ''}), + ] + + response = HtmlResponse(base_url, body=html) + reqx = BaseSgmlRequestExtractor() # default: tag=a, attr=href + + self.failUnless( + self._requests_equals(requests, reqx.extract_requests(response)) + ) + + def test_base_url(self): + reqx = BaseSgmlRequestExtractor() + + html = """<html><head><title>Page title<title> + <base href="http://otherdomain.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('http://otherdomain.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title> + <base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://example.org/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + # base url has no scheme + html = """<html><head><title>Page title<title> + <base href="//noscheme.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://noscheme.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + def test_extraction_encoding(self): + #TODO: use own fixtures + body = get_testdata('link_extractor', 'linkextractor_noenc.html') + response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, + headers={'Content-Type': ['text/html; charset=utf-8']}) + response_noenc = HtmlResponse(url='http://example.com/noenc', + body=body) + body = get_testdata('link_extractor', 'linkextractor_latin1.html') + response_latin1 = HtmlResponse(url='http://example.com/latin1', + body=body) + + reqx = BaseSgmlRequestExtractor() + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_utf8), + [ Request(url='http://example.com/sample_%C3%B1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E2%82%AC.html', + meta={'link_text': + 'sample \xe2\x82\xac text'.decode('utf-8')}) ] + ) + ) + + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_noenc), + [ Request(url='http://example.com/sample_%C3%B1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E2%82%AC.html', + meta={'link_text': + 'sample \xe2\x82\xac text'.decode('utf-8')}) ] + ) + ) + + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_latin1), + [ Request(url='http://example.com/sample_%F1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E1.html', + meta={'link_text': + 'sample \xe1 text'.decode('latin1')}) ] + ) + ) + + +class SgmlRequestExtractorTest(AbstractRequestExtractorTest): + pass + + +class XPathRequestExtractorTest(AbstractRequestExtractorTest): + + def setUp(self): + # TODO: use own fixtures + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + + def test_restrict_xpaths(self): + reqx = XPathRequestExtractor('//div[@id="subwrapper"]') + self.failUnless( + self._requests_equals( + reqx.extract_requests(self.response), + [ Request(url='http://example.com/sample1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample2.html', + meta={'link_text': 'sample 2'}) ] + ) + ) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py b/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py new file mode 100644 index 000000000..67aca2387 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py @@ -0,0 +1,128 @@ +from twisted.internet import defer +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.utils.python import equal_attributes + +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqgen import RequestGenerator +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDomain +from scrapy.contrib_exp.crawlspider.reqproc import FilterUrl +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + +class RequestGeneratorTest(unittest.TestCase): + + def setUp(self): + url = 'http://example.org/somepage/index.html' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + self.response = HtmlResponse(url, body=html) + self.deferred = defer.Deferred() + self.requests = [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/about.html', + meta={'link_text': 'About us'}), + Request('http://example.org/othercat.html', + meta={'link_text': 'Other category'}), + Request('http://example.org/', + meta={'link_text': ''}), + ] + + def _equal_requests_list(self, list1, list2): + list1 = list(list1) + list2 = list(list2) + if not len(list1) == len(list2): + return False + + for (req1, req2) in zip(list1, list2): + if not equal_attributes(req1, req2, ['url']): + return False + return True + + def test_basic(self): + reqgen = RequestGenerator([], [], callback=self.deferred) + # returns generator + requests = reqgen.generate_requests(self.response) + self.failUnlessEqual(list(requests), []) + + def test_request_extractor(self): + extractors = [ + SgmlRequestExtractor() + ] + + # extract all requests + reqgen = RequestGenerator(extractors, [], callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnless(self._equal_requests_list(requests, self.requests)) + + for req in requests: + # check callback + self.failUnlessEqual(req.deferred, self.deferred) + + def test_request_processor(self): + extractors = [ + SgmlRequestExtractor() + ] + + processors = [ + Canonicalize(), + FilterDupes(), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnless(self._equal_requests_list(requests, self.requests)) + + # filter domain + processors = [ + Canonicalize(), + FilterDupes(), + FilterDomain(deny='example.org'), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnlessEqual(list(requests), []) + + # filter url + processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(deny=(r'about', r'othercat')), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + + self.failUnless(self._equal_requests_list(requests, [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/', + meta={'link_text': ''}), + ])) + + processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(allow=r'/somepage/'), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + + self.failUnless(self._equal_requests_list(requests, [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + ])) + + + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py b/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py new file mode 100644 index 000000000..da5db67b2 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py @@ -0,0 +1,144 @@ +from twisted.trial import unittest + +from scrapy.http import Request + +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDomain +from scrapy.contrib_exp.crawlspider.reqproc import FilterUrl +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + +import copy + +class RequestProcessorsTest(unittest.TestCase): + + def test_canonicalize_requests(self): + urls = [ + 'http://example.com/do?&b=1&a=2&c=3', + 'http://example.com/do?123,&q=a space', + ] + urls_after = [ + 'http://example.com/do?a=2&b=1&c=3', + 'http://example.com/do?123%2C=&q=a+space', + ] + + proc = Canonicalize() + results = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(results, urls_after) + + def test_unique_requests(self): + urls = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + ] + urls_unique = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + ] + + proc = FilterDupes() + results = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(results, urls_unique) + + # Check custom attributes + requests = [ + Request('http://example.com', method='GET'), + Request('http://example.com', method='POST'), + ] + proc = FilterDupes('url', 'method') + self.failUnlessEqual(len(list(proc(requests))), 2) + + proc = FilterDupes('url') + self.failUnlessEqual(len(list(proc(requests))), 1) + + def test_filter_domain(self): + urls = [ + 'http://blah1.com/index', + 'http://blah2.com/index', + 'http://blah1.com/section', + 'http://blah2.com/section', + ] + + proc = FilterDomain(allow=('blah1.com'), deny=('blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, [ + 'http://blah1.com/index', + 'http://blah1.com/section', + ]) + + proc = FilterDomain(deny=('blah1.com', 'blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, []) + + proc = FilterDomain(allow=('blah1.com', 'blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, urls) + + def test_filter_url(self): + urls = [ + 'http://blah1.com/index', + 'http://blah2.com/index', + 'http://blah1.com/section', + 'http://blah2.com/section', + ] + + proc = FilterUrl(allow=(r'blah1'), deny=(r'blah2')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, [ + 'http://blah1.com/index', + 'http://blah1.com/section', + ]) + + proc = FilterUrl(deny=('blah1', 'blah2')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, []) + + proc = FilterUrl(allow=('index$', 'section$')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, urls) + + + + def test_all_processors(self): + urls = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/do?&b=1&a=2&c=3', + 'http://example.com/do?123,&q=a space', + ] + urls_processed = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/do?a=2&b=1&c=3', + 'http://example.com/do?123%2C=&q=a+space', + ] + + processors = [ + Canonicalize(), + FilterDupes(), + ] + + def _process(requests): + """Apply all processors""" + # copy list + processed = [copy.copy(req) for req in requests] + for proc in processors: + processed = proc(processed) + return processed + + # empty requests + results1 = [r.url for r in _process([])] + self.failUnlessEquals(results1, []) + + # try urls + requests = (Request(url) for url in urls) + results2 = [r.url for r in _process(requests)] + self.failUnlessEquals(results2, urls_processed) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_rules.py b/scrapy/tests/test_contrib_exp_crawlspider_rules.py new file mode 100644 index 000000000..e04eb7b06 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_rules.py @@ -0,0 +1,262 @@ +from twisted.trial import unittest + +from scrapy.http import HtmlResponse +from scrapy.spider import BaseSpider +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher + +from scrapy.contrib_exp.crawlspider.rules import CompiledRule +from scrapy.contrib_exp.crawlspider.rules import Rule +from scrapy.contrib_exp.crawlspider.rules import RulesManager + +from functools import partial + +class RuleInitializationTest(unittest.TestCase): + + def test_fail_if_rule_null(self): + # fail on empty rule + self.failUnlessRaises(ValueError, Rule) + self.failUnlessRaises(ValueError, Rule, + **dict(callback=None, follow=None)) + self.failUnlessRaises(ValueError, Rule, + **dict(callback=None, follow=False)) + + def test_minimal_arguments_to_instantiation(self): + # not fail if callback set + self.failUnless(Rule(callback=lambda: True)) + # not fail if follow set + self.failUnless(Rule(follow=True)) + + def test_validate_default_attributes(self): + # test null Rule + rule = Rule(follow=True) + self.failUnlessEqual(None, rule.matcher) + self.failUnlessEqual(None, rule.callback) + self.failUnlessEqual({}, rule.cb_kwargs) + # follow default False + self.failUnlessEqual(True, rule.follow) + + def test_validate_attributes_set(self): + matcher = BaseMatcher() + callback = lambda: True + rule = Rule(matcher, callback, True, a=1) + # test attributes + self.failUnlessEqual(matcher, rule.matcher) + self.failUnlessEqual(callback, rule.callback) + self.failUnlessEqual({'a': 1}, rule.cb_kwargs) + self.failUnlessEqual(True, rule.follow) + +class CompiledRuleInitializationTest(unittest.TestCase): + + def test_fail_on_invalid_matcher(self): + # pass with valid matcher + self.failUnless(CompiledRule(BaseMatcher()), + "Failed CompiledRule instantiation") + + # at least needs valid matcher + self.assertRaises(AssertionError, CompiledRule, None) + self.assertRaises(AssertionError, CompiledRule, False) + self.assertRaises(AssertionError, CompiledRule, True) + + def test_fail_on_invalid_callback(self): + # pass with valid callback + callback = lambda: True + self.failUnless(CompiledRule(BaseMatcher(), callback)) + # pass with callback none + self.failUnless(CompiledRule(BaseMatcher(), None)) + + # assert on invalid callback + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + 'myfunc') + + # numeric variable + var = 123 + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + var) + + class A: + pass + + # random instance + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + A()) + + + def test_fail_on_invalid_follow_value(self): + callback = lambda: True + matcher = BaseMatcher() + # pass bool + self.failUnless(CompiledRule(matcher, callback, True)) + self.failUnless(CompiledRule(matcher, callback, False)) + + # assert with non-bool + self.assertRaises(AssertionError, CompiledRule, matcher, + callback, None) + self.assertRaises(AssertionError, CompiledRule, matcher, + callback, 1) + + def test_validate_default_attributes(self): + callback = lambda: True + matcher = BaseMatcher() + rule = CompiledRule(matcher, callback, True) + + # test attributes + self.failUnlessEqual(matcher, rule.matcher) + self.failUnlessEqual(callback, rule.callback) + self.failUnlessEqual(True, rule.follow) + + +class RulesTest(unittest.TestCase): + def test_rules_manager_basic(self): + spider = BaseSpider('foo') + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + rulesman = RulesManager([], spider) + + # should return none + self.failIf(rulesman.get_rule_from_response(response1)) + self.failIf(rulesman.get_rule_from_response(response2)) + + # rules manager with match-all rule + rulesman = RulesManager([ + Rule(BaseMatcher(), follow=True), + ], spider) + + # returns CompiledRule + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnless(isinstance(rule1, CompiledRule)) + self.failUnless(isinstance(rule2, CompiledRule)) + self.assert_(rule1 is rule2) + self.failUnlessEqual(rule1.callback, None) + self.failUnlessEqual(rule1.follow, True) + + def test_rules_manager_empty_rule(self): + spider = BaseSpider('foo') + response = HtmlResponse('http://example.org') + + rulesman = RulesManager([Rule(follow=True)], spider) + + rule = rulesman.get_rule_from_response(response) + # default matcher if None: BaseMatcher + self.failUnless(isinstance(rule.matcher, BaseMatcher)) + + def test_rules_manager_default_matcher(self): + spider = BaseSpider('foo') + response = HtmlResponse('http://example.org') + callback = lambda x: None + + rulesman = RulesManager([ + Rule('http://example.org', callback), + ], spider, default_matcher=UrlMatcher) + + rule = rulesman.get_rule_from_response(response) + self.failUnless(isinstance(rule.matcher, UrlMatcher)) + + def test_rules_manager_matchers(self): + spider = BaseSpider('foo') + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + + urlmatcher = UrlMatcher('http://example.org') + basematcher = BaseMatcher() + # callback needed for Rule + callback = lambda x: None + + # test fail matcher resolve + self.assertRaises(ValueError, RulesManager, + [Rule(False, callback)], spider) + self.assertRaises(ValueError, RulesManager, + [Rule(spider, callback)], spider) + + rulesman = RulesManager([ + Rule(urlmatcher, callback), + Rule(basematcher, callback), + ], spider) + + # response1 matches example.org + rule1 = rulesman.get_rule_from_response(response1) + # response2 is catch by BaseMatcher() + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.matcher, urlmatcher) + self.failUnlessEqual(rule2.matcher, basematcher) + + # reverse order. BaseMatcher should match all + rulesman = RulesManager([ + Rule(basematcher, callback), + Rule(urlmatcher, callback), + ], spider) + + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.matcher, basematcher) + self.failUnlessEqual(rule2.matcher, basematcher) + self.failUnless(rule1 is rule2) + + def test_rules_manager_callbacks(self): + mycallback = lambda: True + + spider = BaseSpider('foo') + spider.parse_item = lambda: True + + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + + rulesman = RulesManager([ + Rule('example', mycallback), + Rule('othersite', 'parse_item'), + ], spider, default_matcher=UrlRegexMatcher) + + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.callback, mycallback) + self.failUnlessEqual(rule2.callback, spider.parse_item) + + # fail unknown callback + self.assertRaises(AttributeError, RulesManager, [ + Rule(BaseMatcher(), 'mycallback') + ], spider) + # fail not callable + spider.not_callable = True + self.assertRaises(AttributeError, RulesManager, [ + Rule(BaseMatcher(), 'not_callable') + ], spider) + + + def test_rules_manager_callback_with_arguments(self): + spider = BaseSpider('foo') + response = HtmlResponse('http://example.org') + + kwargs = {'a': 1} + + def myfunc(**mykwargs): + return mykwargs + + # verify return validation + self.failUnlessEquals(kwargs, myfunc(**kwargs)) + + # test callback w/o arguments + rulesman = RulesManager([ + Rule(BaseMatcher(), myfunc), + ], spider) + rule = rulesman.get_rule_from_response(response) + + # without arguments should return same callback + self.failUnlessEqual(rule.callback, myfunc) + + # test callback w/ arguments + rulesman = RulesManager([ + Rule(BaseMatcher(), myfunc, **kwargs), + ], spider) + rule = rulesman.get_rule_from_response(response) + + # with argument should return partial applied callback + self.failUnless(isinstance(rule.callback, partial)) + self.failUnlessEquals(kwargs, rule.callback()) + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_spider.py b/scrapy/tests/test_contrib_exp_crawlspider_spider.py new file mode 100644 index 000000000..21f7f71e4 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_spider.py @@ -0,0 +1,222 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.item import BaseItem +from scrapy.utils.spider import iterate_spider_output + +# basics +from scrapy.contrib_exp.crawlspider import CrawlSpider +from scrapy.contrib_exp.crawlspider import Rule + +# matchers +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlListMatcher + +# extractors +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor + +# processors +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + + +# mock items +class Item1(BaseItem): + pass + +class Item2(BaseItem): + pass + +class Item3(BaseItem): + pass + + +class CrawlSpiderTest(unittest.TestCase): + + def spider_factory(self, rules=[], + extractors=[], processors=[], + start_urls=[]): + # mock spider + class Spider(CrawlSpider): + def parse_item1(self, response): + return Item1() + + def parse_item2(self, response): + return Item2() + + def parse_item3(self, response): + return Item3() + + def parse_request1(self, response): + return Request('http://example.org/request1') + + def parse_request2(self, response): + return Request('http://example.org/request2') + + Spider.start_urls = start_urls + Spider.rules = rules + Spider.request_extractors = extractors + Spider.request_processors = processors + + return Spider('foo') + + def test_start_url_auto_rule(self): + spider = self.spider_factory() + # zero spider rules + self.failUnlessEqual(len(spider.rules), 0) + self.failUnlessEqual(len(spider._rulesman._rules), 0) + + spider = self.spider_factory(start_urls=['http://example.org']) + + self.failUnlessEqual(len(spider.rules), 0) + self.failUnlessEqual(len(spider._rulesman._rules), 1) + + def test_start_url_matcher(self): + url = 'http://example.org' + spider = self.spider_factory(start_urls=[url]) + + response = HtmlResponse(url) + + rule = spider._rulesman.get_rule_from_response(response) + self.failUnless(isinstance(rule.matcher, UrlListMatcher)) + + response = HtmlResponse(url + '/item.html') + + rule = spider._rulesman.get_rule_from_response(response) + self.failUnless(rule is None) + + # TODO: remove this block + # in previous version get_rule returns rule from response.request + response.request = Request(url) + rule = spider._rulesman.get_rule_from_response(response.request) + self.failUnless(isinstance(rule.matcher, UrlListMatcher)) + self.failUnlessEqual(rule.follow, True) + + def test_parse_callback(self): + response = HtmlResponse('http://example.org') + rules = ( + Rule(BaseMatcher(), 'parse_item1'), + ) + spider = self.spider_factory(rules) + + result = list(spider.parse(response)) + self.failUnlessEqual(len(result), 1) + self.failUnless(isinstance(result[0], Item1)) + + def test_crawling_start_url(self): + url = 'http://example.org/' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + response = HtmlResponse(url, body=html) + + extractors = (SgmlRequestExtractor(), ) + spider = self.spider_factory(start_urls=[url], + extractors=extractors) + result = list(spider.parse(response)) + + # 1 request extracted: example.org/ + # because requests returns only matching + self.failUnlessEqual(len(result), 1) + + # we will add catch-all rule to extract all + callback = lambda x: None + rules = [Rule(r'\.html$', callback=callback)] + spider = self.spider_factory(rules, start_urls=[url], + extractors=extractors) + result = list(spider.parse(response)) + + # 4 requests extracted + # 3 of .html pattern + # 1 of start url patter + self.failUnlessEqual(len(result), 4) + + def test_crawling_simple_rule(self): + url = 'http://example.org/somepage/index.html' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + response = HtmlResponse(url, body=html) + + rules = ( + # first response callback + Rule(r'index\.html', 'parse_item1'), + ) + spider = self.spider_factory(rules) + result = list(spider.parse(response)) + + # should return Item1 + self.failUnlessEqual(len(result), 1) + self.failUnless(isinstance(result[0], Item1)) + + # test request generation + rules = ( + # first response without callback and follow flag + Rule(r'index\.html', follow=True), + Rule(r'(\.html|/)$', 'parse_item1'), + ) + spider = self.spider_factory(rules) + result = list(spider.parse(response)) + + # 0 because spider does not have extractors + self.failUnlessEqual(len(result), 0) + + extractors = (SgmlRequestExtractor(), ) + + # instance spider with extractor + spider = self.spider_factory(rules, extractors) + result = list(spider.parse(response)) + # 4 requests extracted + self.failUnlessEqual(len(result), 4) + + def test_crawling_multiple_rules(self): + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + response = HtmlResponse('http://example.org/index.html', body=html) + response1 = HtmlResponse('http://example.org/1.html') + response2 = HtmlResponse('http://example.org/othercat.html') + + rules = ( + Rule(r'\d+\.html$', 'parse_item1'), + Rule(r'othercat\.html$', 'parse_item2'), + # follow-only rules + Rule(r'index\.html', 'parse_item3', follow=True) + ) + extractors = [SgmlRequestExtractor()] + spider = self.spider_factory(rules, extractors) + + result = list(spider.parse(response)) + # 1 Item 2 Requests + self.failUnlessEqual(len(result), 3) + # parse_item3 + self.failUnless(isinstance(result[0], Item3)) + only_requests = lambda r: isinstance(r, Request) + requests = filter(only_requests, result[1:]) + self.failUnlessEqual(len(requests), 2) + self.failUnless(all(requests)) + + result1 = list(spider.parse(response1)) + # parse_item1 + self.failUnlessEqual(len(result1), 1) + self.failUnless(isinstance(result1[0], Item1)) + + result2 = list(spider.parse(response2)) + # parse_item2 + self.failUnlessEqual(len(result2), 1) + self.failUnless(isinstance(result2[0], Item2)) + + diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 48cc90c46..140108b42 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,10 +1,10 @@ -import cPickle as pickle +import unittest, cPickle as pickle from cStringIO import StringIO -from twisted.trial import unittest - from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode +from scrapy.utils.py26 import json +from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter @@ -149,22 +149,10 @@ class XmlItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - def setUp(self): - try: - import json - except ImportError: - try: - import simplejson - except ImportError: - raise unittest.SkipTest("simplejson module not available") - super(JsonLinesItemExporterTest, self).setUp() - def _get_exporter(self, **kwargs): - from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - from scrapy.contrib.exporter.jsonlines import json exported = json.loads(self.output.getvalue().strip()) self.assertEqual(exported, dict(self.i)) diff --git a/scrapy/tests/test_contrib_ibl/__init__.py b/scrapy/tests/test_contrib_ibl/__init__.py new file mode 100644 index 000000000..f5fa69d7a --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/__init__.py @@ -0,0 +1,2 @@ +import sys +path = sys.modules[__name__].__path__[0] diff --git a/scrapy/tests/test_contrib_ibl/samples_htmlpage.json.gz b/scrapy/tests/test_contrib_ibl/samples_htmlpage.json.gz new file mode 100644 index 000000000..fe4d1894c Binary files /dev/null and b/scrapy/tests/test_contrib_ibl/samples_htmlpage.json.gz differ diff --git a/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz b/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz new file mode 100644 index 000000000..20a2eb09a Binary files /dev/null and b/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz differ diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py new file mode 100644 index 000000000..ee1a69bcc --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/test_extraction.py @@ -0,0 +1,819 @@ +""" +tests for page parsing + +Page parsing effectiveness is measured through the evaluation system. These +tests should focus on specific bits of functionality work correctly. +""" +from unittest import TestCase +from scrapy.contrib.ibl.htmlpage import HtmlPage +from scrapy.contrib.ibl.extraction import InstanceBasedLearningExtractor +from scrapy.contrib.ibl.descriptor import (FieldDescriptor as A, + ItemDescriptor) +from scrapy.contrib.ibl.extractors import (contains_any_numbers, + image_url) + +# simple page with all features + +ANNOTATED_PAGE1 = u""" +<html> +<h1>COMPANY - <ins + data-scrapy-annotate="{"variant": 0, "generated": true, + "annotations": {"content": "title"}}" +>Item Title</ins></h1> +<p>introduction</p> +<div> +<img data-scrapy-annotate="{"variant": 0, + "annotations": {"src": "image_url"}}" + src="img.jpg"/> +<p data-scrapy-annotate="{"variant": 0, + "annotations": {"content": "description"}}"> +This is such a nice item<br/> Everybody likes it. +</p> +<br/> +</div> +<p>click here for other items</p> +</html> +""" + +EXTRACT_PAGE1 = u""" +<html> +<h1>Scrapy - Nice Product</h1> +<p>introduction</p> +<div> +<img src="nice_product.jpg" alt="a nice product image"/> +<p>wonderful product</p> +<br/> +</div> +</html> +""" + +# single tag with multiple items extracted +ANNOTATED_PAGE2 = u""" +<a href="http://example.com/xxx" title="xxx" + data-scrapy-annotate="{"variant": 0, + "annotations": {"content": "description", + "href": "image_url", "title": "name"}}" +>xx</a> +xxx +</a> +""" +EXTRACT_PAGE2 = u"""<a href='http://example.com/product1.jpg' + title="product 1">product 1 is great</a>""" + +# matching must match the second attribute in order to find the first +ANNOTATED_PAGE3 = u""" +<p data-scrapy-annotate="{"variant": 0, + "annotations": {"content": "description"}}">xx</p> +<div data-scrapy-annotate="{"variant": 0, + "annotations": {"content": "delivery"}}">xx</div> +""" +EXTRACT_PAGE3 = u""" +<p>description</p> +<div>delivery</div> +<p>this is not the description</p> +""" + +# test inferring repeated elements +ANNOTATED_PAGE4 = u""" +<ul> +<li data-scrapy-annotate="{"variant": 0, + "annotations": {"content": "features"}}">feature1</li> +<li data-scrapy-annotate="{"variant": 0, + "annotations": {"content": "features"}}">feature2</li> +</ul> +""" + +EXTRACT_PAGE4 = u""" +<ul> +<li>feature1</li> ignore this +<li>feature2</li> +<li>feature3</li> +</ul> +""" + +# test variant handling with identical repeated variant +ANNOTATED_PAGE5 = u""" +<p data-scrapy-annotate="{"annotations": + {"content": "description"}}">description</p> +<table> +<tr> +<td data-scrapy-annotate="{"variant": 1, "annotations": + {"content": "colour"}}" >colour 1</td> +<td data-scrapy-annotate="{"variant": 1, "annotations": + {"content": "price"}}" >price 1</td> +</tr> +<tr> +<td data-scrapy-annotate="{"variant": 2, "annotations": + {"content": "colour"}}" >colour 2</td> +<td data-scrapy-annotate="{"variant": 2, "annotations": + {"content": "price"}}" >price 2</td> +</tr> +</table> +""" + +EXTRACT_PAGE5 = u""" +<p>description</p> +<table> +<tr> +<td>colour 1</td> +<td>price 1</td> +</tr> +<tr> +<td>colour 2</td> +<td>price 2</td> +</tr> +<tr> +<td>colour 3</td> +<td>price 3</td> +</tr> +</table> +""" + +# test variant handling with irregular structure and some non-variant +# attributes +ANNOTATED_PAGE6 = u""" +<p data-scrapy-annotate="{"annotations": + {"content": "description"}}">description</p> +<p data-scrapy-annotate="{"variant": 1, "annotations": + {"content": "name"}}">name 1</p> +<div data-scrapy-annotate="{"variant": 3, "annotations": + {"content": "name"}}" >name 3</div> +<p data-scrapy-annotate="{"variant": 2, "annotations": + {"content": "name"}}" >name 2</p> +""" +EXTRACT_PAGE6 = u""" +<p>description</p> +<p>name 1</p> +<div>name 3</div> +<p>name 2</p> +""" + +# test repeating variants at the table column level +ANNOTATED_PAGE7 = u""" +<table> +<tr> +<td data-scrapy-annotate="{"variant": 1, "annotations": + {"content": "colour"}}" >colour 1</td> +<td data-scrapy-annotate="{"variant": 2, "annotations": + {"content": "colour"}}" >colour 2</td> +</tr> +<tr> +<td data-scrapy-annotate="{"variant": 2, "annotations": + {"content": "price"}}" >price 1</td> +<td data-scrapy-annotate="{"variant": 2, "annotations": + {"content": "price"}}" >price 2</td> +</tr> +</table> +""" + +EXTRACT_PAGE7 = u""" +<table> +<tr> +<td>colour 1</td> +<td>colour 2</td> +<td>colour 3</td> +</tr> +<tr> +<td>price 1</td> +<td>price 2</td> +<td>price 3</td> +</tr> +</table> +""" + +ANNOTATED_PAGE8 = u""" +<html><body> +<h1>A product</h1> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<p>XXXX XXXX xxxxx</p> +<div data-scrapy-ignore="true"> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +</div> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}"> +10.00<p data-scrapy-ignore="true"> 13</p> +</div> +</body></html> +""" + +EXTRACT_PAGE8 = u""" +<html><body> +<h1>A product</h1> +<div> +<p>A very nice product for all intelligent people</p> +<div> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +</div> +<div> +12.00<p> ID 15</p> +(VAT exc.)</div> +</body></html> +""" + +ANNOTATED_PAGE9 = ANNOTATED_PAGE8 + +EXTRACT_PAGE9 = u""" +<html><body> +<img src="logo.jpg" /> +<h1>A product</h1> +<div> +<p>A very nice product for all intelligent people</p> +<div> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +</div> +<div> +12.00<p> ID 16</p> +(VAT exc.)</div> +</body></html> +""" + +ANNOTATED_PAGE10a = u""" +<html><body> +<table><tbody> +<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "site_id"}}"> +<td>SKU</td><td>L345</td> +</tr> +<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "dimensions"}}"> +<td>Size</td><td>10cmx20cm</td> +</tr> +<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "price"}}"> +<td>Price</td><td>£s;99.00</td> +</tr> +</tbody></table> +</body></html> +""" + +ANNOTATED_PAGE10b = u""" +<html><body> +<table><tbody> +<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "site_id"}}"> +<td>SKU</td><td>S220</td> +</tr> +<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "dimensions"}}"> +<td>Size</td><td>20cmx20cm</td> +</tr> +<tr data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "price"}}"> +<td>Price</td><td>£s;85.00</td> +</tr> +</tbody></table> +</body></html> +""" + +EXTRACT_PAGE10a = u""" +<html><body> +<table><tbody> +<tr> +<td>Offer</td><td>From $2500.00</td> +</tr> +<tr> +<td>Description</td><td>Electrorheological Cyborgs</td> +</tr> +<tr> +<td>Series:</td><td>T2000</td> +</tr> +</tbody></table> +</body></html> +""" + +EXTRACT_PAGE10b = u""" +<html><body> +<table><tbody> +<tr> +<td>SKU</td><td>K80</td> +</tr> +<tr> +<td>Size</td><td>50cm</td> +</tr> +<tr> +<td>Price</td><td>&euros;85.00</td> +</tr> +</tbody></table> +</body></html> +""" + +ANNOTATED_PAGE11 = u""" +<html><body> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<ins data-scrapy-annotate="{"variant": 0, "generated": true, + "annotations": {"content": "name"}}"> +SL342 +</ins> +<br/> +Nice product for ladies +<br/><ins data-scrapy-annotate="{"variant": 0, "generated": true, + "annotations": {"content": "price"}}"> +£s;85.00 +</ins> +</p> +<ins data-scrapy-annotate="{"variant": 0, "generated": true, + "annotations": {"content": "price_before_discount"}}"> +£s;100.00 +</ins> +</body></html> +""" + +EXTRACT_PAGE11 = u""" +<html><body> +<p> +SL342 +<br/> +Nice product for ladies +<br/> +£s;85.00 +</p> +£s;100.00 +</body></html> +""" + +ANNOTATED_PAGE12 = u""" +<html><body> +<h1 data-scrapy-ignore-beneath="true">A product</h1> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<p>XXXX XXXX xxxxx</p> +<div data-scrapy-ignore-beneath="true"> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +<div> +10.00<p> 13</p> +</div> +</div> +</body></html> +""" + +EXTRACT_PAGE12a = u""" +<html><body> +<h1>A product</h1> +<div> +<p>A very nice product for all intelligent people</p> +<div> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +<div> +12.00<p> ID 15</p> +(VAT exc.) +</div></div> +</body></html> +""" + +EXTRACT_PAGE12b = u""" +<html><body> +<h1>A product</h1> +<div> +<p>A very nice product for all intelligent people</p> +<div> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +<div> +12.00<p> ID 15</p> +(VAT exc.) +</div> +<ul> +Features +<li>Feature A</li> +<li>Feature B</li> +</ul> +</div> +</body></html> +""" + +# Ex1: nested annotation with token sequence replica outside exterior annotation +# and a possible sequence pattern can be extracted only with +# correct handling of nested annotations +ANNOTATED_PAGE13a = u""" +<html><body> +<span> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<hr/> +<h3 data-scrapy-annotate="{"variant": 0, "annotations": {"content": "name"}}">A product</h3> +<b data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}">$50.00</b> +This product is excelent. Buy it! +</p> +</span> +<span> +<p> +<h3>See other products:</h3> +<b>Product b</b> +</p> +</span> +<hr/> +</body></html> +""" + +EXTRACT_PAGE13a = u""" +<html><body> +<span> +<p> +<h3>A product</h3> +<b>$50.00</b> +This product is excelent. Buy it! +<hr/> +</p> +</span> +<span> +<p> +<h3>See other products:</h3> +<b>Product B</b> +</p> +</span> +</body></html> +""" + +# Ex2: annotation with token sequence replica inside a previous nested annotation +# and a possible sequence pattern can be extracted only with +# correct handling of nested annotations +ANNOTATED_PAGE13b = u""" +<html><body> +<span> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<h3 data-scrapy-annotate="{"variant": 0, "annotations": {"content": "name"}}">A product</h3> +<b>Previous price: $50.00</b> +This product is excelent. Buy it! +</p> +</span> +<span> +<p> +<h3>Save 10%!!</h3> +<b data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}">$45.00</b> +</p> +</span> +</body></html> +""" + +EXTRACT_PAGE13b = u""" +<html><body> +<span> +<p> +<h3>A product</h3> +<b>$50.00</b> +This product is excelent. Buy it! +</p> +</span> +<span> +<hr/> +<p> +<h3>Save 10%!!</h3> +<b>$45.00</b> +</p> +</span> +<hr/> +</body></html> +""" + +ANNOTATED_PAGE14 = u""" +<html><body> +<b data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"></b> +<p data-scrapy-ignore="true"></p> +</body></html> +""" + +EXTRACT_PAGE14 = u""" +<html><body> +</body></html> +""" + +ANNOTATED_PAGE15 = u""" +<html><body> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "short_description"}}">Short +<div data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "annotations": {"content": "site_id"}}">892342</div> +</div> +<hr/> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}">Description +<b data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}">90.00</b> +</p> +</body></html> +""" + +EXTRACT_PAGE15 = u""" +<html><body> +<hr/> +<p>Description +<b>80.00</b> +</p> +</body></html> +""" + +ANNOTATED_PAGE16 = u""" +<html><body> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +Description +<p data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "annotations": {"content": "name"}}"> +name</p> +<p data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}"> +80.00</p> +</div> +</body></html> +""" + +EXTRACT_PAGE16 = u""" +<html><body> +<p>product name</p> +<p>90.00</p> +</body></html> +""" + +ANNOTATED_PAGE17 = u""" +<html><body> +<span> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +This product is excelent. Buy it! +</p> +</span> +<table></table> +<img src="line.jpg" data-scrapy-ignore-beneath="true"/> +<span> +<h3>See other products:</h3> +<p>Product b +</p> +</span> +</body></html> +""" + +EXTRACT_PAGE17 = u""" +<html><body> +<span> +<p> +This product is excelent. Buy it! +</p> +</span> +<img src="line.jpg"/> +<span> +<h3>See other products:</h3> +<p>Product B +</p> +</span> +</body></html> +""" + +ANNOTATED_PAGE18 = u""" +<html><body> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<ins data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "generated": true, "annotations": {"content": "site_id"}}">Item Id</ins> +<br> +Description +</div> +</body></html> +""" + +EXTRACT_PAGE18 = u""" +<html><body> +<div> +Item Id +<br> +Description +</div> +</body></html> +""" + +ANNOTATED_PAGE19 = u""" +<html><body> +<div> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "name"}}">Product name</p> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}">60.00</p> +<img data-scrapy-annotate="{"variant": 0, "annotations": {"src": "image_urls"}}"src="image.jpg" /> +<p data-scrapy-annotate="{"variant": 0, "required": ["description"], "annotations": {"content": "description"}}">description</p> +</div> +</body></html> +""" + +EXTRACT_PAGE19a = u""" +<html><body> +<div> +<p>Product name</p> +<p>60.00</p> +<img src="http://example.com/image.jpg" /> +<p>description</p> +</div> +</body></html> +""" + +EXTRACT_PAGE19b = u""" +<html><body> +<div> +<p>Range</p> +<p>from 20.00</p> +<img src="http://example.com/image1.jpg" /> +<p> +<br/> +</div> +</body></html> +""" + +SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [ + A('name', "Product name", required=True), + A('price', "Product price, including any discounts and tax or vat", + contains_any_numbers, True), + A('image_urls', "URLs for one or more images", image_url, True), + A('description', "The full description of the product", allow_markup=True), + ] +) + +# A list of (test name, [templates], page, extractors, expected_result) +TEST_DATA = [ + # extract from a similar page + ('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, None, + {u'title': [u'Nice Product'], u'description': [u'wonderful product'], + u'image_url': [u'nice_product.jpg']} + ), + # strip the first 5 characters from the title + ('extractor test', [ANNOTATED_PAGE1], EXTRACT_PAGE1, + ItemDescriptor('test', 'product test', + [A('title', "something about a title", lambda x: x[5:])]), + {u'title': [u'Product'], u'description': [u'wonderful product'], + u'image_url': [u'nice_product.jpg']} + ), + # compilicated tag (multiple attributes and annotation) + ('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, None, + {'name': [u'product 1'], 'image_url': [u'http://example.com/product1.jpg'], + 'description': [u'product 1 is great']} + ), + # can only work out correct placement by matching the second attribute first + ('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, None, + {'description': [u'description'], 'delivery': [u'delivery']} + ), + # infer a repeated structure + ('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, None, + {'features': [u'feature1', u'feature2', u'feature3']} + ), + # identical variants with a repeated structure + ('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, None, + { + 'description': [u'description'], + 'variants': [ + {u'colour': [u'colour 1'], u'price': [u'price 1']}, + {u'colour': [u'colour 2'], u'price': [u'price 2']}, + {u'colour': [u'colour 3'], u'price': [u'price 3']} + ] + } + ), + # variants with an irregular structure + ('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, None, + { + 'description': [u'description'], + 'variants': [ + {u'name': [u'name 1']}, + {u'name': [u'name 3']}, + {u'name': [u'name 2']} + ] + } + ), + + # discovering repeated variants in table columns +# ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, None, +# {'variants': [ +# {u'colour': [u'colour 1'], u'price': [u'price 1']}, +# {u'colour': [u'colour 2'], u'price': [u'price 2']}, +# {u'colour': [u'colour 3'], u'price': [u'price 3']} +# ]} +# ), + + + # ignored regions + ( + 'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, None, + { + 'description': [u'\n A very nice product for all intelligent people \n\n'], + 'price': [u'\n12.00\n(VAT exc.)'], + } + ), + # shifted ignored regions (detected by region similarity) + ( + 'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, None, + { + 'description': [u'\n A very nice product for all intelligent people \n\n'], + 'price': [u'\n12.00\n(VAT exc.)'], + } + ), + # detection of common prefixes across templates, all templates marked + (# wrong extraction + 'without_match_common_prefix', [ANNOTATED_PAGE10a], EXTRACT_PAGE10a, None, + { + 'price': [u'\n Series: T2000 \n'], + 'dimensions': [u'\n Description Electrorheological Cyborgs \n'], + 'site_id': [u'\n Offer From $2500.00 \n'], + } + ), + (# right extraction + 'with_match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10a, None, + {} + ), + (# another example + 'match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, None, + { + 'dimensions': [u'50cm'], + 'site_id': [u'K80'], + } + ), + (# common_prefix with allow_markup attribute + 'common_prefix_allow_markup', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, + ItemDescriptor('test', 'product test', + [A('dimensions', "something about dimensions", allow_markup=True)]), + { + 'dimensions': [u'50cm</td>'], + 'site_id': [u'K80'], + } + ), + (# special case with partial annotations + 'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None, + { + 'name': [u'SL342'], + 'description': [u'\nSL342\n \nNice product for ladies\n \n£s;85.00\n'], + 'price': [u'£s;85.00'], + 'price_before_discount': [u'£s;100.00'], + } + ), + (# with ignore-beneath feature + 'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, None, + { + 'description': [u'\n A very nice product for all intelligent people \n'], + } + ), + (# ignore-beneath with extra tags + 'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, None, + { + 'description': [u'\n A very nice product for all intelligent people \n'], + } + ), + ('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, None, + {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n \n'], + 'price': ["$50.00"], + 'name': [u'A product']} + ), + ('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, None, + {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n'], + 'price': ["$45.00"], + 'name': [u'A product']} + ), + ('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, None, + {}, + ), + ('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, None, + {'description': [u'Description\n\n'], + 'price': [u'80.00']}, + ), + ('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, None, + {'price': [u'90.00'], + 'name': [u'product name']}, + ), + ('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, None, + {'description': [u'\nThis product is excelent. Buy it!\n']}, + ), + ('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, None, + {u'site_id': [u'Item Id'], + u'description': [u'\nDescription\n']}, + ), + ('extra required attribute product', [ANNOTATED_PAGE19], EXTRACT_PAGE19a, + SAMPLE_DESCRIPTOR1, + {u'price': [u'60.00'], + u'description': [u'description'], + u'image_urls': [['http://example.com/image.jpg']], + u'name': [u'Product name']}, + ), + ('extra required attribute no product', [ANNOTATED_PAGE19], EXTRACT_PAGE19b, + SAMPLE_DESCRIPTOR1, + None, + ), +] + +class TestExtraction(TestCase): + + def _run(self, name, templates, page, extractors, expected_output): + self.trace = None + template_pages = [HtmlPage(None, {}, t) for t in templates] + extractor = InstanceBasedLearningExtractor(template_pages, extractors, True) + actual_output, _ = extractor.extract(HtmlPage(None, {}, page)) + if not actual_output: + if expected_output is None: + return + assert False, "failed to extract data for test '%s'" % name + actual_output = actual_output[0] + self.trace = ["Extractor:\n%s" % extractor] + actual_output.pop('trace', []) + expected_names = set(expected_output.keys()) + actual_names = set(actual_output.keys()) + + missing_in_output = filter(None, expected_names - actual_names) + error = "attributes '%s' were expected but were not present in test '%s'" % \ + ("', '".join(missing_in_output), name) + assert len(missing_in_output) == 0, error + + unexpected = actual_names - expected_names + error = "unexpected attributes %s in test '%s'" % \ + (', '.join(unexpected), name) + assert len(unexpected) == 0, error + + for k, v in expected_output.items(): + extracted = actual_output[k] + assert v == extracted, "in test '%s' for attribute '%s', " \ + "expected value '%s' but got '%s'" % (name, k, v, extracted) + + def test_expected_outputs(self): + try: + for data in TEST_DATA: + self._run(*data) + except AssertionError: + if self.trace: + print "Trace:" + for line in self.trace: + print "\n---\n%s" % line + raise diff --git a/scrapy/tests/test_contrib_ibl/test_htmlpage.py b/scrapy/tests/test_contrib_ibl/test_htmlpage.py new file mode 100644 index 000000000..f502c6364 --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/test_htmlpage.py @@ -0,0 +1,134 @@ +""" +htmlpage.py tests +""" +import os +from gzip import GzipFile +from unittest import TestCase + +from scrapy.utils.py26 import json +from scrapy.tests.test_contrib_ibl import path +from scrapy.contrib.ibl.htmlpage import parse_html, HtmlTag, HtmlDataFragment +from scrapy.tests.test_contrib_ibl.test_htmlpage_data import * + +SAMPLES_FILE = "samples_htmlpage.json.gz" + +def _encode_element(el): + """ + jsonize parse element + """ + if isinstance(el, HtmlTag): + return {"tag": el.tag, "attributes": el.attributes, + "start": el.start, "end": el.end, "tag_type": el.tag_type} + if isinstance(el, HtmlDataFragment): + return {"start": el.start, "end": el.end} + raise TypeError + +def _decode_element(dct): + """ + dejsonize parse element + """ + if "tag" in dct: + return HtmlTag(dct["tag_type"], dct["tag"], \ + dct["attributes"], dct["start"], dct["end"]) + if "start" in dct: + return HtmlDataFragment(dct["start"], dct["end"]) + return dct + +def add_sample(source): + """ + Method for adding samples to test samples file + (use from console) + """ + samples = [] + if os.path.exists(SAMPLES_FILE): + for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines(): + samples.append(json.loads(line)) + + new_sample = {"source": source} + new_sample["parsed"] = list(parse_html(source)) + samples.append(new_sample) + samples_file = GzipFile(os.path.join(path, SAMPLES_FILE), "wb") + for sample in samples: + samples_file.write(json.dumps(sample, default=_encode_element) + "\n") + samples_file.close() + +class TestParseHtml(TestCase): + """Test for parse_html""" + def _test_sample(self, sample): + source = sample["source"] + expected_parsed = sample["parsed"] + parsed = parse_html(source) + count_element = 0 + count_expected = 0 + for element in parsed: + if type(element) == HtmlTag: + count_element += 1 + expected = expected_parsed.pop(0) + if type(expected) == HtmlTag: + count_expected += 1 + element_text = source[element.start:element.end] + expected_text = source[expected.start:expected.end] + if element.start != expected.start or element.end != expected.end: + assert False, "[%s,%s] %s != [%s,%s] %s" % (element.start, \ + element.end, element_text, expected.start, \ + expected.end, expected_text) + if type(element) != type(expected): + assert False, "(%s) %s != (%s) %s for text\n%s" % (count_element, \ + repr(type(element)), count_expected, repr(type(expected)), element_text) + if type(element) == HtmlTag: + self.assertEqual(element.tag, expected.tag) + self.assertEqual(element.attributes, expected.attributes) + + def test_parse(self): + """simple parse_html test""" + parsed = [_decode_element(d) for d in PARSED] + sample = {"source": PAGE, "parsed": parsed} + self._test_sample(sample) + + def test_site_samples(self): + """test parse_html from real cases""" + samples = [] + for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines(): + samples.append(json.loads(line, object_hook=_decode_element)) + for sample in samples: + self._test_sample(sample) + + def test_bad(self): + """test parsing of bad html layout""" + parsed = [_decode_element(d) for d in PARSED2] + sample = {"source": PAGE2, "parsed": parsed} + self._test_sample(sample) + + def test_comments(self): + """test parsing of tags inside comments""" + parsed = [_decode_element(d) for d in PARSED3] + sample = {"source": PAGE3, "parsed": parsed} + self._test_sample(sample) + + def test_script_text(self): + """test parsing of tags inside scripts""" + parsed = [_decode_element(d) for d in PARSED4] + sample = {"source": PAGE4, "parsed": parsed} + self._test_sample(sample) + + def test_sucessive(self): + """test parsing of sucesive cleaned elements""" + parsed = [_decode_element(d) for d in PARSED5] + sample = {"source": PAGE5, "parsed": parsed} + self._test_sample(sample) + + def test_sucessive2(self): + """test parsing of sucesive cleaned elements (variant 2)""" + parsed = [_decode_element(d) for d in PARSED6] + sample = {"source": PAGE6, "parsed": parsed} + self._test_sample(sample) + + def test_special_cases(self): + """some special cases tests""" + parsed = list(parse_html("<meta http-equiv='Pragma' content='no-cache' />")) + self.assertEqual(parsed[0].attributes, {'content': 'no-cache', 'http-equiv': 'Pragma'}) + parsed = list(parse_html("<html xmlns='http://www.w3.org/1999/xhtml' xml:lang='en' lang='en'>")) + self.assertEqual(parsed[0].attributes, {'xmlns': 'http://www.w3.org/1999/xhtml', 'xml:lang': 'en', 'lang': 'en'}) + parsed = list(parse_html("<IMG SRC='http://images.play.com/banners/SAM550a.jpg' align='left' / hspace=5>")) + self.assertEqual(parsed[0].attributes, {'src': 'http://images.play.com/banners/SAM550a.jpg', \ + 'align': 'left', 'hspace': '5', '/': None}) diff --git a/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py b/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py new file mode 100644 index 000000000..2eac24f26 --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py @@ -0,0 +1,227 @@ +PAGE = u""" +<style id="scrapy-style" type="text/css">@import url(http://localhost:8000/as/site_media/clean.css); +</style> +<body> +<div class="scrapy-selected" id="header"> +<img src="company_logo.jpg" style="margin-left: 68px; padding-top:5px;" alt="Logo" width="530" height="105"> +<div id="vertrule"> +<h1>COMPANY - <ins data-scrapy-annotate="{"variant": "0", "generated": true, "annotations": {"content": "title"}}">Item Title</ins></h1> +<p>introduction</p> +<div> +<img src="/upload/img.jpg" classid="" + data-scrapy-annotate="{"variant": "0", "annotations": {"image_url": "src"}}" +> +<p classid="" data-scrapy-annotate="{"variant": "0", "annotations": {"content": "description"}}" +> +This is such a nice item<br/> Everybody likes it. +</p> +<br></br> +</div> +<p data-scrapy-annotate="{"variant": "0", "annotations": {"content": "features"}}" +class="" >Power: 50W</p> +<!-- A comment --!> +<ul data-scrapy-replacement='select' class='product'> +<li data-scrapy-replacement='option'>Small</li> +<li data-scrapy-replacement='option'>Big</li> +</ul> +<p>click here for other items</p> +<h3>Louis Chair</h3> +<table class="rulet" width="420" cellpadding="0" cellspacing="0"><tbody> +<tr><td>Height</td> +<td><ins data-scrapy-annotate="{"variant": "0", "generated": true, "annotations": {"content": "price"}}">32.00</ins></td> +</tr><tbody></table> +<p onmouseover='xxx' class= style="my style"> +""" + +PARSED = [ +{'start': 0, 'end': 1}, +{'attributes': {'type': 'text/css', 'id': 'scrapy-style'}, 'tag': 'style', 'end': 42, 'start': 1, 'tag_type': 1}, +{'start': 42, 'end': 129}, +{'attributes': {}, 'tag': 'style', 'end': 137, 'start': 129, 'tag_type': 2}, +{'start': 137, 'end': 138}, +{'attributes': {}, 'tag': 'body', 'end': 144, 'start': 138, 'tag_type': 1}, +{'start': 144, 'end': 145}, +{'attributes': {'class': 'scrapy-selected', 'id': 'header'}, 'tag': 'div', 'end': 186, 'start': 145, 'tag_type': 1}, +{'start': 186, 'end': 187}, +{'attributes': {'src': 'company_logo.jpg', 'style': 'margin-left: 68px; padding-top:5px;', 'width': '530', 'alt': 'Logo', 'height': '105'}, 'tag': 'img', 'end': 295, 'start': 187, 'tag_type': 1}, +{'start': 295, 'end': 296}, +{'attributes': {'id': 'vertrule'}, 'tag': 'div', 'end': 315, 'start': 296, 'tag_type': 1}, +{'start': 315, 'end': 316}, +{'attributes': {}, 'tag': 'h1', 'end': 320, 'start': 316, 'tag_type': 1}, +{'start': 320, 'end': 330}, +{'attributes': {'data-scrapy-annotate': '{"variant": "0", "generated": true, "annotations": {"content": "title"}}'}, 'tag': 'ins', 'end': 491, 'start': 330, 'tag_type': 1}, +{'start': 491, 'end': 501}, +{'attributes': {}, 'tag': 'ins', 'end': 507, 'start': 501, 'tag_type': 2}, +{'attributes': {}, 'tag': 'h1', 'end': 512, 'start': 507, 'tag_type': 2}, +{'start': 512, 'end': 513}, +{'attributes': {}, 'tag': 'p', 'end': 516, 'start': 513, 'tag_type': 1}, +{'start': 516, 'end': 528}, +{'attributes': {}, 'tag': 'p', 'end': 532, 'start': 528, 'tag_type': 2}, +{'start': 532, 'end': 533}, +{'attributes': {}, 'tag': 'div', 'end': 538, 'start': 533, 'tag_type': 1}, +{'start': 538, 'end': 539}, +{'attributes': {'classid': None, 'src': '/upload/img.jpg', 'data-scrapy-annotate': '{"variant": "0", "annotations": {"image_url": "src"}}'}, 'tag': 'img', 'end': 709, 'start': 539, 'tag_type': 1}, +{'start': 709, 'end': 710}, +{'attributes': {'classid': None, 'data-scrapy-annotate': '{"variant": "0", "annotations": {"content": "description"}}'}, 'tag': 'p', 'end': 858, 'start': 710, 'tag_type': 1}, +{'start': 858, 'end': 883}, +{'attributes': {}, 'tag': 'br', 'end': 888, 'start': 883, 'tag_type': 3}, +{'start': 888, 'end': 909}, +{'attributes': {}, 'tag': 'p', 'end': 913, 'start': 909, 'tag_type': 2}, +{'start': 913, 'end': 914}, +{'attributes': {}, 'tag': 'br', 'end': 918, 'start': 914, 'tag_type': 1}, +{'attributes': {}, 'tag': 'br', 'end': 923, 'start': 918, 'tag_type': 2}, +{'start': 923, 'end': 924}, +{'attributes': {}, 'tag': 'div', 'end': 930, 'start': 924, 'tag_type': 2}, +{'start': 930, 'end': 931}, +{'attributes': {'data-scrapy-annotate': '{"variant": "0", "annotations": {"content": "features"}}', 'class': None}, 'tag': 'p', 'end': 1074, 'start': 931, 'tag_type': 1}, +{'start': 1074, 'end': 1084}, +{'attributes': {}, 'tag': 'p', 'end': 1088, 'start': 1084, 'tag_type': 2}, +{'start': 1088, 'end': 1109}, +{'attributes': {'data-scrapy-replacement': 'select', 'class': 'product'}, 'tag': 'ul', 'end': 1162, 'start': 1109, 'tag_type': 1}, +{'start': 1162, 'end': 1163}, +{'attributes': {'data-scrapy-replacement': 'option'}, 'tag': 'li', 'end': 1200, 'start': 1163, 'tag_type': 1}, +{'start': 1200, 'end': 1205}, +{'attributes': {}, 'tag': 'li', 'end': 1210, 'start': 1205, 'tag_type': 2}, +{'start': 1210, 'end': 1211}, +{'attributes': {'data-scrapy-replacement': 'option'}, 'tag': 'li', 'end': 1248, 'start': 1211, 'tag_type': 1}, +{'start': 1248, 'end': 1251}, +{'attributes': {}, 'tag': 'li', 'end': 1256, 'start': 1251, 'tag_type': 2}, +{'start': 1256, 'end': 1257}, +{'attributes': {}, 'tag': 'ul', 'end': 1262, 'start': 1257, 'tag_type': 2}, +{'start': 1262, 'end': 1263}, +{'attributes': {}, 'tag': 'p', 'end': 1266, 'start': 1263, 'tag_type': 1}, +{'start': 1266, 'end': 1292}, +{'attributes': {}, 'tag': 'p', 'end': 1296, 'start': 1292, 'tag_type': 2}, +{'start': 1296, 'end': 1297}, +{'attributes': {}, 'tag': 'h3', 'end': 1301, 'start': 1297, 'tag_type': 1}, +{'start': 1301, 'end': 1312}, +{'attributes': {}, 'tag': 'h3', 'end': 1317, 'start': 1312, 'tag_type': 2}, +{'start': 1317, 'end': 1318}, +{'attributes': {'cellpadding': '0', 'width': '420', 'cellspacing': '0', 'class': 'rulet'}, 'tag': 'table', 'end': 1383, 'start': 1318, 'tag_type': 1}, +{'attributes': {}, 'tag': 'tbody', 'end': 1390, 'start': 1383, 'tag_type': 1}, +{'start': 1390, 'end': 1391}, +{'attributes': {}, 'tag': 'tr', 'end': 1395, 'start': 1391, 'tag_type': 1}, +{'attributes': {}, 'tag': 'td', 'end': 1399, 'start': 1395, 'tag_type': 1}, +{'start': 1399, 'end': 1405}, +{'attributes': {}, 'tag': 'td', 'end': 1410, 'start': 1405, 'tag_type': 2}, +{'start': 1410, 'end': 1411}, +{'attributes': {}, 'tag': 'td', 'end': 1415, 'start': 1411, 'tag_type': 1}, +{'attributes': {'data-scrapy-annotate': '{"variant": "0", "generated": true, "annotations": {"content": "price"}}'}, 'tag': 'ins', 'end': 1576, 'start': 1415, 'tag_type': 1}, +{'start': 1576, 'end': 1581}, +{'attributes': {}, 'tag': 'ins', 'end': 1587, 'start': 1581, 'tag_type': 2}, +{'attributes': {}, 'tag': 'td', 'end': 1592, 'start': 1587, 'tag_type': 2}, +{'start': 1592, 'end': 1593}, +{'attributes': {}, 'tag': 'tr', 'end': 1598, 'start': 1593, 'tag_type': 2}, +{'attributes': {}, 'tag': 'tbody', 'end': 1605, 'start': 1598, 'tag_type': 1}, +{'attributes': {}, 'tag': 'table', 'end': 1613, 'start': 1605, 'tag_type': 2}, +{'start': 1613, 'end': 1614}, +{'attributes': {'style': 'my style', 'onmouseover': 'xxx', 'class': None}, 'tag': 'p', 'end': 1659, 'start': 1614, 'tag_type': 1}, +{'start': 1659, 'end': 1660}, +] + +# for testing parsing of some invalid html code (but still managed by browsers) +PAGE2 = u""" +<html> +<body> +<p class="MsoNormal" style="margin: 0cm 0cm 0pt"><span lang="EN-GB"> +Hello world! +</span> +</p> +</body> +</html> +""" + +PARSED2 = [ + {'end': 1, 'start': 0}, + {'attributes': {}, 'end': 7, 'start': 1, 'tag': u'html', 'tag_type': 1}, + {'end': 8, 'start': 7}, + {'attributes': {}, 'end': 14, 'start': 8, 'tag': u'body', 'tag_type': 1}, + {'end': 15, 'start': 14}, + {'attributes': {u'style': u'"margin:', u'0pt"': None, u'class': u'"MsoNormal"', u'0cm': None}, 'end': 80, 'start': 15, 'tag': u'p', 'tag_type': 2}, + {'attributes': {u'lang': u'"EN-GB"'}, 'end': 107, 'start': 80, 'tag': u'span', 'tag_type': 1}, + {'end': 121, 'start': 107}, + {'attributes': {}, 'end': 128, 'start': 121, 'tag': u'span', 'tag_type': 2}, + {'end': 129, 'start': 128}, + {'attributes': {}, 'end': 133, 'start': 129, 'tag': u'p', 'tag_type': 2}, + {'end': 134, 'start': 133}, + {'attributes': {}, 'end': 141, 'start': 134, 'tag': u'body', 'tag_type': 2}, + {'end': 142, 'start': 141}, + {'attributes': {}, 'end': 149, 'start': 142, 'tag': u'html', 'tag_type': 2}, + {'end': 150, 'start': 149}, +] + +# for testing tags inside comments +PAGE3 = u"""<html><body><h1>Helloooo!!</h1><p>Did i say hello??</p><!--<p> +</p>--><script type="text/javascript">bla<!--comment-->blabla</script></body></html>""" + +PARSED3 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1}, + {'end': 26, 'start': 16}, + {'attributes': {}, 'end': 31, 'start': 26, 'tag': u'h1', 'tag_type': 2}, + {'attributes': {}, 'end': 34, 'start': 31, 'tag': u'p', 'tag_type': 1}, + {'end': 51, 'start': 34}, + {'attributes': {}, 'end': 55, 'start': 51, 'tag': u'p', 'tag_type': 2}, + {'end': 70, 'start': 55}, + {'attributes': {u'type': u'text/javascript'}, 'end': 101, 'start': 70, 'tag': u'script', 'tag_type': 1}, + {'end': 124, 'start': 101}, + {'attributes': {}, 'end': 133, 'start': 124, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 140, 'start': 133, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 147, 'start': 140, 'tag': u'html', 'tag_type': 2} +] + +# for testing tags inside scripts +PAGE4 = u"""<html><body><h1>Konnichiwa!!</h1>hello<script type="text/javascript">\ +doc.write("<img src=" + base + "product/" + productid + ">");\ +</script>hello again</body></html>""" + +PARSED4 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1}, + {'end': 28,'start': 16}, + {'attributes': {}, 'end': 33, 'start': 28, 'tag': u'h1', 'tag_type': 2}, + {'end': 38, 'start': 33}, + {'attributes': {u'type': u'text/javascript'}, 'end': 69, 'start': 38, 'tag': u'script', 'tag_type': 1}, + {'end': 130, 'start': 69}, + {'attributes': {}, 'end': 139, 'start': 130, 'tag': u'script', 'tag_type': 2}, + {'end': 150, 'start': 139}, + {'attributes': {}, 'end': 157, 'start': 150, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 164, 'start': 157, 'tag': u'html', 'tag_type': 2}, +] + +# Test sucessive cleaning elements +PAGE5 = u"""<html><body><script>hello</script><script>brb</script></body><!--commentA--><!--commentB--></html>""" + +PARSED5 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1}, + {'end': 25, 'start': 20}, + {'attributes': {}, 'end': 34, 'start': 25, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 42, 'start': 34, 'tag': u'script', 'tag_type': 1}, + {'end': 45, 'start': 42}, + {'attributes': {}, 'end': 54, 'start': 45, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 61, 'start': 54, 'tag': u'body', 'tag_type': 2}, + {'end': 91, 'start': 61}, + {'attributes': {}, 'end': 98, 'start': 91, 'tag': u'html', 'tag_type': 2}, +] + +# Test sucessive cleaning elements variant 2 +PAGE6 = u"""<html><body><script>pss<!--comment-->pss</script>all<script>brb</script>\n\n</body></html>""" + +PARSED6 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1}, + {'end': 40, 'start': 20}, + {'attributes': {}, 'end': 49, 'start': 40, 'tag': u'script', 'tag_type': 2}, + {'end': 52, 'start': 49}, + {'attributes': {}, 'end': 60, 'start': 52, 'tag': u'script', 'tag_type': 1}, + {'end': 63, 'start': 60}, + {'attributes': {}, 'end': 72, 'start': 63, 'tag': u'script', 'tag_type': 2}, + {'end': 74, 'start': 72}, + {'attributes': {}, 'end': 81, 'start': 74, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 88, 'start': 81, 'tag': u'html', 'tag_type': 2}, +] diff --git a/scrapy/tests/test_contrib_ibl/test_pageparsing.py b/scrapy/tests/test_contrib_ibl/test_pageparsing.py new file mode 100644 index 000000000..545e9d46b --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/test_pageparsing.py @@ -0,0 +1,283 @@ +""" +Unit tests for pageparsing +""" +import os +from cStringIO import StringIO +from gzip import GzipFile + +from unittest import TestCase +from scrapy.utils.python import str_to_unicode +from scrapy.utils.py26 import json + +from scrapy.contrib.ibl.htmlpage import HtmlPage +from scrapy.contrib.ibl.extraction.pageparsing import (InstanceLearningParser, + TemplatePageParser, ExtractionPageParser) +from scrapy.contrib.ibl.extraction.pageobjects import TokenDict, TokenType +from scrapy.tests.test_contrib_ibl import path + +SIMPLE_PAGE = u""" +<html> <p some-attr="foo">this is a test</p> </html> +""" + +LABELLED_PAGE1 = u""" +<html> +<h1 data-scrapy-annotate="{"variant": 0, "annotations": {"content": "name"}}">Some Product</h1> +<p> some stuff</p> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +This is such a nice item<br/> +Everybody likes it. +</p> +<p data-scrapy-annotate="{"variant": 0, "common_prefix": true, "annotations": {"content": "price"}}"/> +\xa310.00 +<br/> +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "short_description"}}"> +Old fashioned product +<p data-scrapy-annotate="{"variant": 0, "annotations": {"content": "short_description"}}"> +For exigent individuals +<p>click here for other items</p> +</html> +""" + +BROKEN_PAGE = u""" +<html> <p class="ruleb"align="center">html parser cannot parse this</p></html> +""" + +LABELLED_PAGE2 = u""" +<html><body> +<h1>A product</h1> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<p>A very nice product for all intelligent people</p> +<div data-scrapy-ignore="true"> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +</div> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}"> +\xa310.00<p data-scrapy-ignore="true"> 13 <br></p> +</div> +<table data-scrapy-ignore="true"> +<tr><td data-scrapy-ignore="true"></td></tr> +<tr></tr> +</table> +<img data-scrapy-ignore="true" src="image2.jpg"> +<img data-scrapy-ignore="true" src="image3.jpg" /> +<img data-scrapy-ignore-beneath="true" src="image2.jpg"> +<img data-scrapy-ignore-beneath="true" src="image3.jpg" /> +</body></html> + +""" + +LABELLED_PAGE3 = u""" +<html><body> +<h1>A product</h1> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<p>A very nice product for all intelligent people</p> +<div data-scrapy-ignore="true"> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +</div> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +\xa310.00<p data-scrapy-ignore="true"> 13 <br></p> +<table><tr> +<td>Description 1</td> +<td data-scrapy-ignore-beneath="true">Description 2</td> +<td>Description 3</td> +<td>Description 4</td> +</tr></table> +</div> +</body></html> +""" + +LABELLED_PAGE4 = u""" +<html><body> +<h1>A product</h1> +<div> +<p>A very nice product for all intelligent people</p> +<div> +<img scr="image.jpg" /><br/><a link="back.html">Click here to go back</a> +</div> +</div> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +\xa310.00<p data-scrapy-ignore="true"> 13 <br></p> +<table><tr> +<td>Description 1</td> +<td data-scrapy-ignore-beneath="true">Description 2</td> +<td>Description 3</td> +<td data-scrapy-annotate="{"variant": 0, "annotations": {"content": "price"}}"> +Price \xa310.00</td> +</tr></table> +</div> +</body></html> +""" + +LABELLED_PAGE5 = u""" +<html><body> +<ul data-scrapy-replacement='select'> +<li data-scrapy-replacement='option'>Option A</li> +<li>Option I</li> +<li data-scrapy-replacement='option'>Option B</li> +</ul> +</body></html> +""" + +LABELLED_PAGE6 = u""" +<html><body> +Text A +<p><ins data-scrapy-annotate="{"variant": 0, "generated": true, "annotations": {"content": "price"}}"> +65.00</ins>pounds</p> +<p>Description: <ins data-scrapy-annotate="{"variant": 0, "generated": true, "annotations": {"content": "description"}}"> +Text B</ins></p> +Text C +</body></html> +""" + +LABELLED_PAGE7 = u""" +<html><body> +<div data-scrapy-annotate="{"variant": 0, "annotations": {"content": "description"}}"> +<ins data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "generated": true, "annotations": {"content": "site_id"}}">Item Id</ins> +Description +</div> +</body></html> +""" + +LABELLED_PAGE8 = u""" +<html><body> +<div data-scrapy-annotate="{"required": ["description"], "variant": 0, "annotations": {"content": "description"}}"> +<ins data-scrapy-ignore="true" data-scrapy-annotate="{"variant": 0, "generated": true, "annotations": {"content": "site_id"}}">Item Id</ins> +Description +</div> +</body></html> +""" + +def _parse_page(parser_class, pagetext): + htmlpage = HtmlPage(None, {}, pagetext) + parser = parser_class(TokenDict()) + parser.feed(htmlpage) + return parser + +def _tags(pp, predicate): + return [pp.token_dict.token_string(s) for s in pp.token_list \ + if predicate(s)] + +class TestPageParsing(TestCase): + + def test_instance_parsing(self): + pp = _parse_page(InstanceLearningParser, SIMPLE_PAGE) + # all tags + self.assertEqual(_tags(pp, bool), ['<html>', '<p>', '</p>', '</html>']) + + # open/closing tag handling + openp = lambda x: pp.token_dict.token_type(x) == TokenType.OPEN_TAG + self.assertEqual(_tags(pp, openp), ['<html>', '<p>']) + closep = lambda x: pp.token_dict.token_type(x) == TokenType.CLOSE_TAG + self.assertEqual(_tags(pp, closep), ['</p>', '</html>']) + + def _validate_annotation(self, parser, lable_region, name, start_tag, end_tag): + assert lable_region.surrounds_attribute == name + start_token = parser.token_list[lable_region.start_index] + assert parser.token_dict.token_string(start_token) == start_tag + end_token = parser.token_list[lable_region.end_index] + assert parser.token_dict.token_string(end_token) == end_tag + + def test_template_parsing(self): + lp = _parse_page(TemplatePageParser, LABELLED_PAGE1) + self.assertEqual(len(lp.annotations), 5) + self._validate_annotation(lp, lp.annotations[0], + 'name', '<h1>', '</h1>') + self.assertEqual(lp.annotations[0].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[1], + 'description', '<p>', '</p>') + self.assertEqual(lp.annotations[1].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[2], + 'price', '<p/>', '<p>') + self.assertEqual(lp.annotations[2].match_common_prefix, True) + self._validate_annotation(lp, lp.annotations[3], + 'short_description', '<p>', '<p>') + self.assertEqual(lp.annotations[3].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[4], + 'short_description', '<p>', '<p>') + self.assertEqual(lp.annotations[4].match_common_prefix, False) + + # all tags were closed + self.assertEqual(len(lp.labelled_tag_stacks), 0) + + def test_extraction_page_parsing(self): + epp = _parse_page(ExtractionPageParser, SIMPLE_PAGE) + ep = epp.to_extraction_page() + assert len(ep.page_tokens) == 4 + assert ep.token_html(0) == '<html>' + assert ep.token_html(1) == '<p some-attr="foo">' + + assert ep.html_between_tokens(1, 2) == 'this is a test' + assert ep.html_between_tokens(1, 3) == 'this is a test</p> ' + + def test_invalid_html(self): + p = _parse_page(InstanceLearningParser, BROKEN_PAGE) + assert p + + def test_ignore_region(self): + """Test ignored regions""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE2) + self.assertEqual(p.ignored_regions, [(7,12),(15,17),(19,26),(21,22),(27,28),(28,29),(29,None),(30,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_ignore_regions2(self): + """Test ignore-beneath regions""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE3) + self.assertEqual(p.ignored_regions, [(7,12),(15,17),(22,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_ignore_regions3(self): + """Test ignore-beneath with annotation inside region""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE4) + self.assertEqual(p.ignored_regions, [(15,17),(22,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_replacement(self): + """Test parsing of replacement tags""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE5) + self.assertEqual(_tags(p, bool), ['<html>', '<body>', '<select>', '<option>', + '</option>', '<li>', '</li>', '<option>', '</option>', '</select>', '</body>', '</html>']) + + def test_partial(self): + """Test partial annotation parsing""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE6) + text = p.annotations[0].annotation_text + self.assertEqual(text.start_text, '') + self.assertEqual(text.follow_text, 'pounds') + text = p.annotations[1].annotation_text + self.assertEqual(text.start_text, "Description: ") + self.assertEqual(text.follow_text, '') + + def test_ignored_partial(self): + """Test ignored region declared on partial annotation""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE7) + self.assertEqual(p.ignored_regions, [(2, 3)]) + + def test_extra_required(self): + """Test parsing of extra required attributes""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE8) + self.assertEqual(p.extra_required_attrs, ["description"]) + + def test_site_pages(self): + """ + Tests from real pages. More reliable and easy to build for more complicated structures + """ + samples_file = open(os.path.join(path, "samples_pageparsing.json.gz"), "r") + samples = [] + for line in GzipFile(fileobj=StringIO(samples_file.read())).readlines(): + samples.append(json.loads(line)) + for sample in samples: + source = sample["annotated"] + annotations = sample["annotations"] + template = HtmlPage(body=str_to_unicode(source)) + parser = TemplatePageParser(TokenDict()) + parser.feed(template) + for annotation in parser.annotations: + test_annotation = annotations.pop(0) + for s in annotation.__slots__: + if s == "tag_attributes": + for pair in getattr(annotation, s): + self.assertEqual(list(pair), test_annotation[s].pop(0)) + else: + self.assertEqual(getattr(annotation, s), test_annotation[s]) + self.assertEqual(annotations, []) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 4a60b8a2d..a5e57bde0 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -35,6 +35,20 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title><base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://example.org/item/12.html', text='Item 12')]) + + # base url has no scheme + html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + def test_extraction_encoding(self): body = get_testdata('link_extractor', 'linkextractor_noenc.html') response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) @@ -96,6 +110,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition') ]) + lx = SgmlLinkExtractor(allow=('sample', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) + lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py new file mode 100644 index 000000000..5b0d9e954 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -0,0 +1,62 @@ +import weakref +import unittest + +# just a hack to avoid cyclic imports of scrapy.spider when running this test +# alone +import scrapy.spider +from scrapy.contrib.spidermanager import TwistedPluginSpiderManager +from scrapy.http import Request + +class TwistedPluginSpiderManagerTest(unittest.TestCase): + + def setUp(self): + self.spiderman = TwistedPluginSpiderManager() + assert not self.spiderman.loaded + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager']) + assert self.spiderman.loaded + + def test_list(self): + self.assertEqual(set(self.spiderman.list()), + set(['spider1', 'spider2'])) + + def test_create(self): + spider1 = self.spiderman.create("spider1") + self.assertEqual(spider1.__class__.__name__, 'Spider1') + spider2 = self.spiderman.create("spider2", foo="bar") + self.assertEqual(spider2.__class__.__name__, 'Spider2') + self.assertEqual(spider2.foo, 'bar') + + def test_create_uses_cache(self): + # TwistedPluginSpiderManager uses an internal cache which is + # invalidated in close_spider() but this isn't necessarily the best + # thing to do in all cases. + spider1 = self.spiderman.create("spider1") + spider2 = self.spiderman.create("spider1") + assert spider1 is spider2 + + def test_find_by_request(self): + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), + ['spider1']) + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy2.org/test')), + ['spider2']) + self.assertEqual(set(self.spiderman.find_by_request(Request('http://scrapy3.org/test'))), + set(['spider1', 'spider2'])) + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), + []) + + def test_close_spider_remove_refs(self): + spider = self.spiderman.create("spider1") + wref = weakref.ref(spider) + assert wref() + self.spiderman.close_spider(spider) + del spider + assert not wref() + + def test_close_spider_invalidates_cache(self): + spider1 = self.spiderman.create("spider1") + self.spiderman.close_spider(spider1) + spider2 = self.spiderman.create("spider1") + assert spider1 is not spider2 + +if __name__ == '__main__': + unittest.main() diff --git a/scrapy/tests/test_contrib_spidermanager/dropin.cache b/scrapy/tests/test_contrib_spidermanager/dropin.cache new file mode 100644 index 000000000..81239e0b4 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/dropin.cache @@ -0,0 +1,71 @@ +(dp1 +S'spider2' +p2 +ccopy_reg +_reconstructor +p3 +(ctwisted.plugin +CachedDropin +p4 +c__builtin__ +object +p5 +NtRp6 +(dp7 +S'moduleName' +p8 +S'scrapy.tests.test_contrib_spidermanager.spider2' +p9 +sS'description' +p10 +NsS'plugins' +p11 +(lp12 +g3 +(ctwisted.plugin +CachedPlugin +p13 +g5 +NtRp14 +(dp15 +S'provided' +p16 +(lp17 +cscrapy.spider.models +ISpider +p18 +asS'dropin' +p19 +g6 +sS'name' +p20 +S'SPIDER' +p21 +sg10 +NsbasbsS'spider1' +p22 +g3 +(g4 +g5 +NtRp23 +(dp24 +g8 +S'scrapy.tests.test_contrib_spidermanager.spider1' +p25 +sg10 +Nsg11 +(lp26 +g3 +(g13 +g5 +NtRp27 +(dp28 +g16 +(lp29 +g18 +asg19 +g23 +sg20 +g21 +sg10 +Nsbasbs. \ No newline at end of file diff --git a/scrapy/tests/test_contrib_spidermanager/spider1.py b/scrapy/tests/test_contrib_spidermanager/spider1.py new file mode 100644 index 000000000..0a9b60989 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/spider1.py @@ -0,0 +1,7 @@ +from scrapy.spider import BaseSpider + +class Spider1(BaseSpider): + name = "spider1" + allowed_domains = ["scrapy1.org", "scrapy3.org"] + +SPIDER = Spider1() diff --git a/scrapy/tests/test_contrib_spidermanager/spider2.py b/scrapy/tests/test_contrib_spidermanager/spider2.py new file mode 100644 index 000000000..52023277f --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/spider2.py @@ -0,0 +1,7 @@ +from scrapy.spider import BaseSpider + +class Spider2(BaseSpider): + name = "spider2" + allowed_domains = ["scrapy2.org", "scrapy3.org"] + +SPIDER = Spider2() diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index a23d6c6a4..4c08cf021 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -32,11 +32,11 @@ class FileTestCase(unittest.TestCase): request = Request('file://%s' % self.tmpname + '^') assert request.url.upper().endswith('%5E') - return download_file(request, BaseSpider()).addCallback(_test) + return download_file(request, BaseSpider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = download_file(request, BaseSpider()) + d = download_file(request, BaseSpider('foo')) return self.assertFailure(d, IOError) @@ -66,20 +66,34 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d + def test_download_head(self): + request = Request(self.getURL('file'), method='HEAD') + d = download_http(request, BaseSpider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, '') + return d + def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEquals, 302) + return d + + def test_redirect_status_head(self): + request = Request(self.getURL('redirect'), method='HEAD') + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_timeout_download_from_spider(self): - spider = BaseSpider() + spider = BaseSpider('foo') spider.download_timeout = 0.000001 request = Request(self.getURL('wait')) d = download_http(request, spider) @@ -91,7 +105,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -99,9 +113,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -109,14 +123,14 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d def test_broken_download(self): request = Request(self.getURL('broken')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) return self.assertFailure(d, PartialDownloadError) @@ -152,7 +166,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -161,4 +175,4 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d1b4275e8..a04efd79d 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -10,7 +10,7 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = CookiesMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index 1de53e85c..09dfdbaf8 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -21,7 +21,7 @@ class DecompressionMiddlewareTest(TestCase): def setUp(self): self.mw = DecompressionMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_known_compression_formats(self): for fmt in self.test_formats: diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index cd1624eb4..805289787 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -9,7 +9,7 @@ from scrapy.spider import BaseSpider class TestDefaultHeadersMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() self.default_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) diff --git a/scrapy/tests/test_downloadermiddleware_httpauth.py b/scrapy/tests/test_downloadermiddleware_httpauth.py index 79c815184..747ea9de1 100644 --- a/scrapy/tests/test_downloadermiddleware_httpauth.py +++ b/scrapy/tests/test_downloadermiddleware_httpauth.py @@ -18,7 +18,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): self.mw.default_useragent = 'default_useragent' - spider = TestSpider() + spider = TestSpider('foo') req = Request('http://scrapytest.org/') assert self.mw.process_request(req, spider) is None self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==') diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 9490fb40f..c8fb38d22 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -20,7 +20,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpCompressionMiddleware() def _getresponse(self, coding): diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py index dd9602661..7f6f7ee4e 100644 --- a/scrapy/tests/test_downloadermiddleware_httpproxy.py +++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py @@ -8,7 +8,7 @@ from scrapy.http import Response, Request from scrapy.spider import BaseSpider from scrapy.conf import settings -spider = BaseSpider() +spider = BaseSpider('foo') class TestDefaultHeadersMiddleware(TestCase): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index ba7daece8..8409fec6d 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -3,12 +3,12 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import BaseSpider from scrapy.core.exceptions import IgnoreRequest -from scrapy.http import Request, Response, Headers +from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RedirectMiddleware() def test_priority_adjust(self): @@ -58,7 +58,7 @@ class RedirectMiddlewareTest(unittest.TestCase): <head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head> </html>""" req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -70,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase): <head><meta http-equiv="refresh" content="1000;url=http://example.org/newpage" /></head> </html>""" req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) rsp2 = self.mw.process_response(req, rsp, self.spider) assert rsp is rsp2 @@ -81,7 +81,7 @@ class RedirectMiddlewareTest(unittest.TestCase): </html>""" req = Request(url='http://example.org', method='POST', body='test', headers={'Content-Type': 'text/plain', 'Content-length': '4'}) - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index 101bb01c0..96e7cd297 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -10,7 +10,7 @@ from scrapy.http import Request, Response class RetryTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RetryMiddleware() self.mw.max_retry_times = 2 diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index d12f2bc63..338a63a23 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -9,7 +9,7 @@ from scrapy.conf import settings class UserAgentMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UserAgentMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py index d11595a9d..fa9eef332 100644 --- a/scrapy/tests/test_dupefilter.py +++ b/scrapy/tests/test_dupefilter.py @@ -8,7 +8,7 @@ from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilt class RequestFingerprintDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = RequestFingerprintDupeFilter() filter.open_spider(spider) @@ -28,7 +28,7 @@ class RequestFingerprintDupeFilterTest(unittest.TestCase): class NullDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = NullDupeFilter() filter.open_spider(spider) diff --git a/scrapy/tests/test_encoding_aliases.py b/scrapy/tests/test_encoding_aliases.py deleted file mode 100644 index cc3a0089f..000000000 --- a/scrapy/tests/test_encoding_aliases.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -import scrapy # adds encoding aliases (if not added before) - -class EncodingAliasesTestCase(unittest.TestCase): - - def test_encoding_aliases(self): - """Test common encdoing aliases not included in Python""" - - uni = u'\u041c\u041e\u0421K\u0412\u0410' - str = uni.encode('windows-1251') - self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251')) - self.assertEqual(str.decode('windows-1251'), str.decode('win-1251')) - - text = u'\u8f6f\u4ef6\u540d\u79f0' - str = uni.encode('gb2312') - self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn')) - self.assertEqual(str.decode('gb2312'), str.decode('zh-cn')) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index f205ff264..2d7b5950c 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -22,8 +22,8 @@ class TestItem(Item): price = Field() class TestSpider(BaseSpider): - domain_name = "scrapytest.org" - extra_domain_names = ["localhost"] + name = "scrapytest.org" + allowed_domains = ["scrapytest.org", "localhost"] start_urls = ['http://localhost'] itemurl_re = re.compile("item\d+.html") @@ -68,7 +68,7 @@ def start_test_site(): class CrawlingSession(object): def __init__(self): - self.domain = 'scrapytest.org' + self.name = 'scrapytest.org' self.spider = None self.respplug = [] self.reqplug = [] @@ -97,7 +97,8 @@ class CrawlingSession(object): dispatcher.connect(self.response_downloaded, signals.response_downloaded) scrapymanager.configure() - scrapymanager.runonce(self.spider) + scrapymanager.crawl_spider(self.spider) + scrapymanager.start() self.port.stopListening() self.wasrun = True @@ -138,7 +139,7 @@ class EngineTest(unittest.TestCase): Check the spider is loaded and located properly via the SpiderLocator """ assert session.spider is not None - self.assertEqual(session.spider.domain_name, session.domain) + self.assertEqual(session.spider.name, session.name) def test_visited_urls(self): """ diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index c0ae0cab5..b37eb33e7 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -171,13 +171,6 @@ class RequestTest(unittest.TestCase): self.assertEqual(r4.meta, {}) assert r4.dont_filter is False - # __init__ and replace() signatures must be equal unles *args,**kwargs is used - i_args, i_varargs, i_varkwargs, _ = getargspec(self.request_class.__init__) - self.assertFalse(bool(i_varargs) ^ bool(i_varkwargs)) - if not i_varargs: - r_args, _, _, _ = getargspec(self.request_class.replace) - self.assertEqual(i_args, r_args) - def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" x = self.request_class('http://www.example.com') diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 3b0a144d2..437d75b98 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -2,7 +2,7 @@ import unittest import weakref from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers -from scrapy.conf import settings +from scrapy.utils.encoding import resolve_encoding class BaseResponseTest(unittest.TestCase): @@ -112,10 +112,13 @@ class BaseResponseTest(unittest.TestCase): body_str = body assert isinstance(response.body, str) - self.assertEqual(response.encoding, encoding) + self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_str) self.assertEqual(response.body_as_unicode(), body_unicode) + def _assert_response_encoding(self, response, encoding): + self.assertEqual(response.encoding, resolve_encoding(encoding)) + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): @@ -134,14 +137,14 @@ class TextResponseTest(BaseResponseTest): assert isinstance(r2, self.response_class) self.assertEqual(r2.url, "http://www.example.com/other") - self.assertEqual(r2.encoding, "cp852") + self._assert_response_encoding(r2, "cp852") self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3.encoding, "latin1") + self.assertEqual(r3._declared_encoding(), "latin1") def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) resp = self.response_class(u"http://www.example.com/") - self.assertEqual(resp.encoding, settings['DEFAULT_RESPONSE_ENCODING']) + self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') @@ -175,21 +178,64 @@ class TextResponseTest(BaseResponseTest): r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") + r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") + r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body="\xa8D") + r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body="\xa8D") - self.assertEqual(r1.headers_encoding(), "utf-8") - self.assertEqual(r2.headers_encoding(), None) - self.assertEqual(r2.encoding, 'utf-8') - self.assertEqual(r3.headers_encoding(), "iso-8859-1") - self.assertEqual(r3.encoding, 'iso-8859-1') - self.assertEqual(r4.headers_encoding(), None) - assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii' + self.assertEqual(r1._headers_encoding(), "utf-8") + self.assertEqual(r2._headers_encoding(), None) + self.assertEqual(r2._declared_encoding(), 'utf-8') + self._assert_response_encoding(r2, 'utf-8') + self.assertEqual(r3._headers_encoding(), "iso-8859-1") + self.assertEqual(r3._declared_encoding(), "iso-8859-1") + self.assertEqual(r4._headers_encoding(), None) + self.assertEqual(r5._headers_encoding(), None) + self._assert_response_encoding(r5, "utf-8") + assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3") self._assert_response_values(r3, 'iso-8859-1', u"\xa3") + self._assert_response_values(r6, 'gb18030', u"\u2015") + self._assert_response_values(r7, 'gb18030', u"\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + def test_declared_encoding_invalid(self): + """Check that unknown declared encodings are ignored""" + r = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=UKNOWN"]}, body="\xc2\xa3") + self.assertEqual(r._declared_encoding(), None) + self._assert_response_values(r, 'utf-8', u"\xa3") + + def test_utf16(self): + """Test utf-16 because UnicodeDammit is known to have problems with""" + r = self.response_class("http://www.example.com", body='\xff\xfeh\x00i\x00', encoding='utf-16') + self._assert_response_values(r, 'utf-16', u"hi") + + def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): + r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xef\xbb\xbfWORD\xe3\xab") + self.assertEqual(r6.encoding, 'utf-8') + self.assertEqual(r6.body_as_unicode(), u'\ufeffWORD\ufffd\ufffd') + + def test_replace_wrong_encoding(self): + """Test invalid chars are replaced properly""" + r = self.response_class("http://www.example.com", encoding='utf-8', body='PREFIX\xe3\xabSUFFIX') + # XXX: Policy for replacing invalid chars may suffer minor variations + # but it should always contain the unicode replacement char (u'\ufffd') + assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + + # Do not destroy html tags due to encoding bugs + r = self.response_class("http://example.com", encoding='utf-8', \ + body='\xf0<span>value</span>') + assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode()) + + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse + #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') + #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse @@ -231,7 +277,7 @@ class XmlResponseTest(TextResponseTest): body = "<xml></xml>" r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, settings['DEFAULT_RESPONSE_ENCODING'], body) + self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>""" r2 = self.response_class("http://www.example.com", body=body) diff --git a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index db59de2d5..1384c2237 100644 --- a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -10,7 +10,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = DuplicatesFilterMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw.open_spider(self.spider) def tearDown(self): diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py new file mode 100644 index 000000000..14aa78168 --- /dev/null +++ b/scrapy/tests/test_spider.py @@ -0,0 +1,101 @@ +from __future__ import with_statement + +import sys +import warnings + +from twisted.trial import unittest + +from scrapy.spider import BaseSpider +from scrapy.contrib.spiders.init import InitSpider +from scrapy.contrib.spiders.crawl import CrawlSpider +from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider + + +class BaseSpiderTest(unittest.TestCase): + + spider_class = BaseSpider + + class OldSpider(spider_class): + + domain_name = 'example.com' + extra_domain_names = ('example.org', 'example.net') + + + class OldSpiderWithoutExtradomains(spider_class): + + domain_name = 'example.com' + + + class NewSpider(spider_class): + + name = 'example.com' + allowed_domains = ('example.org', 'example.net') + + + def setUp(self): + warnings.simplefilter("always") + + def tearDown(self): + warnings.resetwarnings() + + def test_sep12_deprecation_warnings(self): + if sys.version_info[:2] < (2, 6): + # warnings.catch_warnings() was added in Python 2.6 + raise unittest.SkipTest("This test requires Python 2.6+") + with warnings.catch_warnings(record=True) as w: + spider = self.OldSpider() + self.assertEqual(len(w), 2) # one for domain_name & one for extra_domain_names + self.assert_(issubclass(w[-1].category, DeprecationWarning)) + + def test_sep12_backwards_compatibility(self): + spider = self.OldSpider() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains) + + spider = self.OldSpiderWithoutExtradomains() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + + spider = self.NewSpider() + self.assertEqual(spider.domain_name, 'example.com') + self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names) + self.assert_('example.net' in spider.extra_domain_names, spider.extra_domain_names) + + def test_base_spider(self): + spider = self.spider_class("example.com") + self.assertEqual(spider.name, 'example.com') + self.assertEqual(spider.start_urls, []) + self.assertEqual(spider.allowed_domains, []) + + def test_spider_args(self): + """Constructor arguments are assigned to spider attributes""" + spider = self.spider_class('example.com', foo='bar') + self.assertEqual(spider.foo, 'bar') + + def test_spider_without_name(self): + """Constructor arguments are assigned to spider attributes""" + self.assertRaises(ValueError, self.spider_class) + self.assertRaises(ValueError, self.spider_class, somearg='foo') + + +class InitSpiderTest(BaseSpiderTest): + + spider_class = InitSpider + +class XMLFeedSpiderTest(BaseSpiderTest): + + spider_class = XMLFeedSpider + +class CSVFeedSpiderTest(BaseSpiderTest): + + spider_class = CSVFeedSpider + +class CrawlSpiderTest(BaseSpiderTest): + + spider_class = CrawlSpider + + +if __name__ == '__main__': + unittest.main() diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index a0dfb492e..3cbe9f3b4 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -2,13 +2,13 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spider import BaseSpider -from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware +from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpErrorException class TestHttpErrorMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpErrorMiddleware() self.req = Request('http://scrapytest.org') @@ -18,21 +18,19 @@ class TestHttpErrorMiddleware(TestCase): self.res404.request = self.req def test_process_spider_input(self): - self.assertEquals(self.mw.process_spider_input(self.res200, self.spider), - None) - - self.assertEquals(self.mw.process_spider_input(self.res404, self.spider), - []) + self.assertEquals(None, + self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpErrorException, + self.mw.process_spider_input, self.res404, self.spider) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - - self.assertEquals(self.mw.process_spider_input(res, self.spider), - None) + self.assertEquals(None, + self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEquals(self.mw.process_spider_input(self.res404, self.spider), - None) + self.assertEquals(None, + self.mw.process_spider_input(self.res404, self.spider)) diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 6c71213aa..861595524 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -8,9 +8,9 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() - self.spider.domain_name = 'scrapytest.org' - self.spider.extra_domain_names = ['scrapy.org'] + self.spider = BaseSpider('foo') + self.spider.name = 'scrapytest.org' + self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] self.mw = OffsiteMiddleware() self.mw.spider_opened(self.spider) diff --git a/scrapy/tests/test_spidermiddleware_referer.py b/scrapy/tests/test_spidermiddleware_referer.py index ec11b0489..467c301ba 100644 --- a/scrapy/tests/test_spidermiddleware_referer.py +++ b/scrapy/tests/test_spidermiddleware_referer.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.referer import RefererMiddleware class TestRefererMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RefererMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urlfilter.py b/scrapy/tests/test_spidermiddleware_urlfilter.py index 21f776f17..45621599a 100644 --- a/scrapy/tests/test_spidermiddleware_urlfilter.py +++ b/scrapy/tests/test_spidermiddleware_urlfilter.py @@ -9,7 +9,7 @@ from scrapy.utils.url import canonicalize_url class TestUrlFilterMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlFilterMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/scrapy/tests/test_spidermiddleware_urllength.py index de0e2eae0..cbd80d488 100644 --- a/scrapy/tests/test_spidermiddleware_urllength.py +++ b/scrapy/tests/test_spidermiddleware_urllength.py @@ -12,7 +12,7 @@ class TestUrlLengthMiddleware(TestCase): settings.disabled = False settings.overrides['URLLENGTH_LIMIT'] = 25 - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlLengthMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index 612ee0eaa..786021898 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -9,7 +9,7 @@ from scrapy.stats.signals import stats_spider_opened, stats_spider_closing, \ class StatsCollectorTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_collector(self): stats = StatsCollector() diff --git a/scrapy/tests/test_utils_encoding.py b/scrapy/tests/test_utils_encoding.py new file mode 100644 index 000000000..b6800cd75 --- /dev/null +++ b/scrapy/tests/test_utils_encoding.py @@ -0,0 +1,25 @@ +import unittest + +from scrapy.utils.encoding import encoding_exists, resolve_encoding + +class UtilsEncodingTestCase(unittest.TestCase): + + _ENCODING_ALIASES = { + 'foo': 'cp1252', + 'bar': 'none', + } + + def test_resolve_encoding(self): + self.assertEqual(resolve_encoding('latin1', self._ENCODING_ALIASES), + 'latin1') + self.assertEqual(resolve_encoding('foo', self._ENCODING_ALIASES), + 'cp1252') + + def test_encoding_exists(self): + assert encoding_exists('latin1', self._ENCODING_ALIASES) + assert encoding_exists('foo', self._ENCODING_ALIASES) + assert not encoding_exists('bar', self._ENCODING_ALIASES) + assert not encoding_exists('none', self._ENCODING_ALIASES) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_markup.py b/scrapy/tests/test_utils_markup.py index f51350d90..5f8a40422 100644 --- a/scrapy/tests/test_utils_markup.py +++ b/scrapy/tests/test_utils_markup.py @@ -87,6 +87,15 @@ class UtilsMarkupTest(unittest.TestCase): self.assertEqual(remove_tags(u'<p align="center" class="one">texty</p>', which_ones=('b',)), u'<p align="center" class="one">texty</p>') + # text with empty tags + self.assertEqual(remove_tags(u'a<br />b<br/>c'), u'abc') + self.assertEqual(remove_tags(u'a<br />b<br/>c', which_ones=('br',)), u'abc') + + # test keep arg + self.assertEqual(remove_tags(u'<p>a<br />b<br/>c</p>', keep=('br',)), u'a<br />b<br/>c') + self.assertEqual(remove_tags(u'<p>a<br />b<br/>c</p>', keep=('p',)), u'<p>abc</p>') + self.assertEqual(remove_tags(u'<p>a<br />b<br/>c</p>', keep=('p','br','div')), u'<p>a<br />b<br/>c</p>') + def test_remove_tags_with_content(self): # make sure it always return unicode assert isinstance(remove_tags_with_content('no tags'), unicode) @@ -105,6 +114,9 @@ class UtilsMarkupTest(unittest.TestCase): self.assertEqual(remove_tags_with_content(u'<b>not will removed</b><i>i will removed</i>', which_ones=('i',)), u'<b>not will removed</b>') + # text with empty tags + self.assertEqual(remove_tags_with_content(u'<br/>a<br />', which_ones=('br',)), u'a') + def test_replace_escape_chars(self): # make sure it always return unicode assert isinstance(replace_escape_chars('no ec'), unicode) diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index 13d8be30c..dc46e1f91 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -1,7 +1,8 @@ +import operator import unittest from scrapy.utils.python import str_to_unicode, unicode_to_str, \ - memoizemethod_noargs, isbinarytext + memoizemethod_noargs, isbinarytext, equal_attributes class UtilsPythonTestCase(unittest.TestCase): def test_str_to_unicode(self): @@ -61,5 +62,52 @@ class UtilsPythonTestCase(unittest.TestCase): # finally some real binary bytes assert isbinarytext("\x02\xa3") + def test_equal_attributes(self): + class Obj: + pass + + a = Obj() + b = Obj() + # no attributes given return False + self.failIf(equal_attributes(a, b, [])) + # not existent attributes + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + a.x = 1 + b.x = 1 + # equal attribute + self.failUnless(equal_attributes(a, b, ['x'])) + + b.y = 2 + # obj1 has no attribute y + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + a.y = 2 + # equal attributes + self.failUnless(equal_attributes(a, b, ['x', 'y'])) + + a.y = 1 + # differente attributes + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + # test callable + a.meta = {} + b.meta = {} + self.failUnless(equal_attributes(a, b, ['meta'])) + + # compare ['meta']['a'] + a.meta['z'] = 1 + b.meta['z'] = 1 + + get_z = operator.itemgetter('z') + get_meta = operator.attrgetter('meta') + compare_z = lambda obj: get_z(get_meta(obj)) + + self.failUnless(equal_attributes(a, b, [compare_z, 'x'])) + # fail z equality + a.meta['z'] = 2 + self.failIf(equal_attributes(a, b, [compare_z, 'x'])) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 9281f4f40..71c14569e 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -1,9 +1,10 @@ import unittest +import urlparse from scrapy.xlib.BeautifulSoup import BeautifulSoup -from scrapy.http import Response, TextResponse +from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, \ - response_httprepr, get_cached_beautifulsoup + response_httprepr, get_cached_beautifulsoup, open_in_browser class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body='dummy_response') @@ -28,30 +29,46 @@ class ResponseUtilsTest(unittest.TestCase): self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode)) def test_get_base_url(self): - response = Response(url='http://example.org', body="""\ + response = HtmlResponse(url='https://example.org', body="""\ <html>\ <head><title>Dummy\ blahablsdfsal&\ """) self.assertEqual(get_base_url(response), 'http://example.org/something') + # relative url with absolute path + response = HtmlResponse(url='https://example.org', body="""\ + \ + Dummy\ + blahablsdfsal&\ + """) + self.assertEqual(get_base_url(response), 'https://example.org/absolutepath') + + # no scheme url + response = HtmlResponse(url='https://example.org', body="""\ + \ + Dummy\ + blahablsdfsal&\ + """) + self.assertEqual(get_base_url(response), 'https://noscheme.com/path') + def test_get_meta_refresh(self): body = """ Dummy blahablsdfsal& """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # refresh without url should return (None, None) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (None, None)) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # meta refresh in multiple lines @@ -59,17 +76,17 @@ class ResponseUtilsTest(unittest.TestCase): """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage')) # entities in the redirect url body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other')) # relative redirects body = """""" - response = Response(url='http://example.com/page/this.html', body=body) + response = TextResponse(url='http://example.com/page/this.html', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html')) # non-standard encodings (utf-16) @@ -80,7 +97,7 @@ class ResponseUtilsTest(unittest.TestCase): # non-ascii chars in the url (default encoding - utf8) body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) # non-ascii chars in the url (custom encoding - latin1) @@ -88,13 +105,8 @@ class ResponseUtilsTest(unittest.TestCase): response = TextResponse(url='http://example.com', body=body, encoding='latin1') self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) - # wrong encodings (possibly caused by truncated chunks) - body = """""" - response = Response(url='http://example.com', body=body) - self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT')) - # responses without refresh tag should return None None - response = Response(url='http://example.org') + response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) @@ -131,5 +143,18 @@ class ResponseUtilsTest(unittest.TestCase): assert soup1 is soup2 assert soup1 is not soup3 + def test_open_in_browser(self): + url = "http:///www.example.com/some/page.html" + body = " test page test body " + def browser_open(burl): + bbody = open(urlparse.urlparse(burl).path).read() + assert '' % url in bbody, " tag not added" + return True + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=browser_open), \ + "Browser not called" + self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \ + debug=True) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 05a22e8db..af61b8e7f 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -136,12 +136,26 @@ class UrlUtilsTest(unittest.TestCase): 'http://rmc-offers.co.uk/productlist.asp?BCat=newvalue&CatID=60') def test_url_query_cleaner(self): - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id'), - 'product.html?id=200') - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']), - 'product.html?id=200&name=wired') - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo']), - 'product.html?id=200&foo=bar') + self.assertEqual('product.html?id=200', + url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'])) + self.assertEqual('product.html?id=200', + url_query_cleaner("product.html?&id=200&&foo=bar&name=wired", ['id'])) + self.assertEqual('product.html', + url_query_cleaner("product.html?foo=bar&name=wired", ['id'])) + self.assertEqual('product.html?id=200&name=wired', + url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name'])) + self.assertEqual('product.html?id', + url_query_cleaner("product.html?id&other=3&novalue=", ['id'])) + self.assertEqual('product.html?d=1&d=2&d=3', + url_query_cleaner("product.html?d=1&e=b&d=2&d=3&other=other", ['d'], unique=False)) + self.assertEqual('product.html?id=200&foo=bar', + url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo'])) + self.assertEqual('product.html?foo=bar&name=wired', + url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'], remove=True)) + self.assertEqual('product.html?name=wired', + url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'foo'], remove=True)) + self.assertEqual('product.html?foo=bar&name=wired', + url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'footo'], remove=True)) def test_canonicalize_url(self): # simplest case diff --git a/scrapy/utils/encoding.py b/scrapy/utils/encoding.py index 9eb06d942..c7d645041 100644 --- a/scrapy/utils/encoding.py +++ b/scrapy/utils/encoding.py @@ -1,11 +1,20 @@ import codecs -def add_encoding_alias(encoding, alias, overwrite=False): +from scrapy.conf import settings + +_ENCODING_ALIASES = dict(settings['ENCODING_ALIASES_BASE']) +_ENCODING_ALIASES.update(settings['ENCODING_ALIASES']) + +def encoding_exists(encoding, _aliases=_ENCODING_ALIASES): + """Returns ``True`` if encoding is valid, otherwise returns ``False``""" try: - codecs.lookup(alias) - alias_exists = True + codecs.lookup(resolve_encoding(encoding, _aliases)) except LookupError: - alias_exists = False - if overwrite or not alias_exists: - codec = codecs.lookup(encoding) - codecs.register(lambda x: codec if x == alias else None) + return False + return True + +def resolve_encoding(alias, _aliases=_ENCODING_ALIASES): + """Return the encoding the given alias maps to, or the alias as passed if + no mapping is found. + """ + return _aliases.get(alias.lower(), alias) diff --git a/scrapy/utils/fetch.py b/scrapy/utils/fetch.py deleted file mode 100644 index b489c75bf..000000000 --- a/scrapy/utils/fetch.py +++ /dev/null @@ -1,17 +0,0 @@ -from scrapy.http import Request -from scrapy.core.manager import scrapymanager - -def fetch(urls): - """Fetch a list of urls and return a list of the downloaded Scrapy - Responses. - - This is a blocking function not suitable for calling from spiders. Instead, - it is indended to be called from outside the framework such as Scrapy - commands or standalone scripts. - """ - responses = [] - requests = [Request(url, callback=responses.append, dont_filter=True) \ - for url in urls] - scrapymanager.runonce(*requests) - return responses - diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index d422fbde5..1e381074f 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -60,10 +60,10 @@ def remove_entities(text, keep=(), remove_illegal=True, encoding='utf-8'): return _ent_re.sub(convert_entity, str_to_unicode(text, encoding)) -def has_entities(text): - return bool(_ent_re.search(str_to_unicode(text))) +def has_entities(text, encoding=None): + return bool(_ent_re.search(str_to_unicode(text, encoding))) -def replace_tags(text, token=''): +def replace_tags(text, token='', encoding=None): """Replace all markup tags found in the given text by the given token. By default token is a null string so it just remove all tags. @@ -71,43 +71,58 @@ def replace_tags(text, token=''): Always returns a unicode string. """ - return _tag_re.sub(token, str_to_unicode(text)) + return _tag_re.sub(token, str_to_unicode(text, encoding)) -def remove_comments(text): +def remove_comments(text, encoding=None): """ Remove HTML Comments. """ - return re.sub('', u'', str_to_unicode(text), re.DOTALL) + return re.sub('', u'', str_to_unicode(text, encoding), re.DOTALL) -def remove_tags(text, which_ones=()): +def remove_tags(text, which_ones=(), keep=(), encoding=None): """ Remove HTML Tags only. - which_ones -- is a tuple of which tags we want to remove. - if is empty remove all tags. + which_ones and keep are both tuples, there are four cases: + + which_ones, keep (1 - not empty, 0 - empty) + 1, 0 - remove all tags in which_ones + 0, 1 - remove all tags except the ones in keep + 0, 0 - remove all tags + 1, 1 - not allowd """ - if which_ones: - tags = ['<%s>|<%s .*?>|' % (tag,tag,tag) for tag in which_ones] - regex = '|'.join(tags) - else: - regex = '<.*?>' + + assert not (which_ones and keep), 'which_ones and keep can not be given at the same time' + + def will_remove(tag): + if which_ones: + return tag in which_ones + else: + return tag not in keep + + def remove_tag(m): + tag = m.group(1) + return u'' if will_remove(tag) else m.group(0) + + regex = '/]+).*?>' retags = re.compile(regex, re.DOTALL | re.IGNORECASE) - return retags.sub(u'', str_to_unicode(text)) + return retags.sub(remove_tag, str_to_unicode(text, encoding)) -def remove_tags_with_content(text, which_ones=()): +def remove_tags_with_content(text, which_ones=(), encoding=None): """ Remove tags and its content. which_ones -- is a tuple of which tags with its content we want to remove. if is empty do nothing. """ - text = str_to_unicode(text) + text = str_to_unicode(text, encoding) if which_ones: - tags = '|'.join(['<%s.*?' % (tag,tag) for tag in which_ones]) + tags = '|'.join([r'<%s.*?|<%s\s*/>' % (tag, tag, tag) for tag in which_ones]) retags = re.compile(tags, re.DOTALL | re.IGNORECASE) text = retags.sub(u'', text) return text -def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): +def replace_escape_chars(text, which_ones=('\n', '\t', '\r'), replace_by=u'', \ + encoding=None): """ Remove escape chars. Default : \\n, \\t, \\r which_ones -- is a tuple of which escape chars we want to remove. @@ -117,10 +132,10 @@ def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): It defaults to '', so the escape chars are removed. """ for ec in which_ones: - text = text.replace(ec, str_to_unicode(replace_by)) - return str_to_unicode(text) + text = text.replace(ec, str_to_unicode(replace_by, encoding)) + return str_to_unicode(text, encoding) -def unquote_markup(text, keep=(), remove_illegal=True): +def unquote_markup(text, keep=(), remove_illegal=True, encoding=None): """ This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following: - removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA @@ -138,7 +153,7 @@ def unquote_markup(text, keep=(), remove_illegal=True): offset = match_e yield txt[offset:] - text = str_to_unicode(text) + text = str_to_unicode(text, encoding) ret_text = u'' for fragment in _get_fragments(text, _cdata_re): if isinstance(fragment, basestring): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3fe1e4122..098bfdfa6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -42,7 +42,7 @@ def load_object(path): return obj -def extract_regex(regex, text, encoding): +def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: * if the regex contains a named group called "extract" that will be returned diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py new file mode 100644 index 000000000..c8a9cd7ea --- /dev/null +++ b/scrapy/utils/py26.py @@ -0,0 +1,107 @@ +""" +This module provides functions added in Python 2.6, which weren't yet available +in Python 2.5. The Python 2.6 function is used when available. +""" + +import sys +import os +import fnmatch +from shutil import copy2, copystat + +__all__ = ['cpu_count', 'copytree', 'ignore_patterns'] + +try: + import multiprocessing + cpu_count = multiprocessing.cpu_count +except ImportError: + def cpu_count(): + ''' + Returns the number of CPUs in the system + ''' + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif 'bsd' in sys.platform or sys.platform == 'darwin': + try: + num = int(os.popen('sysctl -n hw.ncpu').read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + + if num >= 1: + return num + else: + raise NotImplementedError('cannot determine number of cpus') + +if sys.version_info >= (2, 6): + from shutil import copytree, ignore_patterns +else: + try: + WindowsError + except NameError: + WindowsError = None + + class Error(EnvironmentError): + pass + + def ignore_patterns(*patterns): + def _ignore_patterns(path, names): + ignored_names = [] + for pattern in patterns: + ignored_names.extend(fnmatch.filter(names, pattern)) + return set(ignored_names) + return _ignore_patterns + + def copytree(src, dst, symlinks=False, ignore=None): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + os.makedirs(dst) + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + copytree(srcname, dstname, symlinks, ignore) + else: + copy2(srcname, dstname) + # XXX What about devices, sockets etc.? + except (IOError, os.error), why: + errors.append((srcname, dstname, str(why))) + # catch the Error from the recursive copytree so that we can + # continue with other files + except Error, err: + errors.extend(err.args[0]) + try: + copystat(src, dst) + except OSError, why: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) + if errors: + raise Error, errors + +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + import scrapy.xlib.simplejson as json diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7d43c4566..abf3625e3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,12 +6,9 @@ higher than 2.5 which is the lowest version supported by Scrapy. """ import re -import os -import fnmatch import inspect import weakref from functools import wraps -from shutil import copy2, copystat from sgmllib import SGMLParser class FixedSGMLParser(SGMLParser): @@ -64,13 +61,15 @@ def unique(list_, key=lambda x: x): return result -def str_to_unicode(text, encoding='utf-8'): +def str_to_unicode(text, encoding=None): """Return the unicode representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a unicode object without the risk of double-decoding problems (which can happen if you don't use the default 'ascii' encoding) """ + if encoding is None: + encoding = 'utf-8' if isinstance(text, str): return text.decode(encoding) elif isinstance(text, unicode): @@ -78,13 +77,15 @@ def str_to_unicode(text, encoding='utf-8'): else: raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__) -def unicode_to_str(text, encoding='utf-8'): +def unicode_to_str(text, encoding=None): """Return the str representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a str object without the risk of double-decoding problems (which can happen if you don't use the default 'ascii' encoding) """ + if encoding is None: + encoding = 'utf-8' if isinstance(text, unicode): return text.encode(encoding) elif isinstance(text, str): @@ -142,68 +143,6 @@ def isbinarytext(text): assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ return any(c in _BINARYCHARS for c in text) - -# ----- shutil.copytree function from Python 2.6 adds ignore argument ---- # - -try: - WindowsError -except NameError: - WindowsError = None - -class Error(EnvironmentError): - pass - -def ignore_patterns(*patterns): - def _ignore_patterns(path, names): - ignored_names = [] - for pattern in patterns: - ignored_names.extend(fnmatch.filter(names, pattern)) - return set(ignored_names) - return _ignore_patterns - -def copytree(src, dst, symlinks=False, ignore=None): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except (IOError, os.error), why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error, err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except OSError, why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.extend((src, dst, str(why))) - if errors: - raise Error, errors - -# ----- end of shutil.copytree function from Python 2.6 ---- # - - def get_func_args(func): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -216,3 +155,27 @@ def get_func_args(func): else: raise TypeError('%s is not callable' % type(func)) return func_args + +def equal_attributes(obj1, obj2, attributes): + """Compare two objects attributes""" + # not attributes given return False by default + if not attributes: + return False + + for attr in attributes: + # support callables like itemgetter + if callable(attr): + if not attr(obj1) == attr(obj2): + return False + else: + # check that objects has attribute + if not hasattr(obj1, attr): + return False + if not hasattr(obj2, attr): + return False + # compare object attributes + if not getattr(obj1, attr) == getattr(obj2, attr): + return False + # all attributes equal + return True + diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 6ed6f43a3..ca258ef44 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -18,7 +18,8 @@ from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse def body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ + assert isinstance(obj, (Response, basestring)), \ + "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): return obj.body_as_unicode() if unicode else obj.body elif isinstance(obj, str): @@ -26,16 +27,17 @@ def body_or_str(obj, unicode=True): else: return obj if unicode else obj.encode('utf-8') -BASEURL_RE = re.compile(r']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', re.DOTALL | re.IGNORECASE) +META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', \ + re.DOTALL | re.IGNORECASE) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv parameter of the HTML meta element from the given @@ -46,9 +48,7 @@ def get_meta_refresh(response): If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - encoding = getattr(response, 'encoding', None) or 'utf-8' - body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \ - errors='ignore')) + body_chunk = remove_entities(response.body_as_unicode()[0:4096]) match = META_REFRESH_RE.search(body_chunk) if match: interval = int(match.group('int')) @@ -92,7 +92,7 @@ def response_httprepr(response): s += response.body return s -def open_in_browser(response): +def open_in_browser(response, _openfunc=webbrowser.open): """Open the given response in a local web browser, populating the tag for external links to work """ @@ -106,4 +106,4 @@ def open_in_browser(response): fd, fname = tempfile.mkstemp('.html') os.write(fd, body) os.close(fd) - webbrowser.open("file://%s" % fname) + return _openfunc("file://%s" % fname) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 9307fc234..29432f0e9 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,4 +4,3 @@ from scrapy.utils.misc import arg_to_iter def iterate_spider_output(result): return [result] if isinstance(result, BaseItem) else arg_to_iter(result) - diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1c2fe18ae..196c03727 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,9 +22,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - domains = [spider.domain_name] - domains.extend(spider.extra_domain_names) - return url_is_from_any_domain(url, domains) + return url_is_from_any_domain(url, spider.allowed_domains) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref @@ -87,29 +85,28 @@ def url_query_parameter(url, parameter, default=None, keep_blank_values=0): keep_blank_values=keep_blank_values) return queryparams.get(parameter, [default])[0] -def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='): - """Clean url arguments leaving only those passed in the parameterlist""" - try: - url = urlparse.urldefrag(url)[0] - base, query = url.split('?', 1) - parameters = [pair.split(kvsep, 1) for pair in query.split(sep)] - except: - base = url - query = "" - parameters = [] +def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='=', remove=False, unique=True): + """Clean url arguments leaving only those passed in the parameterlist keeping order - # unique parameters while keeping order - unique = {} + If remove is True, leave only those not in parameterlist. + If unique is False, do not remove duplicated keys + """ + url = urlparse.urldefrag(url)[0] + base, _, query = url.partition('?') + seen = set() querylist = [] - for pair in parameters: - k = pair[0] - if not unique.get(k): - querylist += [pair] - unique[k] = 1 - - query = sep.join([kvsep.join(pair) for pair in querylist if pair[0] in \ - parameterlist]) - return '?'.join([base, query]) + for ksv in query.split(sep): + k, _, _ = ksv.partition(kvsep) + if unique and k in seen: + continue + elif remove and k in parameterlist: + continue + elif not remove and k not in parameterlist: + continue + else: + querylist.append(ksv) + seen.add(k) + return '?'.join([base, sep.join(querylist)]) if querylist else base def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False): """Add or remove a parameter to a given url""" @@ -130,7 +127,8 @@ def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False) name+'='+new_value) return next_url -def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): +def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ + encoding=None): """Canonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value @@ -147,7 +145,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): For examples see the tests in scrapy.tests.test_utils_url """ - url = unicode_to_str(url) + url = unicode_to_str(url, encoding) scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) keyvals = cgi.parse_qsl(query, keep_blank_values) keyvals.sort() diff --git a/scrapy/xlib/BeautifulSoup.py b/scrapy/xlib/BeautifulSoup.py index 0e214630c..748e6fe4b 100644 --- a/scrapy/xlib/BeautifulSoup.py +++ b/scrapy/xlib/BeautifulSoup.py @@ -42,7 +42,7 @@ http://www.crummy.com/software/BeautifulSoup/documentation.html Here, have some legalese: -Copyright (c) 2004-2008, Leonard Richardson +Copyright (c) 2004-2010, Leonard Richardson All rights reserved. @@ -79,8 +79,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. from __future__ import generators __author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "3.0.7a" -__copyright__ = "Copyright (c) 2004-2008 Leonard Richardson" +__version__ = "3.0.8.1" +__copyright__ = "Copyright (c) 2004-2010 Leonard Richardson" __license__ = "New-style BSD" from sgmllib import SGMLParser, SGMLParseError @@ -104,9 +104,13 @@ markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match DEFAULT_OUTPUT_ENCODING = "utf-8" +def _match_css_class(str): + """Build a RE to match the given CSS class.""" + return re.compile(r"(^|.*\s)%s($|\s)" % str) + # First, the classes that represent markup elements. -class PageElement: +class PageElement(object): """Contains the navigational information for some part of the page (either a tag or a piece of text)""" @@ -124,10 +128,11 @@ class PageElement: def replaceWith(self, replaceWith): oldParent = self.parent - myIndex = self.parent.contents.index(self) - if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent: + myIndex = self.parent.index(self) + if hasattr(replaceWith, "parent")\ + and replaceWith.parent is self.parent: # We're replacing this element with one of its siblings. - index = self.parent.contents.index(replaceWith) + index = replaceWith.parent.index(replaceWith) if index and index < myIndex: # Furthermore, it comes before this element. That # means that when we extract it, the index of this @@ -136,11 +141,20 @@ class PageElement: self.extract() oldParent.insert(myIndex, replaceWith) + def replaceWithChildren(self): + myParent = self.parent + myIndex = self.parent.index(self) + self.extract() + reversedChildren = list(self.contents) + reversedChildren.reverse() + for child in reversedChildren: + myParent.insert(myIndex, child) + def extract(self): """Destructively rips this element out of the tree.""" if self.parent: try: - self.parent.contents.remove(self) + del self.parent.contents[self.parent.index(self)] except ValueError: pass @@ -173,18 +187,17 @@ class PageElement: return lastChild def insert(self, position, newChild): - if (isinstance(newChild, basestring) - or isinstance(newChild, unicode)) \ + if isinstance(newChild, basestring) \ and not isinstance(newChild, NavigableString): newChild = NavigableString(newChild) position = min(position, len(self.contents)) - if hasattr(newChild, 'parent') and newChild.parent != None: + if hasattr(newChild, 'parent') and newChild.parent is not None: # We're 'inserting' an element that's already one # of this object's children. - if newChild.parent == self: - index = self.find(newChild) - if index and index < position: + if newChild.parent is self: + index = self.index(newChild) + if index > position: # Furthermore we're moving it further down the # list of this object's children. That means that # when we extract this element, our target index @@ -322,8 +335,21 @@ class PageElement: if isinstance(name, SoupStrainer): strainer = name + # (Possibly) special case some findAll*(...) searches + elif text is None and not limit and not attrs and not kwargs: + # findAll*(True) + if name is True: + return [element for element in generator() + if isinstance(element, Tag)] + # findAll*('tag-name') + elif isinstance(name, basestring): + return [element for element in generator() + if isinstance(element, Tag) and + element.name == name] + else: + strainer = SoupStrainer(name, attrs, text, **kwargs) + # Build a SoupStrainer else: - # Build a SoupStrainer strainer = SoupStrainer(name, attrs, text, **kwargs) results = ResultSet(strainer) g = generator() @@ -344,31 +370,31 @@ class PageElement: #NavigableStrings and Tags. def nextGenerator(self): i = self - while i: + while i is not None: i = i.next yield i def nextSiblingGenerator(self): i = self - while i: + while i is not None: i = i.nextSibling yield i def previousGenerator(self): i = self - while i: + while i is not None: i = i.previous yield i def previousSiblingGenerator(self): i = self - while i: + while i is not None: i = i.previousSibling yield i def parentGenerator(self): i = self - while i: + while i is not None: i = i.parent yield i @@ -503,7 +529,7 @@ class Tag(PageElement): self.parserClass = parser.__class__ self.isSelfClosing = parser.isSelfClosingTag(name) self.name = name - if attrs == None: + if attrs is None: attrs = [] self.attrs = attrs self.contents = [] @@ -521,12 +547,49 @@ class Tag(PageElement): val)) self.attrs = map(convert, self.attrs) + def getString(self): + if (len(self.contents) == 1 + and isinstance(self.contents[0], NavigableString)): + return self.contents[0] + + def setString(self, string): + """Replace the contents of the tag with a string""" + self.clear() + self.append(string) + + string = property(getString, setString) + + def getText(self, separator=u""): + if not len(self.contents): + return u"" + stopNode = self._lastRecursiveChild().next + strings = [] + current = self.contents[0] + while current is not stopNode: + if isinstance(current, NavigableString): + strings.append(current.strip()) + current = current.next + return separator.join(strings) + + text = property(getText) + def get(self, key, default=None): """Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that attribute.""" return self._getAttrMap().get(key, default) + def clear(self): + """Extract all children.""" + for child in self.contents[:]: + child.extract() + + def index(self, element): + for i, child in enumerate(self.contents): + if child is element: + return i + raise ValueError("Tag.index: element not in tag") + def has_key(self, key): return self._getAttrMap().has_key(key) @@ -595,6 +658,8 @@ class Tag(PageElement): NOTE: right now this will return false if two tags have the same attributes in a different order. Should this be fixed?""" + if other is self: + return True if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): return False for i in range(0, len(self.contents)): @@ -638,7 +703,7 @@ class Tag(PageElement): if self.attrs: for key, val in self.attrs: fmt = '%s="%s"' - if isString(val): + if isinstance(val, basestring): if self.containsSubstitutions and '%SOUP-ENCODING%' in val: val = self.substituteEncoding(val, encoding) @@ -710,13 +775,20 @@ class Tag(PageElement): def decompose(self): """Recursively destroys the contents of this tree.""" - contents = [i for i in self.contents] - for i in contents: - if isinstance(i, Tag): - i.decompose() - else: - i.extract() self.extract() + if len(self.contents) == 0: + return + current = self.contents[0] + while current is not None: + next = current.next + if isinstance(current, Tag): + del current.contents[:] + current.parent = None + current.previous = None + current.previousSibling = None + current.next = None + current.nextSibling = None + current = next def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): return self.__str__(encoding, True) @@ -795,24 +867,18 @@ class Tag(PageElement): #Generator methods def childGenerator(self): - for i in range(0, len(self.contents)): - yield self.contents[i] - raise StopIteration + # Just use the iterator from the contents + return iter(self.contents) def recursiveChildGenerator(self): - stack = [(self, 0)] - while stack: - tag, start = stack.pop() - if isinstance(tag, Tag): - for i in range(start, len(tag.contents)): - a = tag.contents[i] - yield a - if isinstance(a, Tag) and tag.contents: - if i < len(tag.contents) - 1: - stack.append((tag, i+1)) - stack.append((a, 0)) - break - raise StopIteration + if not len(self.contents): + raise StopIteration + stopNode = self._lastRecursiveChild().next + current = self.contents[0] + while current is not stopNode: + yield current + current = current.next + # Next, a couple classes to represent queries and their results. class SoupStrainer: @@ -821,8 +887,8 @@ class SoupStrainer: def __init__(self, name=None, attrs={}, text=None, **kwargs): self.name = name - if isString(attrs): - kwargs['class'] = attrs + if isinstance(attrs, basestring): + kwargs['class'] = _match_css_class(attrs) attrs = None if kwargs: if attrs: @@ -881,7 +947,8 @@ class SoupStrainer: found = None # If given a list of items, scan it for a text element that # matches. - if isList(markup) and not isinstance(markup, Tag): + if hasattr(markup, "__iter__") \ + and not isinstance(markup, Tag): for element in markup: if isinstance(element, NavigableString) \ and self.search(element): @@ -894,7 +961,7 @@ class SoupStrainer: found = self.searchTag(markup) # If it's text, make sure the text matches. elif isinstance(markup, NavigableString) or \ - isString(markup): + isinstance(markup, basestring): if self._matches(markup, self.text): found = markup else: @@ -905,8 +972,8 @@ class SoupStrainer: def _matches(self, markup, matchAgainst): #print "Matching %s against %s" % (markup, matchAgainst) result = False - if matchAgainst == True and type(matchAgainst) == types.BooleanType: - result = markup != None + if matchAgainst is True: + result = markup is not None elif callable(matchAgainst): result = matchAgainst(markup) else: @@ -914,17 +981,17 @@ class SoupStrainer: #other ways of matching match the tag name as a string. if isinstance(markup, Tag): markup = markup.name - if markup and not isString(markup): + if markup and not isinstance(markup, basestring): markup = unicode(markup) #Now we know that chunk is either a string, or None. if hasattr(matchAgainst, 'match'): # It's a regexp object. result = markup and matchAgainst.search(markup) - elif isList(matchAgainst): + elif hasattr(matchAgainst, '__iter__'): # list-like result = markup in matchAgainst elif hasattr(matchAgainst, 'items'): result = markup.has_key(matchAgainst) - elif matchAgainst and isString(markup): + elif matchAgainst and isinstance(markup, basestring): if isinstance(markup, unicode): matchAgainst = unicode(matchAgainst) else: @@ -943,20 +1010,6 @@ class ResultSet(list): # Now, some helper functions. -def isList(l): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is listlike.""" - return hasattr(l, '__iter__') \ - or (type(l) in (types.ListType, types.TupleType)) - -def isString(s): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is stringlike.""" - try: - return isinstance(s, unicode) or isinstance(s, basestring) - except NameError: - return isinstance(s, str) - def buildTagMap(default, *args): """Turns a list of maps, lists, or scalars into a single map. Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and @@ -967,7 +1020,7 @@ def buildTagMap(default, *args): #It's a map. Merge it. for k,v in portion.items(): built[k] = v - elif isList(portion): + elif hasattr(portion, '__iter__'): # is a list #It's a list. Map each item to the default. for k in portion: built[k] = default @@ -1116,7 +1169,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): self.declaredHTMLEncoding = dammit.declaredHTMLEncoding if markup: if self.markupMassage: - if not isList(self.markupMassage): + if not hasattr(self.markupMassage, "__iter__"): self.markupMassage = self.MARKUP_MASSAGE for fix, m in self.markupMassage: markup = fix.sub(m, markup) @@ -1139,10 +1192,10 @@ class BeautifulStoneSoup(Tag, SGMLParser): superclass or the Tag superclass, depending on the method name.""" #print "__getattr__ called on %s.%s" % (self.__class__, methodName) - if methodName.find('start_') == 0 or methodName.find('end_') == 0 \ - or methodName.find('do_') == 0: + if methodName.startswith('start_') or methodName.startswith('end_') \ + or methodName.startswith('do_'): return SGMLParser.__getattr__(self, methodName) - elif methodName.find('__') != 0: + elif not methodName.startswith('__'): return Tag.__getattr__(self, methodName) else: raise AttributeError @@ -1165,12 +1218,6 @@ class BeautifulStoneSoup(Tag, SGMLParser): def popTag(self): tag = self.tagStack.pop() - # Tags with just one string-owning child get the child as a - # 'string' property, so that soup.tag.string is shorthand for - # soup.tag.contents[0] - if len(self.currentTag.contents) == 1 and \ - isinstance(self.currentTag.contents[0], NavigableString): - self.currentTag.string = self.currentTag.contents[0] #print "Pop", tag.name if self.tagStack: @@ -1259,9 +1306,9 @@ class BeautifulStoneSoup(Tag, SGMLParser): #last occurance. popTo = name break - if (nestingResetTriggers != None + if (nestingResetTriggers is not None and p.name in nestingResetTriggers) \ - or (nestingResetTriggers == None and isResetNesting + or (nestingResetTriggers is None and isResetNesting and self.RESET_NESTING_TAGS.has_key(p.name)): #If we encounter one of the nesting reset triggers @@ -1280,7 +1327,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): if self.quoteStack: #This is not a real tag. #print "<%s> is not real!" % name - attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs)) + attrs = ''.join([' %s="%s"' % (x, y) for x, y in attrs]) self.handle_data('<%s%s>' % (name, attrs)) return self.endData() @@ -1470,8 +1517,8 @@ class BeautifulSoup(BeautifulStoneSoup): BeautifulStoneSoup.__init__(self, *args, **kwargs) SELF_CLOSING_TAGS = buildTagMap(None, - ['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) + ('br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base', 'col')) PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) @@ -1480,13 +1527,13 @@ class BeautifulSoup(BeautifulStoneSoup): #According to the HTML standard, each of these inline tags can #contain another tag of the same type. Furthermore, it's common #to actually use these tags this way. - NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', - 'center'] + NESTABLE_INLINE_TAGS = ('span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', + 'center') #According to the HTML standard, these block tags can contain #another tag of the same type. Furthermore, it's common #to actually use these tags this way. - NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del'] + NESTABLE_BLOCK_TAGS = ('blockquote', 'div', 'fieldset', 'ins', 'del') #Lists can contain other lists, but there are restrictions. NESTABLE_LIST_TAGS = { 'ol' : [], @@ -1506,7 +1553,7 @@ class BeautifulSoup(BeautifulStoneSoup): 'tfoot' : ['table'], } - NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre'] + NON_NESTABLE_BLOCK_TAGS = ('address', 'form', 'p', 'pre') #If one of these tags is encountered, all tags up to the next tag of #this type are popped. @@ -1597,11 +1644,11 @@ class ICantBelieveItsBeautifulSoup(BeautifulSoup): wouldn't be.""" I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ - ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', + ('em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', - 'big'] + 'big') - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript'] + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ('noscript',) NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, @@ -1752,7 +1799,7 @@ class UnicodeDammit: """Changes a MS smart quote character to an XML or HTML entity.""" sub = self.MS_CHARS.get(orig) - if type(sub) == types.TupleType: + if isinstance(sub, tuple): if self.smartQuotesTo == 'xml': sub = '&#x%s;' % sub[1] else: diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py index d5c5b0034..8fe4d1ab6 100644 --- a/scrapy/xlib/pydispatch/dispatcher.py +++ b/scrapy/xlib/pydispatch/dispatcher.py @@ -30,7 +30,7 @@ import types, weakref from scrapy.xlib.pydispatch import saferef, robustapply, errors __author__ = "Patrick K. O'Brien " -__cvsid__ = "$Id$" +__cvsid__ = "$Id: dispatcher.py,v 1.1.1.1 2006/07/07 15:59:38 mcfletch Exp $" __version__ = "$Revision: 1.1.1.1 $"[11:-2] try: @@ -377,28 +377,29 @@ def _removeReceiver(receiver): # During module cleanup the mapping will be replaced with None return False backKey = id(receiver) - for senderkey in sendersBack.get(backKey,()): - try: - signals = connections[senderkey].keys() - except KeyError,err: - pass - else: - for signal in signals: - try: - receivers = connections[senderkey][signal] - except KeyError: - pass - else: - try: - receivers.remove( receiver ) - except Exception, err: - pass - _cleanupConnections(senderkey, signal) try: - del sendersBack[ backKey ] - except KeyError: - pass - + backSet = sendersBack.pop(backKey) + except KeyError, err: + return False + else: + for senderkey in backSet: + try: + signals = connections[senderkey].keys() + except KeyError,err: + pass + else: + for signal in signals: + try: + receivers = connections[senderkey][signal] + except KeyError: + pass + else: + try: + receivers.remove( receiver ) + except Exception, err: + pass + _cleanupConnections(senderkey, signal) + def _cleanupConnections(senderkey, signal): """Delete any empty signals for senderkey. Delete senderkey if empty.""" try: diff --git a/scrapy/xlib/pydispatch/saferef.py b/scrapy/xlib/pydispatch/saferef.py index 6b3eda1d3..d28229a9b 100644 --- a/scrapy/xlib/pydispatch/saferef.py +++ b/scrapy/xlib/pydispatch/saferef.py @@ -22,7 +22,7 @@ def safeRef(target, onDelete = None): onDelete=onDelete ) return reference - if callable(onDelete): + if onDelete is not None: return weakref.ref(target, onDelete) else: return weakref.ref( target ) @@ -120,7 +120,7 @@ class BoundMethodWeakref(object): self.key = self.calculateKey( target ) self.weakSelf = weakref.ref(target.im_self, remove) self.weakFunc = weakref.ref(target.im_func, remove) - self.selfName = str(target.im_self) + self.selfName = target.im_self.__class__.__name__ self.funcName = str(target.im_func.__name__) def calculateKey( cls, target ): """Calculate the reference key for this reference diff --git a/scrapy/xlib/simplejson/__init__.py b/scrapy/xlib/simplejson/__init__.py new file mode 100644 index 000000000..dcfd5413b --- /dev/null +++ b/scrapy/xlib/simplejson/__init__.py @@ -0,0 +1,437 @@ +r"""JSON (JavaScript Object Notation) is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. + +Encoding basic Python object hierarchies:: + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print json.dumps("\"foo\bar") + "\"foo\bar" + >>> print json.dumps(u'\u1234') + "\u1234" + >>> print json.dumps('\\') + "\\" + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> json.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> json.load(io)[0] == 'streaming API' + True + +Specializing JSON object decoding:: + + >>> import simplejson as json + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + >>> from decimal import Decimal + >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') + True + +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) + '[2.0, 1.0]' + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) + '[2.0, 1.0]' + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' + + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) +""" +__version__ = '2.1.1' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', + 'OrderedDict', +] + +__author__ = 'Bob Ippolito ' + +from decimal import Decimal + +from decoder import JSONDecoder, JSONDecodeError +from encoder import JSONEncoder +def _import_OrderedDict(): + import collections + try: + return collections.OrderedDict + except AttributeError: + import ordered_dict + return ordered_dict.OrderedDict +OrderedDict = _import_OrderedDict() + +def _import_c_make_encoder(): + try: + from simplejson._speedups import make_encoder + return make_encoder + except ImportError: + return None + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + use_decimal=False, +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If *indent* is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + default=default, use_decimal=use_decimal, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not use_decimal + and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, default=default, + use_decimal=use_decimal, **kw).encode(obj) + + +_default_decoder = JSONDecoder(encoding=None, object_hook=None, + object_pairs_hook=None) + + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + use_decimal=use_decimal, **kw) + + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and object_pairs_hook is None + and not use_decimal and not kw): + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + if object_pairs_hook is not None: + kw['object_pairs_hook'] = object_pairs_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant + if use_decimal: + if parse_float is not None: + raise TypeError("use_decimal=True implies parse_float=Decimal") + kw['parse_float'] = Decimal + return cls(encoding=encoding, **kw).decode(s) + + +def _toggle_speedups(enabled): + import simplejson.decoder as dec + import simplejson.encoder as enc + import simplejson.scanner as scan + c_make_encoder = _import_c_make_encoder() + if enabled: + dec.scanstring = dec.c_scanstring or dec.py_scanstring + enc.c_make_encoder = c_make_encoder + enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or + enc.py_encode_basestring_ascii) + scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner + else: + dec.scanstring = dec.py_scanstring + enc.c_make_encoder = None + enc.encode_basestring_ascii = enc.py_encode_basestring_ascii + scan.make_scanner = scan.py_make_scanner + dec.make_scanner = scan.make_scanner + global _default_decoder + _default_decoder = JSONDecoder( + encoding=None, + object_hook=None, + object_pairs_hook=None, + ) + global _default_encoder + _default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + ) diff --git a/scrapy/xlib/simplejson/decoder.py b/scrapy/xlib/simplejson/decoder.py new file mode 100644 index 000000000..4cf4015f6 --- /dev/null +++ b/scrapy/xlib/simplejson/decoder.py @@ -0,0 +1,421 @@ +"""Implementation of JSONDecoder +""" +import re +import sys +import struct + +from simplejson.scanner import make_scanner +def _import_c_scanstring(): + try: + from simplejson._speedups import scanstring + return scanstring + except ImportError: + return None +c_scanstring = _import_c_scanstring() + +__all__ = ['JSONDecoder'] + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + # The struct module in Python 2.4 would get frexp() out of range here + # when an endian is specified in the format string. Fixed in Python 2.5+ + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + + +class JSONDecodeError(ValueError): + """Subclass of ValueError with the following additional properties: + + msg: The unformatted error message + doc: The JSON document being parsed + pos: The start index of doc where parsing failed + end: The end index of doc where parsing failed (may be None) + lineno: The line corresponding to pos + colno: The column corresponding to pos + endlineno: The line corresponding to end (may be None) + endcolno: The column corresponding to end (may be None) + + """ + def __init__(self, msg, doc, pos, end=None): + ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) + self.msg = msg + self.doc = doc + self.pos = pos + self.end = end + self.lineno, self.colno = linecol(doc, pos) + if end is not None: + self.endlineno, self.endcolno = linecol(doc, pos) + else: + self.endlineno, self.endcolno = None, None + + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + + +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups + lineno, colno = linecol(doc, pos) + if end is None: + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, +} + +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def py_scanstring(s, end, encoding=None, strict=True, + _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + end = chunk.end() + content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows + if terminator == '"': + break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise JSONDecodeError(msg, s, end) + else: + _append(terminator) + continue + try: + esc = s[end] + except IndexError: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + # If not a unicode escape sequence, must be in the lookup table + if esc != 'u': + try: + char = _b[esc] + except KeyError: + msg = "Invalid \\escape: " + repr(esc) + raise JSONDecodeError(msg, s, end) + end += 1 + else: + # Unicode escape sequence + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise JSONDecodeError(msg, s, end) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise JSONDecodeError(msg, s, end) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise JSONDecodeError(msg, s, end) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) + return u''.join(chunks), end + + +# Use speedup if available +scanstring = c_scanstring or py_scanstring + +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, + object_pairs_hook, memo=None, + _w=WHITESPACE.match, _ws=WHITESPACE_STR): + # Backwards compatibility + if memo is None: + memo = {} + memo_get = memo.setdefault + pairs = [] + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty + nextchar = s[end:end + 1] + # Normally we expect nextchar == '"' + if nextchar != '"': + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + 1 + elif nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end) + end += 1 + while True: + key, end = scanstring(s, end, encoding, strict) + key = memo_get(key, key) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". + if s[end:end + 1] != ':': + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise JSONDecodeError("Expecting : delimiter", s, end) + + end += 1 + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + pairs.append((key, value)) + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 + + if nextchar == '}': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end - 1) + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + + end += 1 + if nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end - 1) + + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = dict(pairs) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + values = [] + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array + if nextchar == ']': + return values, end + 1 + _append = values.append + while True: + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + _append(value) + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end) + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + return values, end + +class JSONDecoder(object): + """Simple JSON decoder + + Performs the following translations in decoding by default: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + + """ + + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True, + object_pairs_hook=None): + """ + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + *strict* controls the parser's behavior when it encounters an + invalid control character in a string. The default setting of + ``True`` means that unescaped control characters are parse errors, if + ``False`` then control characters will be allowed in strings. + + """ + self.encoding = encoding + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.memo = {} + self.scan_once = make_scanner(self) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise JSONDecodeError("Extra data", s, end, len(s)) + return obj + + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` + beginning with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + + """ + try: + obj, end = self.scan_once(s, idx) + except StopIteration: + raise JSONDecodeError("No JSON object could be decoded", s, idx) + return obj, end diff --git a/scrapy/xlib/simplejson/encoder.py b/scrapy/xlib/simplejson/encoder.py new file mode 100644 index 000000000..cab845653 --- /dev/null +++ b/scrapy/xlib/simplejson/encoder.py @@ -0,0 +1,501 @@ +"""Implementation of JSONEncoder +""" +import re +from decimal import Decimal + +def _import_speedups(): + try: + from simplejson import _speedups + return _speedups.encode_basestring_ascii, _speedups.make_encoder + except ImportError: + return None, None +c_encode_basestring_ascii, c_make_encoder = _import_speedups() + +from simplejson.decoder import PosInf + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +FLOAT_REPR = repr + +def encode_basestring(s): + """Return a JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + return ESCAPE_DCT[match.group(0)] + return u'"' + ESCAPE.sub(replace, s) + u'"' + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + #return '\\u{0:04x}'.format(n) + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii) + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8', default=None, + use_decimal=False): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + + If use_decimal is true (not the default), ``decimal.Decimal`` will + be supported directly by the encoder. For the inverse, decode JSON + with ``parse_float=decimal.Decimal``. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.use_decimal = use_decimal + if isinstance(indent, (int, long)): + indent = ' ' * indent + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + if default is not None: + self.default = default + self.encoding = encoding + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from simplejson import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + if self.ensure_ascii: + return ''.join(chunks) + else: + return u''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, + _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on + # the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + key_memo = {} + if (_one_shot and c_make_encoder is not None + and not self.indent and not self.sort_keys): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, key_memo, self.use_decimal) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, self.use_decimal) + try: + return _iterencode(o, 0) + finally: + key_memo.clear() + + +class JSONEncoderForHTML(JSONEncoder): + """An encoder that produces JSON safe to embed in HTML. + + To embed JSON content in, say, a script tag on a web page, the + characters &, < and > should be escaped. They cannot be escaped + with the usual entities (e.g. &) because they are not expanded + within