diff --git a/.travis-workarounds.sh b/.travis-workarounds.sh deleted file mode 100755 index 5c34e54f7..000000000 --- a/.travis-workarounds.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e -set -x - -if [[ "${TOXENV}" == "pypy" ]]; then - sudo add-apt-repository -y ppa:pypy/ppa - sudo apt-get -qy update - sudo apt-get install -y pypy pypy-dev - # This is required because we need to get rid of the Travis installed PyPy - # or it'll take precedence over the PPA installed one. - sudo rm -rf /usr/local/pypy/bin -fi - -# Workaround travis-ci/travis-ci#2065 -pip install -U wheel diff --git a/.travis.yml b/.travis.yml index cdb40bfcf..9a60b6015 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,16 @@ language: python python: 2.7 +sudo: false +branches: + only: + - master + - /^\d\.\d+$/ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 - TOXENV=docs install: -- "./.travis-workarounds.sh" - pip install -U tox twine wheel script: tox notifications: @@ -15,6 +19,9 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy +cache: + directories: + - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/README.rst b/README.rst index 6ca5deb49..7e8d45104 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,10 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: PyPI Version +.. image:: https://img.shields.io/pypi/dm/Scrapy.svg + :target: https://pypi.python.org/pypi/Scrapy + :alt: PyPI Monthly downloads + .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg :target: http://travis-ci.org/scrapy/scrapy :alt: Build Status @@ -13,6 +17,11 @@ Scrapy .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy :alt: Wheel Status + +.. image:: http://static.scrapy.org/py3progress/badge.svg + :target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting + :alt: Python 3 Porting Status + Overview ======== diff --git a/conftest.py b/conftest.py index 5829f2952..23a05dd8d 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ import glob import six import pytest +from twisted import version as twisted_version def _py_files(folder): @@ -21,6 +22,9 @@ collect_ignore = [ "scrapy/spider.py", ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): + collect_ignore += _py_files("scrapy/xlib/tx") + if six.PY3: for line in open('tests/py3-ignores.txt'): diff --git a/docs/conf.py b/docs/conf.py index 2d22c6a64..a593bbd65 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -108,7 +108,10 @@ html_theme = 'sphinx_rtd_theme' #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# Add path to the RTD explicitly to robustify builds (otherwise might +# fail in a clean Debian build env) +import sphinx_rtd_theme +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The style sheet to use for HTML and HTML Help pages. A file of that name diff --git a/docs/contributing.rst b/docs/contributing.rst index ad9a3805a..87cea2a5f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -146,6 +146,14 @@ tests requires `tox`_. Running tests ------------- +Make sure you have a recent enough `tox`_ installation: + + ``tox --version`` + +If your version is older than 1.7.0, please update it first: + + ``pip install -U tox`` + To run all tests go to the root directory of Scrapy source code and run: ``tox`` diff --git a/docs/faq.rst b/docs/faq.rst index 2e61f44ee..3d2bd8d4d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -144,7 +144,7 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those message are thrown by the Offsite Spider Middleware, which is a spider +Those messages are thrown by the Offsite Spider Middleware, which is a spider middleware (enabled by default) whose purpose is to filter out requests to domains outside the ones covered by the spider. diff --git a/docs/index.rst b/docs/index.rst index 0d21f5d40..4cb3eb741 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ First steps =========== .. toctree:: + :caption: First steps :hidden: intro/overview @@ -53,6 +54,7 @@ Basic concepts ============== .. toctree:: + :caption: Basic concepts :hidden: topics/commands @@ -110,6 +112,7 @@ Built-in services ================= .. toctree:: + :caption: Built-in services :hidden: topics/logging @@ -138,6 +141,7 @@ Solving specific problems ========================= .. toctree:: + :caption: Solving specific problems :hidden: faq @@ -203,6 +207,7 @@ Extending Scrapy ================ .. toctree:: + :caption: Extending Scrapy :hidden: topics/architecture @@ -240,6 +245,7 @@ All the rest ============ .. toctree:: + :caption: All the rest :hidden: news diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 67507016d..122de47f6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -14,7 +14,8 @@ The installation steps assume that you have the following things installed: * `Python`_ 2.7 * `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and - installs `setuptools`_ if not installed. + installs `setuptools`_ if not installed. Python 2.7.9 and later include + `pip`_ by default, so you may have it already. * `lxml`_. Most Linux distributions ships prepackaged versions of lxml. Otherwise refer to http://lxml.de/installation.html @@ -23,9 +24,7 @@ The installation steps assume that you have the following things installed: where the Python installer ships it bundled. You can install Scrapy using pip (which is the canonical way to install Python -packages). - -To install using pip:: +packages). To install using ``pip`` run:: pip install Scrapy @@ -34,6 +33,22 @@ To install using pip:: Platform specific installation notes ==================================== +Anaconda +-------- + +.. note:: + + For Windows users, or if you have issues installing through `pip`, this is + the recommended way to install Scrapy. + +If you already have installed `Anaconda`_ or `Miniconda`_, the company +`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X. + +To install Scrapy using ``conda``, run:: + + conda install -c scrapinghub scrapy + + Windows ------- @@ -58,7 +73,8 @@ Windows Be sure you download the architecture (win32 or amd64) that matches your system -* Install `pip`_ from https://pip.pypa.io/en/latest/installing.html +* *(Only required for Python<2.7.9)* Install `pip`_ from + https://pip.pypa.io/en/latest/installing.html Now open a Command prompt to check ``pip`` is installed correctly:: @@ -79,6 +95,21 @@ Instead, use the official :ref:`Ubuntu Packages `, which already solve all dependencies for you and are continuously updated with the latest bug fixes. +If you prefer to build the python dependencies locally instead of relying on +system packages you'll need to install their required non-python dependencies +first:: + + sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev + +You can install Scrapy with ``pip`` after that:: + + pip install Scrapy + +.. note:: + + The same non-python dependencies can be used to install Scrapy in Debian + Wheezy (7.0) and above. + Archlinux --------- @@ -86,6 +117,58 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa yaourt -S scrapy +Mac OS X +-------- + +Building Scrapy's dependencies requires the presence of a C compiler and +development headers. On OS X this is typically provided by Apple’s Xcode +development tools. To install the Xcode command line tools open a terminal +window and run:: + + xcode-select --install + +There's a `known issue `_ that +prevents ``pip`` from updating system packages. This has to be addressed to +successfully install Scrapy and its dependencies. Here are some proposed +solutions: + +* *(Recommended)* **Don't** use system python, install a new, updated version + that doesn't conflict with the rest of your system. Here's how to do it using + the `homebrew`_ package manager: + + * Install `homebrew`_ following the instructions in http://brew.sh/ + + * Update your ``PATH`` variable to state that homebrew packages should be + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + if you're using `zsh`_ as default shell):: + + echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc + + * Reload ``.bashrc`` to ensure the changes have taken place:: + + source ~/.bashrc + + * Install python:: + + brew install python + + * Latest versions of python have ``pip`` bundled with them so you won't need + to install it separately. If this is not the case, upgrade python:: + + brew update; brew upgrade python + +* *(Optional)* Install Scrapy inside an isolated python environment. + + This method is a workaround for the above OS X issue, but it's an overall + good practice for managing dependencies and can complement the first method. + + `virtualenv`_ is a tool you can use to create virtual environments in python. + We recommended reading a tutorial like + http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started. + +After any of these workarounds you should be able to install Scrapy:: + + pip install Scrapy .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing.html @@ -95,3 +178,9 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa .. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ +.. _homebrew: http://brew.sh/ +.. _zsh: http://www.zsh.org/ +.. _virtualenv: https://virtualenv.pypa.io/en/latest/ +.. _Scrapinghub: http://scrapinghub.com +.. _Anaconda: http://docs.continuum.io/anaconda/index +.. _Miniconda: http://conda.pydata.org/docs/install/quick.html diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index aaf46bc92..79f0b3b53 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -34,7 +34,7 @@ These are some common properties often found in broad crawls: As said above, Scrapy default settings are optimized for focused crawls, not broad crawls. However, due to its asynchronous architecture, Scrapy is very -well suited for performing fast broad crawls. This page summarize some things +well suited for performing fast broad crawls. This page summarizes some things you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. @@ -46,7 +46,7 @@ Concurrency is the number of requests that are processed in parallel. There is a global limit and a per-domain limit. The default global concurrency limit in Scrapy is not suitable for crawling -many different domains in parallel, so you will want to increase it. How much +many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU you crawler will have available. A good starting point is ``100``, but the best way to find out is by doing some trials and identifying at what concurrency your Scrapy process gets CPU diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index b7394d5ce..bc48ddce7 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -8,7 +8,7 @@ This section describes the different options you have for deploying your Scrapy spiders to run them on a regular basis. Running Scrapy spiders in your local machine is very convenient for the (early) development stage, but not so much when you need to execute long-running spiders or move spiders to run in -production continously. This is where the solutions for deploying Scrapy +production continuously. This is where the solutions for deploying Scrapy spiders come in. Popular choices for deploying Scrapy spiders are: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 03c5deffe..b304d581d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -736,7 +736,7 @@ RetryMiddleware .. class:: RetryMiddleware - A middlware to retry failed requests that are potentially caused by + A middleware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. Failed pages are collected on the scraping process and rescheduled at the diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 8a10ee796..9f8d16d84 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -57,7 +57,7 @@ remain disabled. Those components include: * Extensions * Item pipelines - * Downloader middlwares + * Downloader middlewares * Spider middlewares The exception must be raised in the component constructor. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d5d985087..c95b76bdf 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ Extensions use the :ref:`Scrapy settings ` to manage their settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, an +avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like `GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. @@ -145,7 +145,7 @@ Here is the code of such extension:: self.items_scraped += 1 if self.items_scraped % self.item_count == 0: logger.info("scraped %d items", self.items_scraped) - + .. _topics-extensions-ref: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 359c99450..b93ee4ff9 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -118,7 +118,7 @@ they work as we expect. As you can see, the page markup is not very descriptive: the elements don't contain ``id``, ``class`` or any attribute that clearly identifies them, so -we''ll use the ranking bars as a reference point to select the data to extract +we'll use the ranking bars as a reference point to select the data to extract when we construct our XPaths. After using FireBug, we can see that each link is inside a ``td`` tag, which is diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index f74400b4d..28969be61 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -95,7 +95,7 @@ contain a price:: Write items to a JSON file -------------------------- -The following pipeline stores all scraped items (from all spiders) into a a +The following pipeline stores all scraped items (from all spiders) into a single ``items.jl`` file, containing one item per line serialized in JSON format:: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 21ec0ed8c..4a8f47e93 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -61,7 +61,7 @@ the example above. You can specify any kind of metadata for each field. There is no restriction on the values accepted by :class:`Field` objects. For this same reason, there is no reference list of all available metadata keys. Each key -defined in :class:`Field` objects could be used by a different components, and +defined in :class:`Field` objects could be used by a different component, and only those components know about it. You can also define and use any other :class:`Field` key in your project too, for your own needs. The main goal of :class:`Field` objects is to provide a way to define all field metadata in one diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 735137ea2..92590c180 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -97,7 +97,7 @@ subclasses): A real example -------------- -Let's see a concrete example of an hypothetical case of memory leaks. +Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4ee4f1758..5ed6ce97d 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -7,7 +7,7 @@ Downloading and processing files and images .. currentmodule:: scrapy.pipelines.images Scrapy provides reusable :doc:`item pipelines ` for -downloading fies attached to a particular item (for example, when you scrape +downloading files attached to a particular item (for example, when you scrape products and also want to download their images locally). These pipelines share a bit of functionality and structure (we refer to them as media pipelines), but typically you'll either use the Files Pipeline or the Images Pipeline. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7666a84cf..60fe2267c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -61,7 +61,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('testspider', domain='scrapinghub.com') + process.crawl('followall', domain='scrapinghub.com') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling @@ -228,7 +228,7 @@ with varying degrees of sophistication. Getting around those measures can be difficult and tricky, and may sometimes require special infrastructure. Please consider contacting `commercial support`_ if in doubt. -Here are some tips to keep in mind when dealing with these kind of sites: +Here are some tips to keep in mind when dealing with these kinds of sites: * rotate your user agent from a pool of well-known ones from browsers (google around to get a list of them) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 688c2b715..8dc82dfe5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -146,12 +146,12 @@ If you want to extract only first matched element, you can call the selector ``. It returns ``None`` if no element was found: - >>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None + >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None True A default return value can be provided as an argument, to be used instead of ``None``: - >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first(default='not-found') + >>> sel.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found') 'not-found' Notice that CSS selectors can select text or attribute nodes using CSS3 @@ -579,7 +579,7 @@ Built-in Selectors reference is used together with ``text``. If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follow: + inferred from the response type as follows: * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type @@ -757,7 +757,7 @@ nodes can be accessed directly by their names:: " diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 81433a657..4bf085c9e 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -11,6 +11,8 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): + default_settings = {'LOG_ENABLED': False} + def syntax(self): return "[-v]" diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 5a63e9d08..9346ce08d 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -10,6 +10,7 @@ class FileDownloadHandler(object): @defers def download_request(self, request, spider): filepath = file_uri_to_path(request.url) - body = open(filepath, 'rb').read() + with open(filepath, 'rb') as fo: + body = fo.read() respcls = responsetypes.from_args(filename=filepath, body=body) return respcls(url=request.url, body=body) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 739ba922f..9d8966b9c 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -36,6 +36,7 @@ class RFPDupeFilter(BaseDupeFilter): self.logger = logging.getLogger(__name__) if path: self.file = open(os.path.join(path, 'requests.seen'), 'a+') + self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) @classmethod diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 58ddf6215..48682f828 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -25,8 +25,10 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + nodename_patt = re.escape(nodename) + + HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) + HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -34,7 +36,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) + r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] diff --git a/sep/sep-003.rst b/sep/sep-003.rst index 282574968..184839525 100644 --- a/sep/sep-003.rst +++ b/sep/sep-003.rst @@ -146,7 +146,7 @@ Default values p['numbers'] # returns [] -Accesing and changing nested item values +Accessing and changing nested item values ---------------------------------------- :: diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 98a31b1aa..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -54,7 +54,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. -This is an enhancemente to ``LinkExtractors`` which returns urls (links), +This is an enhancement to ``LinkExtractors`` which returns urls (links), Request Extractors return Request objects. Request Processors diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e30821917..aca7ac342 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -477,7 +477,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def should_follow(self, request, spider): info = self.spiders[spider] - # hostanme can be None for wrong urls (like javascript links) + # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or '' return bool(info.regex.search(host)) diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 7b2c043b7..49d068479 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -23,9 +23,9 @@ Rationale ========= There are certain markup patterns that lend themselves quite nicely to -automated parsing, for example the ```` tag outlilnes such a pattern +automated parsing, for example the ``
`` tag outlines such a pattern for populating a database table with the embedded ```` elements denoting -the rows and the furthur embedded ``
`` elements denoting the individual +the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index f8f800a72..a9d92a1a2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,5 +1,7 @@ import hashlib +import tempfile import unittest +import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request @@ -23,6 +25,27 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close('finished') + def test_dupefilter_path(self): + r1 = Request('http://scrapytest.org/1') + r2 = Request('http://scrapytest.org/2') + + path = tempfile.mkdtemp() + try: + df = RFPDupeFilter(path) + df.open() + assert not df.request_seen(r1) + assert df.request_seen(r1) + df.close('finished') + + df2 = RFPDupeFilter(path) + df2.open() + assert df2.request_seen(r1) + assert not df2.request_seen(r2) + assert df2.request_seen(r2) + df2.close('finished') + finally: + shutil.rmtree(path) + def test_request_fingerprint(self): """Test if customization of request_fingerprint method will change output of request_seen. diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a7042a6cf..47bc6aa0b 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -33,6 +33,19 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + def test_xmliter_unusual_node(self): + body = b""" + + + + + """ + response = XmlResponse(url="http://example.com", body=body) + nodenames = [e.xpath('name()').extract() + for e in self.xmliter(response, 'matchme...')] + self.assertEqual(nodenames, [['matchme...']]) + + def test_xmliter_text(self): body = u"""onetwo""" diff --git a/tox.ini b/tox.ini index ebc07181a..fc5c2f392 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = -rrequirements.txt # Extras boto - Pillow + Pillow != 3.0.0 leveldb -rtests/requirements.txt commands =