From 1b85bcf3367799300e015d6a20b2f5de2dc6bfa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aivars=20Kalv=C4=81ns?= Date: Wed, 19 Aug 2015 16:54:10 +0300 Subject: [PATCH 01/35] Make common practices sample code match the comments --- docs/topics/practices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7666a84cf..9ae34f423 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -61,7 +61,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('testspider', domain='scrapinghub.com') + process.crawl('followall', domain='scrapinghub.com') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling From 1ddcc7b96ac3fcb9ab6bebb249b6406d802eed7d Mon Sep 17 00:00:00 2001 From: David Tagatac Date: Thu, 27 Aug 2015 17:37:16 -0400 Subject: [PATCH 02/35] minor: scrapy.Spider docs grammar --- docs/topics/spiders.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 9c54a255e..5fd187e4e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -108,7 +108,7 @@ scrapy.Spider .. attribute:: settings - Configuration on which this spider is been ran. This is a + Configuration for running this spider. This is a :class:`~scrapy.settings.Settings` instance, see the :ref:`topics-settings` topic for a detailed introduction on this subject. @@ -122,12 +122,12 @@ scrapy.Spider This is the class method used by Scrapy to create your spiders. - You probably won't need to override this directly, since the default + You probably won't need to override this directly because the default implementation acts as a proxy to the :meth:`__init__` method, calling it with the given arguments `args` and named arguments `kwargs`. Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` - attributes in the new instance, so they can be accessed later inside the + attributes in the new instance so they can be accessed later inside the spider's code. :param crawler: crawler to which the spider will be bound From de73b1a2c543afd48c521e26ce23bb31b6708d8e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 25 Jun 2015 10:46:22 -0400 Subject: [PATCH 03/35] DOC(ENH): specify path to rtd theme explicitly --- docs/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 2d22c6a64..a593bbd65 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -108,7 +108,10 @@ html_theme = 'sphinx_rtd_theme' #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# Add path to the RTD explicitly to robustify builds (otherwise might +# fail in a clean Debian build env) +import sphinx_rtd_theme +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The style sheet to use for HTML and HTML Help pages. A file of that name From d8f4cbae10095377a8e89d619d91ff229798fa2c Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Jul 2015 17:14:50 -0300 Subject: [PATCH 04/35] Add OS X installation section to docs Closes #1342 and (possibly) #1126 --- docs/intro/install.rst | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 67507016d..26631ac1f 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -86,6 +86,67 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa yaourt -S scrapy +Mac OS X +-------- + +Building Scrapy's dependencies requires the presence of a C compiler and +development headers. On OS X this is typically provided by Apple’s Xcode +development tools. To install the Xcode command line tools open a terminal +window and run:: + + xcode-select --install + +There's a `known issue `_ that +prevents ``pip`` from updating system packages. This has to be addressed to +successfully install Scrapy and its dependencies. Here are some proposed +solutions: + +* *(Recommended)* **Don't** use system python, install a new, updated version + that doesn't conflict with the rest of your system. Here's how to do it using + the `homebrew`_ package manager: + + * Install `homebrew`_:: + + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + + * Update your ``PATH`` variable to state that homebrew packages should be + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + if you're using `zsh`_ as default shell):: + + echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc + + * Reload ``.bashrc`` to ensure the changes have taken place:: + + source ~/.bashrc + + * Install python:: + + brew install python + + * Latest versions of python have ``pip`` bundled with them so you won't need + to install it separately. If this is not the case, upgrade python:: + + brew update; brew upgrade python + +* *(Alternative)* Force system python to load the user installed packages + first: + + * Update your ``PYTHONPATH`` variable (Change ``.bashrc`` to ``.zshrc`` + accordantly if you're using `zsh`_ as default shell):: + + echo "export PYTHONPATH=/Library/Python/2.7/site-packages:$PYTHONPATH" >> ~/.bashrc + + * Reload ``.bashrc`` to ensure the changes have taken place:: + + source ~/.bashrc + + * Install ``pip``:: + + sudo easy_install pip + +After any of these workarounds you should be able to install Scrapy:: + + pip install Scrapy .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing.html @@ -95,3 +156,5 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa .. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ +.. _homebrew: http://brew.sh/ +.. _zsh: http://www.zsh.org/ From fbd010d603d61709df6f3e9161655ea39a312497 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Jul 2015 18:52:01 -0300 Subject: [PATCH 05/35] Add non-python dependencies to Ubuntu install section in the docs Closes #1314 and closes #1198. --- docs/intro/install.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 26631ac1f..f1ab90db4 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -79,6 +79,16 @@ Instead, use the official :ref:`Ubuntu Packages `, which already solve all dependencies for you and are continuously updated with the latest bug fixes. +If you prefer to build the python dependencies locally instead of relying on +system packages you'll need to install their required non-python dependencies +first:: + + sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev + +You can install Scrapy with ``pip`` after that:: + + pip install Scrapy + Archlinux --------- From 85c980e2fa5814b0b5e4ab3b14d95d2e3eaf1651 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Jul 2015 19:06:23 -0300 Subject: [PATCH 06/35] Note in install docs about pip being already included in python>=2.7.9 --- docs/intro/install.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index f1ab90db4..246964d3f 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -14,7 +14,8 @@ The installation steps assume that you have the following things installed: * `Python`_ 2.7 * `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and - installs `setuptools`_ if not installed. + installs `setuptools`_ if not installed. Python 2.7.9 and later include + `pip`_ by default, so you may have it already. * `lxml`_. Most Linux distributions ships prepackaged versions of lxml. Otherwise refer to http://lxml.de/installation.html @@ -58,7 +59,8 @@ Windows Be sure you download the architecture (win32 or amd64) that matches your system -* Install `pip`_ from https://pip.pypa.io/en/latest/installing.html +* *(Only required for Python<2.7.9)* Install `pip`_ from + https://pip.pypa.io/en/latest/installing.html Now open a Command prompt to check ``pip`` is installed correctly:: From 5d10d6dcc1a6944fd4e3fc435aee23b6e2d6bf50 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 10 Jul 2015 19:15:14 -0300 Subject: [PATCH 07/35] Add oldest supported tox version to contributing docs Better fix for #1337 --- docs/contributing.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ad9a3805a..87cea2a5f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -146,6 +146,14 @@ tests requires `tox`_. Running tests ------------- +Make sure you have a recent enough `tox`_ installation: + + ``tox --version`` + +If your version is older than 1.7.0, please update it first: + + ``pip install -U tox`` + To run all tests go to the root directory of Scrapy source code and run: ``tox`` From 1925db1345b22cdd78c142e380abe5e5a0cdfe9d Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 19 Aug 2015 02:21:08 -0300 Subject: [PATCH 08/35] Reference Homebrew's homepage for installation instructions --- docs/intro/install.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 246964d3f..918197e43 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -117,9 +117,7 @@ solutions: that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: - * Install `homebrew`_:: - - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + * Install `homebrew`_ following the instructions in http://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly From 98b63eeba442d85fed3e315b60e4af2943d856fb Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 19 Aug 2015 03:19:27 -0300 Subject: [PATCH 09/35] Replace alternative OSX install workaround with virtualenv --- docs/intro/install.rst | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 918197e43..978d89920 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -138,21 +138,14 @@ solutions: brew update; brew upgrade python -* *(Alternative)* Force system python to load the user installed packages - first: +* *(Optional)* Install Scrapy inside an isolated python environment. - * Update your ``PYTHONPATH`` variable (Change ``.bashrc`` to ``.zshrc`` - accordantly if you're using `zsh`_ as default shell):: + This method is a workaround for the above OS X issue, but it's an overall + good practice for managing dependencies and can complement the first method. - echo "export PYTHONPATH=/Library/Python/2.7/site-packages:$PYTHONPATH" >> ~/.bashrc - - * Reload ``.bashrc`` to ensure the changes have taken place:: - - source ~/.bashrc - - * Install ``pip``:: - - sudo easy_install pip + `virtualenv`_ is a tool you can use to create virtual environments in python. + We recommended reading a tutorial like + http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started. After any of these workarounds you should be able to install Scrapy:: @@ -168,3 +161,4 @@ After any of these workarounds you should be able to install Scrapy:: .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: http://brew.sh/ .. _zsh: http://www.zsh.org/ +.. _virtualenv: https://virtualenv.pypa.io/en/latest/ From 23fda69f3acdef5ee66962a706117eb14e89116a Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 8 Sep 2015 18:49:44 -0300 Subject: [PATCH 10/35] Add note to ubuntu install section about debian compatibility --- docs/intro/install.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 978d89920..3adb4e6b0 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -91,6 +91,11 @@ You can install Scrapy with ``pip`` after that:: pip install Scrapy +.. note:: + + The same non-python dependencies can be used to install Scrapy in Debian + Wheezy (7.0) and above. + Archlinux --------- From 32f115c4c67758646f8f8a4711408ef45847529e Mon Sep 17 00:00:00 2001 From: hy Date: Wed, 16 Sep 2015 16:59:23 +0800 Subject: [PATCH 11/35] fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware --- docs/topics/downloader-middleware.rst | 2 +- docs/topics/exceptions.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 03c5deffe..b304d581d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -736,7 +736,7 @@ RetryMiddleware .. class:: RetryMiddleware - A middlware to retry failed requests that are potentially caused by + A middleware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. Failed pages are collected on the scraping process and rescheduled at the diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 8a10ee796..9f8d16d84 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -57,7 +57,7 @@ remain disabled. Those components include: * Extensions * Item pipelines - * Downloader middlwares + * Downloader middlewares * Spider middlewares The exception must be raised in the component constructor. From 7067117a7bbd65e517ca35b2d9ac135723841e5a Mon Sep 17 00:00:00 2001 From: smirecki Date: Fri, 2 Oct 2015 23:48:27 -0400 Subject: [PATCH 12/35] Typos corrections I've made a few small corrections, some spelling changes and typo fixes. I've tried to respect regional spelling differences and avoided proposing hyphenating compound words. Please enter the commit message for your changes. Lines starting --- docs/topics/deploy.rst | 2 +- docs/topics/firebug.rst | 2 +- docs/topics/shell.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index b7394d5ce..bc48ddce7 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -8,7 +8,7 @@ This section describes the different options you have for deploying your Scrapy spiders to run them on a regular basis. Running Scrapy spiders in your local machine is very convenient for the (early) development stage, but not so much when you need to execute long-running spiders or move spiders to run in -production continously. This is where the solutions for deploying Scrapy +production continuously. This is where the solutions for deploying Scrapy spiders come in. Popular choices for deploying Scrapy spiders are: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 359c99450..b93ee4ff9 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -118,7 +118,7 @@ they work as we expect. As you can see, the page markup is not very descriptive: the elements don't contain ``id``, ``class`` or any attribute that clearly identifies them, so -we''ll use the ranking bars as a reference point to select the data to extract +we'll use the ranking bars as a reference point to select the data to extract when we construct our XPaths. After using FireBug, we can see that each link is inside a ``td`` tag, which is diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 9c9411d6d..83a5a378d 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -56,7 +56,7 @@ Available Shortcuts * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order for external links (such as images and style sheets) to display properly. - Note, however,that this will create a temporary file in your computer, + Note, however, that this will create a temporary file in your computer, which won't be removed automatically. .. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base From ca8d60f2670adfba80fa1ef55446ed7f3463edca Mon Sep 17 00:00:00 2001 From: Hoat Le Date: Wed, 7 Oct 2015 15:43:02 +0700 Subject: [PATCH 13/35] fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst >>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None Traceback (most recent call last): File "", line 1, in File "/home/vagrant/.virtualenvs/scrapy/lib/python2.7/site-packages/scrapy/http/response/text.py", line 109, in xpath return self.selector.xpath(query) File "/home/vagrant/.virtualenvs/scrapy/lib/python2.7/site-packages/scrapy/selector/unified.py", line 100, in xpath raise ValueError(msg if six.PY3 else msg.encode("unicode_escape")) ValueError: Invalid XPath: //div/[id="not-exists"]/text() --- docs/topics/selectors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 688c2b715..9d19506d2 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -146,7 +146,7 @@ If you want to extract only first matched element, you can call the selector ``. It returns ``None`` if no element was found: - >>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None + >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None True A default return value can be provided as an argument, to be used instead of ``None``: From 0a1366ed8de2df1fcaafe70c6ff91918c9cf7e96 Mon Sep 17 00:00:00 2001 From: hoatle Date: Wed, 7 Oct 2015 15:51:12 +0700 Subject: [PATCH 14/35] fix another invalid xpath error --- docs/topics/selectors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9d19506d2..273cae0f8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -151,7 +151,7 @@ It returns ``None`` if no element was found: A default return value can be provided as an argument, to be used instead of ``None``: - >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first(default='not-found') + >>> sel.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found') 'not-found' Notice that CSS selectors can select text or attribute nodes using CSS3 From 5456c0e6109773c9a11fea32be08303014ad1305 Mon Sep 17 00:00:00 2001 From: Christian Pedersen Date: Wed, 14 Oct 2015 14:07:34 +0200 Subject: [PATCH 15/35] Version 1 now exists --- docs/versioning.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/versioning.rst b/docs/versioning.rst index fffa68b33..6ae9ce981 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -12,8 +12,7 @@ Scrapy uses the `odd-numbered versions for development releases`_. There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very - large changes. So far, only zero is available for *A* as Scrapy hasn't yet - reached 1.0. + large changes. * *B* is the release number. This will include many changes including features and things that possibly break backwards compatibility. Even Bs will be stable branches, and odd Bs will be development. @@ -27,12 +26,10 @@ For example: API Stability ============= -API stability is one of Scrapy major goals for the *1.0* release, which doesn't -have a due date scheduled yet. +API stability was one of the major goals for the *1.0* release. Methods or functions that start with a single dash (``_``) are private and -should never be relied as stable. Besides those, the plan is to stabilize and -document the entire API, as we approach the 1.0 release. +should never be relied as stable. Also, keep in mind that stable doesn't mean complete: stable APIs could grow new methods or functionality but the existing methods should keep working the From b71f6775c1ea24fe41e332e2a0cea89dd91e5909 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Jr Date: Mon, 26 Oct 2015 00:00:20 -0200 Subject: [PATCH 16/35] fixed a typo in the documentation. --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4ee4f1758..5ed6ce97d 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -7,7 +7,7 @@ Downloading and processing files and images .. currentmodule:: scrapy.pipelines.images Scrapy provides reusable :doc:`item pipelines ` for -downloading fies attached to a particular item (for example, when you scrape +downloading files attached to a particular item (for example, when you scrape products and also want to download their images locally). These pipelines share a bit of functionality and structure (we refer to them as media pipelines), but typically you'll either use the Files Pipeline or the Images Pipeline. From 7f4ddd5d8d2db3b18c983e9a3eb9896748d8f30c Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 16 Nov 2015 07:30:17 +0800 Subject: [PATCH 17/35] Fixed minor grammar issues. --- docs/faq.rst | 2 +- docs/topics/broad-crawls.rst | 4 ++-- docs/topics/extensions.rst | 4 ++-- docs/topics/item-pipeline.rst | 2 +- docs/topics/items.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/selectors.rst | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2e61f44ee..3d2bd8d4d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -144,7 +144,7 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those message are thrown by the Offsite Spider Middleware, which is a spider +Those messages are thrown by the Offsite Spider Middleware, which is a spider middleware (enabled by default) whose purpose is to filter out requests to domains outside the ones covered by the spider. diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index aaf46bc92..79f0b3b53 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -34,7 +34,7 @@ These are some common properties often found in broad crawls: As said above, Scrapy default settings are optimized for focused crawls, not broad crawls. However, due to its asynchronous architecture, Scrapy is very -well suited for performing fast broad crawls. This page summarize some things +well suited for performing fast broad crawls. This page summarizes some things you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. @@ -46,7 +46,7 @@ Concurrency is the number of requests that are processed in parallel. There is a global limit and a per-domain limit. The default global concurrency limit in Scrapy is not suitable for crawling -many different domains in parallel, so you will want to increase it. How much +many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU you crawler will have available. A good starting point is ``100``, but the best way to find out is by doing some trials and identifying at what concurrency your Scrapy process gets CPU diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d5d985087..c95b76bdf 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ Extensions use the :ref:`Scrapy settings ` to manage their settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, an +avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like `GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. @@ -145,7 +145,7 @@ Here is the code of such extension:: self.items_scraped += 1 if self.items_scraped % self.item_count == 0: logger.info("scraped %d items", self.items_scraped) - + .. _topics-extensions-ref: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index f74400b4d..28969be61 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -95,7 +95,7 @@ contain a price:: Write items to a JSON file -------------------------- -The following pipeline stores all scraped items (from all spiders) into a a +The following pipeline stores all scraped items (from all spiders) into a single ``items.jl`` file, containing one item per line serialized in JSON format:: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 21ec0ed8c..4a8f47e93 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -61,7 +61,7 @@ the example above. You can specify any kind of metadata for each field. There is no restriction on the values accepted by :class:`Field` objects. For this same reason, there is no reference list of all available metadata keys. Each key -defined in :class:`Field` objects could be used by a different components, and +defined in :class:`Field` objects could be used by a different component, and only those components know about it. You can also define and use any other :class:`Field` key in your project too, for your own needs. The main goal of :class:`Field` objects is to provide a way to define all field metadata in one diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 735137ea2..92590c180 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -97,7 +97,7 @@ subclasses): A real example -------------- -Let's see a concrete example of an hypothetical case of memory leaks. +Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 9ae34f423..60fe2267c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -228,7 +228,7 @@ with varying degrees of sophistication. Getting around those measures can be difficult and tricky, and may sometimes require special infrastructure. Please consider contacting `commercial support`_ if in doubt. -Here are some tips to keep in mind when dealing with these kind of sites: +Here are some tips to keep in mind when dealing with these kinds of sites: * rotate your user agent from a pool of well-known ones from browsers (google around to get a list of them) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 273cae0f8..8dc82dfe5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -579,7 +579,7 @@ Built-in Selectors reference is used together with ``text``. If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follow: + inferred from the response type as follows: * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type @@ -757,7 +757,7 @@ nodes can be accessed directly by their names:: Date: Tue, 11 Aug 2015 14:09:49 -0300 Subject: [PATCH 22/35] explicit close file on file:// scheme handler --- scrapy/core/downloader/handlers/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 5a63e9d08..9346ce08d 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -10,6 +10,7 @@ class FileDownloadHandler(object): @defers def download_request(self, request, spider): filepath = file_uri_to_path(request.url) - body = open(filepath, 'rb').read() + with open(filepath, 'rb') as fo: + body = fo.read() respcls = responsetypes.from_args(filename=filepath, body=body) return respcls(url=request.url, body=body) From 97f2fb302da06e80bbfb976157061ae51271496f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 28 Jul 2015 14:31:42 +0500 Subject: [PATCH 23/35] TST a test to show that dupefilter persistence is not working --- tests/test_dupefilters.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index f8f800a72..a9d92a1a2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,5 +1,7 @@ import hashlib +import tempfile import unittest +import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request @@ -23,6 +25,27 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close('finished') + def test_dupefilter_path(self): + r1 = Request('http://scrapytest.org/1') + r2 = Request('http://scrapytest.org/2') + + path = tempfile.mkdtemp() + try: + df = RFPDupeFilter(path) + df.open() + assert not df.request_seen(r1) + assert df.request_seen(r1) + df.close('finished') + + df2 = RFPDupeFilter(path) + df2.open() + assert df2.request_seen(r1) + assert not df2.request_seen(r2) + assert df2.request_seen(r2) + df2.close('finished') + finally: + shutil.rmtree(path) + def test_request_fingerprint(self): """Test if customization of request_fingerprint method will change output of request_seen. From 97d080ee2609276e6f0f8c6ad2c28fbd8b2b8d48 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 28 Jul 2015 14:32:14 +0500 Subject: [PATCH 24/35] fixed RFPDupeFilter persistence --- scrapy/dupefilters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 739ba922f..9d8966b9c 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -36,6 +36,7 @@ class RFPDupeFilter(BaseDupeFilter): self.logger = logging.getLogger(__name__) if path: self.file = open(os.path.join(path, 'requests.seen'), 'a+') + self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) @classmethod From 73ac80d3ca118352257b6f78c3ff0081179698d5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Aug 2015 19:13:54 +0500 Subject: [PATCH 25/35] Add Python 3 porting status badge to the README --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 6ca5deb49..7006925ae 100644 --- a/README.rst +++ b/README.rst @@ -13,6 +13,11 @@ Scrapy .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy :alt: Wheel Status + +.. image:: http://static.scrapy.org/py3progress/badge.svg + :target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting + :alt: Python 3 Porting Status + Overview ======== From a83ab41ba0971f7d2123c04678a7afa26d861de4 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 4 Sep 2015 20:55:41 +0500 Subject: [PATCH 26/35] don't run tests twice on Travis if a PR is made from a scrapy/scrapy branch --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index cdb40bfcf..32a42f2c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: python python: 2.7 +branches: + only: + - master + - /^\d\.\d+$/ env: - TOXENV=py27 - TOXENV=precise From df2b9448a97bafff5cc136a777d6d66fbf9966e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 17 Sep 2015 14:07:47 -0300 Subject: [PATCH 27/35] Add PyPI download stats badge --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 7006925ae..7e8d45104 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,10 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: PyPI Version +.. image:: https://img.shields.io/pypi/dm/Scrapy.svg + :target: https://pypi.python.org/pypi/Scrapy + :alt: PyPI Monthly downloads + .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg :target: http://travis-ci.org/scrapy/scrapy :alt: Build Status From db4c9fee4c96fe75c958fffa04b374bcd67c764b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 17 Sep 2015 01:05:52 -0300 Subject: [PATCH 28/35] disable log on startproject command --- scrapy/commands/startproject.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 0c77cad01..4375b6d4c 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -27,6 +27,7 @@ IGNORE = ignore_patterns('*.pyc', '.svn') class Command(ScrapyCommand): requires_project = False + default_settings = {'LOG_ENABLED': False} def syntax(self): return "" From 86fc33060beffda40ee2ab4b60418625d291ad16 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 25 Sep 2015 16:39:21 -0300 Subject: [PATCH 29/35] disable log on version command. closes #1426 --- scrapy/commands/version.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 81433a657..4bf085c9e 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -11,6 +11,8 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): + default_settings = {'LOG_ENABLED': False} + def syntax(self): return "[-v]" From a55078cff299d1d166585298dab585e642152000 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 6 Oct 2015 17:24:52 +0500 Subject: [PATCH 30/35] TST don't use broken Pillow version in tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ebc07181a..fc5c2f392 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = -rrequirements.txt # Extras boto - Pillow + Pillow != 3.0.0 leveldb -rtests/requirements.txt commands = From 4418fc3b862fd0b23062be70ae6e07fcf34ea4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Wed, 7 Oct 2015 14:43:47 +0300 Subject: [PATCH 31/35] test xml nodename with dots --- tests/test_utils_iterators.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a7042a6cf..47bc6aa0b 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -33,6 +33,19 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + def test_xmliter_unusual_node(self): + body = b""" + + + + + """ + response = XmlResponse(url="http://example.com", body=body) + nodenames = [e.xpath('name()').extract() + for e in self.xmliter(response, 'matchme...')] + self.assertEqual(nodenames, [['matchme...']]) + + def test_xmliter_text(self): body = u"""onetwo""" From da3c155b399a7cbdf12989911fcefae81035d092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Wed, 7 Oct 2015 14:47:23 +0300 Subject: [PATCH 32/35] escape nodename in xmliter regex --- scrapy/utils/iterators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 58ddf6215..48682f828 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -25,8 +25,10 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + nodename_patt = re.escape(nodename) + + HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) + HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -34,7 +36,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) + r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] From 823a1cc685a83fe515d01cfd19a4ad71d5606803 Mon Sep 17 00:00:00 2001 From: seales Date: Sun, 13 Dec 2015 19:39:48 -0800 Subject: [PATCH 33/35] Spelling fixes --- sep/sep-003.rst | 2 +- sep/sep-014.rst | 2 +- sep/sep-018.rst | 2 +- sep/sep-020.rst | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sep/sep-003.rst b/sep/sep-003.rst index 282574968..184839525 100644 --- a/sep/sep-003.rst +++ b/sep/sep-003.rst @@ -146,7 +146,7 @@ Default values p['numbers'] # returns [] -Accesing and changing nested item values +Accessing and changing nested item values ---------------------------------------- :: diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 98a31b1aa..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -54,7 +54,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. -This is an enhancemente to ``LinkExtractors`` which returns urls (links), +This is an enhancement to ``LinkExtractors`` which returns urls (links), Request Extractors return Request objects. Request Processors diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e30821917..aca7ac342 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -477,7 +477,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def should_follow(self, request, spider): info = self.spiders[spider] - # hostanme can be None for wrong urls (like javascript links) + # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or '' return bool(info.regex.search(host)) diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 7b2c043b7..49d068479 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -23,9 +23,9 @@ Rationale ========= There are certain markup patterns that lend themselves quite nicely to -automated parsing, for example the ```` tag outlilnes such a pattern +automated parsing, for example the ``
`` tag outlines such a pattern for populating a database table with the embedded ```` elements denoting -the rows and the furthur embedded ``
`` elements denoting the individual +the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader From 6e42f0bf447ebb4a6debabf432d83310ef47441d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 30 Dec 2015 15:20:04 -0300 Subject: [PATCH 34/35] Run on new travis-ci infra --- .travis-workarounds.sh | 15 --------------- .travis.yml | 5 ++++- 2 files changed, 4 insertions(+), 16 deletions(-) delete mode 100755 .travis-workarounds.sh diff --git a/.travis-workarounds.sh b/.travis-workarounds.sh deleted file mode 100755 index 5c34e54f7..000000000 --- a/.travis-workarounds.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e -set -x - -if [[ "${TOXENV}" == "pypy" ]]; then - sudo add-apt-repository -y ppa:pypy/ppa - sudo apt-get -qy update - sudo apt-get install -y pypy pypy-dev - # This is required because we need to get rid of the Travis installed PyPy - # or it'll take precedence over the PPA installed one. - sudo rm -rf /usr/local/pypy/bin -fi - -# Workaround travis-ci/travis-ci#2065 -pip install -U wheel diff --git a/.travis.yml b/.travis.yml index 32a42f2c4..9a60b6015 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: python python: 2.7 +sudo: false branches: only: - master @@ -10,7 +11,6 @@ env: - TOXENV=py33 - TOXENV=docs install: -- "./.travis-workarounds.sh" - pip install -U tox twine wheel script: tox notifications: @@ -19,6 +19,9 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy +cache: + directories: + - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" From 7dfa979197620749128ee062e8771a4283bbf771 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 2 Dec 2015 17:05:44 +0100 Subject: [PATCH 35/35] Ignoring xlib/tx folder, depending on Twisted version. --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index 5829f2952..23a05dd8d 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ import glob import six import pytest +from twisted import version as twisted_version def _py_files(folder): @@ -21,6 +22,9 @@ collect_ignore = [ "scrapy/spider.py", ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): + collect_ignore += _py_files("scrapy/xlib/tx") + if six.PY3: for line in open('tests/py3-ignores.txt'):