diff --git a/.coveragerc b/.coveragerc index 1fde07e7e..914d697a0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,5 +4,3 @@ include = scrapy/* omit = tests/* scrapy/xlib/* - scrapy/conf.py - scrapy/log.py diff --git a/appveyor.yml b/appveyor.yml index 93cfd469e..7fd636864 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,7 +12,8 @@ branches: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" + - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - "pip install -U tox" build: false diff --git a/conftest.py b/conftest.py index 2d015f5e9..d8531d6cc 100644 --- a/conftest.py +++ b/conftest.py @@ -9,10 +9,6 @@ def _py_files(folder): collect_ignore = [ - # deprecated or moved modules - "scrapy/conf.py", - "scrapy/log.py", - # not a test, but looks like a test "scrapy/utils/testsite.py", diff --git a/docs/Makefile b/docs/Makefile index 187f03c4c..ff68bf1ae 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -82,6 +82,9 @@ pydoc-topics: build @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ "into the Lib/ directory" +coverage: BUILDER = coverage +coverage: build + htmlview: html $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ os.path.realpath('build/html/index.html'))" diff --git a/docs/conf.py b/docs/conf.py index a54a6bbe9..61d5b9600 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,8 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'scrapydocs', - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', ] # Add any paths that contain templates here, relative to this directory. @@ -218,3 +219,31 @@ linkcheck_ignore = [ 'http://localhost:\d+', 'http://hg.scrapy.org', 'http://directory.google.com/' ] + + +# Options for the Coverage extension +# ---------------------------------- +coverage_ignore_pyobjects = [ + # Contract’s add_pre_hook and add_post_hook are not documented because + # they should be transparent to contract developers, for whom pre_hook and + # post_hook should be the actual concern. + r'\bContract\.add_(pre|post)_hook$', + + # ContractsManager is an internal class, developers are not expected to + # interact with it directly in any way. + r'\bContractsManager\b$', + + # For default contracts we only want to document their general purpose in + # their constructor, the methods they reimplement to achieve that purpose + # should be irrelevant to developers using those contracts. + r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', + + # Methods of downloader middlewares are not documented, only the classes + # themselves, since downloader middlewares are controlled through Scrapy + # settings. + r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.', + + # Base classes of downloader middlewares are implementation details that + # are not meant for users. + r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware', +] diff --git a/docs/contributing.rst b/docs/contributing.rst index aac0f4496..b4f91ea8d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -99,6 +99,15 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +* if you're adding a private API, please add a regular expression to the + ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new + private API from documentation coverage checks. + + To see if your private API is skipped properly, generate a documentation + coverage report as follows:: + + tox -e docs-coverage + .. _submitting-patches: Submitting patches @@ -167,8 +176,9 @@ Documentation policies For reference documentation of API members (classes, methods, etc.) use docstrings and make sure that the Sphinx documentation uses the autodoc_ -extension to pull the docstrings. API reference documentation should be -IDE-friendly: short, to the point, and it may provide short examples. +extension to pull the docstrings. API reference documentation should follow +docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point, +and it may provide short examples. Other types of documentation, such as tutorials or topics, should be covered in files within the ``docs/`` directory. This includes documentation that is @@ -205,6 +215,29 @@ To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` +To run the tests on a specific tox_ environment, use ``-e `` with an +environment name from ``tox.ini``. For example, to run the tests with Python +3.6 use:: + + tox -e py36 + +You can also specify a comma-separated list of environmets, and use `tox’s +parallel mode`_ to run the tests on multiple environments in parallel:: + + tox -e py27,py36 -p auto + +To pass command-line options to pytest_, add them after ``--`` in your call to +tox_. Using ``--`` overrides the default positional arguments defined in +``tox.ini``, so you must include those default positional arguments +(``scrapy tests``) after ``--`` as well:: + + tox -- scrapy tests -x # stop after first failure + +You can also use the `pytest-xdist`_ plugin. For example, to run all tests on +the Python 3.6 tox_ environment using all your CPU cores:: + + tox -e py36 -- scrapy tests -n auto + To see coverage report install `coverage`_ (``pip install coverage``) and run: ``coverage report`` @@ -237,5 +270,9 @@ And their unit-tests are in:: .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues -.. _pull request: https://help.github.com/send-pull-requests/ +.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ +.. _pull request: https://help.github.com/en/articles/creating-a-pull-request +.. _pytest: https://docs.pytest.org/en/latest/usage.html +.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html .. _tox: https://pypi.python.org/pypi/tox +.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..7105baeef 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -129,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order? By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. If you do want to crawl in true `BFO order`_, you can do it by +in most cases. + +If you do want to crawl in true `BFO order`_, you can do it by setting the following settings:: DEPTH_PRIORITY = 1 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue' SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue' +While pending requests are below the configured values of +:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent +concurrently. As a result, the first few requests of a crawl rarely follow the +desired order. Lowering those settings to ``1`` enforces the desired order, but +it significantly slows down the crawl as a whole. + + My Scrapy crawler has memory leaks. What can I do? -------------------------------------------------- @@ -319,6 +329,29 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. +How to split an item into multiple items in an item pipeline? +------------------------------------------------------------- + +:ref:`Item pipelines ` cannot yield multiple items per +input item. :ref:`Create a spider middleware ` +instead, and use its +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` +method for this puspose. For example:: + + from copy import deepcopy + + from scrapy.item import BaseItem + + + class MultiplyItemsMiddleware: + + def process_spider_output(self, response, result, spider): + for item in result: + if isinstance(item, (BaseItem, dict)): + for _ in range(item['multiply_by']): + yield deepcopy(item) + + .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/index.rst b/docs/index.rst index cedde8f38..6d5f9e77d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,6 +158,7 @@ Solving specific problems topics/practices topics/broad-crawls topics/developer-tools + topics/dynamic-content topics/leaks topics/media-pipeline topics/deploy @@ -183,6 +184,9 @@ Solving specific problems :doc:`topics/developer-tools` Learn how to scrape with your browser's developer tools. +:doc:`topics/dynamic-content` + Read webpage data that is loaded dynamically. + :doc:`topics/leaks` Learn how to find and get rid of memory leaks in your crawler. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 41e61542a..a190ce407 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -205,7 +205,7 @@ Extracting data --------------- The best way to learn how to extract data with Scrapy is trying selectors -using the shell :ref:`Scrapy shell `. Run:: +using the :ref:`Scrapy shell `. Run:: scrapy shell 'http://quotes.toscrape.com/page/1/' @@ -296,8 +296,8 @@ expressions`_:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools to inspect the HTML and come up -with a selector (see section about :ref:`topics-developer-tools`). +You can use your browser's developer tools to inspect the HTML and come up +with a selector (see :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. @@ -379,11 +379,11 @@ variable, so that we can run our CSS selectors directly on a particular quote:: >>> quote = response.css("div.quote")[0] -Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +Now, let's extract ``text``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").get() - >>> title + >>> text = quote.css("span.text::text").get() + >>> text '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' >>> author = quote.css("small.author::text").get() >>> author @@ -511,7 +511,7 @@ We can try extracting it in the shell:: 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, -Scrapy supports a CSS extension that let's you select the attribute contents, +Scrapy supports a CSS extension that lets you select the attribute contents, like this:: >>> response.css('li.next a::attr(href)').get() diff --git a/docs/requirements.txt b/docs/requirements.txt index 8e7611d21..379da9994 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.6 +Sphinx>=2.1 sphinx_rtd_theme \ No newline at end of file diff --git a/docs/topics/api.rst b/docs/topics/api.rst index ba832ab5d..7c8c40b5f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -99,6 +99,8 @@ how you :ref:`configure the downloader middlewares Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop + .. autoclass:: CrawlerRunner :members: @@ -154,7 +156,7 @@ Settings API SpiderLoader API ================ -.. module:: scrapy.loader +.. module:: scrapy.spiderloader :synopsis: The spider loader .. class:: SpiderLoader diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index eb02086dc..b887b98af 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -39,6 +39,17 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. +Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` +================================================= + +Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. +It works best during single-domain crawl. It does not work well with crawling +many different domains in parallel + +To apply the recommended priority queue use:: + + SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + Increase concurrency ==================== @@ -85,7 +96,7 @@ When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you should not use ``DEBUG`` log level when preforming large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may be +production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. To set the log level use:: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 97f8311de..a93bee06b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -461,6 +461,9 @@ Supported options: * ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' +* ``--cbkwargs``: additional keyword arguments that will be passed to the callback. + This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 70f20d4ed..9337375bb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -120,3 +120,23 @@ get the failures pretty printed:: for header in self.args: if header not in response.headers: raise ContractFail('X-CustomHeader not present') + + +Detecting check runs +==================== + +When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is +set to the ``true`` string. You can use `os.environ`_ to perform any change to +your spiders or your settings when ``scrapy check`` is used:: + + import os + import scrapy + + class ExampleSpider(scrapy.Spider): + name = 'example' + + def __init__(self): + if os.environ.get('SCRAPY_CHECK'): + pass # Do some scraper adjustments when a check is running + +.. _os.environ: https://docs.python.org/3/library/os.html#os.environ diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index f93aa2c72..0aaad0c77 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -28,16 +28,15 @@ Consider the following scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item}) + yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) - def parse_details(self, response): - item = response.meta['item'] + def parse_details(self, response, item): # populate more `item` fields return item Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a partially populated item. @@ -100,8 +99,7 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see from scrapy.shell import inspect_response - def parse_details(self, response): - item = response.meta.get('item', None) + def parse_details(self, response, item=None): if item: # populate more `item` fields return item @@ -134,8 +132,7 @@ Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be available in all future runs should they be necessary again:: - def parse_details(self, response): - item = response.meta.get('item', None) + def parse_details(self, response, item=None): if item: # populate more `item` fields return item diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f2f3ef466..d7add4ec4 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -349,7 +349,7 @@ HttpCacheMiddleware * :ref:`httpcache-storage-leveldb` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` - setting. Or you can also implement your own storage backend. + setting. Or you can also :ref:`implement your own storage backend. ` Scrapy ships with two HTTP cache policies: @@ -496,6 +496,61 @@ In order to use this storage backend: .. _LevelDB: https://github.com/google/leveldb .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb +.. _httpcache-storage-custom: + +Writing your own storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can implement a cache storage backend by creating a Python class that +defines the methods described below. + +.. module:: scrapy.extensions.httpcache + +.. class:: CacheStorage + + .. method:: open_spider(spider) + + This method gets called after a spider has been opened for crawling. It handles + the :signal:`open_spider ` signal. + + :param spider: the spider which has been opened + :type spider: :class:`~scrapy.spiders.Spider` object + + .. method:: close_spider(spider) + + This method gets called after a spider has been closed. It handles + the :signal:`close_spider ` signal. + + :param spider: the spider which has been closed + :type spider: :class:`~scrapy.spiders.Spider` object + + .. method:: retrieve_response(spider, request) + + Return response if present in cache, or ``None`` otherwise. + + :param spider: the spider which generated the request + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the request to find cached reponse for + :type request: :class:`~scrapy.http.Request` object + + .. method:: store_response(spider, request, response) + + Store the given response in the cache. + + :param spider: the spider for which the response is intended + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the corresponding request the spider generated + :type request: :class:`~scrapy.http.Request` object + + :param response: the response to store in the cache + :type response: :class:`~scrapy.http.Response` object + +In order to use your storage backend, set: + +* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class. + HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -805,6 +860,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` +* :setting:`METAREFRESH_IGNORE_TAGS` * :setting:`METAREFRESH_MAXDELAY` This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, @@ -826,6 +882,15 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. +.. setting:: METAREFRESH_IGNORE_TAGS + +METAREFRESH_IGNORE_TAGS +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``['script', 'noscript']`` + +Meta tags within these tags are ignored. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst new file mode 100644 index 000000000..8b5dacf56 --- /dev/null +++ b/docs/topics/dynamic-content.rst @@ -0,0 +1,246 @@ +.. _topics-dynamic-content: + +==================================== +Selecting dynamically-loaded content +==================================== + +Some webpages show the desired data when you load them in a web browser. +However, when you download them using Scrapy, you cannot reach the desired data +using :ref:`selectors `. + +When this happens, the recommended approach is to +:ref:`find the data source ` and extract the data +from it. + +If you fail to do that, and you can nonetheless access the desired data through +the :ref:`DOM ` from your web browser, see +:ref:`topics-javascript-rendering`. + +.. _topics-finding-data-source: + +Finding the data source +======================= + +To extract the desired data, you must first find its source location. + +If the data is in a non-text-based format, such as an image or a PDF document, +use the :ref:`network tool ` of your web browser to find +the corresponding request, and :ref:`reproduce it +`. + +If your web browser lets you select the desired data as text, the data may be +defined in embedded JavaScript code, or loaded from an external resource in a +text-based format. + +In that case, you can use a tool like wgrep_ to find the URL of that resource. + +If the data turns out to come from the original URL itself, you must +:ref:`inspect the source code of the webpage ` to +determine where the data is located. + +If the data comes from a different URL, you will need to :ref:`reproduce the +corresponding request `. + +.. _topics-inspecting-source: + +Inspecting the source code of a webpage +======================================= + +Sometimes you need to inspect the source code of a webpage (not the +:ref:`DOM `) to determine where some desired data is located. + +Use Scrapy’s :command:`fetch` command to download the webpage contents as seen +by Scrapy:: + + scrapy fetch --nolog https://example.com > response.html + +If the desired data is in embedded JavaScript code within a ``