diff --git a/appveyor.yml b/appveyor.yml index 93cfd469e..7fd636864 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,7 +12,8 @@ branches: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" + - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - "pip install -U tox" build: false diff --git a/docs/Makefile b/docs/Makefile index 187f03c4c..ff68bf1ae 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -82,6 +82,9 @@ pydoc-topics: build @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ "into the Lib/ directory" +coverage: BUILDER = coverage +coverage: build + htmlview: html $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ os.path.realpath('build/html/index.html'))" diff --git a/docs/conf.py b/docs/conf.py index a54a6bbe9..bf222b361 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,8 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'scrapydocs', - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', ] # Add any paths that contain templates here, relative to this directory. @@ -218,3 +219,22 @@ linkcheck_ignore = [ 'http://localhost:\d+', 'http://hg.scrapy.org', 'http://directory.google.com/' ] + + +# Options for the Coverage extension +# ---------------------------------- +coverage_ignore_pyobjects = [ + # Contract’s add_pre_hook and add_post_hook are not documented because + # they should be transparent to contract developers, for whom pre_hook and + # post_hook should be the actual concern. + r'\bContract\.add_(pre|post)_hook$', + + # ContractsManager is an internal class, developers are not expected to + # interact with it directly in any way. + r'\bContractsManager\b$', + + # For default contracts we only want to document their general purpose in + # their constructor, the methods they reimplement to achieve that purpose + # should be irrelevant to developers using those contracts. + r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', +] diff --git a/docs/contributing.rst b/docs/contributing.rst index aac0f4496..b4f91ea8d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -99,6 +99,15 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +* if you're adding a private API, please add a regular expression to the + ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new + private API from documentation coverage checks. + + To see if your private API is skipped properly, generate a documentation + coverage report as follows:: + + tox -e docs-coverage + .. _submitting-patches: Submitting patches @@ -167,8 +176,9 @@ Documentation policies For reference documentation of API members (classes, methods, etc.) use docstrings and make sure that the Sphinx documentation uses the autodoc_ -extension to pull the docstrings. API reference documentation should be -IDE-friendly: short, to the point, and it may provide short examples. +extension to pull the docstrings. API reference documentation should follow +docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point, +and it may provide short examples. Other types of documentation, such as tutorials or topics, should be covered in files within the ``docs/`` directory. This includes documentation that is @@ -205,6 +215,29 @@ To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` +To run the tests on a specific tox_ environment, use ``-e `` with an +environment name from ``tox.ini``. For example, to run the tests with Python +3.6 use:: + + tox -e py36 + +You can also specify a comma-separated list of environmets, and use `tox’s +parallel mode`_ to run the tests on multiple environments in parallel:: + + tox -e py27,py36 -p auto + +To pass command-line options to pytest_, add them after ``--`` in your call to +tox_. Using ``--`` overrides the default positional arguments defined in +``tox.ini``, so you must include those default positional arguments +(``scrapy tests``) after ``--`` as well:: + + tox -- scrapy tests -x # stop after first failure + +You can also use the `pytest-xdist`_ plugin. For example, to run all tests on +the Python 3.6 tox_ environment using all your CPU cores:: + + tox -e py36 -- scrapy tests -n auto + To see coverage report install `coverage`_ (``pip install coverage``) and run: ``coverage report`` @@ -237,5 +270,9 @@ And their unit-tests are in:: .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues -.. _pull request: https://help.github.com/send-pull-requests/ +.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ +.. _pull request: https://help.github.com/en/articles/creating-a-pull-request +.. _pytest: https://docs.pytest.org/en/latest/usage.html +.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html .. _tox: https://pypi.python.org/pypi/tox +.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode diff --git a/docs/index.rst b/docs/index.rst index cedde8f38..6d5f9e77d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,6 +158,7 @@ Solving specific problems topics/practices topics/broad-crawls topics/developer-tools + topics/dynamic-content topics/leaks topics/media-pipeline topics/deploy @@ -183,6 +184,9 @@ Solving specific problems :doc:`topics/developer-tools` Learn how to scrape with your browser's developer tools. +:doc:`topics/dynamic-content` + Read webpage data that is loaded dynamically. + :doc:`topics/leaks` Learn how to find and get rid of memory leaks in your crawler. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 41e61542a..a190ce407 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -205,7 +205,7 @@ Extracting data --------------- The best way to learn how to extract data with Scrapy is trying selectors -using the shell :ref:`Scrapy shell `. Run:: +using the :ref:`Scrapy shell `. Run:: scrapy shell 'http://quotes.toscrape.com/page/1/' @@ -296,8 +296,8 @@ expressions`_:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools to inspect the HTML and come up -with a selector (see section about :ref:`topics-developer-tools`). +You can use your browser's developer tools to inspect the HTML and come up +with a selector (see :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. @@ -379,11 +379,11 @@ variable, so that we can run our CSS selectors directly on a particular quote:: >>> quote = response.css("div.quote")[0] -Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +Now, let's extract ``text``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").get() - >>> title + >>> text = quote.css("span.text::text").get() + >>> text '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' >>> author = quote.css("small.author::text").get() >>> author @@ -511,7 +511,7 @@ We can try extracting it in the shell:: 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, -Scrapy supports a CSS extension that let's you select the attribute contents, +Scrapy supports a CSS extension that lets you select the attribute contents, like this:: >>> response.css('li.next a::attr(href)').get() diff --git a/docs/requirements.txt b/docs/requirements.txt index 8e7611d21..379da9994 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.6 +Sphinx>=2.1 sphinx_rtd_theme \ No newline at end of file diff --git a/docs/topics/api.rst b/docs/topics/api.rst index ba832ab5d..7c8c40b5f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -99,6 +99,8 @@ how you :ref:`configure the downloader middlewares Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop + .. autoclass:: CrawlerRunner :members: @@ -154,7 +156,7 @@ Settings API SpiderLoader API ================ -.. module:: scrapy.loader +.. module:: scrapy.spiderloader :synopsis: The spider loader .. class:: SpiderLoader diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index eb02086dc..b887b98af 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -39,6 +39,17 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. +Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` +================================================= + +Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. +It works best during single-domain crawl. It does not work well with crawling +many different domains in parallel + +To apply the recommended priority queue use:: + + SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + Increase concurrency ==================== @@ -85,7 +96,7 @@ When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you should not use ``DEBUG`` log level when preforming large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may be +production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. To set the log level use:: diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 70f20d4ed..9337375bb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -120,3 +120,23 @@ get the failures pretty printed:: for header in self.args: if header not in response.headers: raise ContractFail('X-CustomHeader not present') + + +Detecting check runs +==================== + +When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is +set to the ``true`` string. You can use `os.environ`_ to perform any change to +your spiders or your settings when ``scrapy check`` is used:: + + import os + import scrapy + + class ExampleSpider(scrapy.Spider): + name = 'example' + + def __init__(self): + if os.environ.get('SCRAPY_CHECK'): + pass # Do some scraper adjustments when a check is running + +.. _os.environ: https://docs.python.org/3/library/os.html#os.environ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f2f3ef466..fa65f66ed 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -805,6 +805,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` +* :setting:`METAREFRESH_IGNORE_TAGS` * :setting:`METAREFRESH_MAXDELAY` This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, @@ -826,6 +827,15 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. +.. setting:: METAREFRESH_IGNORE_TAGS + +METAREFRESH_IGNORE_TAGS +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``['script', 'noscript']`` + +Meta tags within these tags are ignored. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst new file mode 100644 index 000000000..8b5dacf56 --- /dev/null +++ b/docs/topics/dynamic-content.rst @@ -0,0 +1,246 @@ +.. _topics-dynamic-content: + +==================================== +Selecting dynamically-loaded content +==================================== + +Some webpages show the desired data when you load them in a web browser. +However, when you download them using Scrapy, you cannot reach the desired data +using :ref:`selectors `. + +When this happens, the recommended approach is to +:ref:`find the data source ` and extract the data +from it. + +If you fail to do that, and you can nonetheless access the desired data through +the :ref:`DOM ` from your web browser, see +:ref:`topics-javascript-rendering`. + +.. _topics-finding-data-source: + +Finding the data source +======================= + +To extract the desired data, you must first find its source location. + +If the data is in a non-text-based format, such as an image or a PDF document, +use the :ref:`network tool ` of your web browser to find +the corresponding request, and :ref:`reproduce it +`. + +If your web browser lets you select the desired data as text, the data may be +defined in embedded JavaScript code, or loaded from an external resource in a +text-based format. + +In that case, you can use a tool like wgrep_ to find the URL of that resource. + +If the data turns out to come from the original URL itself, you must +:ref:`inspect the source code of the webpage ` to +determine where the data is located. + +If the data comes from a different URL, you will need to :ref:`reproduce the +corresponding request `. + +.. _topics-inspecting-source: + +Inspecting the source code of a webpage +======================================= + +Sometimes you need to inspect the source code of a webpage (not the +:ref:`DOM `) to determine where some desired data is located. + +Use Scrapy’s :command:`fetch` command to download the webpage contents as seen +by Scrapy:: + + scrapy fetch --nolog https://example.com > response.html + +If the desired data is in embedded JavaScript code within a ``