diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57ff603fa..36484c49f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,29 +1,7 @@ [bumpversion] -current_version = 1.3.0 +current_version = 1.3.2 commit = True tag = True tag_name = {new_version} -parse = ^ - (?P\d+)\.(?P\d+)\.(?P\d+) - (?:(?P[abc]|rc|dev)(?P\d+))? -serialize = - {major}.{minor}.{patch}{prerel}{prerelversion} - {major}.{minor}.{patch} [bumpversion:file:scrapy/VERSION] - -[bumpversion:part:prerel] -optional_value = gamma -values = - dev - rc - gamma - -[bumpversion:part:prerelversion] -values = - 1 - 2 - 3 - 4 - 5 - diff --git a/.gitignore b/.gitignore index b116640b4..406146e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ dist .idea htmlcov/ .coverage +.coverage.* +.cache/ # Windows Thumbs.db diff --git a/.travis.yml b/.travis.yml index 506f3779b..2df02ea43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,46 @@ language: python -python: 3.5 sudo: false branches: only: - master - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ -env: - - TOXENV=py27 - - TOXENV=jessie - - TOXENV=py33 - - TOXENV=py35 - - TOXENV=docs + - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 2.7 + env: TOXENV=jessie + - python: 3.3 + env: TOXENV=py33 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - python: 2.7 + env: TOXENV=pypy + - python: 3.6 + env: TOXENV=docs + allow_failures: + - python: 2.7 + env: TOXENV=pypy install: - - pip install -U tox twine wheel codecov + - | + if [ "$TOXENV" = "pypy" ]; then + export PYENV_ROOT="$HOME/.pyenv" + if [ -f "$PYENV_ROOT/bin/pyenv" ]; then + pushd "$PYENV_ROOT" && git pull && popd + else + rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" + fi + # get latest PyPy from pyenv directly (thanks to natural version sort option -V) + export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1` + "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" + virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + - pip install -U tox twine wheel codecov + script: tox after_success: - codecov @@ -35,4 +62,4 @@ deploy: on: tags: true repo: scrapy/scrapy - condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$" + condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/README.rst b/README.rst index 38dda62e5..4eb36b44a 100644 --- a/README.rst +++ b/README.rst @@ -13,10 +13,6 @@ Scrapy .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy :alt: Wheel Status - -.. image:: http://static.scrapy.org/py3progress/badge.svg - :target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting - :alt: Python 3 Porting Status .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: http://codecov.io/github/scrapy/scrapy?branch=master diff --git a/artwork/README b/artwork/README.rst similarity index 97% rename from artwork/README rename to artwork/README.rst index c185d57da..016462f2c 100644 --- a/artwork/README +++ b/artwork/README.rst @@ -1,3 +1,5 @@ +:orphan: + Scrapy artwork ============== diff --git a/docs/README b/docs/README.rst similarity index 99% rename from docs/README rename to docs/README.rst index cf04965ac..733af2af4 100644 --- a/docs/README +++ b/docs/README.rst @@ -1,3 +1,5 @@ +:orphan: + ====================================== Scrapy documentation quick start guide ====================================== diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7195017ff..1da1a4059 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination:: next_page = response.css('li.next a::attr("href")').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + yield response.follow(next_page, self.parse) Put this in a text file, name it to something like ``quotes_spider.py`` diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8e14d1b7c..3b3bd8d21 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -225,7 +225,7 @@ You will see something like:: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> + >>> Using the shell, you can try selecting elements using `CSS`_ with the response object:: @@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary:: >>> Extracting data in our spider ------------------------------- +----------------------------- Let's get back to our spider. Until now, it doesn't extract any data in particular, just saves the whole HTML page to a local file. Let's integrate the @@ -423,7 +423,7 @@ in the callback, as you can see below:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -522,7 +522,7 @@ page, extracting data from it:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -551,13 +551,65 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. + +.. _response-follow-example: + +A shortcut for creating Requests +-------------------------------- + +As a shortcut for creating Request objects you can use +:meth:`response.follow `:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + ] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } + + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) + +Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no +need to call urljoin. Note that ``response.follow`` just returns a Request +instance; you still have to yield this Request. + +You can also pass a selector to ``response.follow`` instead of a string; +this selector should extract necessary attributes:: + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +For ```` elements there is a shortcut: ``response.follow`` uses their href +attribute automatically. So the code can be shortened further:: + + for a in response.css('li.next a'): + yield response.follow(a, callback=self.parse) + +.. note:: + + ``response.follow(response.css('li.next a'))`` is not valid because + ``response.css`` returns a list-like object with selectors for all results, + not a single selector. A ``for`` loop like in the example above, or + ``response.follow(response.css('li.next a')[0])`` is fine. + More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: - import scrapy @@ -568,15 +620,12 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author+a::attr(href)').extract(): - yield scrapy.Request(response.urljoin(href), - callback=self.parse_author) + for href in response.css('.author + a::attr(href)'): + yield response.follow(href, self.parse_author) # follow pagination links - next_page = response.css('li.next a::attr(href)').extract_first() - if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, self.parse) def parse_author(self, response): def extract_with_css(query): @@ -592,6 +641,9 @@ This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also the pagination links with the ``parse`` callback as we saw before. +Here we're passing callbacks to ``response.follow`` as positional arguments +to make the code shorter; it also works for ``scrapy.Request``. + The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -624,7 +676,7 @@ option when running them:: scrapy crawl quotes -o quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become -spider attributes by default. +spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes @@ -647,13 +699,12 @@ with a specific tag, building the URL based on the argument:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small a::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), } next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, self.parse) + yield response.follow(next_page, self.parse) If you pass the ``tag=humor`` argument to this spider, you'll notice that it diff --git a/docs/news.rst b/docs/news.rst index cce46599b..ff1e4ce03 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,64 @@ Release notes ============= +Scrapy 1.3.2 (2017-02-13) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`). +- Use consistent selectors for author field in tutorial (:issue:`2551`). +- Fix TLS compatibility in Twisted 17+ (:issue:`2558`) + +Scrapy 1.3.1 (2017-02-08) +------------------------- + +New features +~~~~~~~~~~~~ + +- Support ``'True'`` and ``'False'`` string values for boolean settings (:issue:`2519`); + you can now do something like ``scrapy crawl myspider -s REDIRECT_ENABLED=False``. +- Support kwargs with ``response.xpath()`` to use :ref:`XPath variables ` + and ad-hoc namespaces declarations ; + this requires at least Parsel v1.1 (:issue:`2457`). +- Add support for Python 3.6 (:issue:`2485`). +- Run tests on PyPy (warning: some tests still fail, so PyPy is not supported yet). + +Bug fixes +~~~~~~~~~ + +- Enforce ``DNS_TIMEOUT`` setting (:issue:`2496`). +- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`). +- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`). +- Fix name of generated pipeline class when using basic project template (:issue:`2466`). +- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`). +- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). +- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, + ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + +Documentation +~~~~~~~~~~~~~ + +- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4 + (:issue:`2469`). +- Clarify that passing spider arguments converts them to spider attributes + (:issue:`2483`). +- Document ``formid`` argument on ``FormRequest.from_response()`` (:issue:`2497`). +- Add .rst extension to README files (:issue:`2507`). +- Mention LevelDB cache storage backend (:issue:`2525`). +- Use ``yield`` in sample callback code (:issue:`2533`). +- Add note about HTML entities decoding with ``.re()/.re_first()`` (:issue:`1704`). +- Typos (:issue:`2512`, :issue:`2534`, :issue:`2531`). + +Cleanups +~~~~~~~~ + +- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`). +- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`). +- Remove dead code supporting old Twisted versions (:issue:`2544`). + + Scrapy 1.3.0 (2016-12-21) ------------------------- diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6636c30cb..eaeeee113 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -358,6 +358,12 @@ Opens the given URL in a browser, as your Scrapy spider would "see" it. Sometimes spiders see pages differently from regular users, so this can be used to check what the spider "sees" and confirm it's what you expect. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy view http://www.example.com/some/page.html diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3b9a5335a..0ef3fb071 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -318,10 +318,11 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with two HTTP cache storage backends: + Scrapy ships with three HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` + * :ref:`httpcache-storage-leveldb` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own storage backend. @@ -680,7 +681,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -748,7 +751,7 @@ REDIRECT_MAX_TIMES Default: ``20`` -The maximum number of redirections that will be follow for a single request. +The maximum number of redirections that will be followed for a single request. MetaRefreshMiddleware --------------------- @@ -948,8 +951,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index af469eb7b..85c73222d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -225,7 +225,8 @@ XmlItemExporter Exports Items in XML format to the specified file object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param root_element: The name of root element in the exported XML. :type root_element: str @@ -281,7 +282,8 @@ CsvItemExporter CSV columns and their order. The :attr:`export_empty_fields` attribute has no effect on this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from @@ -312,7 +314,8 @@ PickleItemExporter Exports Items in pickle format to the given file-like object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param protocol: The pickle protocol to use. :type protocol: int @@ -333,7 +336,8 @@ PprintItemExporter Exports Items in pretty print format to the specified file object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor. @@ -356,7 +360,8 @@ JsonItemExporter arguments to the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ constructor argument to customize this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) A typical output of this exporter would be:: @@ -386,7 +391,8 @@ JsonLinesItemExporter the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ constructor argument to customize this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) A typical output of this exporter would be:: diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 4636ddb18..75bdb4142 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -51,7 +51,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None) +.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True) LxmlLinkExtractor is the recommended link extractor with handy filtering options. It is implemented using lxml's robust HTMLParser. @@ -132,4 +132,13 @@ LxmlLinkExtractor :type process_value: callable + :param strip: whether to strip whitespaces from extracted attributes. + According to HTML5 standard, leading and trailing whitespaces + must be stripped from ``href`` attributes of ````, ```` + and many other elements, ``src`` attribute of ````, ``