Bump version: 1.8.0 → 1.8.1

.bumpversion.cfg

@@ -1,8 +1,7 @@
 [bumpversion]
-current_version = 1.8.0
+current_version = 1.8.1
 commit = True
 tag = True
 tag_name = {new_version}
 
 [bumpversion:file:scrapy/VERSION]
-

.github/workflows/checks.yml

@@ -0,0 +1,27 @@
+name: Checks
+on: [push, pull_request]
+
+jobs:
+  checks:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: 3.8  # Keep in sync with .readthedocs.yml
+          env:
+            TOXENV: docs
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run check
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox

.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish
+on: [push]
+
+jobs:
+  publish:
+    runs-on: ubuntu-18.04
+    if: startsWith(github.event.ref, 'refs/tags/')
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Check Tag
+      id: check-release-tag
+      run: |
+        if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
+          echo ::set-output name=release_tag::true
+        fi
+
+    - name: Publish to PyPI
+      if: steps.check-release-tag.outputs.release_tag == 'true'
+      run: |
+        pip install --upgrade setuptools wheel twine
+        python setup.py sdist bdist_wheel
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+        twine upload dist/*

.github/workflows/tests.yml

@@ -0,0 +1,81 @@
+name: Ubuntu
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: 2.7
+          env:
+            TOXENV: py27
+        - python-version: 2.7
+          env:
+            TOXENV: py27-pinned
+        - python-version: 2.7
+          env:
+            TOXENV: py27-extra-deps
+        - python-version: pypy3
+          env:
+            TOXENV: pypy
+            PYPY_VERSION: '2.7-v7.3.5'
+        - python-version: pypy3
+          env:
+            TOXENV: pypy3
+            PYPY_VERSION: '3.7-v7.3.5'
+        - python-version: 3.5
+          env:
+            TOXENV: py35
+        - python-version: 3.5
+          env:
+            TOXENV: py35-pinned
+        - python-version: 3.6
+          env:
+            TOXENV: py36
+        - python-version: 3.7
+          env:
+            TOXENV: py37
+        - python-version: 3.8
+          env:
+            TOXENV: py38
+        - python-version: 3.8
+          env:
+            TOXENV: py38-extra-deps
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system libraries
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+      run: |
+        sudo apt-get update
+        # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
+        sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        if [[ ! -z "$PYPY_VERSION" ]]; then
+          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
+          tar -jxf ${PYPY_VERSION}.tar.bz2
+          if [ "${PYPY_VERSION:4:1}" = "2" ]; then
+            pip install -U virtualenv
+            virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
+          else
+            $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
+          fi
+          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        fi
+        pip install -U tox
+        tox
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)

.readthedocs.yml

@@ -0,0 +1,16 @@
+version: 2
+formats: all
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+
+build:
+  image: latest
+
+python:
+  # For available versions, see:
+  # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
+  version: 3.8  # Keep in sync with .github/workflows/checks.yml
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

.travis.yml

@@ -1,73 +0,0 @@
-language: python
-dist: xenial
-branches:
-  only:
-    - master
-    - /^\d\.\d+$/
-    - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
-matrix:
-  include:
-    - env: TOXENV=py27
-      python: 2.7
-    - env: TOXENV=py27-pinned
-      python: 2.7
-    - env: TOXENV=py27-extra-deps
-      python: 2.7
-    - env: TOXENV=pypy
-      python: 2.7
-    - env: TOXENV=pypy3
-      python: 3.5
-    - env: TOXENV=py35
-      python: 3.5
-    - env: TOXENV=py35-pinned
-      python: 3.5
-    - env: TOXENV=py36
-      python: 3.6
-    - env: TOXENV=py37
-      python: 3.7
-    - env: TOXENV=py38
-      python: 3.8
-    - env: TOXENV=py38-extra-deps
-      python: 3.8
-    - env: TOXENV=docs
-      python: 3.6
-install:
-  - |
-      if [ "$TOXENV" = "pypy" ]; then
-        export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
-      if [ "$TOXENV" = "pypy3" ]; then
-        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
-  - pip install -U tox twine wheel codecov
-
-script: tox
-after_success:
-  - codecov
-notifications:
-  irc:
-    use_notice: true
-    skip_join: true
-    channels:
-    - irc.freenode.org#scrapy
-cache:
-  directories:
-    - $HOME/.cache/pip
-deploy:
-  provider: pypi
-  distributions: "sdist bdist_wheel"
-  user: scrapy
-  password:
-    secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
-  on:
-    tags: true
-    repo: scrapy/scrapy
-    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

docs/news.rst

@@ -6,6 +6,44 @@ Release notes
 .. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
           planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
 
+.. _release-1.8.1:
+
+Scrapy 1.8.1 (2021-10-05)
+-------------------------
+
+*   **Security bug fix:**
+
+    If you use
+    :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
+    (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
+    authentication, any request exposes your credentials to the request target.
+
+    To prevent unintended exposure of authentication credentials to unintended
+    domains, you must now additionally set a new, additional spider attribute,
+    ``http_auth_domain``, and point it to the specific domain to which the
+    authentication credentials must be sent.
+
+    If the ``http_auth_domain`` spider attribute is not set, the domain of the
+    first request will be considered the HTTP authentication target, and
+    authentication credentials will only be sent in requests targeting that
+    domain.
+
+    If you need to send the same HTTP authentication credentials to multiple
+    domains, you can use :func:`w3lib.http.basic_auth_header` instead to
+    set the value of the ``Authorization`` header of your requests.
+
+    If you *really* want your spider to send the same HTTP authentication
+    credentials to any domain, set the ``http_auth_domain`` spider attribute
+    to ``None``.
+
+    Finally, if you are a user of `scrapy-splash`_, know that this version of
+    Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
+    need to upgrade scrapy-splash to a greater version for it to continue to
+    work.
+
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+
+
 .. _release-1.8.0:
 
 Scrapy 1.8.0 (2019-10-28)

docs/topics/downloader-middleware.rst

@@ -315,8 +315,21 @@ HttpAuthMiddleware
     This middleware authenticates all requests generated from certain spiders
     using `Basic access authentication`_ (aka. HTTP auth).
 
-    To enable HTTP authentication from certain spiders, set the ``http_user``
+    To enable HTTP authentication for a spider, set the ``http_user`` and
-    and ``http_pass`` attributes of those spiders.
+    ``http_pass`` spider attributes to the authentication data and the
+    ``http_auth_domain`` spider attribute to the domain which requires this
+    authentication (its subdomains will be also handled in the same way).
+    You can set ``http_auth_domain`` to ``None`` to enable the
+    authentication for all requests but you risk leaking your authentication
+    credentials to unrelated domains.
+
+    .. warning::
+        In previous Scrapy versions HttpAuthMiddleware sent the authentication
+        data with all requests, which is a security problem if the spider
+        makes requests to several different domains. Currently if the
+        ``http_auth_domain`` attribute is not set, the middleware will use the
+        domain of the first request, which will work for some spiders but not
+        for others. In the future the middleware will produce an error instead.
 
     Example::
 
@@ -326,6 +339,7 @@ HttpAuthMiddleware
 
             http_user = 'someuser'
             http_pass = 'somepass'
+            http_auth_domain = 'intranet.example.com'
             name = 'intranet.example.com'
 
             # .. rest of the spider code omitted ...
@@ -828,7 +842,9 @@ REDIRECT_MAX_TIMES
 
 Default: ``20``
 
-The maximum number of redirections that will be followed for a single request.
+Defines the maximum times a request can be redirected. After this maximum the
+request's response is returned as is. We used Firefox default value for the
+same task.
 
 MetaRefreshMiddleware
 ---------------------

docs/topics/settings.rst

@@ -1104,17 +1104,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
 DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
 this value if you're experiencing problems with insufficient blocking IO.
 
-.. setting:: REDIRECT_MAX_TIMES
-
-REDIRECT_MAX_TIMES
-------------------
-
-Default: ``20``
-
-Defines the maximum times a request can be redirected. After this maximum the
-request's response is returned as is. We used Firefox default value for the
-same task.
-
 .. setting:: REDIRECT_PRIORITY_ADJUST
 
 REDIRECT_PRIORITY_ADJUST
@@ -1396,17 +1385,6 @@ Default: ``True``
 A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
 will be enabled (provided its extension is also enabled).
 
-.. setting:: TELNETCONSOLE_PORT
-
-TELNETCONSOLE_PORT
-------------------
-
-Default: ``[6023, 6073]``
-
-The port range to use for the telnet console. If set to ``None`` or ``0``, a
-dynamically assigned port is used. For more info see
-:ref:`topics-telnetconsole`.
-
 .. setting:: TEMPLATES_DIR
 
 TEMPLATES_DIR

requirements-py2.txt

@@ -3,9 +3,10 @@ PyDispatcher>=2.0.5
 w3lib>=1.17.0
 protego>=0.1.15
 
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+pyOpenSSL>=16.2.0       # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+queuelib>=1.4.2,<1.6.0  # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
+                        # Greater versions implement peek support, which breaks tests.
+cryptography>=2.0       # Earlier versions would fail to install
 
 # Reference versions taken from
 # https://packages.ubuntu.com/xenial/python/
@@ -14,5 +15,5 @@ cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
-Twisted>=16.0.0
+Twisted>=16.0.0,<21.2.0  # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020
-zope.interface>=4.1.3
+zope.interface>=4.1.3,<5  # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489

requirements-py3.txt

@@ -1,12 +1,13 @@
 parsel>=1.5.0
 PyDispatcher>=2.0.5
-Twisted>=17.9.0
+Twisted>=17.9.0,<21.2.0  # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020
 w3lib>=1.17.0
 protego>=0.1.15
 
-pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+pyOpenSSL>=16.2.0       # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+queuelib>=1.4.2,<1.6.0  # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0  # Earlier versions would fail to install
+                        # Greater versions implement peek support, which breaks tests.
+cryptography>=2.0       # Earlier versions would fail to install
 
 # Reference versions taken from
 # https://packages.ubuntu.com/xenial/python/
@@ -15,4 +16,4 @@ cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
 six>=1.10.0
-zope.interface>=4.1.3
+zope.interface>=4.1.3,<5  # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489

scrapy/VERSION

@@ -1 +1 @@
-1.8.0
+1.8.1

scrapy/downloadermiddlewares/httpauth.py

@@ -3,10 +3,14 @@ HTTP basic auth downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+import warnings
 
 from w3lib.http import basic_auth_header
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.url import url_is_from_any_domain
 
 
 class HttpAuthMiddleware(object):
@@ -24,8 +28,23 @@ class HttpAuthMiddleware(object):
         pwd = getattr(spider, 'http_pass', '')
         if usr or pwd:
             self.auth = basic_auth_header(usr, pwd)
+            if not hasattr(spider, 'http_auth_domain'):
+                warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
+                              'problems if the spider makes requests to several different domains. http_auth_domain '
+                              'will be set to the domain of the first request, please set it to the correct value '
+                              'explicitly.',
+                              category=ScrapyDeprecationWarning)
+                self.domain_unset = True
+            else:
+                self.domain = spider.http_auth_domain
+                self.domain_unset = False
 
     def process_request(self, request, spider):
         auth = getattr(self, 'auth', None)
         if auth and b'Authorization' not in request.headers:
-            request.headers[b'Authorization'] = auth
+            domain = urlparse_cached(request).hostname
+            if self.domain_unset:
+                self.domain = domain
+                self.domain_unset = False
+            if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
+                request.headers[b'Authorization'] = auth

tests/test_downloadermiddleware_httpauth.py

@@ -1,13 +1,60 @@
 import unittest
 
+from w3lib.http import basic_auth_header
+
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spiders import Spider
 
 
+class TestSpiderLegacy(Spider):
+    http_user = 'foo'
+    http_pass = 'bar'
+
+
 class TestSpider(Spider):
     http_user = 'foo'
     http_pass = 'bar'
+    http_auth_domain = 'example.com'
+
+
+class TestSpiderAny(Spider):
+    http_user = 'foo'
+    http_pass = 'bar'
+    http_auth_domain = None
+
+
+class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
+
+    def setUp(self):
+        self.spider = TestSpiderLegacy('foo')
+
+    def test_auth(self):
+        mw = HttpAuthMiddleware()
+        mw.spider_opened(self.spider)
+
+        # initial request, sets the domain and sends the header
+        req = Request('http://example.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+        # subsequent request to the same domain, should send the header
+        req = Request('http://example.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+        # subsequent request to a different domain, shouldn't send the header
+        req = Request('http://example-noauth.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertNotIn('Authorization', req.headers)
+
+    def test_auth_already_set(self):
+        mw = HttpAuthMiddleware()
+        mw.spider_opened(self.spider)
+        req = Request('http://example.com/',
+                      headers=dict(Authorization='Digest 123'))
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
 
 
 class HttpAuthMiddlewareTest(unittest.TestCase):
@@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
     def tearDown(self):
         del self.mw
 
-    def test_auth(self):
+    def test_no_auth(self):
-        req = Request('http://scrapytest.org/')
+        req = Request('http://example-noauth.com/')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
+        self.assertNotIn('Authorization', req.headers)
+
+    def test_auth_domain(self):
+        req = Request('http://example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+    def test_auth_subdomain(self):
+        req = Request('http://foo.example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
 
     def test_auth_already_set(self):
-        req = Request('http://scrapytest.org/',
+        req = Request('http://example.com/',
+                      headers=dict(Authorization='Digest 123'))
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
+
+
+class HttpAuthAnyMiddlewareTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mw = HttpAuthMiddleware()
+        self.spider = TestSpiderAny('foo')
+        self.mw.spider_opened(self.spider)
+
+    def tearDown(self):
+        del self.mw
+
+    def test_auth(self):
+        req = Request('http://example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+    def test_auth_already_set(self):
+        req = Request('http://example.com/',
                       headers=dict(Authorization='Digest 123'))
         assert self.mw.process_request(req, self.spider) is None
         self.assertEqual(req.headers['Authorization'], b'Digest 123')

tests/test_utils_python.py

@@ -5,6 +5,7 @@ import unittest
 from itertools import count
 import platform
 import six
+from sys import version_info
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
@@ -234,10 +235,12 @@ class UtilsPythonTestCase(unittest.TestCase):
             stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
                 get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself), ['obj'])
-
+            if version_info < (3, 6):
+                self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+            else:
+                self.assertEqual(get_func_args(" ".join, stripself), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])