diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..2748bf5fe --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,38 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.8 + env: + TOXENV: security + - python-version: 3.8 + env: + TOXENV: flake8 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.8 + env: + TOXENV: typing + - python-version: 3.7 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 28771216c..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Run test suite -on: [push, pull_request] - -jobs: - test-windows: - name: "Windows Tests" - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest] - python-version: [3.7, 3.8] - env: [TOXENV: py] - include: - - os: windows-latest - python-version: 3.6 - env: - TOXENV: windows-pinned - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Run test suite - env: ${{ matrix.env }} - run: | - pip install -U tox twine wheel codecov - tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..aec6b8696 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 000000000..51d27c405 --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,25 @@ +name: macOS +on: [push, pull_request] + +jobs: + tests: + runs-on: macos-10.15 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 000000000..89c0334e2 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,69 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 + + # pinned deps + - python-version: 3.6.12 + env: + TOXENV: pinned + - python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 + + # extras + - python-version: 3.8 + env: + TOXENV: extra-deps + - python-version: 3.8 + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 000000000..ed2e4075d --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,32 @@ +name: Windows +on: [push, pull_request] + +jobs: + tests: + runs-on: windows-latest + strategy: + matrix: + include: + - python-version: 3.6 + env: + TOXENV: windows-pinned + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 880c8772b..000000000 --- a/.travis.yml +++ /dev/null @@ -1,77 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pylint - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml - - env: TOXENV=typing - python: 3.8 - - - env: TOXENV=pinned - python: 3.6.1 - - env: TOXENV=asyncio-pinned - python: 3.6.1 - - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - dist: bionic - - - env: TOXENV=py - python: 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - dist: bionic - - - env: TOXENV=py - python: 3.7 - - - env: TOXENV=py PYPI_RELEASE_JOB=true - python: 3.8 - dist: bionic - - env: TOXENV=extra-deps - python: 3.8 - dist: bionic - - env: TOXENV=asyncio - python: 3.8 - dist: bionic -install: - - | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..652460383 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/README.rst b/README.rst index a8f2ba52b..5750e2c0f 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu + +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy @@ -34,9 +42,16 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. + +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Zyte: https://www.zyte.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ @@ -81,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 73d7ede42..bf919ce25 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -266,7 +266,6 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.python.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html .. _conda-forge: https://conda-forge.org/ diff --git a/docs/news.rst b/docs/news.rst index 0391506c4..d9fe897ad 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2428,7 +2428,7 @@ Bug fixes - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, - ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). Documentation ~~~~~~~~~~~~~ diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 55065a1a3..c3445d40e 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -101,7 +101,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -117,7 +117,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..502fd5fcd 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -63,7 +63,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('followall', domain='scrapinghub.com') + process.crawl('followall', domain='scrapy.org') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling @@ -244,7 +244,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is - `Crawlera`_ + `Zyte Smart Proxy Manager`_ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -254,5 +254,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b576fde91..c7ec2e0cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet, you may want to take a look first at this `XPath tutorial`_. .. note:: - Some of the tips are based on `this post from ScrapingHub's blog`_. + Some of the tips are based on `this post from Zyte's blog`_. .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ +.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ .. _topics-selectors-relative-xpaths: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c9154bec1..fae34fb15 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -280,11 +280,14 @@ class ScrapyAgent: proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams if omitConnectTunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + warnings.warn( + "Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Zyte Smart Proxy Manager (formerly Crawlera), " + "it doesn't require this mode anymore, so you should " + "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "from the Zyte Smart Proxy Manager URL.", + ScrapyDeprecationWarning, + ) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 68b856a88..a86c4ae4f 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -14,6 +14,6 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython brotlipy # optional for HTTP compress downloader middleware tests -zstandard # optional for HTTP compress downloader middleware tests +zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index a1451ae66..d8e900e06 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 @@ -26,9 +26,9 @@ passenv = GCS_TEST_FILE_URI GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools -download = true +download = true commands = - py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} + py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} [testenv:typing] basepython = python3 @@ -89,10 +89,6 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 -install_command = - # --use-feature=2020-resolver is required, otherwise the latest verion of - # Twisted gets installed. - pip install --use-feature=2020-resolver {opts} {packages} setenv = _SCRAPY_PINNED=true @@ -101,11 +97,9 @@ deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} - + [testenv:windows-pinned] basepython = python3 deps = @@ -114,8 +108,6 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} @@ -124,10 +116,6 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser -install_command = - # Test --use-feature=2020-resolver for the latest version of all - # dependencies. - pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = @@ -135,8 +123,6 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -install_command = - {[pinned]install_command} commands = {[testenv:asyncio]commands} setenv = {[pinned]setenv} @@ -152,8 +138,6 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 -install_command = - {[pinned]install_command} commands = {[testenv:pypy3]commands} setenv = {[pinned]setenv}