RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into undent-rst-doctests

This commit is contained in:
Adrián Chaves 2019-12-19 12:39:17 +01:00 committed by GitHub
parent 83b8046fdc c841a1f3e7
commit b4a2d985a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
154 changed files with 1075 additions and 783 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.bandit.yml
View File

@ -1,13 +1,15 @@
skips:
- B101
- B105
- B301
- B303
- B306
- B307
- B311
- B320
- B321
- B402
- B402 # https://github.com/scrapy/scrapy/issues/4180
- B403
- B404
- B406
- B410

7
.readthedocs.yml Normal file
View File

@ -0,0 +1,7 @@
version: 2
sphinx:
configuration: docs/conf.py
python:
version: 3.8
install:
- requirements: docs/requirements.txt

7
.travis.yml
View File

@ -12,10 +12,9 @@ matrix:
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
python: 3.5
- env: TOXENV=py35
python: 3.5
- env: TOXENV=py35-pinned
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py36
python: 3.6
@ -23,10 +22,10 @@ matrix:
python: 3.7
- env: TOXENV=py38
python: 3.8
- env: TOXENV=py38-extra-deps
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=docs
python: 3.6
python: 3.8
install:
- |
if [ "$TOXENV" = "pypy3" ]; then

9
conftest.py
View File

@ -1,12 +1,19 @@
from pathlib import Path
import pytest
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess")
]
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':

281
docs/_tests/quotes.html Normal file
View File

@ -0,0 +1,281 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Quotes to Scrape</title>
<link rel="stylesheet" href="/static/bootstrap.min.css">
<link rel="stylesheet" href="/static/main.css">
</head>
<body>
<div class="container">
<div class="row header-box">
<div class="col-md-8">
<h1>
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
</h1>
</div>
<div class="col-md-4">
<p>
<a href="/login">Login</a>
</p>
</div>
</div>
<div class="row">
<div class="col-md-8">
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
<a class="tag" href="/tag/change/page/1/">change</a>
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
<a class="tag" href="/tag/world/page/1/">world</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
<a href="/author/J-K-Rowling">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
<a class="tag" href="/tag/choices/page/1/">choices</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/live/page/1/">live</a>
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
<span>by <small class="author" itemprop="author">Jane Austen</small>
<a href="/author/Jane-Austen">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
<a class="tag" href="/tag/books/page/1/">books</a>
<a class="tag" href="/tag/classic/page/1/">classic</a>
<a class="tag" href="/tag/humor/page/1/">humor</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
<a href="/author/Marilyn-Monroe">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
<a class="tag" href="/tag/success/page/1/">success</a>
<a class="tag" href="/tag/value/page/1/">value</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
<span>by <small class="author" itemprop="author">André Gide</small>
<a href="/author/Andre-Gide">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="life,love" / >
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/love/page/1/">love</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
<a href="/author/Thomas-A-Edison">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
<a class="tag" href="/tag/edison/page/1/">edison</a>
<a class="tag" href="/tag/failure/page/1/">failure</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
<a href="/author/Eleanor-Roosevelt">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
<span>by <small class="author" itemprop="author">Steve Martin</small>
<a href="/author/Steve-Martin">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
<a class="tag" href="/tag/humor/page/1/">humor</a>
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
<a class="tag" href="/tag/simile/page/1/">simile</a>
</div>
</div>
<nav>
<ul class="pager">
<li class="next">
<a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
</li>
</ul>
</nav>
</div>
<div class="col-md-4 tags-box">
<h2>Top Ten tags</h2>
<span class="tag-item">
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
</span>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="text-muted">
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
</p>
</div>
</footer>
</body>
</html>

9
docs/conf.py
View File

@ -12,6 +12,7 @@
# serve to show the default.
import sys
from datetime import datetime
from os import path
# If your extensions are in another directory, add it here. If the directory
@ -49,8 +50,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'Scrapy'
copyright = u'2008–2018, Scrapy developers'
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -194,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', u'Scrapy Documentation',
u'Scrapy developers', 'manual'),
('index', 'Scrapy.tex', 'Scrapy Documentation',
'Scrapy developers', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of

14
docs/contributing.rst
View File

@ -203,17 +203,9 @@ Tests are implemented using the :doc:`Twisted unit-testing framework
Running tests
-------------
Make sure you have a recent enough :doc:`tox <tox:index>` installation:
To run all tests::
``tox --version``
If your version is older than 1.7.0, please update it first:
``pip install -U tox``
To run all tests go to the root directory of Scrapy source code and run:
``tox``
tox
To run a specific test (say ``tests/test_loader.py``) use:
@ -229,7 +221,7 @@ You can also specify a comma-separated list of environmets, and use :ref:`tox’
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py27,py36 -p auto
tox -e py36,py38 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the

1
docs/requirements.txt
View File

@ -1,4 +1,3 @@
-r ../requirements-py3.txt
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page

36
docs/topics/developer-tools.rst
View File

@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things:
.. _topics-inspector:
Inspecting a website
===================================
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can
expand and collapse a tag by clicking on the arrow in front of it or by double
clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
copy XPaths to selected elements. Let's try it out: Right-click on the ``span``
tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
copy XPaths to selected elements. Let's try it out.
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
(...)
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
Then, back to your web browser, right-click on the ``span`` tag, select
``Copy > XPath`` and paste it in the Scrapy shell like so:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@ -115,10 +125,10 @@ we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the scrapy shell::
we'll check another quite useful command from the scrapy shell:
.. skip: next
.. code-block:: none
$ scrapy shell "quotes.toscrape.com/scroll"
(...)

6
docs/topics/spiders.rst
View File

@ -414,6 +414,12 @@ Crawling rules
from which the request originated as second argument. It must return a
``Request`` object or ``None`` (to filter out the request).
``errback`` is a callable or a string (in which case a method from the spider
object with that name will be used) to be called if any exception is
raised while processing a request generated by the rule.
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~

1
extras/qps-bench-server.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python
from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET

91
pytest.ini
View File

@ -8,7 +8,6 @@ addopts =
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
--ignore=docs/topics/developer-tools.rst
--ignore=docs/topics/dynamic-content.rst
--ignore=docs/topics/items.rst
--ignore=docs/topics/leaks.rst
@ -26,52 +25,52 @@ flake8-ignore =
scrapy/http/__init__.py F401
# Issues pending a review:
# extras
extras/qps-bench-server.py E261 E501
extras/qpsclient.py E501 E261 E501
extras/qps-bench-server.py E501
extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E502 E731
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731 E226
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/startproject.py E502 E127 E501 E128
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E502 E128
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
scrapy/core/engine.py E501 E128 E127 E306 E502
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E306 E261 E128 W504
scrapy/core/spidermw.py E501 E731 E502 E126 E226
scrapy/core/scraper.py E501 E306 E128 W504
scrapy/core/spidermw.py E501 E731 E126 E226
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E305 E241
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E502 E128
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
scrapy/extensions/closespider.py E501 E502 E128 E123
scrapy/extensions/closespider.py E501 E128 E123
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501 E303
@ -85,13 +84,13 @@ flake8-ignore =
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128 W293 W291
scrapy/http/response/text.py E501 W293 E128 E124
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E502 E501 E402
scrapy/linkextractors/__init__.py E731 E501 E402
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
# scrapy/loader
scrapy/loader/__init__.py E501 E502 E128
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/files.py E116 E501 E266
@ -102,7 +101,7 @@ flake8-ignore =
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
scrapy/settings/default_settings.py E501 E114 E116 E226
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
@ -112,25 +111,24 @@ flake8-ignore =
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
scrapy/spiders/feed.py E501 E261
scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/benchserver.py E501
scrapy/utils/conf.py E402 E502 E501
scrapy/utils/console.py E261 E306 E305
scrapy/utils/conf.py E402 E501
scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/engine.py E261
scrapy/utils/gz.py E305 E501 W504
scrapy/utils/http.py F403 E226
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/log.py E128 W503
scrapy/utils/markup.py F403 W292
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501 E226
scrapy/utils/multipart.py F403 W292
scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E226
@ -145,18 +143,17 @@ flake8-ignore =
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/_monkeypatches.py W293
scrapy/cmdline.py E502 E501
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501 E261 E226
scrapy/exporters.py E501 E226
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501 W293
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E502 E128 E501
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/responsetypes.py E128 E501 E305
scrapy/robotstxt.py E501
@ -171,15 +168,15 @@ flake8-ignore =
tests/pipelines.py F841 E226
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_fetch.py E501 E261
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128 W293
tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py F841 E501 E305
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
@ -190,18 +187,18 @@ flake8-ignore =
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
tests/test_engine.py E401 E501 E502 E128 E261
tests/test_dupefilters.py E221 E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
tests/test_http_response.py E501 E301 E502 E128 E265
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E301 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
@ -210,28 +207,28 @@ flake8-ignore =
tests/test_mail.py E128 E501 E305
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
tests/test_pipeline_files.py E501 W293 E303 E272 E226
tests/test_pipeline_files.py E501 E303 E272 E226
tests/test_pipeline_images.py F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E305
tests/test_robotstxt_interface.py E501 W291 E501
tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
tests/test_spidermiddleware_output_chain.py E501 W293 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
tests/test_utils_defer.py E306 E261 E501 F841 E226
tests/test_utils_defer.py E306 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
tests/test_utils_http.py E501 E502 E128 W504
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E303 E241
tests/test_utils_log.py E741 E226
tests/test_utils_python.py E501 E303 E731 E701 E305
@ -240,11 +237,11 @@ flake8-ignore =
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731 E226
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_spider.py E261 E305
tests/test_utils_spider.py E305
tests/test_utils_template.py E305
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
tests/test_cmdline/__init__.py E502 E501
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501

18
requirements-py3.txt
View File

@ -1,18 +0,0 @@
parsel>=1.5.0
PyDispatcher>=2.0.5
Twisted>=17.9.0
w3lib>=1.17.0
protego>=0.1.15
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
cryptography>=2.0 # Earlier versions would fail to install
# Reference versions taken from
# https://packages.ubuntu.com/xenial/python/
# https://packages.ubuntu.com/xenial/zope/
cssselect>=0.9.1
lxml>=3.5.0
service_identity>=16.0.0
six>=1.10.0
zope.interface>=4.1.3

2
scrapy/_monkeypatches.py
View File

@ -1,4 +1,4 @@
from six.moves import copyreg
import copyreg
# Undo what Twisted's perspective broker adds to pickle register

5
scrapy/cmdline.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import sys
import os
import optparse
@ -68,7 +67,7 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
@ -124,7 +123,7 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)

3
scrapy/commands/bench.py
View File

@ -1,8 +1,7 @@
import sys
import time
import subprocess
from six.moves.urllib.parse import urlencode
from urllib.parse import urlencode
import scrapy
from scrapy.commands import ScrapyCommand

12
scrapy/commands/fetch.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import sys
from w3lib.url import is_url
@ -25,12 +24,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true", \
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():

1
scrapy/commands/genspider.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import os
import shutil
import string

1
scrapy/commands/list.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
from scrapy.commands import ScrapyCommand

1
scrapy/commands/parse.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import json
import logging

1
scrapy/commands/settings.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import json
from scrapy.commands import ScrapyCommand

5
scrapy/commands/startproject.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import re
import os
import string
@ -44,8 +43,8 @@ class Command(ScrapyCommand):
return False
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print('Error: Project names must begin with a letter and contain'\
' only\nletters, numbers and underscores')
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
else:

2
scrapy/commands/version.py
View File

@ -1,5 +1,3 @@
from __future__ import print_function
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions

4
scrapy/contracts/default.py
View File

@ -86,8 +86,8 @@ class ReturnsContract(Contract):
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
raise ContractFail("Returned %s %s, expected %s" % \
(occurrences, self.obj_name, expected))
raise ContractFail("Returned %s %s, expected %s" %
(occurrences, self.obj_name, expected))
class ScrapesContract(Contract):

3
scrapy/core/downloader/__init__.py
View File

@ -3,7 +3,6 @@ from time import time
from datetime import datetime
from collections import deque
import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
@ -188,7 +187,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
for slot in six.itervalues(self.slots):
for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):

5
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,8 +1,9 @@
"""Download handlers for different schemes"""
import logging
from twisted.internet import defer
import six
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
@ -22,7 +23,7 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
for scheme, clspath in six.iteritems(handlers):
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)

2
scrapy/core/downloader/handlers/ftp.py
View File

@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
from six.moves.urllib.parse import unquote
from urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed

11
scrapy/core/downloader/handlers/http11.py
View File

@ -2,10 +2,10 @@
import re
import logging
import warnings
from io import BytesIO
from time import time
import warnings
from six.moves.urllib.parse import urldefrag
from urllib.parse import urldefrag
from zope.interface import implementer
from twisted.internet import defer, reactor, protocol
@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
@ -285,6 +286,12 @@ class ScrapyAgent(object):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Crawlera, it doesn't require this mode anymore, "
"so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.",
ScrapyDeprecationWarning)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)

6
scrapy/core/downloader/handlers/s3.py
View File

@ -1,4 +1,4 @@
from six.moves.urllib.parse import unquote
from urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
@ -32,8 +32,8 @@ def _get_boto_connection():
class S3DownloadHandler(object):
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
httpdownloadhandler=HTTPDownloadHandler, **kw):
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']

8
scrapy/core/downloader/middleware.py
View File

@ -3,8 +3,6 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
import six
from twisted.internet import defer
from scrapy.exceptions import _InvalidOutput
@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
(six.get_method_self(method).__class__.__name__, response.__class__.__name__))
(method.__self__.__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, response=response, spider=spider)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
(six.get_method_self(method).__class__.__name__, type(response)))
(method.__self__.__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, exception=exception, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
(six.get_method_self(method).__class__.__name__, type(response)))
(method.__self__.__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)

10
scrapy/core/downloader/webclient.py
View File

@ -1,5 +1,5 @@
from time import time
from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
def connectionMade(self):
self.headers = Headers() # bucket for response headers
self.headers = Headers() # bucket for response headers
# Method command
self.sendCommand(self.factory.method, self.factory.path)
@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
if self.factory.url.startswith(b'https'):
self.transport.stopProducing()
self.factory.noPage(\
defer.TimeoutError("Getting %s took longer than %s seconds." % \
(self.factory.url, self.factory.timeout)))
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds." %
(self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):

2
scrapy/core/engine.py
View File

@ -25,7 +25,7 @@ class Slot(object):
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
self.inprogress = set() # requests in progress
self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall

2
scrapy/core/scraper.py
View File

@ -123,7 +123,7 @@ class Scraper(object):
callback/errback"""
assert isinstance(response, (Response, Failure))
dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
return dfd

11
scrapy/core/spidermw.py
View File

@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
"""
from itertools import chain, islice
import six
from twisted.python.failure import Failure
from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
@ -37,15 +36,15 @@ class SpiderMiddlewareManager(MiddlewareManager):
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f: '%s.%s' % (
six.get_method_self(f).__class__.__name__,
six.get_method_function(f).__name__)
f.__self__.__class__.__name__,
f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
if result is not None:
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
.format(fname(method), type(result)))
except _InvalidOutput:
raise
@ -70,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
.format(fname(method), type(result)))
return _failure
@ -104,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = evaluate_iterable(result, method_index)
else:
raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
raise _InvalidOutput('Middleware {} must return an iterable, got {}'
.format(fname(method), type(result)))
return chain(result, recovered)

8
scrapy/crawler.py
View File

@ -1,7 +1,6 @@
import pprint
import six
import signal
import logging
import pprint
import signal
import warnings
from twisted.internet import reactor, defer
@ -22,6 +21,7 @@ from scrapy.utils.log import (
get_scrapy_root_handler, install_scrapy_root_handler)
from scrapy import signals
logger = logging.getLogger(__name__)
@ -206,7 +206,7 @@ class CrawlerRunner(object):
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
if isinstance(spidercls, six.string_types):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)

4
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import re
import logging
import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@ -67,7 +65,7 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
def _has_ajaxcrawlable_meta(text):

9
scrapy/downloadermiddlewares/cookies.py
View File

@ -1,13 +1,12 @@
import logging
from collections import defaultdict
import six
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@ -82,8 +81,10 @@ class CookiesMiddleware(object):
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
cookie_list = [{'name': k, 'value': v} for k, v in \
six.iteritems(request.cookies)]
cookie_list = [
{'name': k, 'value': v}
for k, v in request.cookies.items()
]
else:
cookie_list = request.cookies

11
scrapy/downloadermiddlewares/decompression.py
View File

@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
from io import BytesIO
import zipfile
import tarfile
import logging
import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
import six
from scrapy.responsetypes import responsetypes
logger = logging.getLogger(__name__)
@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
if not response.body:
return response
for fmt, func in six.iteritems(self._formats):
for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',

5
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
respcls = responsetypes.from_args(headers=response.headers, \
url=response.url, body=decoded_body)
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the

6
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -1,8 +1,6 @@
import base64
from urllib.request import _parse_proxy
from six.moves.urllib.parse import unquote, urlunparse
from six.moves.urllib.request import getproxies, proxy_bypass
from urllib.parse import unquote, urlunparse
from urllib.request import getproxies, proxy_bypass, _parse_proxy
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached

3
scrapy/downloadermiddlewares/redirect.py
View File

@ -1,5 +1,5 @@
import logging
from six.moves.urllib.parse import urljoin, urlparse
from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)

1
scrapy/dupefilters.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import os
import logging

17
scrapy/exporters.py
View File

@ -6,15 +6,14 @@ import csv
import io
import pprint
import marshal
import six
from six.moves import cPickle as pickle
import warnings
import pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@ -60,9 +59,9 @@ class BaseItemExporter(object):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
field_iter = six.iterkeys(item.fields)
field_iter = item.fields.keys()
else:
field_iter = six.iterkeys(item)
field_iter = item.keys()
else:
if include_empty:
field_iter = self.fields_to_export
@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter):
for value in serialized_value:
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
elif isinstance(serialized_value, str):
self.xg.characters(serialized_value)
else:
self.xg.characters(str(serialized_value))
@ -200,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
)
self.csv_writer = csv.writer(self.stream, **kwargs)
self._headers_not_written = True
@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (six.text_type, bytes)):
if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
def _serialize_dict(self, value):
for key, val in six.iteritems(value):
for key, val in value.items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)

6
scrapy/extensions/closespider.py
View File

@ -54,9 +54,9 @@ class CloseSpider(object):
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
def spider_opened(self, spider):
self.task = reactor.callLater(self.close_on['timeout'], \
self.crawler.engine.close_spider, spider, \
reason='closespider_timeout')
self.task = reactor.callLater(self.close_on['timeout'],
self.crawler.engine.close_spider, spider,
reason='closespider_timeout')
def item_scraped(self, item, spider):
self.counter['itemcount'] += 1

2
scrapy/extensions/feedexport.py
View File

@ -10,7 +10,7 @@ import logging
import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
from six.moves.urllib.parse import urlparse, unquote
from urllib.parse import urlparse, unquote
from ftplib import FTP
from zope.interface import Interface, implementer

4
scrapy/extensions/httpcache.py
View File

@ -1,14 +1,12 @@
from __future__ import print_function
import gzip
import logging
import os
import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from time import time
from weakref import WeakKeyDictionary
from six.moves import cPickle as pickle
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response

3
scrapy/extensions/memdebug.py
View File

@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
"""
import gc
import six
from scrapy import signals
from scrapy.exceptions import NotConfigured
@ -28,7 +27,7 @@ class MemoryDebugger(object):
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
for cls, wdict in six.iteritems(live_refs):
for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)

2
scrapy/extensions/spiderstate.py
View File

@ -1,5 +1,5 @@
import os
from six.moves import cPickle as pickle
import pickle
from scrapy import signals
from scrapy.exceptions import NotConfigured

5
scrapy/http/cookies.py
View File

@ -1,7 +1,6 @@
import time
from six.moves.http_cookiejar import (
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
)
from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode

10
scrapy/http/headers.py
View File

@ -1,4 +1,3 @@
import six
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@ -19,7 +18,7 @@ class Headers(CaselessDict):
"""Normalize values to bytes"""
if value is None:
value = []
elif isinstance(value, (six.text_type, bytes)):
elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
@ -29,10 +28,10 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
elif isinstance(x, six.text_type):
elif isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
return six.text_type(x).encode(self.encoding)
return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
@ -68,9 +67,6 @@ class Headers(CaselessDict):
self[key] = lst
def items(self):
return list(self.iteritems())
def iteritems(self):
return ((k, self.getlist(k)) for k in self.keys())
def values(self):

3
scrapy/http/request/__init__.py
View File

@ -4,7 +4,6 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
import six
from w3lib.url import safe_url_string
from scrapy.http.headers import Headers
@ -60,7 +59,7 @@ class Request(object_ref):
return self._url
def _set_url(self, url):
if not isinstance(url, six.string_types):
if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
s = safe_url_string(url, self.encoding)

5
scrapy/http/request/form.py
View File

@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
import six
from six.moves.urllib.parse import urljoin, urlencode
from urllib.parse import urljoin, urlencode
import lxml.html
from parsel.selector import create_root_node
@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + \
u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')

2
scrapy/http/request/rpc.py
View File

@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
from six.moves import xmlrpc_client as xmlrpclib
import xmlrpc.client as xmlrpclib
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args

6
scrapy/http/response/__init__.py
View File

@ -4,7 +4,7 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
from scrapy.http.request import Request
from scrapy.http.headers import Headers
@ -113,8 +113,8 @@ class Response(object_ref):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""

13
scrapy/http/response/text.py
View File

@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
import six
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
@ -31,14 +30,14 @@ class TextResponse(Response):
super(TextResponse, self).__init__(*args, **kwargs)
def _set_url(self, url):
if isinstance(url, six.text_type):
if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
if isinstance(body, six.text_type):
if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
@ -126,7 +125,7 @@ class TextResponse(Response):
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
@ -134,7 +133,7 @@ class TextResponse(Response):
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` or ``<link>`` element, e.g.
``response.css('a.my_link')[0]``.
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
@ -158,7 +157,7 @@ class TextResponse(Response):
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
if isinstance(sel.root, six.string_types):
if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):

7
scrapy/item.py
View File

@ -10,8 +10,6 @@ from copy import deepcopy
from pprint import pformat
from warnings import warn
import six
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
@ -78,7 +76,7 @@ class DictItem(MutableMapping, BaseItem):
def __init__(self, *args, **kwargs):
self._values = {}
if args or kwargs: # avoid creating dict for most common case
for k, v in six.iteritems(dict(*args, **kwargs)):
for k, v in dict(*args, **kwargs).items():
self[k] = v
def __getitem__(self, key):
@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
return deepcopy(self)
@six.add_metaclass(ItemMeta)
class Item(DictItem):
class Item(DictItem, metaclass=ItemMeta):
pass

5
scrapy/linkextractors/__init__.py
View File

@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
For more info see docs/topics/link-extractors.rst
"""
import re
from urllib.parse import urlparse
from six.moves.urllib.parse import urlparse
from parsel.csstranslator import HTMLTranslator
from w3lib.url import canonicalize_url
@ -44,8 +44,7 @@ IGNORED_EXTENSIONS = [
_re_type = type(re.compile("", 0))
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \
'file', 'ftp'}
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor(object):

7
scrapy/linkextractors/htmlparser.py
View File

@ -2,9 +2,8 @@
HTMLParser-based link extractor
"""
import warnings
import six
from six.moves.html_parser import HTMLParser
from six.moves.urllib.parse import urljoin
from html.parser import HTMLParser
from urllib.parse import urljoin
from w3lib.url import safe_url_string
from w3lib.html import strip_html5_whitespace
@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser):
ret = []
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in links:
if isinstance(link.url, six.text_type):
if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)

5
scrapy/linkextractors/lxmlhtml.py
View File

@ -1,8 +1,7 @@
"""
Link extractor based on lxml.html
"""
import six
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()")
def _nons(tag):
if isinstance(tag, six.string_types):
if isinstance(tag, str):
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
return tag.split('}')[-1]
return tag

3
scrapy/linkextractors/regex.py
View File

@ -1,11 +1,12 @@
import re
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
from scrapy.link import Link
from scrapy.linkextractors.sgml import SgmlLinkExtractor
linkre = re.compile(
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
re.DOTALL | re.IGNORECASE)

5
scrapy/linkextractors/sgml.py
View File

@ -1,9 +1,8 @@
"""
SGMLParser-based Link extractors
"""
import six
from six.moves.urllib.parse import urljoin
import warnings
from urllib.parse import urljoin
from sgmllib import SGMLParser
from w3lib.url import safe_url_string, canonicalize_url
@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
if isinstance(link.url, six.text_type):
if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)

4
scrapy/logformatter.py
View File

@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'"
class LogFormatter(object):
"""Class for generating log messages for different actions.
All methods must return a dictionary listing the parameters ``level``, ``msg``
and ``args`` which are going to be used for constructing the log message when
calling ``logging.log``.
@ -48,7 +48,7 @@ class LogFormatter(object):
}
}
"""
def crawled(self, request, response, spider):
"""Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else ''

17
scrapy/mail.py
View File

@ -3,21 +3,21 @@ Mail sending helpers
See documentation in docs/topics/email.rst
"""
from io import BytesIO
import logging
from email.utils import COMMASPACE, formatdate
from six.moves.email_mime_multipart import MIMEMultipart
from six.moves.email_mime_text import MIMEText
from six.moves.email_mime_base import MIMEBase
from email.mime.nonmultipart import MIMENonMultipart
from email import encoders as Encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.nonmultipart import MIMENonMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
from io import BytesIO
from twisted.internet import defer, reactor, ssl
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
logger = logging.getLogger(__name__)
@ -73,8 +73,7 @@ class MailSender(object):
part = MIMEBase(*mimetype.split('/'))
part.set_payload(f.read())
Encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="%s"' \
% attach_name)
part.add_header('Content-Disposition', 'attachment', filename=attach_name)
msg.attach(part)
else:
msg.set_payload(body)

4
scrapy/middleware.py
View File

@ -65,8 +65,8 @@ class MiddlewareManager(object):
return process_chain(self.methods[methodname], obj, *args)
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
return process_chain_both(self.methods[cb_methodname], \
self.methods[eb_methodname], obj, *args)
return process_chain_both(self.methods[cb_methodname],
self.methods[eb_methodname], obj, *args)
def open_spider(self, spider):
return self._process_parallel('open_spider', spider)

19
scrapy/pipelines/files.py
View File

@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
from io import BytesIO
import logging
import mimetypes
import os
import os.path
import time
import logging
from email.utils import parsedate_tz, mktime_tz
from six.moves.urllib.parse import urlparse
from collections import defaultdict
import six
from email.utils import parsedate_tz, mktime_tz
from io import BytesIO
from urllib.parse import urlparse
from twisted.internet import defer, threads
@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
logger = logging.getLogger(__name__)
@ -153,14 +152,14 @@ class S3FilesStore(object):
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
for metakey, metavalue in six.iteritems(meta):
for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
@ -201,7 +200,7 @@ class S3FilesStore(object):
'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
})
extra = {}
for key, value in six.iteritems(headers):
for key, value in headers.items():
try:
kwarg = mapping[key]
except KeyError:
@ -249,7 +248,7 @@ class GCSFilesStore(object):
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),

3
scrapy/pipelines/images.py
View File

@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
import functools
import hashlib
from io import BytesIO
import six
from PIL import Image
@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
image, buf = self.convert_image(orig_image)
yield path, image, buf
for thumb_id, size in six.iteritems(self.thumbs):
for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf

2
scrapy/pipelines/media.py
View File

@ -1,5 +1,3 @@
from __future__ import print_function
import functools
import logging
from collections import defaultdict

4
scrapy/responsetypes.py
View File

@ -2,11 +2,9 @@
This module implements a class which returns the appropriate Response class
based on different criteria.
"""
from __future__ import absolute_import
from mimetypes import MimeTypes
from pkgutil import get_data
from io import StringIO
import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
@ -37,7 +35,7 @@ class ResponseTypes(object):
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in six.iteritems(self.CLASSES):
for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):

5
scrapy/robotstxt.py
View File

@ -1,7 +1,6 @@
import sys
import logging
from abc import ABCMeta, abstractmethod
from six import with_metaclass
from scrapy.utils.python import to_unicode
@ -26,7 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
return robotstxt_body
class RobotParser(with_metaclass(ABCMeta)):
class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):
@ -56,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
from six.moves.urllib_robotparser import RobotFileParser
from urllib.robotparser import RobotFileParser
self.spider = spider
robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp = RobotFileParser()

19
scrapy/settings/__init__.py
View File

@ -1,4 +1,3 @@
import six
import json
import copy
from collections.abc import MutableMapping
@ -23,7 +22,7 @@ def get_settings_priority(priority):
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
numerical value, or directly returns a given numerical priority.
"""
if isinstance(priority, six.string_types):
if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
@ -173,7 +172,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or [])
if isinstance(value, six.string_types):
if isinstance(value, str):
value = value.split(',')
return list(value)
@ -194,7 +193,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or {})
if isinstance(value, six.string_types):
if isinstance(value, str):
value = json.loads(value)
return dict(value)
@ -284,7 +283,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
if isinstance(module, six.string_types):
if isinstance(module, str):
module = import_module(module)
for key in dir(module):
if key.isupper():
@ -313,14 +312,14 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
if isinstance(values, six.string_types):
if isinstance(values, str):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
for name, value in six.iteritems(values):
for name, value in values.items():
self.set(name, value, values.getpriority(name))
else:
for name, value in six.iteritems(values):
for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority='project'):
@ -377,7 +376,7 @@ class BaseSettings(MutableMapping):
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in six.iteritems(self)}
for k, v in self.items()}
def copy_to_dict(self):
"""
@ -445,7 +444,7 @@ class Settings(BaseSettings):
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities
for name, val in six.iteritems(self):
for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)

4
scrapy/settings/default_settings.py
View File

@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
# also allowing negotiation
# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
DOWNLOADER_MIDDLEWARES = {}

2
scrapy/shell.py
View File

@ -3,8 +3,6 @@
See documentation in docs/topics/shell.rst
"""
from __future__ import print_function
import os
import signal
import warnings

1
scrapy/signalmanager.py
View File

@ -1,4 +1,3 @@
from __future__ import absolute_import
from pydispatch import dispatcher
from scrapy.utils import signal as _signal

1
scrapy/spiderloader.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from collections import defaultdict
import traceback
import warnings

2
scrapy/spidermiddlewares/referer.py
View File

@ -2,8 +2,8 @@
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from six.moves.urllib.parse import urlparse
import warnings
from urllib.parse import urlparse
from w3lib.url import safe_url_string

64
scrapy/spiders/crawl.py
View File

@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst
import copy
import warnings
import six
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
from scrapy.linkextractors import LinkExtractor
@ -18,14 +16,18 @@ from scrapy.utils.python import get_func_args
from scrapy.utils.spider import iterate_spider_output
def _identity(request, response):
def _identity(x):
return x
def _identity_process_request(request, response):
return request
def _get_method(method, spider):
if callable(method):
return method
elif isinstance(method, six.string_types):
elif isinstance(method, str):
return getattr(spider, method, None)
@ -34,17 +36,20 @@ _default_link_extractor = LinkExtractor()
class Rule(object):
def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
process_links=None, process_request=None, errback=None):
self.link_extractor = link_extractor or _default_link_extractor
self.callback = callback
self.errback = errback
self.cb_kwargs = cb_kwargs or {}
self.process_links = process_links
self.process_request = process_request or _identity
self.process_links = process_links or _identity
self.process_request = process_request or _identity_process_request
self.process_request_argcount = None
self.follow = follow if follow is not None else not callback
def _compile(self, spider):
self.callback = _get_method(self.callback, spider)
self.errback = _get_method(self.errback, spider)
self.process_links = _get_method(self.process_links, spider)
self.process_request = _get_method(self.process_request, spider)
self.process_request_argcount = len(get_func_args(self.process_request))
@ -78,48 +83,59 @@ class CrawlSpider(Spider):
def process_results(self, response, results):
return results
def _build_request(self, rule, link):
r = Request(url=link.url, callback=self._response_downloaded)
r.meta.update(rule=rule, link_text=link.text)
return r
def _build_request(self, rule_index, link):
return Request(
url=link.url,
callback=self._callback,
errback=self._errback,
meta=dict(rule=rule_index, link_text=link.text),
)
def _requests_to_follow(self, response):
if not isinstance(response, HtmlResponse):
return
seen = set()
for n, rule in enumerate(self._rules):
for rule_index, rule in enumerate(self._rules):
links = [lnk for lnk in rule.link_extractor.extract_links(response)
if lnk not in seen]
if links and rule.process_links:
links = rule.process_links(links)
for link in links:
for link in rule.process_links(links):
seen.add(link)
request = self._build_request(n, link)
request = self._build_request(rule_index, link)
yield rule._process_request(request, response)
def _response_downloaded(self, response):
def _callback(self, response):
rule = self._rules[response.meta['rule']]
return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
def _errback(self, failure):
rule = self._rules[failure.request.meta['rule']]
return self._handle_failure(failure, rule.errback)
def _parse_response(self, response, callback, cb_kwargs, follow=True):
if callback:
cb_res = callback(response, **cb_kwargs) or ()
cb_res = self.process_results(response, cb_res)
for requests_or_item in iterate_spider_output(cb_res):
yield requests_or_item
for request_or_item in iterate_spider_output(cb_res):
yield request_or_item
if follow and self._follow_links:
for request_or_item in self._requests_to_follow(response):
yield request_or_item
def _handle_failure(self, failure, errback):
if errback:
results = errback(failure) or ()
for request_or_item in iterate_spider_output(results):
yield request_or_item
def _compile_rules(self):
self._rules = [copy.copy(r) for r in self.rules]
for rule in self._rules:
rule._compile(self)
self._rules = []
for rule in self.rules:
self._rules.append(copy.copy(rule))
self._rules[-1]._compile(self)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool(
'CRAWLSPIDER_FOLLOW_LINKS', True)
spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
return spider

4
scrapy/spiders/feed.py
View File

@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):
and the file's headers.
"""
delimiter = None # When this is None, python's csv module's default delimiter is used
quotechar = None # When this is None, python's csv module's default quotechar is used
delimiter = None # When this is None, python's csv module's default delimiter is used
quotechar = None # When this is None, python's csv module's default quotechar is used
headers = None
def process_results(self, response, results):

5
scrapy/spiders/sitemap.py
View File

@ -1,6 +1,5 @@
import re
import logging
import six
from scrapy.spiders import Spider
from scrapy.http import Request, XmlResponse
@ -22,7 +21,7 @@ class SitemapSpider(Spider):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
if isinstance(c, six.string_types):
if isinstance(c, str):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
@ -86,7 +85,7 @@ class SitemapSpider(Spider):
def regex(x):
if isinstance(x, six.string_types):
if isinstance(x, str):
return re.compile(x)
return x

2
scrapy/squeues.py
View File

@ -3,7 +3,7 @@ Scheduler queues
"""
import marshal
from six.moves import cPickle as pickle
import pickle
from queuelib import queue

3
scrapy/utils/benchserver.py
View File

@ -1,5 +1,6 @@
import random
from six.moves.urllib.parse import urlencode
from urllib.parse import urlencode
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor

2
scrapy/utils/boto.py
View File

@ -1,7 +1,5 @@
"""Boto/botocore helpers"""
from __future__ import absolute_import
from scrapy.exceptions import NotConfigured

14
scrapy/utils/conf.py
View File

@ -1,11 +1,9 @@
from configparser import ConfigParser
import os
import sys
import numbers
from configparser import ConfigParser
from operator import itemgetter
import six
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
for k, v in six.iteritems(compdict):
for k, v in compdict.items():
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
@ -33,13 +31,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
return compbs
else:
_check_components(compdict)
return {convert(k): v for k, v in six.iteritems(compdict)}
return {convert(k): v for k, v in compdict.items()}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
for name, value in six.iteritems(compdict):
for name, value in compdict.items():
if value is not None and not isinstance(value, numbers.Real):
raise ValueError('Invalid value {} for component {}, please provide ' \
raise ValueError('Invalid value {} for component {}, please provide '
'a real number or None instead'.format(value, name))
# BEGIN Backward compatibility for old (base, custom) call signature
@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
def arglist_to_dict(arglist):

8
scrapy/utils/console.py
View File

@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
def _embed_standard_shell(namespace={}, banner=''):
"""Start a standard python shell"""
import code
try: # readline module is only available on unix systems
try: # readline module is only available on unix systems
import readline
except ImportError:
pass
@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None):
"""Return the first acceptable shell-embed function
from a given list of shell names.
"""
if shells is None: # list, preference order of shells
if shells is None: # list, preference order of shells
shells = DEFAULT_PYTHON_SHELLS.keys()
if known_shells is None: # available embeddable shells
if known_shells is None: # available embeddable shells
known_shells = DEFAULT_PYTHON_SHELLS.copy()
for shell in shells:
if shell in known_shells:
@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):
shell = get_shell_embed_func(shells)
if shell is not None:
shell(namespace=namespace, banner=banner)
except SystemExit: # raised when using exit() in python code.interact
except SystemExit: # raised when using exit() in python code.interact
pass

7
scrapy/utils/curl.py
View File

@ -1,10 +1,9 @@
import argparse
import warnings
from shlex import split
from http.cookies import SimpleCookie
from urllib.parse import urlparse
from six.moves.http_cookies import SimpleCookie
from six.moves.urllib.parse import urlparse
from six import iteritems
from w3lib.http import basic_auth_header
@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
name = name.strip()
val = val.strip()
if name.title() == 'Cookie':
for name, morsel in iteritems(SimpleCookie(val)):
for name, morsel in SimpleCookie(val).items():
cookies[name] = morsel.value
else:
headers.append((name, val))

67
scrapy/utils/datatypes.py
View File

@ -5,12 +5,10 @@ Python Standard Library.
This module must not depend on any module outside the Standard Library.
"""
import copy
import collections
from collections.abc import Mapping
import copy
import warnings
import six
from collections.abc import Mapping
from scrapy.exceptions import ScrapyDeprecationWarning
@ -151,7 +149,7 @@ class MultiValueDict(dict):
self.setlistdefault(key, []).append(value)
except TypeError:
raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
for key, value in six.iteritems(kwargs):
for key, value in kwargs.items():
self.setlistdefault(key, []).append(value)
@ -238,65 +236,6 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
class MergeDict(object):
"""
A simple class for creating new "virtual" dictionaries that actually look
up values in more than one dictionary, passed in the ``__init__`` method.
If a key appears in more than one of the given dictionaries, only the
first occurrence will be used.
"""
def __init__(self, *dicts):
warnings.warn(
"scrapy.utils.datatypes.MergeDict is deprecated in favor "
"of collections.ChainMap (introduced in Python 3.3)",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.dicts = dicts
def __getitem__(self, key):
for dict_ in self.dicts:
try:
return dict_[key]
except KeyError:
pass
raise KeyError
def __copy__(self):
return self.__class__(*self.dicts)
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
def getlist(self, key):
for dict_ in self.dicts:
if key in dict_.keys():
return dict_.getlist(key)
return []
def items(self):
item_list = []
for dict_ in self.dicts:
item_list.extend(dict_.items())
return item_list
def has_key(self, key):
for dict_ in self.dicts:
if key in dict_:
return True
return False
__contains__ = has_key
def copy(self):
"""Returns a copy of this object."""
return self.__copy__()
class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.

1
scrapy/utils/display.py
View File

@ -2,7 +2,6 @@
pprint and pformat wrappers with colorization support
"""
from __future__ import print_function
import sys
from pprint import pformat as pformat_

3
scrapy/utils/httpobj.py
View File

@ -1,8 +1,7 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
import weakref
from six.moves.urllib.parse import urlparse
from urllib.parse import urlparse
_urlparse_cache = weakref.WeakKeyDictionary()

12
scrapy/utils/iterators.py
View File

@ -1,13 +1,13 @@
import re
import csv
from io import StringIO
import logging
import six
import re
from io import StringIO
from scrapy.http import TextResponse, Response
from scrapy.selector import Selector
from scrapy.utils.python import re_rsearch, to_unicode
logger = logging.getLogger(__name__)
@ -60,7 +60,7 @@ class _StreamReader(object):
self._text, self.encoding = obj.body, obj.encoding
else:
self._text, self.encoding = obj, 'utf-8'
self._is_unicode = isinstance(self._text, six.text_type)
self._is_unicode = isinstance(self._text, str)
def read(self, n=65535):
self.read = self._read_unicode if self._is_unicode else self._read_string
@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
expected_types = (Response, six.text_type, six.binary_type)
expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
return obj.text
else:
return obj.body.decode('utf-8')
elif isinstance(obj, six.text_type):
elif isinstance(obj, str):
return obj if unicode else obj.encode('utf-8')
else:
return obj.decode('utf-8') if unicode else obj

2
scrapy/utils/markup.py
View File

@ -11,4 +11,4 @@ from w3lib.html import * # noqa: F401
warnings.warn("Module `scrapy.utils.markup` is deprecated. "
"Please import from `w3lib.html` instead.",
ScrapyDeprecationWarning, stacklevel=2)
ScrapyDeprecationWarning, stacklevel=2)

7
scrapy/utils/misc.py
View File

@ -6,14 +6,13 @@ from contextlib import contextmanager
from importlib import import_module
from pkgutil import iter_modules
import six
from w3lib.html import replace_entities
from scrapy.utils.python import flatten, to_unicode
from scrapy.item import BaseItem
_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
def arg_to_iter(arg):
@ -83,7 +82,7 @@ def extract_regex(regex, text, encoding='utf-8'):
* if the regex doesn't contain any group the entire regex matching is returned
"""
if isinstance(regex, six.string_types):
if isinstance(regex, str):
regex = re.compile(regex, re.UNICODE)
try:
@ -92,7 +91,7 @@ def extract_regex(regex, text, encoding='utf-8'):
strings = regex.findall(text) # full regex or numbered groups
strings = flatten(strings)
if isinstance(text, six.text_type):
if isinstance(text, str):
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])

2
scrapy/utils/multipart.py
View File

@ -12,4 +12,4 @@ from w3lib.form import * # noqa: F401
warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
"If you're using `encode_multipart` function, please use "
"`urllib3.filepost.encode_multipart_formdata` instead",
ScrapyDeprecationWarning, stacklevel=2)
ScrapyDeprecationWarning, stacklevel=2)

2
scrapy/utils/ossignal.py
View File

@ -1,5 +1,3 @@
from __future__ import absolute_import
import signal
from twisted.internet import reactor

6
scrapy/utils/project.py
View File

@ -1,5 +1,5 @@
import os
from six.moves import cPickle as pickle
import pickle
import warnings
from importlib import import_module
@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
ENVVAR = 'SCRAPY_SETTINGS_MODULE'
DATADIR_CFG_SECTION = 'datadir'

23
scrapy/utils/python.py
View File

@ -7,7 +7,6 @@ import re
import inspect
import weakref
import errno
import six
from functools import partial, wraps
from itertools import chain
import sys
@ -65,10 +64,10 @@ def is_listlike(x):
True
>>> is_listlike((x for x in range(3)))
True
>>> is_listlike(six.moves.xrange(5))
>>> is_listlike(range(5))
True
"""
return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
def unique(list_, key=lambda x: x):
@ -87,9 +86,9 @@ def unique(list_, key=lambda x: x):
def to_unicode(text, encoding=None, errors='strict'):
"""Return the unicode representation of a bytes object ``text``. If
``text`` is already an unicode object, return it as-is."""
if isinstance(text, six.text_type):
if isinstance(text, str):
return text
if not isinstance(text, (bytes, six.text_type)):
if not isinstance(text, (bytes, str)):
raise TypeError('to_unicode must receive a bytes or str '
'object, got %s' % type(text).__name__)
if encoding is None:
@ -102,7 +101,7 @@ def to_bytes(text, encoding=None, errors='strict'):
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
if not isinstance(text, six.string_types):
if not isinstance(text, str):
raise TypeError('to_bytes must receive a str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
@ -138,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
yield (text[offset:], offset)
yield (text, 0)
if isinstance(pattern, six.string_types):
if isinstance(pattern, str):
pattern = re.compile(pattern)
for chunk, offset in _chunk_iter():
@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
return new_method
_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
@ -301,10 +300,10 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
dict or a list of tuples, like any dict ``__init__`` method supports.
"""
d = {}
for k, v in six.iteritems(dict(dct_or_tuples)):
k = k.encode(encoding) if isinstance(k, six.text_type) else k
for k, v in dict(dct_or_tuples).items():
k = k.encode(encoding) if isinstance(k, str) else k
if not keys_only:
v = v.encode(encoding) if isinstance(v, six.text_type) else v
v = v.encode(encoding) if isinstance(v, str) else v
d[k] = v
return d
@ -346,7 +345,7 @@ def without_none_values(iterable):
value ``None`` have been removed.
"""
try:
return {k: v for k, v in six.iteritems(iterable) if v is not None}
return {k: v for k, v in iterable.items() if v is not None}
except AttributeError:
return type(iterable)((v for v in iterable if v is not None))

6
scrapy/utils/reqser.py
View File

@ -1,8 +1,6 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
import six
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@ -87,12 +85,12 @@ def _mangle_private_name(obj, func, name):
def _find_method(obj, func):
if obj:
try:
func_self = six.get_method_self(func)
func_self = func.__self__
except AttributeError: # func has no __self__
pass
else:
if func_self is obj:
name = six.get_method_function(func).__name__
name = func.__func__.__name__
if _is_private_method(name):
return _mangle_private_name(obj, func, name)
return name

7
scrapy/utils/request.py
View File

@ -3,16 +3,15 @@ This module provides some useful functions for working with
scrapy.http.Request objects
"""
from __future__ import print_function
import hashlib
import weakref
from six.moves.urllib.parse import urlunparse
from urllib.parse import urlunparse
from w3lib.http import basic_auth_header
from scrapy.utils.python import to_bytes, to_unicode
from w3lib.url import canonicalize_url
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
_fingerprint_cache = weakref.WeakKeyDictionary()

3
scrapy/utils/sitemap.py
View File

@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
from urllib.parse import urljoin
import lxml.etree
from six.moves.urllib.parse import urljoin
class Sitemap(object):

5
scrapy/utils/spider.py
View File

@ -1,11 +1,10 @@
import logging
import inspect
import six
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
logger = logging.getLogger(__name__)
@ -21,7 +20,7 @@ def iter_spider_classes(module):
# singleton in scrapy.spider.spiders
from scrapy.spiders import Spider
for obj in six.itervalues(vars(module)):
for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, Spider) and \
obj.__module__ == module.__name__ and \

1
scrapy/utils/test.py
View File

@ -2,7 +2,6 @@
This module contains some assorted functions used in tests
"""
from __future__ import absolute_import
import os
from importlib import import_module

1
scrapy/utils/testproc.py
View File

@ -1,4 +1,3 @@
from __future__ import absolute_import
import sys
import os

Some files were not shown because too many files have changed in this diff Show More