RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into flake8-max-line-length

This commit is contained in:
Eugenio Lacuesta 2020-05-15 11:11:20 -03:00
parent 25eeb77ba6 fffb0a5b6a
commit 3eeecb4273
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
108 changed files with 1134 additions and 846 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.travis.yml
View File

@ -11,23 +11,31 @@ matrix:
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pylint
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=pypy3
- python: 3.5
- env: TOXENV=py
python: 3.5
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=asyncio
python: 3.5.2
- python: 3.6
- python: 3.7
- env: PYPI_RELEASE_JOB=true
- env: TOXENV=py
python: 3.6
- env: TOXENV=py
python: 3.7
- env: TOXENV=py PYPI_RELEASE_JOB=true
python: 3.8
dist: bionic
- env: TOXENV=extra-deps
python: 3.8
dist: bionic
- env: TOXENV=asyncio
python: 3.8
dist: bionic
install:
- |
if [ "$TOXENV" = "pypy3" ]; then

2
docs/conf.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
#
# Scrapy documentation build configuration file, created by
# sphinx-quickstart on Mon Nov 24 12:02:52 2008.
#

13
docs/topics/dynamic-content.rst
View File

@ -184,6 +184,18 @@ data from it:
>>> json.loads(json_data)
{'field': 'value'}
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
For example, if the JavaScript code contains
``var data = {field: "value", secondField: "second value"};``
you can extract that data as follows:
>>> import chompjs
>>> javascript = response.css('script::text').get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration.
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
.. _chompjs: https://github.com/Nykakin/chompjs
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
.. _curl: https://curl.haxx.se/
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser

5
docs/topics/request-response.rst
View File

@ -834,11 +834,6 @@ TextResponse objects
.. automethod:: TextResponse.follow_all
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
kept for backward compatibility; please prefer ``response.text``.
HtmlResponse objects
--------------------

51
docs/topics/signals.rst
View File

@ -112,7 +112,7 @@ engine_started
Sent when the Scrapy engine has started crawling.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
.. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
depending on how the spider was started. So **don't** rely on this signal
@ -127,7 +127,7 @@ engine_stopped
Sent when the Scrapy engine is stopped (for example, when a crawling
process has finished).
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
Item signals
------------
@ -149,7 +149,7 @@ item_scraped
Sent when an item has been scraped, after it has passed all the
:ref:`topics-item-pipeline` stages (without being dropped).
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item scraped
:type item: dict or :class:`~scrapy.item.Item` object
@ -169,7 +169,7 @@ item_dropped
Sent after an item has been dropped from the :ref:`topics-item-pipeline`
when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
@ -194,7 +194,7 @@ item_error
Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
@ -220,7 +220,7 @@ spider_closed
Sent after a spider has been closed. This can be used to release per-spider
resources reserved on :signal:`spider_opened`.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.spiders.Spider` object
@ -244,7 +244,7 @@ spider_opened
reserve per-spider resources, but can be used for any task that needs to be
performed when a spider is opened.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.spiders.Spider` object
@ -268,7 +268,7 @@ spider_idle
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param spider: the spider which has gone idle
:type spider: :class:`~scrapy.spiders.Spider` object
@ -287,7 +287,7 @@ spider_error
Sent when a spider callback generates an error (i.e. raises an exception).
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
@ -310,7 +310,7 @@ request_scheduled
Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
downloaded later.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
@ -327,7 +327,7 @@ request_dropped
Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
downloaded later, is rejected by the scheduler.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
@ -343,7 +343,7 @@ request_reached_downloader
Sent when a :class:`~scrapy.http.Request` reached downloader.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached downloader
:type request: :class:`~scrapy.http.Request` object
@ -370,6 +370,29 @@ request_left_downloader
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
bytes_received
~~~~~~~~~~~~~~
.. signal:: bytes_received
.. function:: bytes_received(data, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
received for a specific request. This signal might be fired multiple
times for the same request, with partial data each time. For instance,
a possible scenario for a 25 kb response would be two signals fired
with 10 kb of data, and a final one with 5 kb of data.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type spider: :class:`bytes` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
Response signals
----------------
@ -382,7 +405,7 @@ response_received
Sent when the engine receives a new :class:`~scrapy.http.Response` from the
downloader.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param response: the response received
:type response: :class:`~scrapy.http.Response` object
@ -401,7 +424,7 @@ response_downloaded
Sent by the downloader right after a ``HTTPResponse`` is downloaded.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param response: the response downloaded
:type response: :class:`~scrapy.http.Response` object

85
docs/utils/linkfix.py
View File

@ -14,50 +14,57 @@ Author: dufferzafar
import re
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
def main():
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# For every line, fix the respective file
for line in output_lines:
match = re.match(line_re, line)
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
if match:
newfilename = match.group(1)
errortype = match.group(2)
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
# Broken links can't be fixed and
# I am not sure what do with the local ones.
if errortype.lower() in ["broken", "local"]:
print("Not Fixed: " + line)
# For every line, fix the respective file
for line in output_lines:
match = re.match(line_re, line)
if match:
newfilename = match.group(1)
errortype = match.group(2)
# Broken links can't be fixed and
# I am not sure what do with the local ones.
if errortype.lower() in ["broken", "local"]:
print("Not Fixed: " + line)
else:
# If this is a new file
if newfilename != _filename:
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4))
else:
# If this is a new file
if newfilename != _filename:
# We don't understand what the current line means!
print("Not Understood: " + line)
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4))
else:
# We don't understand what the current line means!
print("Not Understood: " + line)
if __name__ == '__main__':
main()

113
pylintrc Normal file
View File

@ -0,0 +1,113 @@
[MASTER]
persistent=no
jobs=1 # >1 hides results
[MESSAGES CONTROL]
disable=abstract-method,
anomalous-backslash-in-string,
arguments-differ,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-continuation,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bad-whitespace,
bare-except,
blacklisted-name,
broad-except,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
fixme,
function-redefined,
global-statement,
import-error,
import-outside-toplevel,
import-self,
inconsistent-return-statements,
inherit-non-class,
invalid-name,
invalid-overridden-method,
isinstance-second-argument-not-valid-type,
keyword-arg-before-vararg,
line-too-long,
logging-format-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
misplaced-comparison-constant,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-init,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-self-use,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-statement,
pointless-string-statement,
protected-access,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-branches,
too-many-format-args,
too-many-function-args,
too-many-instance-attributes,
too-many-lines,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-lambda,
unnecessary-pass,
unreachable,
unsubscriptable-object,
unused-argument,
unused-import,
unused-variable,
unused-wildcard-import,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

125
pytest.ini
View File

@ -23,125 +23,22 @@ markers =
flake8-max-line-length = 119
flake8-ignore =
W503
# Files that are only meant to provide top-level imports are expected not
# to use any of their imports:
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
# scrapy/commands
scrapy/commands/__init__.py E128
scrapy/commands/fetch.py E128
scrapy/commands/genspider.py E128
scrapy/commands/parse.py E128
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128
scrapy/commands/startproject.py E128
scrapy/commands/version.py E128
# scrapy/contracts
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E128
scrapy/core/scraper.py E128
scrapy/core/spidermw.py E126
scrapy/core/downloader/contextfactory.py E128 E126
scrapy/core/downloader/webclient.py E128 E126
scrapy/core/downloader/handlers/ftp.py E128
scrapy/core/downloader/handlers/s3.py E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/httpcache.py E126
scrapy/downloadermiddlewares/httpcompression.py E128
scrapy/downloadermiddlewares/retry.py E126
# scrapy/extensions
scrapy/extensions/closespider.py E128
scrapy/extensions/feedexport.py E128
scrapy/extensions/httpcache.py E128
# scrapy/http
scrapy/http/response/__init__.py E128
scrapy/http/response/text.py E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E402
# scrapy/loader
scrapy/loader/__init__.py E128
# scrapy/pipelines
scrapy/pipelines/files.py E116
# scrapy/selector
scrapy/__init__.py E402
scrapy/selector/__init__.py F403
scrapy/selector/unified.py E111
# scrapy/settings
scrapy/settings/default_settings.py E114 E116
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/referer.py E129
# scrapy/spiders
scrapy/spiders/__init__.py E402
# scrapy/utils
scrapy/utils/conf.py E402
scrapy/utils/defer.py E128
scrapy/utils/http.py F403
scrapy/utils/log.py E128
scrapy/utils/markup.py F403
scrapy/utils/multipart.py F403
scrapy/utils/response.py E128
scrapy/utils/signal.py E128
scrapy/utils/url.py F403 E128 F405
# scrapy
scrapy/__init__.py E402
scrapy/item.py E128
scrapy/mail.py E402 E128
scrapy/middleware.py E128
scrapy/responsetypes.py E128
scrapy/spiderloader.py F841 E126
scrapy/squeues.py E128
# tests
tests/__init__.py E402
tests/mockserver.py E126
tests/pipelines.py F841
tests/test_command_parse.py E128
tests/test_command_shell.py E128
tests/test_commands.py E128
tests/test_contracts.py E128
tests/test_crawl.py E741
tests/test_crawler.py F841
tests/test_dependencies.py F841
tests/test_downloader_handlers.py E124 E128 E126
tests/test_downloadermiddleware_cookies.py E741 E128 E126
tests/test_downloadermiddleware_httpcompression.py E126
tests/test_downloadermiddleware_httpproxy.py E128
tests/test_downloadermiddleware_redirect.py E128
tests/test_downloadermiddleware_retry.py E128 E126
tests/test_dupefilters.py E741 E128 E124
tests/test_engine.py E128
tests/test_exporters.py E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py F841
tests/test_http_request.py E402 E128 E128 E126
tests/test_http_response.py E128
tests/test_item.py E128 F841
tests/test_linkextractors.py E128 E124
tests/test_loader.py E741 E128 E117
tests/test_logformatter.py E128 E122
tests/test_mail.py E128
tests/test_middleware.py E128
tests/test_pipeline_crawl.py E128 E126
tests/test_pipeline_images.py F841
tests/test_pipeline_media.py E741 E128
tests/test_proxy_connect.py E741
tests/test_scheduler.py E126
tests/test_spidermiddleware_httperror.py E128 E121
tests/test_spidermiddleware_offsite.py E128 E111
tests/test_spidermiddleware_referer.py F841 E124 E121
tests/test_squeues.py E741
tests/test_utils_conf.py E128
tests/test_utils_datatypes.py E402
tests/test_utils_defer.py F841
tests/test_utils_deprecate.py F841
tests/test_utils_http.py E128
tests/test_utils_iterators.py E128 E129
tests/test_utils_log.py E741
tests/test_utils_reqser.py E128
tests/test_utils_request.py E128
tests/test_utils_signal.py E741 F841
tests/test_utils_sitemap.py E128 E124
tests/test_utils_url.py E126
tests/test_webclient.py E128 E122 E402 E126
tests/test_settings/__init__.py E128
tests/test_spiderloader/__init__.py E128
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
tests/test_webclient.py E402

12
scrapy/commands/__init__.py
View File

@ -59,17 +59,17 @@ class ScrapyCommand:
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
group.add_option("--nolog", action="store_true",
help="disable logging completely")
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE",
help="write process ID to FILE")
help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)

4
scrapy/commands/fetch.py
View File

@ -27,8 +27,8 @@ class Command(ScrapyCommand):
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():

10
scrapy/commands/genspider.py
View File

@ -36,15 +36,15 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:

24
scrapy/commands/parse.py
View File

@ -33,29 +33,29 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
help="print each depth level one by one")
@property
def max_level(self):

10
scrapy/commands/settings.py
View File

@ -19,15 +19,15 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings

8
scrapy/commands/shell.py
View File

@ -34,11 +34,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be

6
scrapy/commands/startproject.py
View File

@ -102,10 +102,8 @@ class Command(ScrapyCommand):
move(join(project_dir, 'module'), join(project_dir, project_name))
for paths in TEMPLATES_TO_RENDER:
path = join(*paths)
tplfile = join(project_dir,
string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name,
ProjectName=string_camelcase(project_name))
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))

2
scrapy/commands/version.py
View File

@ -17,7 +17,7 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
if opts.verbose:

21
scrapy/core/downloader/contextfactory.py
View File

@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
#
# * getattr() for `_ssl_method` attribute for context factories
# not calling super(..., self).__init__
return CertificateOptions(verify=False,
method=getattr(self, 'method',
getattr(self, '_ssl_method', None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers)
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
)
# kept for old-style HTTP/1.0 downloader context twisted calls,
# e.g. connectSSL()
@ -86,8 +87,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
#
# This means that a website like https://www.cacert.org will be rejected
# by default, since CAcert.org CA certificate is seldom shipped.
return optionsForClientTLS(hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={
'method': self._ssl_method,
})
return optionsForClientTLS(
hostname=hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)

7
scrapy/core/downloader/handlers/ftp.py
View File

@ -86,10 +86,9 @@ class FTPDownloadHandler:
password = request.meta.get("ftp_password", self.default_password)
passive_mode = 1 if bool(request.meta.get("ftp_passive",
self.passive_mode)) else 0
creator = ClientCreator(reactor, FTPClient, user, password,
passive=passive_mode)
return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
request, unquote(parsed_url.path))
creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
def gotClient(self, client, request, filepath):
self.client = client

27
scrapy/core/downloader/handlers/http11.py
View File

@ -18,6 +18,7 @@ from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.tls import openssl_methods
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning
@ -34,6 +35,8 @@ class HTTP11DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
self._crawler = crawler
from twisted.internet import reactor
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
@ -79,6 +82,7 @@ class HTTP11DownloadHandler:
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
fail_on_dataloss=self._fail_on_dataloss,
crawler=self._crawler,
)
return agent.download_request(request)
@ -276,7 +280,7 @@ class ScrapyAgent:
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
maxsize=0, warnsize=0, fail_on_dataloss=True):
maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
self._contextFactory = contextFactory
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
@ -285,6 +289,7 @@ class ScrapyAgent:
self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._txresponse = None
self._crawler = crawler
def _get_agent(self, request, timeout):
from twisted.internet import reactor
@ -407,7 +412,15 @@ class ScrapyAgent:
d = defer.Deferred(_cancel)
txresponse.deliverBody(
_ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
_ResponseReader(
finished=d,
txresponse=txresponse,
request=request,
maxsize=maxsize,
warnsize=warnsize,
fail_on_dataloss=fail_on_dataloss,
crawler=self._crawler,
)
)
# save response for timeouts
@ -449,7 +462,7 @@ class _RequestBodyProducer:
class _ResponseReader(protocol.Protocol):
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
self._finished = finished
self._txresponse = txresponse
self._request = request
@ -462,6 +475,7 @@ class _ResponseReader(protocol.Protocol):
self._bytes_received = 0
self._certificate = None
self._ip_address = None
self._crawler = crawler
def connectionMade(self):
if self._certificate is None:
@ -479,6 +493,13 @@ class _ResponseReader(protocol.Protocol):
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
self._crawler.signals.send_catch_log(
signal=signals.bytes_received,
data=bodyBytes,
request=self._request,
spider=self._crawler.spider,
)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",

13
scrapy/core/downloader/handlers/s3.py
View File

@ -100,11 +100,12 @@ class S3DownloadHandler:
url=url, headers=awsrequest.headers.items())
else:
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body)
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body,
)
request = request.replace(url=url, headers=signed_headers)
return self._download_http(request, spider)

4
scrapy/core/downloader/webclient.py
View File

@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing()
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds." %
(self.factory.url, self.factory.timeout)))
defer.TimeoutError("Getting %s took longer than %s seconds."
% (self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):

13
scrapy/core/engine.py
View File

@ -217,11 +217,9 @@ class ExecutionEngine:
self.slot.nextcall.schedule()
def schedule(self, request, spider):
self.signals.send_catch_log(signal=signals.request_scheduled,
request=request, spider=spider)
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
if not self.slot.scheduler.enqueue_request(request):
self.signals.send_catch_log(signal=signals.request_dropped,
request=request, spider=spider)
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
def download(self, request, spider):
d = self._download(request, spider)
@ -247,8 +245,8 @@ class ExecutionEngine:
logkws = self.logformatter.crawled(request, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
self.signals.send_catch_log(signal=signals.response_received,
response=response, request=request, spider=spider)
self.signals.send_catch_log(signals.response_received,
response=response, request=request, spider=spider)
return response
def _on_complete(_):
@ -286,8 +284,7 @@ class ExecutionEngine:
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = self.signals.send_catch_log(signal=signals.spider_idle,
spider=spider, dont_log=DontCloseSpider)
res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
return

2
scrapy/core/spidermw.py
View File

@ -19,7 +19,7 @@ def _isiterable(possible_iterator):
def _fname(f):
return "%s.%s".format(
return "{}.{}".format(
f.__self__.__class__.__name__,
f.__func__.__name__
)

1
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import re
import logging

12
scrapy/downloadermiddlewares/retry.py
View File

@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
import logging
from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
ConnectionRefusedError, ConnectionDone, ConnectError, \
ConnectionLost, TCPTimedOutError
from twisted.internet.error import (
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
DNSLookupError,
TCPTimedOutError,
TimeoutError,
)
from twisted.web.client import ResponseFailed
from scrapy.exceptions import NotConfigured

2
scrapy/exporters.py
View File

@ -250,7 +250,7 @@ class CsvItemExporter(BaseItemExporter):
class PickleItemExporter(BaseItemExporter):
def __init__(self, file, protocol=2, **kwargs):
def __init__(self, file, protocol=4, **kwargs):
super().__init__(**kwargs)
self.file = file
self.protocol = protocol

9
scrapy/extensions/httpcache.py
View File

@ -46,9 +46,10 @@ class RFC2616Policy:
def __init__(self, settings):
self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.ignore_response_cache_controls = [to_bytes(cc) for cc in
settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')]
self._cc_parsed = WeakKeyDictionary()
self.ignore_response_cache_controls = [
to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
]
def _parse_cachecontrol(self, r):
if r not in self._cc_parsed:
@ -250,7 +251,7 @@ class DbmCacheStorage:
'headers': dict(response.headers),
'body': response.body,
}
self.db['%s_data' % key] = pickle.dumps(data, protocol=2)
self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
self.db['%s_time' % key] = str(time())
def _read_data(self, spider, request):
@ -317,7 +318,7 @@ class FilesystemCacheStorage:
with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
f.write(to_bytes(repr(metadata)))
with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
pickle.dump(metadata, f, protocol=2)
pickle.dump(metadata, f, protocol=4)
with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
f.write(headers_dict_to_raw(response.headers))
with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:

2
scrapy/extensions/spiderstate.py
View File

@ -26,7 +26,7 @@ class SpiderState:
def spider_closed(self, spider):
if self.jobdir:
with open(self.statefn, 'wb') as f:
pickle.dump(spider.state, f, protocol=2)
pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider):
if self.jobdir and os.path.exists(self.statefn):

5
scrapy/http/response/text.py
View File

@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
import warnings
from contextlib import suppress
from typing import Generator
from urllib.parse import urljoin
@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
http_content_type_encoding, resolve_encoding)
from w3lib.html import strip_html5_whitespace
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import memoizemethod_noargs, to_unicode
@ -61,6 +63,9 @@ class TextResponse(Response):
def body_as_unicode(self):
"""Return body as unicode"""
warnings.warn('Response.body_as_unicode() is deprecated, '
'please use Response.text instead.',
ScrapyDeprecationWarning)
return self.text
@property

6
scrapy/item.py
View File

@ -86,8 +86,7 @@ class DictItem(MutableMapping, BaseItem):
if key in self.fields:
self._values[key] = value
else:
raise KeyError("%s does not support field: %s" %
(self.__class__.__name__, key))
raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
def __delitem__(self, key):
del self._values[key]
@ -99,8 +98,7 @@ class DictItem(MutableMapping, BaseItem):
def __setattr__(self, name, value):
if not name.startswith('_'):
raise AttributeError("Use item[%r] = %r to set field value" %
(name, value))
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
super(DictItem, self).__setattr__(name, value)
def __len__(self):

80
scrapy/linkextractors/lxmlhtml.py
View File

@ -1,6 +1,8 @@
"""
Link extractor based on lxml.html
"""
import operator
from functools import partial
from urllib.parse import urljoin
import lxml.etree as etree
@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
from scrapy.link import Link
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
from scrapy.linkextractors import FilteringLinkExtractor
# from lxml/src/lxml/html/__init__.py
@ -27,19 +29,24 @@ def _nons(tag):
return tag
def _identity(x):
return x
def _canonicalize_link_url(link):
return canonicalize_url(link.url, keep_fragments=True)
class LxmlParserLinkExtractor:
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True, canonicalized=False):
self.scan_tag = tag if callable(tag) else lambda t: t == tag
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.process_attr = process if callable(process) else lambda v: v
def __init__(
self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
):
self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
self.process_attr = process if callable(process) else _identity
self.unique = unique
self.strip = strip
if canonicalized:
self.link_key = lambda link: link.url
else:
self.link_key = lambda link: canonicalize_url(link.url,
keep_fragments=True)
self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
def _iter_links(self, document):
for el in document.iter(etree.Element):
@ -93,25 +100,44 @@ class LxmlParserLinkExtractor:
class LxmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=False,
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
strip=True, restrict_text=None):
def __init__(
self,
allow=(),
deny=(),
allow_domains=(),
deny_domains=(),
restrict_xpaths=(),
tags=('a', 'area'),
attrs=('href',),
canonicalize=False,
unique=True,
process_value=None,
deny_extensions=None,
restrict_css=(),
strip=True,
restrict_text=None,
):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
lx = LxmlParserLinkExtractor(
tag=lambda x: x in tags,
attr=lambda x: x in attrs,
tag=partial(operator.contains, tags),
attr=partial(operator.contains, attrs),
unique=unique,
process=process_value,
strip=strip,
canonicalized=canonicalize
)
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,
restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
canonicalize=canonicalize, deny_extensions=deny_extensions,
restrict_text=restrict_text)
super(LxmlLinkExtractor, self).__init__(
link_extractor=lx,
allow=allow,
deny=deny,
allow_domains=allow_domains,
deny_domains=deny_domains,
restrict_xpaths=restrict_xpaths,
restrict_css=restrict_css,
canonicalize=canonicalize,
deny_extensions=deny_extensions,
restrict_text=restrict_text,
)
def extract_links(self, response):
"""Returns a list of :class:`~scrapy.link.Link` objects from the
@ -124,9 +150,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
"""
base_url = get_base_url(response)
if self.restrict_xpaths:
docs = [subdoc
for x in self.restrict_xpaths
for subdoc in response.xpath(x)]
docs = [
subdoc
for x in self.restrict_xpaths
for subdoc in response.xpath(x)
]
else:
docs = [response.selector]
all_links = []

32
scrapy/mail.py
View File

@ -28,8 +28,10 @@ def _to_bytes_or_none(text):
class MailSender:
def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
def __init__(
self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None,
smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False
):
self.smtphost = smtphost
self.smtpport = smtpport
self.smtpuser = _to_bytes_or_none(smtpuser)
@ -41,9 +43,15 @@ class MailSender:
@classmethod
def from_settings(cls, settings):
return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'],
settings['MAIL_PASS'], settings.getint('MAIL_PORT'),
settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
return cls(
smtphost=settings['MAIL_HOST'],
mailfrom=settings['MAIL_FROM'],
smtpuser=settings['MAIL_USER'],
smtppass=settings['MAIL_PASS'],
smtpport=settings.getint('MAIL_PORT'),
smtptls=settings.getbool('MAIL_TLS'),
smtpssl=settings.getbool('MAIL_SSL'),
)
def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
from twisted.internet import reactor
@ -89,9 +97,12 @@ class MailSender:
return
dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
dfd.addCallbacks(self._sent_ok, self._sent_failed,
dfd.addCallbacks(
callback=self._sent_ok,
errback=self._sent_failed,
callbackArgs=[to, cc, subject, len(attachs)],
errbackArgs=[to, cc, subject, len(attachs)])
errbackArgs=[to, cc, subject, len(attachs)],
)
reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd)
return dfd
@ -115,9 +126,10 @@ class MailSender:
from twisted.mail.smtp import ESMTPSenderFactory
msg = BytesIO(msg)
d = defer.Deferred()
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
requireTransportSecurity=self.smtptls)
factory = ESMTPSenderFactory(
self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
)
factory.noisy = False
if self.smtpssl:

3
scrapy/pipelines/files.py
View File

@ -83,8 +83,7 @@ class S3FilesStore:
AWS_USE_SSL = None
AWS_VERIFY = None
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in
# FilesPipeline.from_settings.
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
HEADERS = {
'Cache-Control': 'max-age=172800',
}

6
scrapy/responsetypes.py
View File

@ -58,9 +58,9 @@ class ResponseTypes:
def from_content_disposition(self, content_disposition):
try:
filename = to_unicode(content_disposition,
encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
filename = filename.strip('"\'')
filename = to_unicode(
content_disposition, encoding='latin-1', errors='replace'
).split(';')[1].split('=')[1].strip('"\'')
return self.from_filename(filename)
except IndexError:
return Response

6
scrapy/selector/unified.py
View File

@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref):
selectorlist_cls = SelectorList
def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
if not(response is None or text is None):
raise ValueError('%s.__init__() received both response and text'
% self.__class__.__name__)
if response is not None and text is not None:
raise ValueError('%s.__init__() received both response and text'
% self.__class__.__name__)
st = _st(response, type or self._default_type)

5
scrapy/shell.py
View File

@ -146,14 +146,13 @@ class Shell:
b.append("Useful shortcuts:")
if self.inthread:
b.append(" fetch(url[, redirect=True]) "
"Fetch URL and update local objects "
"(by default, redirects are followed)")
"Fetch URL and update local objects (by default, redirects are followed)")
b.append(" fetch(req) "
"Fetch a scrapy.Request and update local objects ")
b.append(" shelp() Shell help (print this help)")
b.append(" view(response) View response in a browser")
return "\n".join("[s] %s" % l for l in b)
return "\n".join("[s] %s" % line for line in b)
def _is_relevant(self, value):
return isinstance(value, self.relevant_classes)

1
scrapy/signals.py
View File

@ -17,6 +17,7 @@ request_reached_downloader = object()
request_left_downloader = object()
response_received = object()
response_downloaded = object()
bytes_received = object()
item_scraped = object()
item_dropped = object()
item_error = object()

46
scrapy/spiderloader.py
View File

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import traceback
import warnings
from collections import defaultdict
from zope.interface import implementer
@ -16,6 +15,7 @@ class SpiderLoader:
SpiderLoader is a class which locates and loads spiders
in a Scrapy project.
"""
def __init__(self, settings):
self.spider_modules = settings.getlist('SPIDER_MODULES')
self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
@ -24,16 +24,21 @@ class SpiderLoader:
self._load_all_spiders()
def _check_name_duplicates(self):
dupes = ["\n".join(" {cls} named {name!r} (in {module})".format(
module=mod, cls=cls, name=name)
for (mod, cls) in locations)
for name, locations in self._found.items()
if len(locations) > 1]
dupes = []
for name, locations in self._found.items():
dupes.extend([
" {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
for mod, cls in locations
if len(locations) > 1
])
if dupes:
msg = ("There are several spiders with the same name:\n\n"
"{}\n\n This can cause unexpected behavior.".format(
"\n\n".join(dupes)))
warnings.warn(msg, UserWarning)
dupes_string = "\n\n".join(dupes)
warnings.warn(
"There are several spiders with the same name:\n\n"
"{}\n\n This can cause unexpected behavior.".format(dupes_string),
category=UserWarning,
)
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
@ -45,12 +50,15 @@ class SpiderLoader:
try:
for module in walk_modules(name):
self._load_spiders(module)
except ImportError as e:
except ImportError:
if self.warn_only:
msg = ("\n{tb}Could not load spiders from module '{modname}'. "
"See above traceback for details.".format(
modname=name, tb=traceback.format_exc()))
warnings.warn(msg, RuntimeWarning)
warnings.warn(
"\n{tb}Could not load spiders from module '{modname}'. "
"See above traceback for details.".format(
modname=name, tb=traceback.format_exc()
),
category=RuntimeWarning,
)
else:
raise
self._check_name_duplicates()
@ -73,8 +81,10 @@ class SpiderLoader:
"""
Return the list of spider names that can handle the given request.
"""
return [name for name, cls in self._spiders.items()
if cls.handles_request(request)]
return [
name for name, cls in self._spiders.items()
if cls.handles_request(request)
]
def list(self):
"""

3
scrapy/spiders/sitemap.py
View File

@ -96,5 +96,4 @@ def iterloc(it, alt=False):
# Also consider alternate URLs (xhtml:link rel="alternate")
if alt and 'alternate' in d:
for l in d['alternate']:
yield l
yield from d['alternate']

9
scrapy/squeues.py
View File

@ -81,12 +81,11 @@ def _scrapy_non_serialization_queue(queue_class):
def _pickle_serialize(obj):
try:
return pickle.dumps(obj, protocol=2)
# Python <= 3.4 raises pickle.PicklingError here while
# 3.5 <= Python < 3.6 raises AttributeError and
# Python >= 3.6 raises TypeError
return pickle.dumps(obj, protocol=4)
# Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
# TypeError is raised from parsel.Selector
except (pickle.PicklingError, AttributeError, TypeError) as e:
raise ValueError(str(e))
raise ValueError(str(e)) from e
PickleFifoDiskQueueNonRequest = _serializable_queue(

2
scrapy/templates/project/module/items.py.tmpl
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:

2
scrapy/templates/project/module/middlewares.py.tmpl
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:

2
scrapy/templates/project/module/pipelines.py.tmpl
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting

2
scrapy/templates/project/module/settings.py.tmpl
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Scrapy settings for $project_name project
#
# For simplicity, this file contains only settings considered important or

1
scrapy/templates/spiders/basic.tmpl
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import scrapy

1
scrapy/templates/spiders/crawl.tmpl
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

1
scrapy/templates/spiders/csvfeed.tmpl
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
from scrapy.spiders import CSVFeedSpider

1
scrapy/templates/spiders/xmlfeed.tmpl
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
from scrapy.spiders import XMLFeedSpider

2
scrapy/utils/console.py
View File

@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
def _embed_bpython_shell(namespace={}, banner=''):
"""Start a bpython shell"""
import bpython
@wraps(_embed_bpython_shell)
def wrapper(namespace=namespace, banner=''):
bpython.embed(locals_=namespace, banner=banner)
@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
def _embed_ptpython_shell(namespace={}, banner=''):
"""Start a ptpython shell"""
import ptpython.repl
@wraps(_embed_ptpython_shell)
def wrapper(namespace=namespace, banner=''):
print(banner)

7
scrapy/utils/defer.py
View File

@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks and errbacks"""
d = defer.Deferred()
for cb, eb in zip(callbacks, errbacks):
d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw,
errbackArgs=a, errbackKeywords=kw)
d.addCallbacks(
callback=cb, errback=eb,
callbackArgs=a, callbackKeywords=kw,
errbackArgs=a, errbackKeywords=kw,
)
if isinstance(input, failure.Failure):
d.errback(input)
else:

14
scrapy/utils/log.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
import logging
import sys
import warnings
@ -39,7 +37,7 @@ class TopLevelFormatter(logging.Filter):
self.loggers = loggers or []
def filter(self, record):
if any(record.name.startswith(l + '.') for l in self.loggers):
if any(record.name.startswith(logger + '.') for logger in self.loggers):
record.name = record.name.split('.', 1)[0]
return True
@ -144,10 +142,12 @@ def _get_handler(settings):
def log_scrapy_info(settings):
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
logger.info("Versions: %(versions)s",
{'versions': ", ".join("%s %s" % (name, version)
for name, version in scrapy_components_versions()
if name != "Scrapy")})
versions = [
"%s %s" % (name, version)
for name, version in scrapy_components_versions()
if name != "Scrapy"
]
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
from twisted.internet import reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)

15
scrapy/utils/misc.py
View File

@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
``*args`` and ``**kwargs`` are forwarded to the constructors.
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
extension has not been implemented correctly).
"""
if settings is None:
if crawler is None:
raise ValueError("Specify at least one of settings and crawler.")
settings = crawler.settings
if crawler and hasattr(objcls, 'from_crawler'):
return objcls.from_crawler(crawler, *args, **kwargs)
instance = objcls.from_crawler(crawler, *args, **kwargs)
method_name = 'from_crawler'
elif hasattr(objcls, 'from_settings'):
return objcls.from_settings(settings, *args, **kwargs)
instance = objcls.from_settings(settings, *args, **kwargs)
method_name = 'from_settings'
else:
return objcls(*args, **kwargs)
instance = objcls(*args, **kwargs)
method_name = '__new__'
if instance is None:
raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
return instance
@contextmanager

1
scrapy/utils/python.py
View File

@ -152,6 +152,7 @@ def memoizemethod_noargs(method):
weak reference to its object
"""
cache = weakref.WeakKeyDictionary()
@wraps(method)
def new_method(self, *args, **kwargs):
if self not in cache:

7
scrapy/utils/response.py
View File

@ -19,8 +19,7 @@ def get_base_url(response):
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
text = response.text[0:4096]
_baseurl_cache[response] = html.get_base_url(text, response.url,
response.encoding)
_baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding)
return _baseurl_cache[response]
@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
_metaref_cache[response] = html.get_meta_refresh(text, response.url,
response.encoding, ignore_tags=ignore_tags)
_metaref_cache[response] = html.get_meta_refresh(
text, response.url, response.encoding, ignore_tags=ignore_tags)
return _metaref_cache[response]

6
scrapy/utils/signal.py
View File

@ -28,8 +28,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(receiver, signal=signal, sender=sender,
*arguments, **named)
response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
if isinstance(response, Deferred):
logger.error("Cannot return deferreds from signal handler: %(receiver)s",
{'receiver': receiver}, extra={'spider': spider})
@ -63,8 +62,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
spider = named.get('spider', None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
*arguments, **named)
d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
d.addErrback(logerror, receiver)
d.addBoth(lambda result: (receiver, result))
dfds.append(d)

2
scrapy/utils/ssl.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
import OpenSSL
import OpenSSL._util as pyOpenSSLutil

3
scrapy/utils/url.py
View File

@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains):
def url_is_from_spider(url, spider):
"""Return True if the url belongs to the given spider"""
return url_is_from_any_domain(url,
[spider.name] + list(getattr(spider, 'allowed_domains', [])))
return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', [])))
def url_has_any_extension(url, extensions):

2
tests/pipelines.py
View File

@ -6,7 +6,7 @@ Some pipelines used for testing
class ZeroDivisionErrorPipeline:
def open_spider(self, spider):
a = 1 / 0
1 / 0
def process_item(self, item, spider):
return item

4
tests/test_command_parse.py
View File

@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
@defer.inlineCallbacks
def test_request_without_meta(self):
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-c', 'parse_request_without_meta',
'--nolinks',
'-c', 'parse_request_without_meta',
'--nolinks',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))

6
tests/test_command_shell.py
View File

@ -103,15 +103,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_local_nofile(self):
filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
errcode, out, err = yield self.execute([filepath, '-c', 'item'],
check_code=False)
errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'No such file or directory', err)
@defer.inlineCallbacks
def test_dns_failures(self):
url = 'www.somedomainthatdoesntexi.st'
errcode, out, err = yield self.execute([url, '-c', 'item'],
check_code=False)
errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'DNS lookup failed', err)

6
tests/test_command_version.py
View File

@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase):
def test_verbose_output(self):
encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
_, out, _ = yield self.execute(['-v'])
headers = [l.partition(":")[0].strip()
for l in out.strip().decode(encoding).splitlines()]
headers = [
line.partition(":")[0].strip()
for line in out.strip().decode(encoding).splitlines()
]
self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
'cssselect', 'parsel', 'w3lib',
'Twisted', 'Python', 'pyOpenSSL',

3
tests/test_contracts.py
View File

@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase):
# extract contracts correctly
contracts = self.conman.extract_contracts(spider.returns_request)
self.assertEqual(len(contracts), 2)
self.assertEqual(frozenset(type(x) for x in contracts),
self.assertEqual(
frozenset(type(x) for x in contracts),
frozenset([UrlContract, ReturnsContract]))
# returns request for valid method

36
tests/test_crawl.py
View File

@ -104,44 +104,44 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_retry_503(self):
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_failed(self):
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_dns_error(self):
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
# try to fetch the homepage of a non-existent domain
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_start_requests_bug_before_yield(self):
with LogCapture('scrapy', level=logging.ERROR) as l:
with LogCapture('scrapy', level=logging.ERROR) as log:
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
self.assertEqual(len(l.records), 1)
record = l.records[0]
self.assertEqual(len(log.records), 1)
record = log.records[0]
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
@defer.inlineCallbacks
def test_start_requests_bug_yielding(self):
with LogCapture('scrapy', level=logging.ERROR) as l:
with LogCapture('scrapy', level=logging.ERROR) as log:
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
self.assertEqual(len(l.records), 1)
record = l.records[0]
self.assertEqual(len(log.records), 1)
record = log.records[0]
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
@ -187,25 +187,25 @@ foo body
with multiples lines
'''})
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)
self.assertEqual(str(l).count("Got response 200"), 1)
self.assertEqual(str(log).count("Got response 200"), 1)
@defer.inlineCallbacks
def test_retry_conn_lost(self):
# connection lost after receiving data
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_aborted(self):
# connection lost before receiving data
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
def _assert_retried(self, log):
self.assertEqual(str(log).count("Retrying"), 2)

15
tests/test_crawler.py
View File

@ -87,7 +87,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
class MySpider(scrapy.Spider):
name = 'spider'
crawler = Crawler(MySpider, {})
Crawler(MySpider, {})
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
@ -240,13 +240,13 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
def test_crawler_runner_asyncio_enabled_true(self):
if self.reactor_pytest == 'asyncio':
runner = CrawlerRunner(settings={
CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerRunner(settings={
CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
@ -313,14 +313,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
def test_ipv6_alternative_name_resolver(self):
log = self.run_script('alternative_name_resolver.py')
self.assertIn('Spider closed (finished)', log)
self.assertTrue(any([
"twisted.internet.error.ConnectionRefusedError" in log,
"twisted.internet.error.ConnectError" in log,
]))
self.assertTrue(any([
"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
]))
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
def test_reactor_select(self):
log = self.run_script("twisted_reactor_select.py")

2
tests/test_dependencies.py
View File

@ -6,7 +6,7 @@ class ScrapyUtilsTest(unittest.TestCase):
def test_required_openssl_version(self):
try:
module = import_module('OpenSSL')
except ImportError as ex:
except ImportError:
raise unittest.SkipTest("OpenSSL is not available")
if hasattr(module, '__version__'):

11
tests/test_downloader_handlers.py
View File

@ -736,6 +736,9 @@ class Http11ProxyTestCase(HttpProxyTestCase):
class HttpDownloadHandlerMock:
def __init__(self, *args, **kwargs):
pass
def download_request(self, request, spider):
return request
@ -859,8 +862,7 @@ class S3TestCase(unittest.TestCase):
def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
date = 'Tue, 27 Mar 2007 19:44:46 +0000'
req = Request('s3://johnsmith/?acl',
method='GET', headers={'Date': date})
req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date})
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'],
@ -885,8 +887,9 @@ class S3TestCase(unittest.TestCase):
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
# botocore does not override Date with x-amz-date
self.assertEqual(httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
self.assertEqual(
httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.

18
tests/test_downloadermiddleware_cookies.py
View File

@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase):
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture('scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG) as l:
level=logging.DEBUG) as log:
req = Request('http://scrapytest.org/')
res = Response('http://scrapytest.org/',
headers={'Set-Cookie': 'C1=value1; path=/'})
@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase):
req2 = Request('http://scrapytest.org/sub1/')
mw.process_request(req2, crawler.spider)
l.check(
log.check(
('scrapy.downloadermiddlewares.cookies',
'DEBUG',
'Received cookies from: <200 http://scrapytest.org/>\n'
@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase):
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture('scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG) as l:
level=logging.DEBUG) as log:
req = Request('http://scrapytest.org/')
res = Response('http://scrapytest.org/',
headers={'Set-Cookie': 'C1=value1; path=/'})
@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase):
req2 = Request('http://scrapytest.org/sub1/')
mw.process_request(req2, crawler.spider)
l.check()
log.check()
def test_do_not_break_on_non_utf8_header(self):
req = Request('http://scrapytest.org/')
@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):
def test_complex_cookies(self):
# merge some cookies into jar
cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
{'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
{'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
{'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
cookies = [
{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
{'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
{'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
{'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
]
req = Request('http://scrapytest.org/', cookies=cookies)
self.mw.process_request(req, self.spider)

7
tests/test_downloadermiddleware_httpproxy.py
View File

@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase):
os.environ.pop('file_proxy', None)
mw = HttpProxyMiddleware()
for url, proxy in [('http://e.com', http_proxy),
('https://e.com', https_proxy), ('file://tmp/a', None)]:
for url, proxy in [
('http://e.com', http_proxy),
('https://e.com', https_proxy),
('file://tmp/a', None),
]:
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEqual(req.url, url)

4
tests/test_downloadermiddleware_redirect.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
import unittest
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
@ -72,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
url = 'http://www.example.com/302'
url2 = 'http://www.example.com/redirected2'
req = Request(url, method='POST', body='test',
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
rsp = Response(url, headers={'Location': url2}, status=302)
req2 = self.mw.process_response(req, rsp, self.spider)

26
tests/test_downloadermiddleware_retry.py
View File

@ -1,8 +1,14 @@
import unittest
from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
ConnectionRefusedError, ConnectionDone, ConnectError, \
ConnectionLost, TCPTimedOutError
from twisted.internet.error import (
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
DNSLookupError,
TCPTimedOutError,
TimeoutError,
)
from twisted.web.client import ResponseFailed
from scrapy.downloadermiddlewares.retry import RetryMiddleware
@ -75,9 +81,17 @@ class RetryTest(unittest.TestCase):
assert self.crawler.stats.get_value('retry/count') == 2
def test_twistederrors(self):
exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
DNSLookupError, ConnectionRefusedError, ConnectionDone,
ConnectError, ConnectionLost, ResponseFailed]
exceptions = [
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
defer.TimeoutError,
DNSLookupError,
ResponseFailed,
TCPTimedOutError,
TimeoutError,
]
for exc in exceptions:
req = Request('http://www.scrapytest.org/%s' % exc.__name__)

1
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
from unittest import mock
from twisted.internet import reactor, error

40
tests/test_dupefilters.py
View File

@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase):
shutil.rmtree(path)
def test_log(self):
with LogCapture() as l:
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
@ -177,15 +177,19 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' - no more duplicates will be shown'
' (see DUPEFILTER_DEBUG to show all duplicates)')))
log.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more'
' duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
)
)
dupefilter.close('finished')
def test_log_debug(self):
with LogCapture() as l:
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
@ -197,18 +201,26 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html',
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
)
headers={'Referer': 'http://scrapytest.org/INDEX.html'})
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: None)')))
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: http://scrapytest.org/INDEX.html)')))
log.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
)
)
log.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: http://scrapytest.org/INDEX.html)'
)
)
dupefilter.close('finished')

112
tests/test_engine.py
View File

@ -13,22 +13,24 @@ module with the ``runserver`` argument::
import os
import re
import sys
from collections import defaultdict
from urllib.parse import urlparse
from twisted.internet import reactor, defer
from twisted.web import server, static, util
from twisted.trial import unittest
from twisted.web import server, static, util
from pydispatch import dispatcher
from scrapy import signals
from scrapy.core.engine import ExecutionEngine
from scrapy.utils.test import get_crawler
from pydispatch import dispatcher
from tests import tests_datadir
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.linkextractors import LinkExtractor
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler
from tests import tests_datadir, get_testdata
class TestItem(Item):
@ -88,6 +90,8 @@ def start_test_site(debug=False):
r = static.File(root_dir)
r.putChild(b"redirect", util.Redirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**14)]
r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
if debug:
@ -107,15 +111,20 @@ class CrawlerRun:
self.reqreached = []
self.itemerror = []
self.itemresp = []
self.signals_catched = {}
self.bytes = defaultdict(lambda: list())
self.signals_caught = {}
self.spider_class = spider_class
def run(self):
self.port = start_test_site()
self.portno = self.port.getHost().port
start_urls = [self.geturl("/"), self.geturl("/redirect"),
self.geturl("/redirect")] # a duplicate
start_urls = [
self.geturl("/"),
self.geturl("/redirect"),
self.geturl("/redirect"), # duplicate
self.geturl("/numbers"),
]
for name, signal in vars(signals).items():
if not name.startswith('_'):
@ -124,6 +133,7 @@ class CrawlerRun:
self.crawler = get_crawler(self.spider_class)
self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
self.crawler.signals.connect(self.item_error, signals.item_error)
self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
@ -155,6 +165,9 @@ class CrawlerRun:
def item_scraped(self, item, spider, response):
self.itemresp.append((item, response))
def bytes_received(self, data, request, spider):
self.bytes[request].append(data)
def request_scheduled(self, request, spider):
self.reqplug.append((request, spider))
@ -172,7 +185,7 @@ class CrawlerRun:
signalargs = kwargs.copy()
sig = signalargs.pop('signal')
signalargs.pop('sender', None)
self.signals_catched[sig] = signalargs
self.signals_caught[sig] = signalargs
class EngineTest(unittest.TestCase):
@ -183,16 +196,17 @@ class EngineTest(unittest.TestCase):
self.run = CrawlerRun(spider)
yield self.run.run()
self._assert_visited_urls()
self._assert_scheduled_requests(urls_to_visit=8)
self._assert_scheduled_requests(urls_to_visit=9)
self._assert_downloaded_responses()
self._assert_scraped_items()
self._assert_signals_catched()
self._assert_signals_caught()
self._assert_bytes_received()
@defer.inlineCallbacks
def test_crawler_dupefilter(self):
self.run = CrawlerRun(TestDupeFilterSpider)
yield self.run.run()
self._assert_scheduled_requests(urls_to_visit=7)
self._assert_scheduled_requests(urls_to_visit=8)
self._assert_dropped_requests()
@defer.inlineCallbacks
@ -204,8 +218,8 @@ class EngineTest(unittest.TestCase):
def _assert_visited_urls(self):
must_be_visited = ["/", "/redirect", "/redirected",
"/item1.html", "/item2.html", "/item999.html"]
urls_visited = set([rp[0].url for rp in self.run.respplug])
urls_expected = set([self.run.geturl(p) for p in must_be_visited])
urls_visited = {rp[0].url for rp in self.run.respplug}
urls_expected = {self.run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited)
def _assert_scheduled_requests(self, urls_to_visit=None):
@ -213,8 +227,8 @@ class EngineTest(unittest.TestCase):
paths_expected = ['/item999.html', '/item2.html', '/item1.html']
urls_requested = set([rq[0].url for rq in self.run.reqplug])
urls_expected = set([self.run.geturl(p) for p in paths_expected])
urls_requested = {rq[0].url for rq in self.run.reqplug}
urls_expected = {self.run.geturl(p) for p in paths_expected}
assert urls_expected <= urls_requested
scheduled_requests_count = len(self.run.reqplug)
dropped_requests_count = len(self.run.reqdropped)
@ -229,8 +243,8 @@ class EngineTest(unittest.TestCase):
def _assert_downloaded_responses(self):
# response tests
self.assertEqual(8, len(self.run.respplug))
self.assertEqual(8, len(self.run.reqreached))
self.assertEqual(9, len(self.run.respplug))
self.assertEqual(9, len(self.run.reqreached))
for response, _ in self.run.respplug:
if self.run.getpath(response.url) == '/item999.html':
@ -263,19 +277,61 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
def _assert_signals_catched(self):
assert signals.engine_started in self.run.signals_catched
assert signals.engine_stopped in self.run.signals_catched
assert signals.spider_opened in self.run.signals_catched
assert signals.spider_idle in self.run.signals_catched
assert signals.spider_closed in self.run.signals_catched
def _assert_bytes_received(self):
self.assertEqual(9, len(self.run.bytes))
for request, data in self.run.bytes.items():
joined_data = b"".join(data)
if self.run.getpath(request.url) == "/":
self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
elif self.run.getpath(request.url) == "/item1.html":
self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
elif self.run.getpath(request.url) == "/item2.html":
self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
elif self.run.getpath(request.url) == "/redirected":
self.assertEqual(joined_data, b"Redirected here")
elif self.run.getpath(request.url) == '/redirect':
self.assertEqual(
joined_data,
b"\n<html>\n"
b" <head>\n"
b" <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
b" </head>\n"
b" <body bgcolor=\"#FFFFFF\" text=\"#000000\">\n"
b" <a href=\"/redirected\">click here</a>\n"
b" </body>\n"
b"</html>\n"
)
elif self.run.getpath(request.url) == "/tem999.html":
self.assertEqual(
joined_data,
b"\n<html>\n"
b" <head><title>404 - No Such Resource</title></head>\n"
b" <body>\n"
b" <h1>No Such Resource</h1>\n"
b" <p>File not found.</p>\n"
b" </body>\n"
b"</html>\n"
)
elif self.run.getpath(request.url) == "/numbers":
# signal was fired multiple times
self.assertTrue(len(data) > 1)
# bytes were received in order
numbers = [str(x).encode("utf8") for x in range(2**14)]
self.assertEqual(joined_data, b"".join(numbers))
def _assert_signals_caught(self):
assert signals.engine_started in self.run.signals_caught
assert signals.engine_stopped in self.run.signals_caught
assert signals.spider_opened in self.run.signals_caught
assert signals.spider_idle in self.run.signals_caught
assert signals.spider_closed in self.run.signals_caught
self.assertEqual({'spider': self.run.spider},
self.run.signals_catched[signals.spider_opened])
self.run.signals_caught[signals.spider_opened])
self.assertEqual({'spider': self.run.spider},
self.run.signals_catched[signals.spider_idle])
self.run.signals_caught[signals.spider_idle])
self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
self.run.signals_catched[signals.spider_closed])
self.run.signals_caught[signals.spider_closed])
@defer.inlineCallbacks
def test_close_downloader(self):

76
tests/test_exporters.py
View File

@ -357,20 +357,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
i2 = dict(name=u'bar', age=i1)
i3 = TestItem(name=u'buz', age=i2)
self.assertExportResult(i3,
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items>'
b'<item>'
b'<age>'
b'<age>'
b'<age>22</age>'
b'<name>foo\xc2\xa3hoo</name>'
b'</age>'
b'<name>bar</name>'
b'</age>'
b'<name>buz</name>'
b'</item>'
b'</items>'
self.assertExportResult(
i3,
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<age>
<age>
<age>22</age>
<name>foo\xc2\xa3hoo</name>
</age>
<name>bar</name>
</age>
<name>buz</name>
</item>
</items>
"""
)
def test_nested_list_item(self):
@ -378,31 +380,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
i3 = TestItem(name=u'buz', age=[i1, i2])
self.assertExportResult(i3,
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items>'
b'<item>'
b'<age>'
b'<value><name>foo</name></value>'
b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
b'</age>'
b'<name>buz</name>'
b'</item>'
b'</items>'
self.assertExportResult(
i3,
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<age>
<value><name>foo</name></value>
<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
</age>
<name>buz</name>
</item>
</items>
"""
)
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
self.assertExportResult(item,
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items>'
b'<item>'
b'<float>3.14</float>'
b'<boolean>False</boolean>'
b'<number>22</number>'
b'<time>2015-01-01 01:01:01</time>'
b'</item>'
b'</items>'
self.assertExportResult(
item,
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<float>3.14</float>
<boolean>False</boolean>
<number>22</number>
<time>2015-01-01 01:01:01</time>
</item>
</items>
"""
)

2
tests/test_extension_telnet.py
View File

@ -11,8 +11,6 @@ class TelnetExtensionTest(unittest.TestCase):
def _get_console_and_portal(self, settings=None):
crawler = get_crawler(settings_dict=settings)
console = TelnetConsole(crawler)
username = console.username
password = console.password
# This function has some side effects we don't need for this test
console._get_telnet_vars = lambda: {}

1
tests/test_feedexport.py
View File

@ -715,7 +715,6 @@ class FeedExportTest(unittest.TestCase):
@defer.inlineCallbacks
def test_export_encoding(self):
items = [dict({'foo': u'Test\xd6'})]
header = ['foo']
formats = {
'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),

56
tests/test_http_request.py
View File

@ -415,8 +415,7 @@ class FormRequestTest(RequestTest):
# using multiples values for a single key
data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertQueryEqual(r3.body,
b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
def test_from_response_post(self):
response = _buildresponse(
@ -426,8 +425,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx">
</form>""",
url="http://www.example.com/this/list.html")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -446,8 +444,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx \xc2\xb5">
</form>""",
url="http://www.example.com/this/list.html")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -468,8 +465,7 @@ class FormRequestTest(RequestTest):
url="http://www.example.com/this/list.html",
encoding='latin1',
)
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -488,8 +484,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx µ">
</form>""",
url="http://www.example.com/this/list.html")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -502,11 +497,13 @@ class FormRequestTest(RequestTest):
def test_from_response_duplicate_form_key(self):
response = _buildresponse(
'<form></form>',
url='http://www.example.com')
req = self.request_class.from_response(response,
method='GET',
formdata=(('foo', 'bar'), ('foo', 'baz')))
'<form></form>',
url='http://www.example.com')
req = self.request_class.from_response(
response=response,
method='GET',
formdata=(('foo', 'bar'), ('foo', 'baz')),
)
self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
@ -530,9 +527,11 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test" value="val2">
<input type="hidden" name="test2" value="xxx">
</form>""")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'},
headers={"Accept-Encoding": "gzip,deflate"})
req = self.request_class.from_response(
response=response,
formdata={'one': ['two', 'three'], 'six': 'seven'},
headers={"Accept-Encoding": "gzip,deflate"},
)
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
@ -545,14 +544,13 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx">
</form>""",
url="http://www.example.com/this/list.html")
r1 = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(r1.method, 'GET')
self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
self.assertEqual(urlparse(r1.url).path, "/this/get.php")
fs = _qs(r1)
self.assertEqual(set(fs[b'test']), set([b'val1', b'val2']))
self.assertEqual(set(fs[b'one']), set([b'two', b'three']))
self.assertEqual(set(fs[b'test']), {b'val1', b'val2'})
self.assertEqual(set(fs[b'one']), {b'two', b'three'})
self.assertEqual(fs[b'test2'], [b'xxx'])
self.assertEqual(fs[b'six'], [b'seven'])
@ -580,9 +578,9 @@ class FormRequestTest(RequestTest):
def test_from_response_override_method(self):
response = _buildresponse(
'''<html><body>
<form action="/app"></form>
</body></html>''')
'''<html><body>
<form action="/app"></form>
</body></html>''')
request = FormRequest.from_response(response)
self.assertEqual(request.method, 'GET')
request = FormRequest.from_response(response, method='POST')
@ -590,9 +588,9 @@ class FormRequestTest(RequestTest):
def test_from_response_override_url(self):
response = _buildresponse(
'''<html><body>
<form action="/app"></form>
</body></html>''')
'''<html><body>
<form action="/app"></form>
</body></html>''')
request = FormRequest.from_response(response)
self.assertEqual(request.url, 'http://example.com/app')
request = FormRequest.from_response(response, url='http://foo.bar/absolute')
@ -1049,7 +1047,7 @@ class FormRequestTest(RequestTest):
</form>''')
req = self.request_class.from_response(res)
fs = _qs(req)
self.assertEqual(set(fs), set([b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4']))
self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'})
def test_from_response_xpath(self):
response = _buildresponse(

30
tests/test_http_response.py
View File

@ -1,8 +1,9 @@
# -*- coding: utf-8 -*-
import unittest
from warnings import catch_warnings
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
XmlResponse, Headers)
from scrapy.selector import Selector
@ -497,8 +498,10 @@ class TextResponseTest(BaseResponseTest):
response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
)
self.assertEqual(
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content", pcount=1).getall(),
response.xpath(
"//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content", pcount=1,
).getall(),
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
)
@ -578,12 +581,14 @@ class TextResponseTest(BaseResponseTest):
'http://example.com',
body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
)
self._assert_followed_url(resp.css('a')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(resp.css('a::attr(href)')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(
resp.css('a')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(
resp.css('a::attr(href)')[0],
'http://example.com/foo',
response=resp)
def test_follow_encoding(self):
resp1 = self.response_class(
@ -673,6 +678,13 @@ class TextResponseTest(BaseResponseTest):
with self.assertRaises(ValueError):
response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
def test_body_as_unicode_deprecation_warning(self):
with catch_warnings(record=True) as warnings:
r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
self.assertEqual(r1.body_as_unicode(), u'Hello')
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
class HtmlResponseTest(TextResponseTest):

30
tests/test_item.py
View File

@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase):
item = D(save='X', load='Y')
self.assertEqual(item['save'], 'X')
self.assertEqual(item['load'], 'Y')
self.assertEqual(D.fields, {'load': {'default': 'A'},
'save': {'default': 'A'}})
self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})
# D class inverted
class E(C, B):
@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase):
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
self.assertEqual(E.fields, {'load': {'default': 'C'},
'save': {'default': 'C'}})
self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})
def test_metaclass_multiple_inheritance_diamond(self):
class A(Item):
@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase):
self.assertEqual(D(save='X')['save'], 'X')
self.assertEqual(D(load='X')['load'], 'X')
self.assertEqual(D.fields, {'save': {'default': 'C'},
'load': {'default': 'D'}, 'update': {'default': 'D'}})
self.assertEqual(
D.fields,
{'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})
# D class inverted
class E(C, B):
@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase):
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
self.assertEqual(E.fields, {'save': {'default': 'C'},
'load': {'default': 'E'}, 'update': {'default': 'C'}})
self.assertEqual(
E.fields,
{'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})
def test_metaclass_multiple_inheritance_without_metaclass(self):
class A(Item):
@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase):
self.assertRaises(KeyError, D, not_allowed='value')
self.assertEqual(D(save='X')['save'], 'X')
self.assertEqual(D.fields, {'save': {'default': 'A'},
'load': {'default': 'A'}})
self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
# D class inverted
class E(C, B):
@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase):
self.assertRaises(KeyError, E, not_allowed='value')
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E.fields, {'save': {'default': 'A'},
'load': {'default': 'A'}})
self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
def test_to_dict(self):
class TestItem(Item):
@ -264,12 +262,12 @@ class ItemTest(unittest.TestCase):
"""Make sure the DictItem deprecation warning is not issued for
Item"""
with catch_warnings(record=True) as warnings:
item = Item()
Item()
self.assertEqual(len(warnings), 0)
class SubclassedItem(Item):
pass
subclassed_item = SubclassedItem()
SubclassedItem()
self.assertEqual(len(warnings), 0)
@ -321,13 +319,13 @@ class DictItemTest(unittest.TestCase):
def test_deprecation_warning(self):
with catch_warnings(record=True) as warnings:
dict_item = DictItem()
DictItem()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
with catch_warnings(record=True) as warnings:
class SubclassedDictItem(DictItem):
pass
subclassed_dict_item = SubclassedDictItem()
SubclassedDictItem()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)

27
tests/test_linkextractors.py
View File

@ -1,3 +1,4 @@
import pickle
import re
import unittest
from warnings import catch_warnings
@ -171,9 +172,9 @@ class Base:
self.assertEqual(lx.matches(url1), False)
self.assertEqual(lx.matches(url2), True)
lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
allow_domains=('blah1.com',),
deny_domains=('blah2.com',))
lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],
allow_domains=['blah1.com'],
deny_domains=['blah2.com'])
self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
@ -421,10 +422,10 @@ class Base:
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
]
)
@ -437,10 +438,10 @@ class Base:
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
]
)
@ -469,6 +470,10 @@ class Base:
Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
])
def test_pickle_extractor(self):
lx = self.extractor_cls()
self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
extractor_cls = LxmlLinkExtractor

21
tests/test_logformatter.py
View File

@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> (referer: None)")
self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
def test_crawled_without_referer(self):
req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
res = Response("http://www.example.com", flags=['cached'])
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
def test_dropped(self):
item = {}
@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: None) []")
def test_crawled_without_referer(self):
@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
class SkipMessagesLogFormatter(LogFormatter):

2
tests/test_mail.py
View File

@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase):
mailsender = MailSender(debug=True)
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
attachs=attachs, _callback=self._catch_mail_sent)
attachs=attachs, _callback=self._catch_mail_sent)
assert self.catched_msg
self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])

9
tests/test_middleware.py
View File

@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())
self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['open_spider']],
[M1, M2])
self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['close_spider']],
[M2, M1])
self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['process']],
[M1, M3])
def test_enabled(self):

21
tests/test_pipeline_crawl.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import os
import shutil
@ -44,9 +43,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
name = 'redirectedmedia'
def _process_url(self, url):
return add_or_replace_parameter(
self.mockserver.url('/redirect-to'),
'goto', url)
return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
class FileDownloadCrawlTestCase(TestCase):
@ -54,10 +51,10 @@ class FileDownloadCrawlTestCase(TestCase):
store_setting_key = 'FILES_STORE'
media_key = 'files'
media_urls_key = 'file_urls'
expected_checksums = set([
expected_checksums = {
'5547178b89448faf0015a13f904c936e',
'c2281c83670e31d8aaab7cb642b824db',
'ed3f6538dc15d4d9179dae57319edc5f'])
'ed3f6538dc15d4d9179dae57319edc5f'}
def setUp(self):
self.mockserver = MockServer()
@ -134,7 +131,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media(self):
crawler = self._create_crawler(MediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key)
self._assert_files_downloaded(self.items, str(log))
@ -143,7 +141,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_wrong_urls(self):
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key)
self._assert_files_download_failure(crawler, self.items, 404, str(log))
@ -152,7 +151,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_redirected_default_failure(self):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
mockserver=self.mockserver)
@ -166,7 +166,8 @@ class FileDownloadCrawlTestCase(TestCase):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
mockserver=self.mockserver)

4
tests/test_pipeline_images.py
View File

@ -15,10 +15,10 @@ from scrapy.utils.python import to_bytes
skip = False
try:
from PIL import Image
except ImportError as e:
except ImportError:
skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
else:
encoders = set(('jpeg_encoder', 'jpeg_decoder'))
encoders = {'jpeg_encoder', 'jpeg_decoder'}
if not encoders.issubset(set(Image.core.__dict__)):
skip = 'Missing JPEG encoders'

22
tests/test_pipeline_media.py
View File

@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
fail = Failure(Exception())
results = [(True, 1), (False, fail)]
with LogCapture() as l:
with LogCapture() as log:
new_item = self.pipe.item_completed(results, item, self.info)
assert new_item is item
assert len(l.records) == 1
record = l.records[0]
assert len(log.records) == 1
record = log.records[0]
assert record.levelname == 'ERROR'
self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
# disable failure logging and check again
self.pipe.LOG_FAILED_RESULTS = False
with LogCapture() as l:
with LogCapture() as log:
new_item = self.pipe.item_completed(results, item, self.info)
assert new_item is item
assert len(l.records) == 0
assert len(log.records) == 0
@inlineCallbacks
def test_default_process_item(self):
@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, rsp)])
self.assertEqual(self.pipe._mockcalled,
['get_media_requests', 'media_to_download',
'media_downloaded', 'request_callback', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
@inlineCallbacks
def test_result_failure(self):
@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(False, fail)])
self.assertEqual(self.pipe._mockcalled,
['get_media_requests', 'media_to_download',
'media_failed', 'request_errback', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
@inlineCallbacks
def test_mix_of_success_and_failure(self):

16
tests/test_proxy_connect.py
View File

@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase):
@defer.inlineCallbacks
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, log)
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl("https://localhost:99999/status?n=200")
self._assert_got_tunnel_error(l)
self._assert_got_tunnel_error(log)
@defer.inlineCallbacks
def test_https_tunnel_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
# The proxy returns a 407 error code but it does not reach the client;
# he just sees a TunnelError.
self._assert_got_tunnel_error(l)
self._assert_got_tunnel_error(log)
@defer.inlineCallbacks
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, log)
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])

1
tests/test_responsetypes.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import unittest
from scrapy.responsetypes import responsetypes

18
tests/test_settings/__init__.py
View File

@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase):
'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
self.assertDictEqual(s.copy_to_dict(),
{'HASNOBASE': {3: 3000},
'TEST': {1: 10, 3: 30},
'TEST_BASE': {1: 1, 2: 2},
'TEST_BOOLEAN': False,
'TEST_LIST': [1, 2],
'TEST_STRING': 'a string'})
self.assertDictEqual(
s.copy_to_dict(),
{
'HASNOBASE': {3: 3000},
'TEST': {1: 10, 3: 30},
'TEST_BASE': {1: 1, 2: 2},
'TEST_LIST': [1, 2],
'TEST_BOOLEAN': False,
'TEST_STRING': 'a string',
}
)
def test_freeze(self):
self.settings.freeze()

40
tests/test_spiderloader/__init__.py
View File

@ -40,25 +40,32 @@ class SpiderLoaderTest(unittest.TestCase):
verifyObject(ISpiderLoader, self.spider_loader)
def test_list(self):
self.assertEqual(set(self.spider_loader.list()),
set(['spider1', 'spider2', 'spider3', 'spider4']))
self.assertEqual(
set(self.spider_loader.list()),
{'spider1', 'spider2', 'spider3', 'spider4'})
def test_load(self):
spider1 = self.spider_loader.load("spider1")
self.assertEqual(spider1.__name__, 'Spider1')
def test_find_by_request(self):
self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
['spider1'])
self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
['spider2'])
self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
set(['spider1', 'spider2']))
self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
self.assertEqual(
set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
{'spider1', 'spider2'})
self.assertEqual(
self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
[])
self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://spider3.com')),
[])
self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
['spider3'])
def test_load_spider_module(self):
@ -137,9 +144,14 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
msg = str(w[0].message)
self.assertIn("several spiders with the same name", msg)
self.assertIn("'spider3'", msg)
self.assertTrue(msg.count("'spider3'") == 2)
self.assertNotIn("'spider1'", msg)
self.assertNotIn("'spider2'", msg)
self.assertNotIn("'spider4'", msg)
spiders = set(spider_loader.list())
self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
def test_multiple_dupename_warning(self):
# copy 2 spider modules so as to have duplicate spider name
@ -156,7 +168,13 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
msg = str(w[0].message)
self.assertIn("several spiders with the same name", msg)
self.assertIn("'spider1'", msg)
self.assertTrue(msg.count("'spider1'") == 2)
self.assertIn("'spider2'", msg)
self.assertTrue(msg.count("'spider2'") == 2)
self.assertNotIn("'spider3'", msg)
self.assertNotIn("'spider4'", msg)
spiders = set(spider_loader.list())
self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})

63
tests/test_spidermiddleware_httperror.py
View File

@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider):
def __init__(self, *args, **kwargs):
super(_HttpErrorSpider, self).__init__(*args, **kwargs)
self.start_urls = [
self.mockserver.url("/status?n=200"),
self.mockserver.url("/status?n=404"),
self.mockserver.url("/status?n=402"),
self.mockserver.url("/status?n=500"),
self.mockserver.url("/status?n=200"),
self.mockserver.url("/status?n=404"),
self.mockserver.url("/status?n=402"),
self.mockserver.url("/status?n=500"),
]
self.failed = set()
self.skipped = set()
@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase):
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEqual([],
self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEqual(None,
self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
self.assertEqual(
[],
self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
self.assertEqual(None,
self.mw.process_spider_input(res, self.spider))
self.assertIsNone(self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
class TestHttpErrorMiddlewareSettings(TestCase):
@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase):
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertEqual(None,
self.mw.process_spider_input(self.res402, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase):
res402 = self.res402.copy()
res402.request = request
self.assertEqual(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
class TestHttpErrorMiddlewareHandleAll(TestCase):
@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402 = self.res402.copy()
res402.request = request
self.assertEqual(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):

32
tests/test_spidermiddleware_offsite.py
View File

@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase):
def test_process_spider_output(self):
res = Response('http://scrapytest.org')
onsite_reqs = [Request('http://scrapytest.org/1'),
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
Request('http://scrapy.test.org/'),
Request('http://scrapy.test.org:8000/')]
offsite_reqs = [Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org'),
Request('http://test.org/'),
Request('http://notscrapy.test.org/')]
onsite_reqs = [
Request('http://scrapytest.org/1'),
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
Request('http://scrapy.test.org/'),
Request('http://scrapy.test.org:8000/'),
]
offsite_reqs = [
Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org'),
Request('http://test.org/'),
Request('http://notscrapy.test.org/'),
]
reqs = onsite_reqs + offsite_reqs
out = list(self.mw.process_spider_output(res, reqs, self.spider))

3
tests/test_spidermiddleware_referer.py
View File

@ -495,7 +495,6 @@ class TestRequestMetaSettingFallback(TestCase):
target = 'http://www.example.com'
for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
spider = Spider('foo')
mw = RefererMiddleware(Settings(settings))
response = Response(origin, headers=response_headers)
@ -547,7 +546,7 @@ class TestSettingsPolicyByName(TestCase):
def test_invalid_name(self):
settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
with self.assertRaises(RuntimeError):
mw = RefererMiddleware(settings)
RefererMiddleware(settings)
class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):

84
tests/test_squeues.py
View File

@ -1,4 +1,5 @@
import pickle
import sys
from queuelib.tests import test_queue as t
from scrapy.squeues import (
@ -28,31 +29,13 @@ class TestLoader(ItemLoader):
def nonserializable_object_test(self):
q = self.queue()
try:
pickle.dumps(lambda x: x)
except Exception:
# Trigger Twisted bug #7989
import twisted.persisted.styles # NOQA
self.assertRaises(ValueError, q.push, lambda x: x)
else:
# Use a different unpickleable object
class A:
pass
a = A()
a.__reduce__ = a.__reduce_ex__ = None
self.assertRaises(ValueError, q.push, a)
self.assertRaises(ValueError, q.push, lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text='<html><body><p>some text</p></body></html>')
self.assertRaises(ValueError, q.push, sel)
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
chunksize = 100000
def queue(self):
return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
class FifoDiskQueueTestMixin:
def test_serialize(self):
q = self.queue()
@ -66,6 +49,13 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
test_nonserializable_object = nonserializable_object_test
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
chunksize = 100000
def queue(self):
return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 1
@ -82,7 +72,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 4
class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
chunksize = 100000
@ -99,12 +89,12 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
def test_serialize_loader(self):
q = self.queue()
l = TestLoader()
q.push(l)
l2 = q.pop()
assert isinstance(l2, TestLoader)
assert l2.default_item_class is TestItem
self.assertEqual(l2.name_out('x'), 'xx')
loader = TestLoader()
q.push(loader)
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
self.assertEqual(loader2.name_out('x'), 'xx')
def test_serialize_request_recursive(self):
q = self.queue()
@ -116,6 +106,21 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
self.assertEqual(r.url, r2.url)
assert r2.meta['request'] is r2
def test_non_pickable_object(self):
q = self.queue()
try:
q.push(lambda x: x)
except ValueError as exc:
if hasattr(sys, "pypy_version_info"):
self.assertIsInstance(exc.__context__, pickle.PicklingError)
else:
self.assertIsInstance(exc.__context__, AttributeError)
sel = Selector(text='<html><body><p>some text</p></body></html>')
try:
q.push(sel)
except ValueError as exc:
self.assertIsInstance(exc.__context__, TypeError)
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 1
@ -133,10 +138,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 4
class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
def queue(self):
return MarshalLifoDiskQueue(self.qpath)
class LifoDiskQueueTestMixin:
def test_serialize(self):
q = self.queue()
@ -150,7 +152,13 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
test_nonserializable_object = nonserializable_object_test
class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):
class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def queue(self):
return MarshalLifoDiskQueue(self.qpath)
class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def queue(self):
return PickleLifoDiskQueue(self.qpath)
@ -165,12 +173,12 @@ class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):
def test_serialize_loader(self):
q = self.queue()
l = TestLoader()
q.push(l)
l2 = q.pop()
assert isinstance(l2, TestLoader)
assert l2.default_item_class is TestItem
self.assertEqual(l2.name_out('x'), 'xx')
loader = TestLoader()
q.push(loader)
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
self.assertEqual(loader2.name_out('x'), 'xx')
def test_serialize_request_recursive(self):
q = self.queue()

3
tests/test_utils_conf.py
View File

@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase):
class UtilsConfTestCase(unittest.TestCase):
def test_arglist_to_dict(self):
self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
self.assertEqual(
arglist_to_dict(['arg1=val1', 'arg2=val2']),
{'arg1': 'val1', 'arg2': 'val2'})

2
tests/test_utils_datatypes.py
View File

@ -217,7 +217,7 @@ class SequenceExcludeTest(unittest.TestCase):
def test_set(self):
"""Anything that is not in the supplied sequence will evaluate as 'in' the container."""
seq = set([-3, "test", 1.1])
seq = {-3, "test", 1.1}
d = SequenceExclude(seq)
self.assertIn(0, d)
self.assertIn("foo", d)

4
tests/test_utils_defer.py
View File

@ -64,7 +64,7 @@ class DeferUtilsTest(unittest.TestCase):
gotexc = False
try:
yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
except TypeError as e:
except TypeError:
gotexc = True
self.assertTrue(gotexc)
@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase):
def iterbad():
for x in range(10):
if x == 5:
a = 1 / 0
1 / 0
yield x
errors = []

5
tests/test_utils_deprecate.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import inspect
import unittest
from unittest import mock
@ -26,7 +25,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
def test_no_warning_on_definition(self):
with warnings.catch_warnings(record=True) as w:
Deprecated = create_deprecated_class('Deprecated', NewName)
create_deprecated_class('Deprecated', NewName)
w = self._mywarnings(w)
self.assertEqual(w, [])
@ -218,7 +217,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
def test_deprecate_a_class_with_custom_metaclass(self):
Meta1 = type('Meta1', (type,), {})
New = Meta1('New', (), {})
Deprecated = create_deprecated_class('Deprecated', New)
create_deprecated_class('Deprecated', New)
def test_deprecate_subclass_of_deprecated_class(self):
with warnings.catch_warnings(record=True) as w:

28
tests/test_utils_iterators.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import os
from twisted.trial import unittest
@ -48,8 +47,7 @@ class XmliterTestCase(unittest.TestCase):
</root>
"""
response = XmlResponse(url="http://example.com", body=body)
nodenames = [e.xpath('name()').getall()
for e in self.xmliter(response, 'matchme...')]
nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
self.assertEqual(nodenames, [['matchme...']])
def test_xmliter_unicode(self):
@ -94,8 +92,8 @@ class XmliterTestCase(unittest.TestCase):
# with bytes
XmlResponse(url="http://example.com", body=body.encode('utf-8')),
# Unicode body needs encoding information
XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
):
attrs = []
for x in self.xmliter(r, u'þingflokkur'):
attrs.append((x.attrib['id'],
@ -374,15 +372,23 @@ class UtilsCsvTestCase(unittest.TestCase):
response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'latin1', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}])
self.assertEqual(
list(csv),
[
{u'id': u'1', u'name': u'latin1', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
]
)
response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'cp852', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}])
self.assertEqual(
list(csv),
[
{u'id': u'1', u'name': u'cp852', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
]
)
class TestHelper(unittest.TestCase):

25
tests/test_utils_log.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import sys
import logging
import unittest
@ -35,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase):
def test_top_level_logger(self):
logger = logging.getLogger('test')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('test', 'WARNING', 'test log msg'))
log.check(('test', 'WARNING', 'test log msg'))
def test_children_logger(self):
logger = logging.getLogger('test.test1')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('test', 'WARNING', 'test log msg'))
log.check(('test', 'WARNING', 'test log msg'))
def test_overlapping_name_logger(self):
logger = logging.getLogger('test2')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('test2', 'WARNING', 'test log msg'))
log.check(('test2', 'WARNING', 'test log msg'))
def test_different_name_logger(self):
logger = logging.getLogger('different')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('different', 'WARNING', 'test log msg'))
log.check(('different', 'WARNING', 'test log msg'))
class LogCounterHandlerTest(unittest.TestCase):
@ -108,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase):
sys.stdout = self.stdout
def test_redirect(self):
with LogCapture() as l:
with LogCapture() as log:
print('test log msg')
l.check(('test', 'ERROR', 'test log msg'))
log.check(('test', 'ERROR', 'test log msg'))

Some files were not shown because too many files have changed in this diff Show More