mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into documentation-coverage
This commit is contained in:
Adrián Chaves
GitHub
commit
607815dfc2
|
|
@ -8,7 +8,7 @@ skips:
|
|||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B402
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
- B406
|
||||
|
|
|
|||
|
|
@ -1,12 +1,19 @@
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _py_files(folder):
|
||||
return (str(p) for p in Path(folder).rglob('*.py'))
|
||||
|
||||
|
||||
collect_ignore = [
|
||||
# not a test, but looks like a test
|
||||
"scrapy/utils/testsite.py",
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
||||
*_py_files("tests/CrawlerProcess")
|
||||
]
|
||||
|
||||
|
||||
for line in open('tests/ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
# serve to show the default.
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from os import path
|
||||
|
||||
# If your extensions are in another directory, add it here. If the directory
|
||||
|
|
@ -49,8 +50,8 @@ source_suffix = '.rst'
|
|||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'Scrapy'
|
||||
copyright = u'2008–2018, Scrapy developers'
|
||||
project = 'Scrapy'
|
||||
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
|
|
@ -194,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
|
|||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, document class [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'Scrapy.tex', u'Scrapy Documentation',
|
||||
u'Scrapy developers', 'manual'),
|
||||
('index', 'Scrapy.tex', 'Scrapy Documentation',
|
||||
'Scrapy developers', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
|
|
|
|||
|
|
@ -229,7 +229,7 @@ You can also specify a comma-separated list of environmets, and use :ref:`tox’
|
|||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py27,py36 -p auto
|
||||
tox -e py36,py38 -p auto
|
||||
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
|
|
|
|||
67
pytest.ini
67
pytest.ini
|
|
@ -26,52 +26,52 @@ flake8-ignore =
|
|||
scrapy/http/__init__.py F401
|
||||
# Issues pending a review:
|
||||
# extras
|
||||
extras/qps-bench-server.py E261 E501
|
||||
extras/qpsclient.py E501 E261 E501
|
||||
extras/qps-bench-server.py E501
|
||||
extras/qpsclient.py E501 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128 E502 E731
|
||||
scrapy/commands/fetch.py E401 E501 E128 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/startproject.py E502 E127 E501 E128
|
||||
scrapy/commands/startproject.py E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E502 E128
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
|
||||
scrapy/core/engine.py E501 E128 E127 E306 E502
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E261 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E502 E126 E226
|
||||
scrapy/core/scraper.py E501 E306 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E126 E226
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
|
||||
scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E502 E128
|
||||
scrapy/downloadermiddlewares/httpcompression.py E501 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E502 E128 E123
|
||||
scrapy/extensions/closespider.py E501 E128 E123
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303
|
||||
|
|
@ -88,10 +88,10 @@ flake8-ignore =
|
|||
scrapy/http/response/__init__.py E501 E128 W293 W291
|
||||
scrapy/http/response/text.py E501 W293 E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402 W504
|
||||
scrapy/linkextractors/__init__.py E731 E501 E402 W504
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E502 E128
|
||||
scrapy/loader/__init__.py E501 E128
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
|
|
@ -102,7 +102,7 @@ flake8-ignore =
|
|||
scrapy/selector/unified.py E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
|
||||
scrapy/settings/default_settings.py E501 E114 E116 E226
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
|
|
@ -112,17 +112,16 @@ flake8-ignore =
|
|||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501 E261
|
||||
scrapy/spiders/feed.py E501
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/conf.py E402 E502 E501
|
||||
scrapy/utils/console.py E261 E306 E305
|
||||
scrapy/utils/conf.py E402 E501
|
||||
scrapy/utils/console.py E306 E305
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/engine.py E261
|
||||
scrapy/utils/gz.py E305 E501 W504
|
||||
scrapy/utils/http.py F403 E226
|
||||
scrapy/utils/httpobj.py E501
|
||||
|
|
@ -146,17 +145,17 @@ flake8-ignore =
|
|||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/_monkeypatches.py W293
|
||||
scrapy/cmdline.py E502 E501
|
||||
scrapy/cmdline.py E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E501 E202
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501 E261 E226
|
||||
scrapy/exporters.py E501 E226
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501 W293
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/middleware.py E502 E128 E501
|
||||
scrapy/middleware.py E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/robotstxt.py E501
|
||||
|
|
@ -171,7 +170,7 @@ flake8-ignore =
|
|||
tests/pipelines.py F841 E226
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501 E261
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128 E303 E226
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py E128 E501
|
||||
|
|
@ -179,7 +178,7 @@ flake8-ignore =
|
|||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
|
|
@ -190,18 +189,18 @@ flake8-ignore =
|
|||
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E502 E128 E261
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_response.py E501 E301 E502 E128 E265
|
||||
tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123
|
||||
tests/test_http_response.py E501 E301 E128 E265
|
||||
tests/test_item.py E701 E128 F841 E306
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
|
|
@ -212,7 +211,7 @@ flake8-ignore =
|
|||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E305
|
||||
|
|
@ -224,14 +223,14 @@ flake8-ignore =
|
|||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py E501 W293 E226
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E701 E741
|
||||
tests/test_utils_conf.py E501 E303 E128
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E261 E501 F841 E226
|
||||
tests/test_utils_defer.py E306 E501 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_http.py E501 E502 E128 W504
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
|
|
@ -240,11 +239,11 @@ flake8-ignore =
|
|||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E731 E226
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E305
|
||||
tests/test_utils_spider.py E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/test_cmdline/__init__.py E502 E501
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ def _pop_command_name(argv):
|
|||
|
||||
def _print_header(settings, inproject):
|
||||
if inproject:
|
||||
print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
|
||||
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
|
||||
settings['BOT_NAME']))
|
||||
else:
|
||||
print("Scrapy %s - no active project\n" % scrapy.__version__)
|
||||
|
|
@ -123,7 +123,7 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
|
|
|
|||
|
|
@ -24,12 +24,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true", \
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -43,8 +43,8 @@ class Command(ScrapyCommand):
|
|||
return False
|
||||
|
||||
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
|
||||
print('Error: Project names must begin with a letter and contain'\
|
||||
' only\nletters, numbers and underscores')
|
||||
print('Error: Project names must begin with a letter and contain'
|
||||
' only\nletters, numbers and underscores')
|
||||
elif _module_exists(project_name):
|
||||
print('Error: Module %r already exists' % project_name)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -86,8 +86,8 @@ class ReturnsContract(Contract):
|
|||
else:
|
||||
expected = '%s..%s' % (self.min_bound, self.max_bound)
|
||||
|
||||
raise ContractFail("Returned %s %s, expected %s" % \
|
||||
(occurrences, self.obj_name, expected))
|
||||
raise ContractFail("Returned %s %s, expected %s" %
|
||||
(occurrences, self.obj_name, expected))
|
||||
|
||||
|
||||
class ScrapesContract(Contract):
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ def _get_boto_connection():
|
|||
|
||||
class S3DownloadHandler(object):
|
||||
|
||||
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
|
||||
httpdownloadhandler=HTTPDownloadHandler, **kw):
|
||||
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
|
||||
httpdownloadhandler=HTTPDownloadHandler, **kw):
|
||||
|
||||
if not aws_access_key_id:
|
||||
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
delimiter = b'\n'
|
||||
|
||||
def connectionMade(self):
|
||||
self.headers = Headers() # bucket for response headers
|
||||
self.headers = Headers() # bucket for response headers
|
||||
|
||||
# Method command
|
||||
self.sendCommand(self.factory.method, self.factory.path)
|
||||
|
|
@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
if self.factory.url.startswith(b'https'):
|
||||
self.transport.stopProducing()
|
||||
|
||||
self.factory.noPage(\
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." % \
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
self.factory.noPage(
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." %
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class Slot(object):
|
|||
|
||||
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
|
||||
self.closing = False
|
||||
self.inprogress = set() # requests in progress
|
||||
self.inprogress = set() # requests in progress
|
||||
self.start_requests = iter(start_requests)
|
||||
self.close_if_idle = close_if_idle
|
||||
self.nextcall = nextcall
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ class Scraper(object):
|
|||
callback/errback"""
|
||||
assert isinstance(response, (Response, Failure))
|
||||
|
||||
dfd = self._scrape2(response, request, spider) # returns spiders processed output
|
||||
dfd = self._scrape2(response, request, spider) # returns spiders processed output
|
||||
dfd.addErrback(self.handle_spider_error, request, response, spider)
|
||||
dfd.addCallback(self.handle_spider_output, request, response, spider)
|
||||
return dfd
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
try:
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
|
||||
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
|
||||
.format(fname(method), type(result)))
|
||||
except _InvalidOutput:
|
||||
raise
|
||||
|
|
@ -69,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
elif result is None:
|
||||
continue
|
||||
else:
|
||||
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
|
||||
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
|
||||
.format(fname(method), type(result)))
|
||||
return _failure
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
if _isiterable(result):
|
||||
result = evaluate_iterable(result, method_index)
|
||||
else:
|
||||
raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
|
||||
raise _InvalidOutput('Middleware {} must return an iterable, got {}'
|
||||
.format(fname(method), type(result)))
|
||||
|
||||
return chain(result, recovered)
|
||||
|
|
|
|||
|
|
@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
|
|||
if content_encoding:
|
||||
encoding = content_encoding.pop()
|
||||
decoded_body = self._decode(response.body, encoding.lower())
|
||||
respcls = responsetypes.from_args(headers=response.headers, \
|
||||
url=response.url, body=decoded_body)
|
||||
respcls = responsetypes.from_args(
|
||||
headers=response.headers, url=response.url, body=decoded_body
|
||||
)
|
||||
kwargs = dict(cls=respcls, body=decoded_body)
|
||||
if issubclass(respcls, TextResponse):
|
||||
# force recalculating the encoding until we make sure the
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
line_buffering=False,
|
||||
write_through=True,
|
||||
encoding=self.encoding,
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
)
|
||||
self.csv_writer = csv.writer(self.stream, **kwargs)
|
||||
self._headers_not_written = True
|
||||
|
|
|
|||
|
|
@ -54,9 +54,9 @@ class CloseSpider(object):
|
|||
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.task = reactor.callLater(self.close_on['timeout'], \
|
||||
self.crawler.engine.close_spider, spider, \
|
||||
reason='closespider_timeout')
|
||||
self.task = reactor.callLater(self.close_on['timeout'],
|
||||
self.crawler.engine.close_spider, spider,
|
||||
reason='closespider_timeout')
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
self.counter['itemcount'] += 1
|
||||
|
|
|
|||
|
|
@ -46,8 +46,7 @@ IGNORED_EXTENSIONS = [
|
|||
|
||||
_re_type = type(re.compile("", 0))
|
||||
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \
|
||||
'file', 'ftp'}
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
|
||||
|
||||
|
||||
class FilteringLinkExtractor(object):
|
||||
|
|
|
|||
|
|
@ -73,8 +73,7 @@ class MailSender(object):
|
|||
part = MIMEBase(*mimetype.split('/'))
|
||||
part.set_payload(f.read())
|
||||
Encoders.encode_base64(part)
|
||||
part.add_header('Content-Disposition', 'attachment; filename="%s"' \
|
||||
% attach_name)
|
||||
part.add_header('Content-Disposition', 'attachment', filename=attach_name)
|
||||
msg.attach(part)
|
||||
else:
|
||||
msg.set_payload(body)
|
||||
|
|
|
|||
|
|
@ -65,8 +65,8 @@ class MiddlewareManager(object):
|
|||
return process_chain(self.methods[methodname], obj, *args)
|
||||
|
||||
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
|
||||
return process_chain_both(self.methods[cb_methodname], \
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
return process_chain_both(self.methods[cb_methodname],
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
|
||||
def open_spider(self, spider):
|
||||
return self._process_parallel('open_spider', spider)
|
||||
|
|
|
|||
|
|
@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
|
|||
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
|
||||
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
|
||||
DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
|
||||
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
|
||||
# also allowing negotiation
|
||||
# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
|
||||
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
|
||||
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {}
|
||||
|
|
|
|||
|
|
@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):
|
|||
and the file's headers.
|
||||
"""
|
||||
|
||||
delimiter = None # When this is None, python's csv module's default delimiter is used
|
||||
quotechar = None # When this is None, python's csv module's default quotechar is used
|
||||
delimiter = None # When this is None, python's csv module's default delimiter is used
|
||||
quotechar = None # When this is None, python's csv module's default quotechar is used
|
||||
headers = None
|
||||
|
||||
def process_results(self, response, results):
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
|
|||
"""Fail if a value in the components dict is not a real number or None."""
|
||||
for name, value in compdict.items():
|
||||
if value is not None and not isinstance(value, numbers.Real):
|
||||
raise ValueError('Invalid value {} for component {}, please provide ' \
|
||||
raise ValueError('Invalid value {} for component {}, please provide '
|
||||
'a real number or None instead'.format(value, name))
|
||||
|
||||
# BEGIN Backward compatibility for old (base, custom) call signature
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
|
|||
def _embed_standard_shell(namespace={}, banner=''):
|
||||
"""Start a standard python shell"""
|
||||
import code
|
||||
try: # readline module is only available on unix systems
|
||||
try: # readline module is only available on unix systems
|
||||
import readline
|
||||
except ImportError:
|
||||
pass
|
||||
|
|
@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None):
|
|||
"""Return the first acceptable shell-embed function
|
||||
from a given list of shell names.
|
||||
"""
|
||||
if shells is None: # list, preference order of shells
|
||||
if shells is None: # list, preference order of shells
|
||||
shells = DEFAULT_PYTHON_SHELLS.keys()
|
||||
if known_shells is None: # available embeddable shells
|
||||
if known_shells is None: # available embeddable shells
|
||||
known_shells = DEFAULT_PYTHON_SHELLS.copy()
|
||||
for shell in shells:
|
||||
if shell in known_shells:
|
||||
|
|
@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):
|
|||
shell = get_shell_embed_func(shells)
|
||||
if shell is not None:
|
||||
shell(namespace=namespace, banner=banner)
|
||||
except SystemExit: # raised when using exit() in python code.interact
|
||||
except SystemExit: # raised when using exit() in python code.interact
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = 'no_request'
|
||||
|
||||
def start_requests(self):
|
||||
return []
|
||||
|
||||
|
||||
process = CrawlerProcess(settings={})
|
||||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
|
@ -25,17 +25,15 @@ class CmdlineTest(unittest.TestCase):
|
|||
return comm.decode(encoding)
|
||||
|
||||
def test_default_settings(self):
|
||||
self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
|
||||
'default')
|
||||
self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default')
|
||||
|
||||
def test_override_settings_using_set_arg(self):
|
||||
self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), \
|
||||
'override')
|
||||
self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s',
|
||||
'TEST1=override'), 'override')
|
||||
|
||||
def test_override_settings_using_envvar(self):
|
||||
self.env['SCRAPY_TEST1'] = 'override'
|
||||
self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
|
||||
'override')
|
||||
self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
|
||||
|
||||
def test_profiling(self):
|
||||
path = tempfile.mkdtemp()
|
||||
|
|
|
|||
|
|
@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_headers(self):
|
||||
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
|
||||
out = out.replace(b'\r', b'') # required on win32
|
||||
out = out.replace(b'\r', b'') # required on win32
|
||||
assert b'Server: TwistedWeb' in out, out
|
||||
assert b'Content-Type: text/plain' in out
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from twisted.internet import defer
|
||||
|
|
@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider
|
|||
from scrapy.utils.misc import load_object
|
||||
from scrapy.extensions.throttle import AutoThrottle
|
||||
from scrapy.extensions import telnet
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
|
||||
class BaseCrawlerTest(unittest.TestCase):
|
||||
|
|
@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
yield runner.crawl(NoRequestsSpider)
|
||||
|
||||
self.assertEqual(runner.bootstrap_failed, True)
|
||||
|
||||
|
||||
class CrawlerProcessSubprocess(unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
|
||||
|
||||
def run_script(self, script_name):
|
||||
script_path = os.path.join(self.script_dir, script_name)
|
||||
args = (sys.executable, script_path)
|
||||
p = subprocess.Popen(args, env=get_testenv(),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
return stderr.decode('utf-8')
|
||||
|
||||
def test_simple(self):
|
||||
log = self.run_script('simple.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
|
|
|
|||
|
|
@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase):
|
|||
return self.download_request(request, Spider('foo')).addCallback(_test)
|
||||
|
||||
def test_payload(self):
|
||||
body = b'1'*100 # PayloadResource requires body length to be 100
|
||||
body = b'1'*100 # PayloadResource requires body length to be 100
|
||||
request = Request(self.getURL('payload'), method='POST', body=body)
|
||||
d = self.download_request(request, Spider('foo'))
|
||||
d.addCallback(lambda r: r.body)
|
||||
|
|
@ -800,8 +800,8 @@ class S3TestCase(unittest.TestCase):
|
|||
req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
|
||||
|
||||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
|
|
@ -813,21 +813,22 @@ class S3TestCase(unittest.TestCase):
|
|||
})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
|
||||
|
||||
def test_request_signing3(self):
|
||||
# lists the content of the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
|
||||
method='GET', headers={
|
||||
'User-Agent': 'Mozilla/5.0',
|
||||
'Date': date,
|
||||
})
|
||||
req = Request(
|
||||
's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy',
|
||||
method='GET', headers={
|
||||
'User-Agent': 'Mozilla/5.0',
|
||||
'Date': date,
|
||||
})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
|
||||
|
||||
def test_request_signing4(self):
|
||||
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
|
||||
|
|
@ -836,8 +837,8 @@ class S3TestCase(unittest.TestCase):
|
|||
method='GET', headers={'Date': date})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
|
||||
def test_request_signing5(self):
|
||||
try:
|
||||
|
|
@ -850,11 +851,11 @@ class S3TestCase(unittest.TestCase):
|
|||
# deletes an object from the 'johnsmith' bucket using the
|
||||
# path-style and Date alternative.
|
||||
date = 'Tue, 27 Mar 2007 21:20:27 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', \
|
||||
method='DELETE', headers={
|
||||
'Date': date,
|
||||
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
|
||||
})
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
|
||||
'Date': date,
|
||||
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
|
||||
})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
# botocore does not override Date with x-amz-date
|
||||
|
|
@ -864,25 +865,26 @@ class S3TestCase(unittest.TestCase):
|
|||
def test_request_signing6(self):
|
||||
# uploads an object to a CNAME style virtual hosted bucket with metadata.
|
||||
date = 'Tue, 27 Mar 2007 21:06:08 +0000'
|
||||
req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
|
||||
method='PUT', headers={
|
||||
'User-Agent': 'curl/7.15.5',
|
||||
'Host': 'static.johnsmith.net:8080',
|
||||
'Date': date,
|
||||
'x-amz-acl': 'public-read',
|
||||
'content-type': 'application/x-download',
|
||||
'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
|
||||
'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
|
||||
'X-Amz-Meta-FileChecksum': '0x02661779',
|
||||
'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
|
||||
'Content-Disposition': 'attachment; filename=database.dat',
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Length': '5913339',
|
||||
})
|
||||
req = Request(
|
||||
's3://static.johnsmith.net:8080/db-backup.dat.gz',
|
||||
method='PUT', headers={
|
||||
'User-Agent': 'curl/7.15.5',
|
||||
'Host': 'static.johnsmith.net:8080',
|
||||
'Date': date,
|
||||
'x-amz-acl': 'public-read',
|
||||
'content-type': 'application/x-download',
|
||||
'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
|
||||
'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
|
||||
'X-Amz-Meta-FileChecksum': '0x02661779',
|
||||
'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
|
||||
'Content-Disposition': 'attachment; filename=database.dat',
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Length': '5913339',
|
||||
})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
|
||||
|
||||
def test_request_signing7(self):
|
||||
# ensure that spaces are quoted properly before signing
|
||||
|
|
|
|||
|
|
@ -165,12 +165,12 @@ class MaxRetryTimesTest(unittest.TestCase):
|
|||
# SETTINGS: meta(max_retry_times) = 4
|
||||
meta_max_retry_times = 4
|
||||
|
||||
req = Request(self.invalid_url, meta= \
|
||||
{'max_retry_times': meta_max_retry_times, 'dont_retry': True})
|
||||
req = Request(self.invalid_url, meta={
|
||||
'max_retry_times': meta_max_retry_times, 'dont_retry': True
|
||||
})
|
||||
|
||||
self._test_retry(req, DNSLookupError('foo'), 0)
|
||||
|
||||
|
||||
def _test_retry(self, req, exception, max_retry_times):
|
||||
|
||||
for i in range(0, max_retry_times):
|
||||
|
|
|
|||
|
|
@ -91,8 +91,8 @@ def start_test_site(debug=False):
|
|||
|
||||
port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
|
||||
if debug:
|
||||
print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \
|
||||
% port.getHost().port)
|
||||
print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish."
|
||||
% port.getHost().port)
|
||||
return port
|
||||
|
||||
|
||||
|
|
@ -271,7 +271,6 @@ class EngineTest(unittest.TestCase):
|
|||
self.run.signals_catched[signals.spider_opened])
|
||||
self.assertEqual({'spider': self.run.spider},
|
||||
self.run.signals_catched[signals.spider_idle])
|
||||
self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17
|
||||
self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
|
||||
self.run.signals_catched[signals.spider_closed])
|
||||
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ class RequestTest(unittest.TestCase):
|
|||
|
||||
r2 = self.request_class(url="http://www.example.com/", body=b"")
|
||||
assert isinstance(r2.body, bytes)
|
||||
self.assertEqual(r2.encoding, 'utf-8') # default encoding
|
||||
self.assertEqual(r2.encoding, 'utf-8') # default encoding
|
||||
|
||||
r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
|
||||
assert isinstance(r3.body, bytes)
|
||||
|
|
@ -622,8 +622,9 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="two" value="3">
|
||||
<input type="submit" name="clickable2" value="clicked2">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, formdata={'two': '2'}, \
|
||||
clickdata={'name': 'clickable2'})
|
||||
req = self.request_class.from_response(
|
||||
response, formdata={'two': '2'}, clickdata={'name': 'clickable2'}
|
||||
)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs[b'clickable2'], [b'clicked2'])
|
||||
self.assertFalse(b'clickable1' in fs, fs)
|
||||
|
|
@ -671,8 +672,9 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="one" value="clicked1">
|
||||
<input type="hidden" name="two" value="clicked2">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, \
|
||||
clickdata={u'name': u'clickable', u'value': u'clicked2'})
|
||||
req = self.request_class.from_response(
|
||||
response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
|
||||
)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs[b'clickable'], [b'clicked2'])
|
||||
self.assertEqual(fs[b'one'], [b'clicked1'])
|
||||
|
|
@ -686,8 +688,9 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="poundsign" value="\u00a3">
|
||||
<input type="hidden" name="eurosign" value="\u20ac">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response, \
|
||||
clickdata={u'name': u'price in \u00a3'})
|
||||
req = self.request_class.from_response(
|
||||
response, clickdata={u'name': u'price in \u00a3'}
|
||||
)
|
||||
fs = _qs(req, to_unicode=True)
|
||||
self.assertTrue(fs[u'price in \u00a3'])
|
||||
|
||||
|
|
@ -700,8 +703,9 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="yensign" value="\u00a5">
|
||||
</form>""",
|
||||
encoding='latin1')
|
||||
req = self.request_class.from_response(response, \
|
||||
clickdata={u'name': u'price in \u00a5'})
|
||||
req = self.request_class.from_response(
|
||||
response, clickdata={u'name': u'price in \u00a5'}
|
||||
)
|
||||
fs = _qs(req, to_unicode=True, encoding='latin1')
|
||||
self.assertTrue(fs[u'price in \u00a5'])
|
||||
|
||||
|
|
@ -716,8 +720,9 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="field2" value="value2">
|
||||
</form>
|
||||
""")
|
||||
req = self.request_class.from_response(response, formname='form2', \
|
||||
clickdata={u'name': u'clickable'})
|
||||
req = self.request_class.from_response(
|
||||
response, formname='form2', clickdata={u'name': u'clickable'}
|
||||
)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs[b'clickable'], [b'clicked2'])
|
||||
self.assertEqual(fs[b'field2'], [b'value2'])
|
||||
|
|
@ -725,8 +730,9 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_from_response_override_clickable(self):
|
||||
response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''')
|
||||
req = self.request_class.from_response(response, \
|
||||
formdata={'clickme': 'two'}, clickdata={'name': 'clickme'})
|
||||
req = self.request_class.from_response(
|
||||
response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}
|
||||
)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs[b'clickme'], [b'two'])
|
||||
|
||||
|
|
@ -853,7 +859,7 @@ class FormRequestTest(RequestTest):
|
|||
<form name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>""")
|
||||
self.assertRaises(IndexError, self.request_class.from_response, \
|
||||
self.assertRaises(IndexError, self.request_class.from_response,
|
||||
response, formname="form3", formnumber=2)
|
||||
|
||||
def test_from_response_formid_exists(self):
|
||||
|
|
@ -907,7 +913,7 @@ class FormRequestTest(RequestTest):
|
|||
<form id="form2" name="form2" action="post.php" method="POST">
|
||||
<input type="hidden" name="two" value="2">
|
||||
</form>""")
|
||||
self.assertRaises(IndexError, self.request_class.from_response, \
|
||||
self.assertRaises(IndexError, self.request_class.from_response,
|
||||
response, formid="form3", formnumber=2)
|
||||
|
||||
def test_from_response_select(self):
|
||||
|
|
|
|||
|
|
@ -316,8 +316,8 @@ class TextResponseTest(BaseResponseTest):
|
|||
assert u'SUFFIX' in r.text, repr(r.text)
|
||||
|
||||
# Do not destroy html tags due to encoding bugs
|
||||
r = self.response_class("http://example.com", encoding='utf-8', \
|
||||
body=b'\xf0<span>value</span>')
|
||||
r = self.response_class("http://example.com", encoding='utf-8',
|
||||
body=b'\xf0<span>value</span>')
|
||||
assert u'<span>value</span>' in r.text, repr(r.text)
|
||||
|
||||
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
|
||||
|
|
|
|||
|
|
@ -238,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
|
||||
m = self.pipe._mockcalled
|
||||
# only once
|
||||
self.assertEqual(m[0], 'get_media_requests') # first hook called
|
||||
self.assertEqual(m[0], 'get_media_requests') # first hook called
|
||||
self.assertEqual(m.count('get_media_requests'), 1)
|
||||
self.assertEqual(m.count('item_completed'), 1)
|
||||
self.assertEqual(m[-1], 'item_completed') # last hook called
|
||||
self.assertEqual(m[-1], 'item_completed') # last hook called
|
||||
# twice, one per request
|
||||
self.assertEqual(m.count('media_to_download'), 2)
|
||||
# one to handle success and other for failure
|
||||
|
|
@ -252,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
def test_get_media_requests(self):
|
||||
# returns single Request (without callback)
|
||||
req = Request('http://url')
|
||||
item = dict(requests=req) # pass a single item
|
||||
item = dict(requests=req) # pass a single item
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
assert new_item is item
|
||||
assert request_fingerprint(req) in self.info.downloaded
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy import Spider, signals, Request
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
|
||||
class SimplePipeline:
|
||||
def process_item(self, item, spider):
|
||||
item['pipeline_passed'] = True
|
||||
return item
|
||||
|
||||
|
||||
class DeferredPipeline:
|
||||
def cb(self, item):
|
||||
item['pipeline_passed'] = True
|
||||
return item
|
||||
|
||||
def process_item(self, item, spider):
|
||||
d = Deferred()
|
||||
d.addCallback(self.cb)
|
||||
d.callback(item)
|
||||
return d
|
||||
|
||||
|
||||
class ItemSpider(Spider):
|
||||
name = 'itemspider'
|
||||
|
||||
def start_requests(self):
|
||||
yield Request(self.mockserver.url('/status?n=200'))
|
||||
|
||||
def parse(self, response):
|
||||
return {'field': 42}
|
||||
|
||||
|
||||
class PipelineTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.mockserver = MockServer()
|
||||
self.mockserver.__enter__()
|
||||
|
||||
def tearDown(self):
|
||||
self.mockserver.__exit__(None, None, None)
|
||||
|
||||
def _on_item_scraped(self, item):
|
||||
self.assertIsInstance(item, dict)
|
||||
self.assertTrue(item.get('pipeline_passed'))
|
||||
self.items.append(item)
|
||||
|
||||
def _create_crawler(self, pipeline_class):
|
||||
settings = {
|
||||
'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
|
||||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
|
||||
self.items = []
|
||||
return crawler
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_simple_pipeline(self):
|
||||
crawler = self._create_crawler(SimplePipeline)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(len(self.items), 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_deferred_pipeline(self):
|
||||
crawler = self._create_crawler(DeferredPipeline)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(len(self.items), 1)
|
||||
|
|
@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
(301, 'http://scrapytest.org/3'),
|
||||
(301, 'http://scrapytest.org/4'),
|
||||
),
|
||||
b'http://scrapytest.org/1', # expected initial referer
|
||||
b'http://scrapytest.org/1', # expected referer for the redirection request
|
||||
b'http://scrapytest.org/1', # expected initial referer
|
||||
b'http://scrapytest.org/1', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/1',
|
||||
'https://scrapytest.org/2',
|
||||
|
|
@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
|
|||
(301, 'http://scrapytest.org/3'),
|
||||
(301, 'http://scrapytest.org/4'),
|
||||
),
|
||||
None, # expected initial "Referer"
|
||||
None, # expected "Referer" for the redirection request
|
||||
None, # expected initial "Referer"
|
||||
None, # expected "Referer" for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/1',
|
||||
'https://scrapytest.org/2',
|
||||
|
|
@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
|
|||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/101', # expected initial "Referer"
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
b'http://scrapytest.org/101', # expected initial "Referer"
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
|
|
@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
|
|||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/101', # expected initial referer
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
b'http://scrapytest.org/101', # expected initial referer
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
|
|
@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
|
|||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/101', # expected initial referer
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
b'http://scrapytest.org/101', # expected initial referer
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase):
|
|||
return steps
|
||||
|
||||
dfd = mustbe_deferred(_append, 1)
|
||||
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
|
||||
steps.append(2) # add another value, that should be catched by assertEqual
|
||||
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
|
||||
steps.append(2) # add another value, that should be catched by assertEqual
|
||||
return dfd
|
||||
|
||||
def test_unfired_deferred(self):
|
||||
|
|
@ -27,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase):
|
|||
return dfd
|
||||
|
||||
dfd = mustbe_deferred(_append, 1)
|
||||
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
|
||||
steps.append(2) # add another value, that should be catched by assertEqual
|
||||
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
|
||||
steps.append(2) # add another value, that should be catched by assertEqual
|
||||
return dfd
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
|
|||
chunked_body += "8\r\n" + "sequence\r\n"
|
||||
chunked_body += "0\r\n\r\n"
|
||||
body = decode_chunked_transfer(chunked_body)
|
||||
self.assertEqual(body, \
|
||||
"This is the data in the first chunk\r\n" +
|
||||
"and this is the second one\r\n" +
|
||||
"consequence")
|
||||
self.assertEqual(body,
|
||||
"This is the data in the first chunk\r\n" +
|
||||
"and this is the second one\r\n" +
|
||||
"consequence")
|
||||
|
|
|
|||
|
|
@ -1,20 +1,16 @@
|
|||
import unittest
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
|
||||
|
||||
from scrapy.spiders import CrawlSpider
|
||||
|
||||
|
||||
class MyBaseSpider(CrawlSpider):
|
||||
pass # abstract spider
|
||||
|
||||
|
||||
class MySpider1(MyBaseSpider):
|
||||
class MySpider1(Spider):
|
||||
name = 'myspider1'
|
||||
|
||||
|
||||
class MySpider2(MyBaseSpider):
|
||||
class MySpider2(Spider):
|
||||
name = 'myspider2'
|
||||
|
||||
|
||||
|
|
@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase):
|
|||
it = iter_spider_classes(tests.test_utils_spider)
|
||||
self.assertEqual(set(it), {MySpider1, MySpider2})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue