RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add missing blank lines between functions and classes 

Also fixed 2 unrelated Flake8 issues
This commit is contained in:
Adrián Chaves 2019-11-18 17:16:09 +01:00
parent a78e58afe0
commit f261cf65e9
70 changed files with 199 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
pytest.ini
View File

@ -30,16 +30,15 @@ flake8-ignore =
scrapy/commands/check.py F401 E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
scrapy/commands/fetch.py E401 E501 E128 E502 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/list.py E302
scrapy/commands/parse.py E128 E501 E731 E226
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E302 E128
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/startproject.py E502 E127 E501 E128
scrapy/commands/version.py E501 E128
scrapy/commands/view.py F401 E302
scrapy/commands/view.py F401
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E502 E128
@ -60,7 +59,7 @@ flake8-ignore =
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
@ -72,11 +71,11 @@ flake8-ignore =
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
scrapy/extensions/closespider.py E501 E502 E128 E123
scrapy/extensions/corestats.py E302 E501
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501 E303 F401
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E302 E501
scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/throttle.py E501
# scrapy/http
@ -87,18 +86,14 @@ flake8-ignore =
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128 W293 W291
scrapy/http/response/html.py E302
scrapy/http/response/text.py E501 W293 E128 E124
scrapy/http/response/xml.py E302
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
# scrapy/loader
scrapy/loader/__init__.py E501 E502 E128
scrapy/loader/common.py E302
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/__init__.py E302
scrapy/pipelines/files.py E116 E501 E266
scrapy/pipelines/images.py E265 E501
scrapy/pipelines/media.py E125 E501 E266
@ -123,56 +118,50 @@ flake8-ignore =
scrapy/utils/benchserver.py E501
scrapy/utils/boto.py F401
scrapy/utils/conf.py E402 E502 E501
scrapy/utils/console.py E302 E261 F401 E306 E305
scrapy/utils/console.py E261 F401 E306 E305
scrapy/utils/curl.py F401
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501 E302
scrapy/utils/defer.py E501 E302 E128
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/display.py E302
scrapy/utils/engine.py F401 E261 E302
scrapy/utils/ftp.py E302
scrapy/utils/gz.py E305 E501 E302 W504
scrapy/utils/engine.py F401 E261
scrapy/utils/gz.py E305 E501 W504
scrapy/utils/http.py F403 F401 E226
scrapy/utils/httpobj.py E302 E501
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/job.py E302
scrapy/utils/log.py E128 W503
scrapy/utils/markup.py F403 F401 W292
scrapy/utils/misc.py E501 E226
scrapy/utils/multipart.py F403 F401 W292
scrapy/utils/project.py E501
scrapy/utils/python.py E501 E302
scrapy/utils/reactor.py E302 E226
scrapy/utils/python.py E501
scrapy/utils/reactor.py E226
scrapy/utils/reqser.py E501
scrapy/utils/request.py E302 E127 E501
scrapy/utils/response.py E501 E302 E128
scrapy/utils/request.py E127 E501
scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
scrapy/utils/spider.py E271 E302 E501
scrapy/utils/spider.py E271 E501
scrapy/utils/ssl.py E501
scrapy/utils/template.py E302
scrapy/utils/test.py E302 E501
scrapy/utils/test.py E501
scrapy/utils/url.py E501 F403 F401 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/_monkeypatches.py W293
scrapy/cmdline.py E502 E501
scrapy/crawler.py E501
scrapy/dupefilters.py E302 E501 E202
scrapy/exceptions.py E302 E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501 E261 E226
scrapy/extension.py E302
scrapy/interfaces.py E302 E501
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501 W293
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E502 E128 E501
scrapy/pqueues.py E501
scrapy/resolver.py E302
scrapy/responsetypes.py E128 E501 E305
scrapy/robotstxt.py E302 E501
scrapy/robotstxt.py E501
scrapy/shell.py E501
scrapy/signalmanager.py E501
scrapy/spiderloader.py E225 F841 E501 E126
@ -181,91 +170,82 @@ flake8-ignore =
# tests
tests/__init__.py F401 E402 E501
tests/mockserver.py E401 E501 E126 E123 F401
tests/pipelines.py E302 F841 E226
tests/spiders.py E302 E501 E127
tests/pipelines.py F841 E226
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_fetch.py E501 E261
tests/test_command_parse.py F401 E302 E501 E128 E303 E226
tests/test_command_parse.py F401 E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py F401 E128 E501
tests/test_contracts.py E501 E128 W293
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py E302 F841 E501 E305
tests/test_dependencies.py F841 E501 E305
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401
tests/test_downloadermiddleware_httpcache.py E501 E305 F401
tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
tests/test_engine.py E401 E501 E502 E128 E261
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F401 F841
tests/test_feedexport.py E501 F401 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E302 E501
tests/test_http_headers.py E501
tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
tests/test_http_response.py E501 E301 E502 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
tests/test_logformatter.py E128 E501 E122 E302
tests/test_mail.py E302 E128 E501 E305
tests/test_middleware.py E302 E501 E128
tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501 E305
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
tests/test_pipeline_images.py F401 F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E302 E305
tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
tests/test_responsetypes.py E501 E305
tests/test_robotstxt_interface.py F401 E501 W291 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py F401 E501 E127
tests/test_spider.py E501 F401
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
tests/test_squeues.py E501 E302 E701 E741
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226
tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_console.py E302
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
tests/test_utils_defer.py E306 E261 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
tests/test_utils_http.py E302 E501 E502 E128 W504
tests/test_utils_httpobj.py E302
tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
tests/test_utils_http.py E501 E502 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E303 E241
tests/test_utils_log.py E741 E226
tests/test_utils_python.py E501 E303 E731 E701 E305
tests/test_utils_reqser.py F401 E501 E128
tests/test_utils_request.py E302 E501 E128 E305
tests/test_utils_request.py E501 E128 E305
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E302 E731 E226
tests/test_utils_sitemap.py E302 E128 E501 E124
tests/test_utils_spider.py E261 E302 E305
tests/test_utils_signal.py E741 F841 E731 E226
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_spider.py E261 E305
tests/test_utils_template.py E305
tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
tests/mocks/dummydbm.py E302
tests/test_cmdline/__init__.py E502 E501
tests/test_cmdline/extensions.py E302
tests/test_settings/__init__.py F401 E501 E128
tests/test_spiderloader/__init__.py E128 E501 E302
tests/test_spiderloader/test_spiders/spider0.py E302
tests/test_spiderloader/test_spiders/spider1.py E302
tests/test_spiderloader/test_spiders/spider2.py E302
tests/test_spiderloader/test_spiders/spider3.py E302
tests/test_spiderloader/test_spiders/nested/spider4.py E302
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501

1
scrapy/commands/fetch.py
View File

@ -8,6 +8,7 @@ from scrapy.exceptions import UsageError
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
class Command(ScrapyCommand):
requires_project = False

1
scrapy/commands/list.py
View File

@ -1,6 +1,7 @@
from __future__ import print_function
from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_project = True

1
scrapy/commands/settings.py
View File

@ -4,6 +4,7 @@ import json
from scrapy.commands import ScrapyCommand
from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False

1
scrapy/commands/view.py
View File

@ -1,6 +1,7 @@
from scrapy.commands import fetch, ScrapyCommand
from scrapy.utils.response import open_in_browser
class Command(fetch.Command):
def short_desc(self):

4
scrapy/core/downloader/handlers/datauri.py
View File

@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse) and
uri.media_type.split('/')[0] == 'text'):
if (issubclass(respcls, TextResponse)
and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset

2
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
def _has_ajaxcrawlable_meta(text):
"""
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')

1
scrapy/dupefilters.py
View File

@ -5,6 +5,7 @@ import logging
from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, request_fingerprint
class BaseDupeFilter(object):
@classmethod

13
scrapy/exceptions.py
View File

@ -7,10 +7,12 @@ new exceptions here without documenting them there.
# Internal
class NotConfigured(Exception):
"""Indicates a missing configuration situation"""
pass
class _InvalidOutput(TypeError):
"""
Indicates an invalid value has been returned by a middleware's processing method.
@ -18,15 +20,19 @@ class _InvalidOutput(TypeError):
"""
pass
# HTTP and crawling
class IgnoreRequest(Exception):
"""Indicates a decision was made not to process a request"""
class DontCloseSpider(Exception):
"""Request the spider not to be closed yet"""
pass
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
@ -34,30 +40,37 @@ class CloseSpider(Exception):
super(CloseSpider, self).__init__()
self.reason = reason
# Items
class DropItem(Exception):
"""Drop item from the item pipeline"""
pass
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
pass
# Commands
class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)
class ScrapyDeprecationWarning(Warning):
"""Warning category for deprecated features, since the default
DeprecationWarning is silenced on Python 2.7+
"""
pass
class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""
pass

1
scrapy/extension.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
class ExtensionManager(MiddlewareManager):
component_name = 'extension'

1
scrapy/extensions/spiderstate.py
View File

@ -5,6 +5,7 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
class SpiderState(object):
"""Store and load spider state during a scraping job"""

1
scrapy/http/response/html.py
View File

@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
from scrapy.http.response.text import TextResponse
class HtmlResponse(TextResponse):
pass

1
scrapy/http/response/xml.py
View File

@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
from scrapy.http.response.text import TextResponse
class XmlResponse(TextResponse):
pass

1
scrapy/interfaces.py
View File

@ -1,5 +1,6 @@
from zope.interface import Interface
class ISpiderLoader(Interface):
def from_settings(settings):

1
scrapy/loader/common.py
View File

@ -3,6 +3,7 @@
from functools import partial
from scrapy.utils.python import get_func_args
def wrap_loader_context(function, context):
"""Wrap functions that receive loader_context to contain the context
"pre-loaded" and expose a interface that receives only one argument

1
scrapy/pipelines/__init__.py
View File

@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rst
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
class ItemPipelineManager(MiddlewareManager):
component_name = 'item pipeline'

1
scrapy/resolver.py
View File

@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCache
dnscache = LocalCache(10000)
class CachingThreadedResolver(ThreadedResolver):
def __init__(self, reactor, cache_size, timeout):
super(CachingThreadedResolver, self).__init__(reactor)

3
scrapy/robotstxt.py
View File

@ -5,8 +5,10 @@ from six import with_metaclass
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
try:
if to_native_str_type:
@ -23,6 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
robotstxt_body = ''
return robotstxt_body
class RobotParser(with_metaclass(ABCMeta)):
@classmethod
@abstractmethod

7
scrapy/utils/console.py
View File

@ -1,6 +1,7 @@
from functools import wraps
from collections import OrderedDict
def _embed_ipython_shell(namespace={}, banner=''):
"""Start an IPython Shell"""
try:
@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
shell()
return wrapper
def _embed_bpython_shell(namespace={}, banner=''):
"""Start a bpython shell"""
import bpython
@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
bpython.embed(locals_=namespace, banner=banner)
return wrapper
def _embed_ptpython_shell(namespace={}, banner=''):
"""Start a ptpython shell"""
import ptpython.repl
@ -40,6 +43,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
ptpython.repl.embed(locals=namespace)
return wrapper
def _embed_standard_shell(namespace={}, banner=''):
"""Start a standard python shell"""
import code
@ -55,6 +59,7 @@ def _embed_standard_shell(namespace={}, banner=''):
code.interact(banner=banner, local=namespace)
return wrapper
DEFAULT_PYTHON_SHELLS = OrderedDict([
('ptpython', _embed_ptpython_shell),
('ipython', _embed_ipython_shell),
@ -62,6 +67,7 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([
('python', _embed_standard_shell),
])
def get_shell_embed_func(shells=None, known_shells=None):
"""Return the first acceptable shell-embed function
from a given list of shell names.
@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None):
except ImportError:
continue
def start_python_console(namespace=None, banner='', shells=None):
"""Start Python console bound to the given namespace.
Readline support and tab completion will be used on Unix, if available.

1
scrapy/utils/decorators.py
View File

@ -34,6 +34,7 @@ def defers(func):
return defer.maybeDeferred(func, *a, **kw)
return wrapped
def inthread(func):
"""Decorator to call a function in a thread and return a deferred with the
result

9
scrapy/utils/defer.py
View File

@ -7,6 +7,7 @@ from twisted.python import failure
from scrapy.exceptions import IgnoreRequest
def defer_fail(_failure):
"""Same as twisted.internet.defer.fail but delay calling errback until
next reactor loop
@ -18,6 +19,7 @@ def defer_fail(_failure):
reactor.callLater(0.1, d.errback, _failure)
return d
def defer_succeed(result):
"""Same as twisted.internet.defer.succeed but delay calling callback until
next reactor loop
@ -29,6 +31,7 @@ def defer_succeed(result):
reactor.callLater(0.1, d.callback, result)
return d
def defer_result(result):
if isinstance(result, defer.Deferred):
return result
@ -37,6 +40,7 @@ def defer_result(result):
else:
return defer_succeed(result)
def mustbe_deferred(f, *args, **kw):
"""Same as twisted.internet.defer.maybeDeferred, but delay calling
callback/errback to next reactor loop
@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw):
else:
return defer_result(result)
def parallel(iterable, count, callable, *args, **named):
"""Execute a callable over the objects in the given iterable, in parallel,
using no more than ``count`` concurrent calls.
@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named):
work = (callable(elem, *args, **named) for elem in iterable)
return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
def process_chain(callbacks, input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks"""
d = defer.Deferred()
@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw):
d.callback(input)
return d
def process_chain_both(callbacks, errbacks, input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks and errbacks"""
d = defer.Deferred()
@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
d.callback(input)
return d
def process_parallel(callbacks, input, *a, **kw):
"""Return a Deferred with the output of all successful calls to the given
callbacks
@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw):
d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
return d
def iter_errback(iterable, errback, *a, **kw):
"""Wraps an iterable calling an errback if an error is caught while
iterating it.

3
scrapy/utils/display.py
View File

@ -6,6 +6,7 @@ from __future__ import print_function
import sys
from pprint import pformat as pformat_
def _colorize(text, colorize=True):
if not colorize or not sys.stdout.isatty():
return text
@ -17,8 +18,10 @@ def _colorize(text, colorize=True):
except ImportError:
return text
def pformat(obj, *args, **kwargs):
return _colorize(pformat_(obj), kwargs.pop('colorize', True))
def pprint(obj, *args, **kwargs):
print(pformat(obj, *args, **kwargs))

3
scrapy/utils/engine.py
View File

@ -3,6 +3,7 @@
from __future__ import print_function
from time import time # used in global tests code
def get_engine_status(engine):
"""Return a report of the current engine status"""
tests = [
@ -32,6 +33,7 @@ def get_engine_status(engine):
return checks
def format_engine_status(engine=None):
checks = get_engine_status(engine)
s = "Execution engine status\n\n"
@ -41,5 +43,6 @@ def format_engine_status(engine=None):
return s
def print_engine_status(engine):
print(format_engine_status(engine))

1
scrapy/utils/ftp.py
View File

@ -1,6 +1,7 @@
from ftplib import error_perm
from posixpath import dirname
def ftp_makedirs_cwd(ftp, path, first_call=True):
"""Set the current directory of the FTP connection given in the ``ftp``
argument (as a ftplib.FTP object), creating all parent directories if they

1
scrapy/utils/gz.py
View File

@ -45,6 +45,7 @@ def gunzip(data):
_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
@deprecated
def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""

3
scrapy/utils/httpobj.py
View File

@ -4,7 +4,10 @@ import weakref
from six.moves.urllib.parse import urlparse
_urlparse_cache = weakref.WeakKeyDictionary()
def urlparse_cached(request_or_response):
"""Return urlparse.urlparse caching the result, where the argument can be a
Request or Response object

1
scrapy/utils/job.py
View File

@ -1,5 +1,6 @@
import os
def job_dir(settings):
path = settings['JOBDIR']
if path and not os.path.exists(path):

1
scrapy/utils/python.py
View File

@ -165,6 +165,7 @@ def memoizemethod_noargs(method):
_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
@deprecated("scrapy.utils.python.binary_is_text")
def isbinarytext(text):
""" This function is deprecated.

1
scrapy/utils/reactor.py
View File

@ -1,5 +1,6 @@
from twisted.internet import reactor, error
def listen_tcp(portrange, host, factory):
"""Like reactor.listenTCP but tries different ports in a range."""
assert len(portrange) <= 2, "invalid portrange: %s" % portrange

2
scrapy/utils/request.py
View File

@ -16,6 +16,8 @@ from scrapy.utils.httpobj import urlparse_cached
_fingerprint_cache = weakref.WeakKeyDictionary()
def request_fingerprint(request, include_headers=None, keep_fragments=False):
"""
Return the request fingerprint.

4
scrapy/utils/response.py
View File

@ -13,6 +13,8 @@ from w3lib import html
_baseurl_cache = weakref.WeakKeyDictionary()
def get_base_url(response):
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
@ -23,6 +25,8 @@ def get_base_url(response):
_metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:

1
scrapy/utils/spider.py
View File

@ -28,6 +28,7 @@ def iter_spider_classes(module):
getattr(obj, 'name', None):
yield obj
def spidercls_for_request(spider_loader, request, default_spidercls=None,
log_none=False, log_multiple=False):
"""Return a spider class that handles the given Request.

2
scrapy/utils/template.py
View File

@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs):
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
def string_camelcase(string):
""" Convert a word to its CamelCase version and remove invalid chars

6
scrapy/utils/test.py
View File

@ -32,6 +32,7 @@ def skip_if_no_boto():
except NotConfigured as e:
raise SkipTest(e)
def get_s3_content_and_delete(bucket, path, with_key=False):
""" Get content from s3 key, and delete key afterwards.
"""
@ -51,6 +52,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
bucket.delete_key(path)
return (content, key) if with_key else content
def get_gcs_content_and_delete(bucket, path):
from google.cloud import storage
client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
@ -61,6 +63,7 @@ def get_gcs_content_and_delete(bucket, path):
bucket.delete_blob(path)
return content, acl, blob
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
@ -72,12 +75,14 @@ def get_crawler(spidercls=None, settings_dict=None):
runner = CrawlerRunner(settings_dict)
return runner.create_crawler(spidercls or Spider)
def get_pythonpath():
"""Return a PYTHONPATH suitable to use in processes so that they find this
installation of Scrapy"""
scrapy_path = import_module('scrapy').__path__[0]
return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
def get_testenv():
"""Return a OS environment dict suitable to fork processes that need to import
this installation of Scrapy, instead of a system installed one.
@ -86,6 +91,7 @@ def get_testenv():
env['PYTHONPATH'] = get_pythonpath()
return env
def assert_samelines(testcase, text1, text2, msg=None):
"""Asserts text1 and text2 have the same lines, ignoring differences in
line endings between platforms

2
scrapy/utils/versions.py
View File

@ -27,5 +27,5 @@ def scrapy_components_versions():
("Python", sys.version.replace("\n", "- ")),
("pyOpenSSL", get_openssl_version()),
("cryptography", cryptography.__version__),
("Platform", platform.platform()),
("Platform", platform.platform()),
]

1
tests/mocks/dummydbm.py
View File

@ -13,6 +13,7 @@ error = KeyError
_DATABASES = collections.defaultdict(DummyDB)
def open(file, flag='r', mode=0o666):
"""Open or create a dummy database compatible.

1
tests/pipelines.py
View File

@ -2,6 +2,7 @@
Some pipelines used for testing
"""
class ZeroDivisionErrorPipeline(object):
def open_spider(self, spider):

1
tests/spiders.py
View File

@ -16,6 +16,7 @@ class MockServerSpider(Spider):
super(MockServerSpider, self).__init__(*args, **kwargs)
self.mockserver = mockserver
class MetaSpider(MockServerSpider):
name = 'meta'

1
tests/test_cmdline/extensions.py
View File

@ -1,5 +1,6 @@
"""A test extension used to check the settings loading order"""
class TestExtension(object):
def __init__(self, settings):

1
tests/test_command_parse.py
View File

@ -12,6 +12,7 @@ def _textmode(bstr):
and reading from it in text mode"""
return to_unicode(bstr).replace(os.linesep, '\n')
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
command = 'parse'

1
tests/test_dependencies.py
View File

@ -1,6 +1,7 @@
from importlib import import_module
from twisted.trial import unittest
class ScrapyUtilsTest(unittest.TestCase):
def test_required_openssl_version(self):
try:

2
tests/test_downloadermiddleware_ajaxcrawlable.py
View File

@ -5,8 +5,10 @@ from scrapy.spiders import Spider
from scrapy.http import Request, HtmlResponse, Response
from scrapy.utils.test import get_crawler
__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl']
class AjaxCrawlMiddlewareTest(unittest.TestCase):
def setUp(self):
crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True})

1
tests/test_downloadermiddleware_httpcache.py
View File

@ -149,6 +149,7 @@ class FilesystemStorageTest(DefaultStorageTest):
storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
class FilesystemStorageGzipTest(FilesystemStorageTest):
def _get_settings(self, **new_settings):

1
tests/test_dupefilters.py
View File

@ -12,6 +12,7 @@ from scrapy.utils.job import job_dir
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@classmethod

1
tests/test_http_headers.py
View File

@ -3,6 +3,7 @@ import copy
from scrapy.http import Headers
class HeadersTest(unittest.TestCase):
def assertSortedEqual(self, first, second, msg=None):

1
tests/test_logformatter.py
View File

@ -118,6 +118,7 @@ class DropSomeItemsPipeline(object):
else:
self.drop = True
class ShowOrSkipMessagesTestCase(TwistedTestCase):
def setUp(self):
self.mockserver = MockServer()

1
tests/test_mail.py
View File

@ -6,6 +6,7 @@ from email.charset import Charset
from scrapy.mail import MailSender
class MailSenderTest(unittest.TestCase):
def test_send(self):

4
tests/test_middleware.py
View File

@ -4,6 +4,7 @@ from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured
from scrapy.middleware import MiddlewareManager
class M1(object):
def open_spider(self, spider):
@ -15,6 +16,7 @@ class M1(object):
def process(self, response, request, spider):
pass
class M2(object):
def open_spider(self, spider):
@ -25,6 +27,7 @@ class M2(object):
pass
class M3(object):
def process(self, response, request, spider):
@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process'):
self.methods['process'].append(mw.process)
class MiddlewareManagerTest(unittest.TestCase):
def test_init(self):

1
tests/test_responsetypes.py
View File

@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers
class ResponseTypesTest(unittest.TestCase):
def test_from_filename(self):

2
tests/test_robotstxt_interface.py
View File

@ -19,6 +19,7 @@ def rerp_available():
return False
return True
def protego_available():
# check if protego parser is installed
try:
@ -27,6 +28,7 @@ def protego_available():
return False
return True
class BaseRobotParserTest:
def _setUp(self, parser_cls):
self.parser_cls = parser_cls

1
tests/test_spiderloader/__init__.py
View File

@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase):
spiders = spider_loader.list()
self.assertEqual(spiders, [])
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self):

1
tests/test_spiderloader/test_spiders/nested/spider4.py
View File

@ -1,5 +1,6 @@
from scrapy.spiders import Spider
class Spider4(Spider):
name = "spider4"
allowed_domains = ['spider4.com']

1
tests/test_spiderloader/test_spiders/spider0.py
View File

@ -1,4 +1,5 @@
from scrapy.spiders import Spider
class Spider0(Spider):
allowed_domains = ["scrapy1.org", "scrapy3.org"]

1
tests/test_spiderloader/test_spiders/spider1.py
View File

@ -1,5 +1,6 @@
from scrapy.spiders import Spider
class Spider1(Spider):
name = "spider1"
allowed_domains = ["scrapy1.org", "scrapy3.org"]

1
tests/test_spiderloader/test_spiders/spider2.py
View File

@ -1,5 +1,6 @@
from scrapy.spiders import Spider
class Spider2(Spider):
name = "spider2"
allowed_domains = ["scrapy2.org", "scrapy3.org"]

1
tests/test_spiderloader/test_spiders/spider3.py
View File

@ -1,5 +1,6 @@
from scrapy.spiders import Spider
class Spider3(Spider):
name = "spider3"
allowed_domains = ['spider3.com']

2
tests/test_spidermiddleware_offsite.py
View File

@ -9,6 +9,7 @@ from scrapy.spidermiddlewares.offsite import URLWarning
from scrapy.utils.test import get_crawler
import warnings
class TestOffsiteMiddleware(TestCase):
def setUp(self):
@ -53,6 +54,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
out = list(self.mw.process_spider_output(res, reqs, self.spider))
self.assertEqual(out, reqs)
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
def _get_spider(self):

4
tests/test_spidermiddleware_output_chain.py
View File

@ -34,6 +34,7 @@ class RecoverySpider(Spider):
if not response.meta.get('dont_fail'):
raise TabError()
class RecoveryMiddleware:
def process_spider_exception(self, response, exception, spider):
spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
@ -50,6 +51,7 @@ class FailProcessSpiderInputMiddleware:
spider.logger.info('Middleware: will raise IndexError')
raise IndexError()
class ProcessSpiderInputSpiderWithoutErrback(Spider):
name = 'ProcessSpiderInputSpiderWithoutErrback'
custom_settings = {
@ -177,6 +179,7 @@ class GeneratorRecoverMiddleware:
spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
yield {'processed': [method]}
class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
pass
@ -247,6 +250,7 @@ class NotGeneratorRecoverMiddleware:
spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
return [{'processed': [method]}]
class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
pass

4
tests/test_spidermiddleware_referer.py
View File

@ -349,6 +349,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware):
]
# --- Tests using Request meta dict to set policy
class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
@ -518,14 +519,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
"""
The empty string means "no-referrer-when-downgrade"

13
tests/test_squeues.py
View File

@ -7,16 +7,20 @@ from scrapy.http import Request
from scrapy.loader import ItemLoader
from scrapy.selector import Selector
class TestItem(Item):
name = Field()
def _test_procesor(x):
return x + x
class TestLoader(ItemLoader):
default_item_class = TestItem
name_out = staticmethod(_test_procesor)
def nonserializable_object_test(self):
q = self.queue()
try:
@ -35,6 +39,7 @@ def nonserializable_object_test(self):
sel = Selector(text='<html><body><p>some text</p></body></html>')
self.assertRaises(ValueError, q.push, sel)
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
chunksize = 100000
@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
test_nonserializable_object = nonserializable_object_test
class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 1
class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 2
class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 3
class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 4
@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
self.assertEqual(r.url, r2.url)
assert r2.meta['request'] is r2
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 1
class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 2
class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 3
class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 4

1
tests/test_utils_console.py
View File

@ -14,6 +14,7 @@ try:
except ImportError:
ipy = False
class UtilsConsoleTestCase(unittest.TestCase):
def test_get_shell_embed_func(self):

9
tests/test_utils_defer.py
View File

@ -33,14 +33,23 @@ class MustbeDeferredTest(unittest.TestCase):
steps.append(2) # add another value, that should be catched by assertEqual
return dfd
def cb1(value, arg1, arg2):
return "(cb1 %s %s %s)" % (value, arg1, arg2)
def cb2(value, arg1, arg2):
return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2))
def cb3(value, arg1, arg2):
return "(cb3 %s %s %s)" % (value, arg1, arg2)
def cb_fail(value, arg1, arg2):
return Failure(TypeError())
def eb1(failure, arg1, arg2):
return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2)

1
tests/test_utils_http.py
View File

@ -2,6 +2,7 @@ import unittest
from scrapy.utils.http import decode_chunked_transfer
class ChunkedTest(unittest.TestCase):
def test_decode_chunked_transfer(self):

1
tests/test_utils_httpobj.py
View File

@ -4,6 +4,7 @@ from six.moves.urllib.parse import urlparse
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
class HttpobjUtilsTest(unittest.TestCase):
def test_urlparse_cached(self):

1
tests/test_utils_iterators.py
View File

@ -235,6 +235,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
i = self.xmliter(42, 'product')
self.assertRaises(TypeError, next, i)
class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')

1
tests/test_utils_request.py
View File

@ -4,6 +4,7 @@ from scrapy.http import Request
from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
request_authenticate, request_httprepr
class UtilsRequestTest(unittest.TestCase):
def test_request_fingerprint(self):

1
tests/test_utils_signal.py
View File

@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
def _get_result(self, signal, *a, **kw):
return send_catch_log_deferred(signal, *a, **kw)
class SendCatchLogTest2(unittest.TestCase):
def test_error_logged_if_deferred_not_supported(self):

1
tests/test_utils_sitemap.py
View File

@ -2,6 +2,7 @@ import unittest
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
class SitemapTest(unittest.TestCase):
def test_sitemap(self):

3
tests/test_utils_spider.py
View File

@ -9,12 +9,15 @@ from scrapy.spiders import CrawlSpider
class MyBaseSpider(CrawlSpider):
pass # abstract spider
class MySpider1(MyBaseSpider):
name = 'myspider1'
class MySpider2(MyBaseSpider):
name = 'myspider2'
class UtilsSpidersTestCase(unittest.TestCase):
def test_iterate_spider_output(self):

2
tests/test_utils_url.py
View File

@ -187,6 +187,7 @@ class AddHttpIfNoScheme(unittest.TestCase):
class GuessSchemeTest(unittest.TestCase):
pass
def create_guess_scheme_t(args):
def do_expected(self):
url = guess_scheme(args[0])
@ -195,6 +196,7 @@ def create_guess_scheme_t(args):
args[0], url, args[1])
return do_expected
def create_skipped_scheme_t(args):
def do_expected(self):
raise unittest.SkipTest(args[2])