RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Address some issues reported by Pylint (#5677) 

Co-authored-by: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com>
Co-authored-by: Andrey Rahmatullin <wrar@wrar.name>
This commit is contained in:
Mark Mayo 2022-11-27 23:00:13 +13:00 committed by GitHub
parent e2db624204
commit f9a29f03d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
87 changed files with 219 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conftest.py
View File

@ -21,7 +21,7 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
with Path('tests/ignores.txt').open() as reader:
with Path('tests/ignores.txt').open(encoding="utf-8") as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':

2
docs/_ext/scrapydocs.py
View File

@ -1,8 +1,8 @@
from operator import itemgetter
from docutils.parsers.rst.roles import set_classes
from docutils import nodes
from docutils.parsers.rst import Directive
from sphinx.util.nodes import make_refnode
from operator import itemgetter
class settingslist_node(nodes.General, nodes.Element):

8
docs/conftest.py
View File

@ -1,15 +1,17 @@
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
from scrapy.http.response.html import HtmlResponse
def load_response(url: str, filename: str) -> HtmlResponse:

9
docs/utils/linkfix.py Executable file → Normal file
View File

@ -13,6 +13,7 @@ Author: dufferzafar
"""
import re
import sys
from pathlib import Path
@ -28,11 +29,11 @@ def main():
# Read lines from the linkcheck output file
try:
with Path("build/linkcheck/output.txt").open() as out:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
sys.exit(1)
# For every line, fix the respective file
for line in output_lines:
@ -52,12 +53,12 @@ def main():
# Update the previous file
if _filename:
Path(_filename).write_text(_contents)
Path(_filename).write_text(_contents, encoding="utf-8")
_filename = newfilename
# Read the new file to memory
_contents = Path(_filename).read_text()
_contents = Path(_filename).read_text(encoding="utf-8")
_contents = _contents.replace(match.group(3), match.group(4))
else:

27
pylintrc
View File

@ -9,27 +9,19 @@ disable=abstract-method,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bare-except,
blacklisted-name,
broad-except,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-dict-items,
consider-using-from-import,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
disallowed-name,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
@ -52,18 +44,12 @@ disable=abstract-method,
lost-exception,
method-hidden,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-statement,
pointless-string-statement,
@ -74,10 +60,7 @@ disable=abstract-method,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
@ -89,31 +72,23 @@ disable=abstract-method,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-dunder-call,
unnecessary-lambda,
unnecessary-pass,
unreachable,
unspecified-encoding,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

2
scrapy/commands/__init__.py
View File

@ -95,7 +95,7 @@ class ScrapyCommand:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile:
Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep)
Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8")
if opts.pdb:
failure.startDebugMode()

2
scrapy/commands/genspider.py
View File

@ -63,7 +63,7 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
print(template_file.read_text())
print(template_file.read_text(encoding="utf-8"))
return
if len(args) != 2:
raise UsageError()

2
scrapy/commands/parse.py
View File

@ -6,14 +6,12 @@ from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from twisted.internet.defer import maybeDeferred
from scrapy.commands import BaseRunSpiderCommand
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)

3
scrapy/core/downloader/handlers/http10.py
View File

@ -33,5 +33,4 @@ class HTTP10DownloadHandler:
crawler=self._crawler,
)
return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)
return reactor.connectTCP(host, port, factory)

22
scrapy/core/downloader/handlers/http11.py
View File

@ -26,7 +26,6 @@ from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.python import to_bytes, to_unicode
logger = logging.getLogger(__name__)
@ -289,16 +288,15 @@ class ScrapyAgent:
bindAddress=bindaddress,
pool=self._pool,
)
else:
proxyScheme = proxyScheme or b'http'
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
proxyScheme = proxyScheme or b'http'
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
return self._Agent(
reactor=reactor,
@ -567,7 +565,7 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["dataloss"])
return
elif not self._fail_on_dataloss_warned:
if not self._fail_on_dataloss_warned:
logger.warning("Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",

1
scrapy/core/downloader/tls.py
View File

@ -7,7 +7,6 @@ from twisted.internet.ssl import AcceptableCiphers
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)

4
scrapy/core/downloader/webclient.py
View File

@ -1,8 +1,8 @@
import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.http import HTTPClient
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
@ -185,7 +185,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
@param version: The HTTP version.
@type version: L{bytes}
@param status: The HTTP status code, an integer represented as a
bytestring.
bytestring.
@type status: L{bytes}
@param message: The HTTP status message.
@type message: L{bytes}

2
scrapy/core/http2/agent.py
View File

@ -140,7 +140,7 @@ class ScrapyProxyH2Agent(H2Agent):
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
super().__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,

2
scrapy/core/http2/protocol.py
View File

@ -42,7 +42,7 @@ class InvalidNegotiatedProtocol(H2Error):
self.negotiated_protocol = negotiated_protocol
def __str__(self) -> str:
return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
return f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}"
class RemoteTerminatedConnection(H2Error):

4
scrapy/core/scheduler.py
View File

@ -333,9 +333,9 @@ class Scheduler(BaseScheduler):
path = Path(dqdir, 'active.json')
if not path.exists():
return []
with path.open() as f:
with path.open(encoding="utf-8") as f:
return json.load(f)
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with Path(dqdir, 'active.json').open('w') as f:
with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f:
json.dump(state, f)

30
scrapy/core/scraper.py
View File

@ -152,9 +152,9 @@ class Scraper:
"""
if isinstance(result, Response):
return self.spidermw.scrape_response(self.call_spider, result, request, spider)
else: # result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
# else result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
if isinstance(result, Response):
@ -276,17 +276,15 @@ class Scraper:
return self.signals.send_catch_log_deferred(
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
else:
logkws = self.logformatter.scraped(output, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
logkws = self.logformatter.scraped(output, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)

21
scrapy/core/spidermw.py
View File

@ -121,11 +121,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return dfd.result
else:
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
raise _InvalidOutput(msg)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
raise _InvalidOutput(msg)
elif result is None:
continue
else:
@ -213,8 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if last_result_is_async:
return MutableAsyncChain(result, recovered)
else:
return MutableChain(result, recovered) # type: ignore[arg-type]
return MutableChain(result, recovered) # type: ignore[arg-type]
async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
@ -227,11 +225,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
if isinstance(result, AsyncIterable):
return MutableAsyncChain(result, recovered)
else:
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered) # type: ignore[arg-type]
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered) # type: ignore[arg-type]
def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Deferred:

3
scrapy/downloadermiddlewares/cookies.py
View File

@ -9,7 +9,6 @@ from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@ -129,7 +128,7 @@ class CookiesMiddleware:
"""
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies

1
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -8,7 +8,6 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
try:

8
scrapy/downloadermiddlewares/redirect.py
View File

@ -8,7 +8,6 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
@ -56,10 +55,9 @@ class BaseRedirectMiddleware:
{'reason': reason, 'redirected': redirected, 'request': request},
extra={'spider': spider})
return redirected
else:
logger.debug("Discarding %(request)s: max redirections reached",
{'request': request}, extra={'spider': spider})
raise IgnoreRequest("max redirections reached")
logger.debug("Discarding %(request)s: max redirections reached",
{'request': request}, extra={'spider': spider})
raise IgnoreRequest("max redirections reached")
def _redirect_request_using_get(self, request, redirect_url):
redirect_request = _build_redirect_request(

17
scrapy/downloadermiddlewares/retry.py
View File

@ -113,15 +113,14 @@ def get_retry_request(
stats.inc_value(f'{stats_base_key}/count')
stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
return new_request
else:
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None
class RetryMiddleware:

3
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -81,8 +81,7 @@ class RobotsTxtMiddleware:
return result
self._parsers[netloc].addCallback(cb)
return d
else:
return self._parsers[netloc]
return self._parsers[netloc]
def _logerror(self, failure, request, spider):
if failure.type is not IgnoreRequest:

4
scrapy/downloadermiddlewares/stats.py
View File

@ -1,9 +1,9 @@
from twisted.web import http
from scrapy.exceptions import NotConfigured
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
from twisted.web import http
def get_header_size(headers):
size = 0

2
scrapy/dupefilters.py
View File

@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
self.file = Path(path, 'requests.seen').open('a+')
self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8")
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)

4
scrapy/exporters.py
View File

@ -334,9 +334,9 @@ class PythonItemExporter(BaseItemExporter):
def _serialize_value(self, value):
if isinstance(value, Item):
return self.export_item(value)
elif is_item(value):
if is_item(value):
return dict(self._serialize_item(value))
elif is_listlike(value):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (str, bytes)):

14
scrapy/extensions/httpcache.py
View File

@ -17,7 +17,6 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
logger = logging.getLogger(__name__)
@ -81,25 +80,24 @@ class RFC2616Policy:
if b'no-store' in cc:
return False
# Never cache 304 (Not Modified) responses
elif response.status == 304:
if response.status == 304:
return False
# Cache unconditionally if configured to do so
elif self.always_store:
if self.always_store:
return True
# Any hint on response expiration is good
elif b'max-age' in cc or b'Expires' in response.headers:
if b'max-age' in cc or b'Expires' in response.headers:
return True
# Firefox fallbacks this statuses to one year expiration if none is set
elif response.status in (300, 301, 308):
if response.status in (300, 301, 308):
return True
# Other statuses without expiration requires at least one validator
elif response.status in (200, 203, 401):
if response.status in (200, 203, 401):
return b'Last-Modified' in response.headers or b'ETag' in response.headers
# Any other is probably not eligible for caching
# Makes no sense to cache responses that does not contain expiration
# info and can not be revalidated
else:
return False
return False
def is_cached_response_fresh(self, cachedresponse, request):
cc = self._parse_cachecontrol(cachedresponse)

2
scrapy/extensions/telnet.py
View File

@ -11,6 +11,7 @@ import binascii
import os
from twisted.internet import protocol
try:
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
@ -26,7 +27,6 @@ from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
from scrapy.utils.decorators import defers
logger = logging.getLogger(__name__)
# signal to update telnet variables

2
scrapy/extensions/throttle.py
View File

@ -56,7 +56,7 @@ class AutoThrottle:
{
'slot': key, 'concurrency': conc,
'delay': slot.delay * 1000, 'delaydiff': diff * 1000,
'latency': latency * 1000, 'size': size
'latency': latency * 1000, 'size': size,
},
extra={'spider': spider}
)

7
scrapy/http/headers.py
View File

@ -37,12 +37,11 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
elif isinstance(x, str):
if isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
if isinstance(x, int):
return str(x).encode(self.encoding)
else:
raise TypeError(f'Unsupported value type: {type(x)}')
raise TypeError(f'Unsupported value type: {type(x)}')
def __getitem__(self, key):
try:

2
scrapy/http/request/__init__.py
View File

@ -185,7 +185,7 @@ class Request(object_ref):
}
for attr in self.attributes:
d.setdefault(attr, getattr(self, attr))
if type(self) is not Request:
if type(self) is not Request: # pylint: disable=unidiomatic-typecheck
d["_class"] = self.__module__ + '.' + self.__class__.__name__
return d

4
scrapy/http/request/form.py
View File

@ -190,7 +190,7 @@ def _select_value(ele: SelectElement, n: str, v: str):
# And for select tags without options
o = ele.value_options
return (n, o[0]) if o else (None, None)
elif v is not None and multiple:
if v is not None and multiple:
# This is a workround to bug in lxml fixed 2.3.1
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
selected_options = ele.xpath('.//option[@selected]')
@ -236,7 +236,7 @@ def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tup
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
elif len(el) > 1:
if len(el) > 1:
raise ValueError(f"Multiple elements found ({el!r}) matching the "
f"criteria in clickdata: {clickdata!r}")
else:

2
scrapy/linkextractors/lxmlhtml.py
View File

@ -5,7 +5,7 @@ import operator
from functools import partial
from urllib.parse import urljoin, urlparse
import lxml.etree as etree
from lxml import etree
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string

2
scrapy/pipelines/__init__.py
View File

@ -18,7 +18,7 @@ class ItemPipelineManager(MiddlewareManager):
return build_component_list(settings.getwithbase('ITEM_PIPELINES'))
def _add_middleware(self, pipe):
super(ItemPipelineManager, self)._add_middleware(pipe)
super()._add_middleware(pipe)
if hasattr(pipe, 'process_item'):
self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))

6
scrapy/pipelines/files.py
View File

@ -221,16 +221,14 @@ class GCSFilesStore:
checksum = blob.md5_hash
last_modified = time.mktime(blob.updated.timetuple())
return {'checksum': checksum, 'last_modified': last_modified}
else:
return {}
return {}
blob_path = self._get_blob_path(path)
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
def _get_content_type(self, headers):
if headers and 'Content-Type' in headers:
return headers['Content-Type']
else:
return 'application/octet-stream'
return 'application/octet-stream'
def _get_blob_path(self, path):
return self.prefix + path

10
scrapy/responsetypes.py
View File

@ -42,11 +42,10 @@ class ResponseTypes:
"""Return the most appropriate Response class for the given mimetype"""
if mimetype is None:
return Response
elif mimetype in self.classes:
if mimetype in self.classes:
return self.classes[mimetype]
else:
basetype = f"{mimetype.split('/')[0]}/*"
return self.classes.get(basetype, Response)
basetype = f"{mimetype.split('/')[0]}/*"
return self.classes.get(basetype, Response)
def from_content_type(self, content_type, content_encoding=None):
"""Return the most appropriate Response class from an HTTP Content-Type
@ -83,8 +82,7 @@ class ResponseTypes:
mimetype, encoding = self.mimetypes.guess_type(filename)
if mimetype and not encoding:
return self.from_mimetype(mimetype)
else:
return Response
return Response
def from_body(self, body):
"""Try to guess the appropriate response based on the body content.

1
scrapy/robotstxt.py
View File

@ -4,7 +4,6 @@ from abc import ABCMeta, abstractmethod
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)

6
scrapy/settings/__init__.py
View File

@ -24,8 +24,7 @@ def get_settings_priority(priority):
"""
if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
return priority
class SettingsAttribute:
@ -260,8 +259,7 @@ class BaseSettings(MutableMapping):
"""
if len(self) > 0:
return max(self.getpriority(name) for name in self)
else:
return get_settings_priority('default')
return get_settings_priority('default')
def __setitem__(self, name, value):
self.set(name, value)

2
scrapy/spidermiddlewares/offsite.py
View File

@ -65,7 +65,7 @@ class OffsiteMiddleware:
for domain in allowed_domains:
if domain is None:
continue
elif url_pattern.match(domain):
if url_pattern.match(domain):
message = ("allowed_domains accepts only domains, not URLs. "
f"Ignoring URL entry {domain} in allowed_domains.")
warnings.warn(message, URLWarning)

5
scrapy/spidermiddlewares/referer.py
View File

@ -189,8 +189,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
else:
return origin
return origin
class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
@ -216,7 +215,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
elif (
if (
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):

2
scrapy/spiders/crawl.py
View File

@ -26,7 +26,7 @@ def _identity_process_request(request, response):
def _get_method(method, spider):
if callable(method):
return method
elif isinstance(method, str):
if isinstance(method, str):
return getattr(spider, method, None)

4
scrapy/spiders/sitemap.py
View File

@ -69,7 +69,7 @@ class SitemapSpider(Spider):
"""
if isinstance(response, XmlResponse):
return response.body
elif gzip_magic_number(response):
if gzip_magic_number(response):
return gunzip(response.body)
# actual gzipped sitemap files are decompressed above ;
# if we are here (response body is not gzipped)
@ -80,7 +80,7 @@ class SitemapSpider(Spider):
# without actually being a .xml.gz file in the first place,
# merely XML gzip-compressed on the fly,
# in other word, here, we have plain XML
elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'):
if response.url.endswith('.xml') or response.url.endswith('.xml.gz'):
return response.body

14
scrapy/utils/conf.py
View File

@ -35,9 +35,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
else:
compbs.set(convert(k), v, priority=prio)
return compbs
else:
_check_components(compdict)
return {convert(k): v for k, v in compdict.items()}
_check_components(compdict)
return {convert(k): v for k, v in compdict.items()}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
@ -181,11 +180,10 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {'format': output_format}}
else:
raise UsageError(
'The -t command-line option cannot be used if multiple output '
'URIs are specified'
)
raise UsageError(
'The -t command-line option cannot be used if multiple output '
'URIs are specified'
)
result: Dict[str, Dict[str, Any]] = {}
for element in output:

22
scrapy/utils/defer.py
View File

@ -58,10 +58,9 @@ def defer_succeed(result) -> Deferred:
def defer_result(result) -> Deferred:
if isinstance(result, Deferred):
return result
elif isinstance(result, failure.Failure):
if isinstance(result, failure.Failure):
return defer_fail(result)
else:
return defer_succeed(result)
return defer_succeed(result)
def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
@ -267,10 +266,9 @@ def deferred_from_coro(o) -> Any:
# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
# that use asyncio, e.g. "await asyncio.sleep(1)"
return ensureDeferred(o)
else:
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
event_loop = get_asyncio_event_loop_policy().get_event_loop()
return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
event_loop = get_asyncio_event_loop_policy().get_event_loop()
return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
return o
@ -295,12 +293,11 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
if isinstance(result, Deferred):
return result
elif asyncio.isfuture(result) or inspect.isawaitable(result):
if asyncio.isfuture(result) or inspect.isawaitable(result):
return deferred_from_coro(result)
elif isinstance(result, failure.Failure):
if isinstance(result, failure.Failure):
return defer.fail(result)
else:
return defer.succeed(result)
return defer.succeed(result)
def deferred_to_future(d: Deferred) -> Future:
@ -352,5 +349,4 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
"""
if not is_asyncio_reactor_installed():
return d
else:
return deferred_to_future(d)
return deferred_to_future(d)

2
scrapy/utils/display.py
View File

@ -5,8 +5,8 @@ pprint and pformat wrappers with colorization support
import ctypes
import platform
import sys
from packaging.version import Version as parse_version
from pprint import pformat as pformat_
from packaging.version import Version as parse_version
def _enable_windows_terminal_processing():

1
scrapy/utils/ftp.py
View File

@ -1,5 +1,4 @@
import posixpath
from ftplib import error_perm, FTP
from posixpath import dirname

13
scrapy/utils/iterators.py
View File

@ -138,8 +138,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
{'csvlnum': csv_r.line_num, 'csvrow': len(row),
'csvheader': len(headers)})
continue
else:
yield dict(zip(headers, row))
yield dict(zip(headers, row))
def _body_or_str(obj, unicode=True):
@ -152,11 +151,9 @@ def _body_or_str(obj, unicode=True):
if isinstance(obj, Response):
if not unicode:
return obj.body
elif isinstance(obj, TextResponse):
if isinstance(obj, TextResponse):
return obj.text
else:
return obj.body.decode('utf-8')
elif isinstance(obj, str):
return obj.body.decode('utf-8')
if isinstance(obj, str):
return obj if unicode else obj.encode('utf-8')
else:
return obj.decode('utf-8') if unicode else obj
return obj.decode('utf-8') if unicode else obj

15
scrapy/utils/misc.py
View File

@ -30,10 +30,9 @@ def arg_to_iter(arg):
"""
if arg is None:
return []
elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
return arg
else:
return [arg]
return [arg]
def load_object(path):
@ -49,9 +48,8 @@ def load_object(path):
if not isinstance(path, str):
if callable(path):
return path
else:
raise TypeError("Unexpected argument type, expected string "
f"or object, got: {type(path)}")
raise TypeError("Unexpected argument type, expected string "
f"or object, got: {type(path)}")
try:
dot = path.rindex('.')
@ -115,9 +113,8 @@ def extract_regex(regex, text, encoding='utf-8'):
if isinstance(text, str):
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
for s in strings]
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
for s in strings]
def md5sum(file):

5
scrapy/utils/python.py
View File

@ -187,10 +187,9 @@ def get_func_args(func, stripself=False):
elif hasattr(func, '__call__'):
if inspect.isroutine(func):
return []
elif getattr(func, '__name__', None) == '__call__':
if getattr(func, '__name__', None) == '__call__':
return []
else:
return get_func_args(func.__call__, True)
return get_func_args(func.__call__, True)
else:
raise TypeError(f'{type(func)} is not callable')
if stripself:

6
scrapy/utils/response.py
View File

@ -9,14 +9,14 @@ import webbrowser
from typing import Any, Callable, Iterable, Optional, Tuple, Union
from weakref import WeakKeyDictionary
from twisted.web import http
from w3lib import html
import scrapy
from scrapy.http.response import Response
from twisted.web import http
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.decorators import deprecated
from w3lib import html
_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()

19
scrapy/utils/serialize.py
View File

@ -16,24 +16,23 @@ class ScrapyJSONEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, set):
return list(o)
elif isinstance(o, datetime.datetime):
if isinstance(o, datetime.datetime):
return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}")
elif isinstance(o, datetime.date):
if isinstance(o, datetime.date):
return o.strftime(self.DATE_FORMAT)
elif isinstance(o, datetime.time):
if isinstance(o, datetime.time):
return o.strftime(self.TIME_FORMAT)
elif isinstance(o, decimal.Decimal):
if isinstance(o, decimal.Decimal):
return str(o)
elif isinstance(o, defer.Deferred):
if isinstance(o, defer.Deferred):
return str(o)
elif is_item(o):
if is_item(o):
return ItemAdapter(o).asdict()
elif isinstance(o, Request):
if isinstance(o, Request):
return f"<{type(o).__name__} {o.method} {o.url}>"
elif isinstance(o, Response):
if isinstance(o, Response):
return f"<{type(o).__name__} {o.status} {o.url}>"
else:
return super().default(o)
return super().default(o)
class ScrapyJSONDecoder(json.JSONDecoder):

5
scrapy/utils/spider.py
View File

@ -12,12 +12,11 @@ logger = logging.getLogger(__name__)
def iterate_spider_output(result):
if inspect.isasyncgen(result):
return result
elif inspect.iscoroutine(result):
if inspect.iscoroutine(result):
d = deferred_from_coro(result)
d.addCallback(iterate_spider_output)
return d
else:
return arg_to_iter(deferred_from_coro(result))
return arg_to_iter(deferred_from_coro(result))
def iter_spider_classes(module):

2
setup.py
View File

@ -58,7 +58,7 @@ setup(
'Tracker': 'https://github.com/scrapy/scrapy/issues',
},
description='A high-level Web Crawling and Web Scraping framework',
long_description=open('README.rst').read(),
long_description=open('README.rst', encoding="utf-8").read(),
author='Scrapy developers',
maintainer='Pablo Hoffman',
maintainer_email='pablo@pablohoffman.com',

3
tests/CrawlerProcess/reactor_default.py
View File

@ -1,6 +1,7 @@
from twisted.internet import reactor # noqa: F401
import scrapy
from scrapy.crawler import CrawlerProcess
from twisted.internet import reactor # noqa: F401
class NoRequestsSpider(scrapy.Spider):

3
tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
View File

@ -1,6 +1,7 @@
from twisted.internet import reactor # noqa: F401
import scrapy
from scrapy.crawler import CrawlerProcess
from twisted.internet import reactor # noqa: F401
class NoRequestsSpider(scrapy.Spider):

4
tests/CrawlerProcess/reactor_select.py
View File

@ -1,6 +1,8 @@
from twisted.internet import selectreactor
import scrapy
from scrapy.crawler import CrawlerProcess
from twisted.internet import selectreactor
selectreactor.install()

4
tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
View File

@ -1,7 +1,7 @@
import scrapy
from scrapy.crawler import CrawlerProcess
from twisted.internet.main import installReactor
from twisted.internet.selectreactor import SelectReactor
import scrapy
from scrapy.crawler import CrawlerProcess
class SelectReactorSubclass(SelectReactor):

4
tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
View File

@ -1,6 +1,8 @@
from twisted.internet import selectreactor
import scrapy
from scrapy.crawler import CrawlerProcess
from twisted.internet import selectreactor
selectreactor.install()

8
tests/CrawlerRunner/ip_address.py
View File

@ -8,7 +8,6 @@ from twisted.python.runtime import platform
from scrapy import Spider, Request
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from tests.mockserver import MockServer, MockDNSServer
@ -30,9 +29,10 @@ class LocalhostSpider(Spider):
def parse(self, response):
netloc = urlparse(response.url).netloc
self.logger.info("Host: %s" % netloc.split(":")[0])
self.logger.info("Type: %s" % type(response.ip_address))
self.logger.info("IP address: %s" % response.ip_address)
host = netloc.split(":")[0]
self.logger.info(f"Host: {host}")
self.logger.info(f"Type: {type(response.ip_address)}")
self.logger.info(f"IP address: {response.ip_address}")
if __name__ == "__main__":

3
tests/mockserver.py
View File

@ -29,8 +29,7 @@ def getarg(request, name, default=None, type=None):
if type is not None:
value = type(value)
return value
else:
return default
return default
# most of the following resources are copied from twisted.web.test.test_webclient

4
tests/test_command_check.py
View File

@ -6,7 +6,7 @@ class CheckCommandTest(CommandTest):
command = 'check'
def setUp(self):
super(CheckCommandTest, self).setUp()
super().setUp()
self.spider_name = 'check_spider'
self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()
@ -24,7 +24,7 @@ class CheckSpider(scrapy.Spider):
{contracts}
\"\"\"
{parse_def}
""")
""", encoding="utf-8")
def _test_contract(self, contracts='', parse_def='pass'):
self._write_contract(contracts, parse_def)

8
tests/test_command_parse.py
View File

@ -92,7 +92,7 @@ class MyBadCrawlSpider(CrawlSpider):
def parse(self, response):
return [scrapy.Item(), dict(foo='bar')]
""")
""", encoding="utf-8")
(self.proj_mod_path / 'pipelines.py').write_text("""
import logging
@ -103,9 +103,9 @@ class MyPipeline:
def process_item(self, item, spider):
logging.info('It Works!')
return item
""")
""", encoding="utf-8")
with (self.proj_mod_path / 'settings.py').open("a") as f:
with (self.proj_mod_path / 'settings.py').open("a", encoding="utf-8") as f:
f.write(f"""
ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
""")
@ -256,7 +256,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.assertTrue(file_path.is_file())
content = '[\n{},\n{"foo": "bar"}\n]'
self.assertEqual(file_path.read_text(), content)
self.assertEqual(file_path.read_text(encoding="utf-8"), content)
def test_parse_add_options(self):
command = parse.Command()

28
tests/test_commands.py
View File

@ -107,7 +107,7 @@ class ProjectTest(unittest.TestCase):
def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
with Path(filename).open("r") as f:
with Path(filename).open("r", encoding="utf-8") as f:
for line in f:
match = pattern.search(line)
if match is not None:
@ -475,7 +475,7 @@ class GenspiderCommandTest(CommandTest):
assert file_path.exists()
# change name of spider but not its file name
with file_path.open('r+') as spider_file:
with file_path.open('r+', encoding="utf-8") as spider_file:
file_data = spider_file.read()
file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
spider_file.seek(0)
@ -489,14 +489,14 @@ class GenspiderCommandTest(CommandTest):
self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out)
modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before)
file_contents_after = file_path.read_text()
file_contents_after = file_path.read_text(encoding="utf-8")
self.assertNotEqual(file_contents_after, file_contents_before)
else:
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{file_path.resolve()} already exists", out)
modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
file_contents_after = file_path.read_text()
file_contents_after = file_path.read_text(encoding="utf-8")
self.assertEqual(file_contents_after, file_contents_before)
def test_same_filename_as_existing_spider_force(self):
@ -536,7 +536,7 @@ class GenspiderStandaloneCommandTest(ProjectTest):
self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out)
assert file_path.exists()
modify_time_before = file_path.stat().st_mtime
file_contents_before = file_path.read_text()
file_contents_before = file_path.read_text(encoding="utf-8")
if force:
# use different template to ensure contents were changed
@ -544,14 +544,14 @@ class GenspiderStandaloneCommandTest(ProjectTest):
self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out)
modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before)
file_contents_after = file_path.read_text()
file_contents_after = file_path.read_text(encoding="utf-8")
self.assertNotEqual(file_contents_after, file_contents_before)
else:
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out)
modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
file_contents_after = file_path.read_text()
file_contents_after = file_path.read_text(encoding="utf-8")
self.assertEqual(file_contents_after, file_contents_before)
def test_same_name_as_existing_file_force(self):
@ -596,7 +596,7 @@ class BadSpider(scrapy.Spider):
fname = (tmpdir / name).resolve()
else:
fname = (tmpdir / self.spider_filename).resolve()
fname.write_text(content)
fname.write_text(content, encoding="utf-8")
try:
yield str(fname)
finally:
@ -754,11 +754,11 @@ class MySpider(scrapy.Spider):
)
return []
"""
Path(self.cwd, "example.json").write_text("not empty")
Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8")
args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
with Path(self.cwd, "example.json").open() as f2:
with Path(self.cwd, "example.json").open(encoding="utf-8") as f2:
first_line = f2.readline()
self.assertNotEqual(first_line, "not empty")
@ -798,7 +798,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
spider_filename = 'myspider.pyw'
def setUp(self):
super(WindowsRunSpiderCommandTest, self).setUp()
super().setUp()
def test_start_requests_errors(self):
log = self.get_log(self.badspider, name='badspider.pyw')
@ -860,7 +860,7 @@ class ViewCommandTest(CommandTest):
class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()):
Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code)
Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code, encoding="utf-8")
return self.proc('crawl', 'myspider', *args)
def get_log(self, code, args=()):
@ -912,11 +912,11 @@ class MySpider(scrapy.Spider):
)
return []
"""
Path(self.cwd, "example.json").write_text("not empty")
Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8")
args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
with Path(self.cwd, "example.json").open() as f2:
with Path(self.cwd, "example.json").open(encoding="utf-8") as f2:
first_line = f2.readline()
self.assertNotEqual(first_line, "not empty")

5
tests/test_crawler.py
View File

@ -11,6 +11,9 @@ from twisted.internet import defer
from twisted.python.versions import Version
from twisted.trial import unittest
from pkg_resources import parse_version
from w3lib import __version__ as w3lib_version
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
from scrapy.exceptions import ScrapyDeprecationWarning
@ -23,8 +26,6 @@ from scrapy.utils.test import get_crawler
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
from scrapy.utils.test import get_testenv
from pkg_resources import parse_version
from w3lib import __version__ as w3lib_version
from tests.mockserver import MockServer

5
tests/test_downloader_handlers.py
View File

@ -110,7 +110,7 @@ class FileTestCase(unittest.TestCase):
def setUp(self):
# add a special char to check that they are handled correctly
self.tmpname = Path(self.mktemp() + '^')
Path(self.tmpname).write_text('0123456789')
Path(self.tmpname).write_text("0123456789", encoding="utf-8")
handler = create_instance(FileDownloadHandler, None, get_crawler())
self.download_request = handler.download_request
@ -722,8 +722,7 @@ class UriResource(resource.Resource):
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b'CONNECT':
return request.uri
else:
return b''
return b''
class HttpProxyTestCase(unittest.TestCase):

6
tests/test_downloader_handlers_http2.py
View File

@ -171,7 +171,7 @@ class Https2InvalidDNSId(Https2TestCase):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
def setUp(self):
super(Https2InvalidDNSId, self).setUp()
super().setUp()
self.host = '127.0.0.1'
@ -190,7 +190,7 @@ class Https2InvalidDNSPattern(Https2TestCase):
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
)
super(Https2InvalidDNSPattern, self).setUp()
super().setUp()
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
@ -245,4 +245,4 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
with self.assertRaises(NotImplementedError):
yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout()
yield super().test_download_with_proxy_https_timeout()

2
tests/test_downloadermiddleware_cookies.py
View File

@ -1,6 +1,6 @@
import logging
from testfixtures import LogCapture
from unittest import TestCase
from testfixtures import LogCapture
import pytest

2
tests/test_downloadermiddleware_decompression.py
View File

@ -2,8 +2,8 @@ from unittest import TestCase, main
from scrapy.http import Response, XmlResponse
from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
from scrapy.spiders import Spider
from tests import get_testdata
from scrapy.utils.test import assert_samelines
from tests import get_testdata
def _test_data(formats):

7
tests/test_downloadermiddleware_httpcache.py
View File

@ -272,10 +272,9 @@ class RFC2616PolicyTest(DefaultStorageTest):
if result:
assert isinstance(result, (Request, Response))
return result
else:
result = mw.process_response(request, response, self.spider)
assert isinstance(result, Response)
return result
result = mw.process_response(request, response, self.spider)
assert isinstance(result, Response)
return result
except Exception:
print('Request', request)
print('Response', response)

3
tests/test_downloadermiddleware_httpcompression.py
View File

@ -4,6 +4,7 @@ from pathlib import Path
from unittest import TestCase, SkipTest
from warnings import catch_warnings
from w3lib.encoding import resolve_encoding
from scrapy.spiders import Spider
from scrapy.http import Response, Request, HtmlResponse
from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
@ -12,8 +13,6 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from w3lib.encoding import resolve_encoding
SAMPLEDIR = Path(tests_datadir, 'compressed')

6
tests/test_exporters.py
View File

@ -400,8 +400,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
children = list(elem.iterchildren())
if children:
return [(child.tag, sorted(xmltuple(child))) for child in children]
else:
return [(elem.tag, [(elem.text, ())])]
return [(elem.tag, [(elem.text, ())])]
def xmlsplit(xmlcontent):
doc = lxml.etree.fromstring(xmlcontent)
@ -621,8 +620,7 @@ class CustomExporterItemTest(unittest.TestCase):
def serialize_field(self, field, name, value):
if name == 'age':
return str(int(value) + 1)
else:
return super().serialize_field(field, name, value)
return super().serialize_field(field, name, value)
i = self.item_class(name='John', age='22')
a = ItemAdapter(i)

2
tests/test_http2_client_protocol.py
View File

@ -164,7 +164,7 @@ class RequestHeaders(LeafResource):
def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate:
pem = key_file.read_text() + certificate_file.read_text()
pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8")
return PrivateCertificate.loadPEM(pem)

2
tests/test_http_request.py
View File

@ -223,7 +223,7 @@ class RequestTest(unittest.TestCase):
r1 = CustomRequest('http://www.example.com')
r2 = r1.copy()
assert type(r2) is CustomRequest
assert isinstance(r2, CustomRequest)
def test_replace(self):
"""Test Request.replace() method"""

2
tests/test_http_response.py
View File

@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase):
r1 = CustomResponse('http://www.example.com')
r2 = r1.copy()
assert type(r2) is CustomResponse
assert isinstance(r2, CustomResponse)
def test_replace(self):
"""Test Response.replace() method"""

5
tests/test_loader_deprecated.py
View File

@ -391,10 +391,9 @@ class BasicItemLoaderTest(unittest.TestCase):
def join(values, sep=None, loader_context=None, ignored=None):
if sep is not None:
return sep.join(values)
elif loader_context and 'sep' in loader_context:
if loader_context and 'sep' in loader_context:
return loader_context['sep'].join(values)
else:
return ''.join(values)
return ''.join(values)
class TestItemLoader(NameItemLoader):
name_out = Compose(partial(join, sep='+'))

4
tests/test_pipeline_media.py
View File

@ -388,11 +388,11 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def thumb_path(self, request, thumb_id, response=None, info=None):
self._mockcalled.append('thumb_path')
return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info)
return super().thumb_path(request, thumb_id, response, info)
def get_images(self, response, request, info):
self._mockcalled.append('get_images')
return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info)
return super().get_images(response, request, info)
def image_downloaded(self, response, request, info):
self._mockcalled.append('image_downloaded')

4
tests/test_request_attribute_binding.py
View File

@ -1,12 +1,12 @@
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from testfixtures import LogCapture
from scrapy import Request, signals
from scrapy.http.response import Response
from scrapy.utils.test import get_crawler
from testfixtures import LogCapture
from tests.mockserver import MockServer
from tests.spiders import SingleRequestSpider

2
tests/test_request_cb_kwargs.py
View File

@ -91,7 +91,7 @@ class KeywordArgumentsSpider(MockServerSpider):
self.checks.append(kwargs['callback'] == 'some_callback')
self.crawler.stats.inc_value('boolean_checks', 3)
elif response.url.endswith('/general_without'):
self.checks.append(kwargs == {})
self.checks.append(kwargs == {}) # pylint: disable=use-implicit-booleaness-not-comparison
self.crawler.stats.inc_value('boolean_checks')
def parse_no_kwargs(self, response):

2
tests/test_scheduler_base.py
View File

@ -11,10 +11,8 @@ from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.request import fingerprint
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
PATHS = ["/a", "/b", "/c"]
URLS = [urljoin("https://example.org", p) for p in PATHS]

2
tests/test_spider.py
View File

@ -7,6 +7,7 @@ from unittest import mock
from testfixtures import LogCapture
from twisted.trial import unittest
from w3lib.url import safe_url_string
from scrapy import signals
from scrapy.settings import Settings
from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
@ -22,7 +23,6 @@ from scrapy.spiders import (
from scrapy.linkextractors import LinkExtractor
from scrapy.utils.test import get_crawler
from tests import get_testdata
from w3lib.url import safe_url_string
class SpiderTest(unittest.TestCase):

2
tests/test_spiderloader/__init__.py
View File

@ -3,6 +3,7 @@ import shutil
import warnings
from pathlib import Path
import tempfile
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@ -10,7 +11,6 @@ from twisted.trial import unittest
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
# alone
import scrapy
import tempfile
from scrapy.interfaces import ISpiderLoader
from scrapy.spiderloader import SpiderLoader
from scrapy.settings import Settings

2
tests/test_spidermiddleware_httperror.py
View File

@ -6,11 +6,11 @@ from twisted.trial.unittest import TestCase as TrialTestCase
from twisted.internet import defer
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError
from scrapy.settings import Settings
from tests.mockserver import MockServer
from tests.spiders import MockServerSpider

1
tests/test_spidermiddleware_output_chain.py
View File

@ -4,7 +4,6 @@ from twisted.trial.unittest import TestCase
from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer

4
tests/test_spidermiddleware_referer.py
View File

@ -1,8 +1,8 @@
from urllib.parse import urlparse
from unittest import TestCase
import warnings
from scrapy.http import Response, Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
@ -380,7 +380,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
scheme = urlparse(request).scheme
if scheme == 'https':
return b'https://python.org/'
elif scheme == 'http':
if scheme == 'http':
return b'http://python.org/'

1
tests/test_squeues_request.py
View File

@ -16,7 +16,6 @@ from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
"""
Queues that handle requests
"""

5
tests/test_utils_defer.py
View File

@ -193,9 +193,8 @@ class AsyncCooperatorTest(unittest.TestCase):
delay = random.random() / 8
reactor.callLater(delay, dfd.callback, None)
return dfd
else:
# simulate trivial sync processing
results.append(o)
# simulate trivial sync processing
results.append(o)
@staticmethod
def get_async_iterable(length):

1
tests/test_utils_deprecate.py
View File

@ -2,6 +2,7 @@ import inspect
import unittest
from unittest import mock
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath

2
tests/test_webclient.py
View File

@ -47,7 +47,7 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
from twisted.web.client import _makeGetterFactory
return _makeGetterFactory(
to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs
to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs,
).deferred