RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use f-strings where appropriate (#5246)

This commit is contained in:
Georgiy Zatserklianyi 2021-10-07 17:27:20 +03:00 committed by GitHub
parent 029cab72e8
commit d3f1bf79e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 51 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/__init__.py
View File

@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 6):
print("Scrapy %s requires Python 3.6+" % __version__)
print(f"Scrapy {__version__} requires Python 3.6+")
sys.exit(1)

12
scrapy/core/downloader/contextfactory.py
View File

@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
msg = (
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
"a `method` argument (type OpenSSL.SSL method, e.g. "
"OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
"argument and/or a `tls_ciphers` argument. Please, upgrade your "
"context factory class to handle them or ignore them."
)
warnings.warn(msg)
return context_factory

8
scrapy/core/downloader/tls.py
View File

@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
verifyHostname(connection, self._hostnameASCII)
except (CertificateError, VerificationError) as e:
logger.warning(
'Remote certificate is not valid for hostname "{}"; {}'.format(
self._hostnameASCII, e))
'Remote certificate is not valid for hostname "%s"; %s',
self._hostnameASCII, e)
except ValueError as e:
logger.warning(
'Ignoring error while verifying certificate '
'from host "{}" (exception: {})'.format(
self._hostnameASCII, repr(e)))
'from host "%s" (exception: %r)',
self._hostnameASCII, e)
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')

2
scrapy/downloadermiddlewares/httpcache.py
View File

@ -70,7 +70,7 @@ class HttpCacheMiddleware:
self.stats.inc_value('httpcache/miss', spider=spider)
if self.ignore_missing:
self.stats.inc_value('httpcache/ignore', spider=spider)
raise IgnoreRequest("Ignored request not in cache: %s" % request)
raise IgnoreRequest(f"Ignored request not in cache: {request}")
return None # first time request
# Return cached response only if not expired

33
scrapy/extensions/feedexport.py
View File

@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
kwargs['feed_options'] = feed_options
else:
warnings.warn(
"{} does not support the 'feed_options' keyword argument. Add a "
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy."
.format(builder.__qualname__),
"version of Scrapy.",
category=ScrapyDeprecationWarning
)
return builder(*preargs, uri, *args, **kwargs)
@ -356,32 +355,28 @@ class FeedExporter:
# properly closed.
return defer.maybeDeferred(slot.storage.store, slot.file)
slot.finish_exporting()
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
log_args = {'format': slot.format,
'itemcount': slot.itemcount,
'uri': slot.uri}
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d = defer.maybeDeferred(slot.storage.store, slot.file)
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
)
d.addErrback(
self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
self._handle_store_error, logmsg, spider, type(slot.storage).__name__
)
return d
def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
def _handle_store_error(self, f, logmsg, spider, slot_type):
logger.error(
logfmt % "Error storing", largs,
"Error storing %s", logmsg,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
def _handle_store_success(self, f, logmsg, spider, slot_type):
logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
"Stored %s", logmsg,
extra={'spider': spider}
)
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
@ -474,10 +469,10 @@ class FeedExporter:
for uri_template, values in self.feeds.items():
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
logger.error(
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
'%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
''.format(uri_template)
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
uri_template
)
return False
return True
@ -526,7 +521,7 @@ class FeedExporter:
instance = build_instance(feedcls)
method_name = '__new__'
if instance is None:
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
return instance
def _get_uri_params(self, spider, uri_params, slot=None):

4
scrapy/extensions/httpcache.py
View File

@ -226,7 +226,7 @@ class DbmCacheStorage:
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
def close_spider(self, spider):
self.db.close()
@ -280,7 +280,7 @@ class FilesystemCacheStorage:
self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider):
logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
def close_spider(self, spider):

2
scrapy/utils/misc.py
View File

@ -50,7 +50,7 @@ def load_object(path):
return path
else:
raise TypeError("Unexpected argument type, expected string "
"or object, got: %s" % type(path))
f"or object, got: {type(path)}")
try:
dot = path.rindex('.')

16
tests/spiders.py
View File

@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider):
self.start_urls = [url]
def parse(self, response):
self.logger.info("Got response %d" % response.status)
self.logger.info(f"Got response {response.status}")
class AsyncDefSpider(SimpleSpider):
@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider):
async def parse(self, response):
await defer.succeed(42)
self.logger.info("Got response %d" % response.status)
self.logger.info(f"Got response {response.status}")
class AsyncDefAsyncioSpider(SimpleSpider):
@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):
async def parse(self, response):
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d" % status)
self.logger.info(f"Got response {status}")
class AsyncDefAsyncioReturnSpider(SimpleSpider):
@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
async def parse(self, response):
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d" % status)
self.logger.info(f"Got response {status}")
return [{'id': 1}, {'id': 2}]
@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
async def parse(self, response):
await asyncio.sleep(0.1)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d" % status)
self.logger.info(f"Got response {status}")
return {"foo": 42}
@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
await asyncio.sleep(0.2)
req_id = response.meta.get('req_id', 0)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d, req_id %d" % (status, req_id))
self.logger.info(f"Got response {status}, req_id {req_id}")
if req_id > 0:
return
reqs = []
@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
async def parse(self, response):
await asyncio.sleep(0.2)
yield {'foo': 42}
self.logger.info("Got response %d" % response.status)
self.logger.info(f"Got response {response.status}")
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
for i in range(10):
await asyncio.sleep(0.1)
yield {'foo': i}
self.logger.info("Got response %d" % response.status)
self.logger.info(f"Got response {response.status}")
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):

2
tests/test_closespider.py
View File

@ -41,7 +41,7 @@ class TestCloseSpider(TestCase):
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
reason = crawler.spider.meta['close_reason']
self.assertEqual(reason, 'closespider_errorcount')
key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}'
errorcount = crawler.stats.get_value(key)
self.assertTrue(errorcount >= close_on)

4
tests/test_commands.py
View File

@ -498,7 +498,7 @@ class GenspiderCommandTest(CommandTest):
self.find_in_file(join(self.proj_mod_path,
'spiders', 'test_name.py'),
r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
self.assertEqual('http://%s/' % domain,
self.assertEqual(f'http://{domain}/',
self.find_in_file(join(self.proj_mod_path,
'spiders', 'test_name.py'),
r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
@ -708,7 +708,7 @@ class MySpider(scrapy.Spider):
])
import asyncio
loop = asyncio.new_event_loop()
self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log)
self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
def test_output(self):
spider_code = """

2
tests/test_downloader_handlers_http2.py
View File

@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
self.assertEqual(response.url, request.url)
self.assertEqual(response.body, b'/')
http_proxy = '%s?noconnect' % self.getURL('')
http_proxy = f"{self.getURL('')}?noconnect"
request = Request('https://example.com', meta={'proxy': http_proxy})
with self.assertWarnsRegex(
Warning,

19
tests/test_http_request.py
View File

@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest):
response, formcss="input[name='abc']")
def test_from_response_valid_form_methods(self):
body = """<form action="post.php" method="%s">
<input type="hidden" name="one" value="1">
</form>"""
form_methods = [[method, method] for method in self.request_class.valid_form_methods]
form_methods.append(['UNKNOWN', 'GET'])
for method in self.request_class.valid_form_methods:
response = _buildresponse(body % method)
for method, expected in form_methods:
response = _buildresponse(
f'<form action="post.php" method="{method}">'
'<input type="hidden" name="one" value="1">'
'</form>'
)
r = self.request_class.from_response(response)
self.assertEqual(r.method, method)
response = _buildresponse(body % 'UNKNOWN')
r = self.request_class.from_response(response)
self.assertEqual(r.method, 'GET')
self.assertEqual(r.method, expected)
def _buildresponse(body, **kwargs):