RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use f-strings (#4307)

This commit is contained in:
Ammar Najjar 2020-08-22 22:32:03 +02:00
parent 0ccaf89a6f
commit 58ca8bbf6d
133 changed files with 561 additions and 568 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/conf.py
View File

@ -49,7 +49,7 @@ master_doc = 'index'
# General information about the project. # General information about the project.
project = 'Scrapy' project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The version info for the project you're documenting, acts as replacement for # The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the # |version| and |release|, also used in various other places throughout the

6
docs/intro/tutorial.rst
View File

@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response): def parse(self, response):
page = response.url.split("/")[-2] page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page filename = f'quotes-{page}.html'
with open(filename, 'wb') as f: with open(filename, 'wb') as f:
f.write(response.body) f.write(response.body)
self.log('Saved file %s' % filename) self.log(f'Saved file {filename}')
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>` As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
@ -190,7 +190,7 @@ for your spider::
def parse(self, response): def parse(self, response):
page = response.url.split("/")[-2] page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page filename = f'quotes-{page}.html'
with open(filename, 'wb') as f: with open(filename, 'wb') as f:
f.write(response.body) f.write(response.body)

138
docs/topics/developer-tools.rst
View File

@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping
================================================= =================================================
Here is a general guide on how to use your browser's Developer Tools Here is a general guide on how to use your browser's Developer Tools
to ease the scraping process. Today almost all browsers come with to ease the scraping process. Today almost all browsers come with
built in `Developer Tools`_ and although we will use Firefox in this built in `Developer Tools`_ and although we will use Firefox in this
guide, the concepts are applicable to any other browser. guide, the concepts are applicable to any other browser.
In this guide we'll introduce the basic tools to use from a browser's In this guide we'll introduce the basic tools to use from a browser's
Developer Tools by scraping `quotes.toscrape.com`_. Developer Tools by scraping `quotes.toscrape.com`_.
@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things:
Inspecting a website Inspecting a website
==================== ====================
By far the most handy feature of the Developer Tools is the `Inspector` By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the any webpage. To demonstrate the Inspector, let's look at the
`quotes.toscrape.com`_-site. `quotes.toscrape.com`_-site.
On the site we have a total of ten quotes from various authors with specific On the site we have a total of ten quotes from various authors with specific
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc. on this page, without any meta-information about authors, tags, etc.
Instead of viewing the whole source code for the page, we can simply right click Instead of viewing the whole source code for the page, we can simply right click
on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
In it you should see something like this: In it you should see something like this:
@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is basic selector. But this XPath is not really that clever. All it does is
go down a desired path in the source code starting from ``html``. So let's go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit: see if we can refine our XPath a bit:
If we check the `Inspector` again we'll see that directly beneath our If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same same attributes as our first. If we expand any of them, we'll see the same
structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
see each quote: see each quote:
.. code-block:: html .. code-block:: html
@ -121,7 +121,7 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow, With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_: the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall() >>> response.xpath('//span[has-class("text")]/text()').getall()
@ -130,45 +130,45 @@ the `has-class-extension`_:
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...] ...]
And with one simple, cleverer XPath we are able to extract all quotes from And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily the number of the last ``div``, but this would have been unnecessarily
complex and by simply constructing an XPath with ``has-class("text")`` complex and by simply constructing an XPath with ``has-class("text")``
we were able to extract all quotes in one line. we were able to extract all quotes in one line.
The `Inspector` has a lot of other helpful features, such as searching in the The `Inspector` has a lot of other helpful features, such as searching in the
source code or directly scrolling to an element you selected. Let's demonstrate source code or directly scrolling to an element you selected. Let's demonstrate
a use case: a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results. search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be follow the pagination. On a simple site such as this, there may not be
the need to find an element visually but the ``Scroll into View`` function the need to find an element visually but the ``Scroll into View`` function
can be quite useful on complex sites. can be quite useful on complex sites.
Note that the search bar can also be used to search for and test CSS Note that the search bar can also be used to search for and test CSS
selectors. For example, you could search for ``span.text`` to find selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page. exactly the ``span`` tag with the ``class="text"`` in the page.
.. _topics-network-tool: .. _topics-network-tool:
The Network-tool The Network-tool
================ ================
While scraping you may come across dynamic webpages where some parts While scraping you may come across dynamic webpages where some parts
of the page are loaded dynamically through multiple requests. While of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools this can be quite tricky, the `Network`-tool in the Developer Tools
greatly facilitates this task. To demonstrate the Network-tool, let's greatly facilitates this task. To demonstrate the Network-tool, let's
take a look at the page `quotes.toscrape.com/scroll`_. take a look at the page `quotes.toscrape.com/scroll`_.
The page is quite similar to the basic `quotes.toscrape.com`_-page, The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the Scrapy shell: we'll check another quite useful command from the Scrapy shell:
.. skip: next .. skip: next
@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell:
(...) (...)
>>> view(response) >>> view(response)
A browser window should open with the webpage but with one A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``. bar with the word ``Loading...``.
.. image:: _images/network_01.png .. image:: _images/network_01.png
:width: 777 :width: 777
@ -189,21 +189,21 @@ bar with the word ``Loading...``.
:alt: Response from quotes.toscrape.com/scroll :alt: Response from quotes.toscrape.com/scroll
The ``view(response)`` command let's us view the response our The ``view(response)`` command let's us view the response our
shell or later our spider receives from the server. Here we see shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title, that some basic template is loaded which includes the title,
the login-button and the footer, but the quotes are missing. This the login-button and the footer, but the quotes are missing. This
tells us that the quotes are being loaded from a different request tells us that the quotes are being loaded from a different request
than ``quotes.toscrape/scroll``. than ``quotes.toscrape/scroll``.
If you click on the ``Network`` tab, you will probably only see If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the clicking on ``Persist Logs``. If this option is disabled, the
log is automatically cleared each time you navigate to a different log is automatically cleared each time you navigate to a different
page. Enabling this option is a good default, since it gives us page. Enabling this option is a good default, since it gives us
control on when to clear the logs. control on when to clear the logs.
If we reload the page now, you'll see the log get populated with six If we reload the page now, you'll see the log get populated with six
new requests. new requests.
.. image:: _images/network_02.png .. image:: _images/network_02.png
:width: 777 :width: 777
@ -212,31 +212,31 @@ new requests.
Here we see every request that has been made when reloading the page Here we see every request that has been made when reloading the page
and can inspect each request and its response. So let's find out and can inspect each request and its response. So let's find out
where our quotes are coming from: where our quotes are coming from:
First click on the request with the name ``scroll``. On the right First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address, about the request headers, such as the URL, the method, the IP-address,
and so on. We'll ignore the other tabs and click directly on ``Response``. and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code, What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``. shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the interests us is the one request called ``quotes?page=1`` with the
type ``json``. type ``json``.
If we click on this request, we see that the request URL is If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response ``http://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview. on the request and open ``Open in new tab`` to get a better overview.
.. image:: _images/network_03.png .. image:: _images/network_03.png
:width: 777 :width: 777
:height: 375 :height: 375
:alt: JSON-object returned from the quotes.toscrape API :alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site:: also request each page to get every quote on the site::
import scrapy import scrapy
@ -255,17 +255,17 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]} yield {"quote": quote["text"]}
if data["has_next"]: if data["has_next"]:
self.page += 1 self.page += 1
url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse) yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``. response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary. This lets us operate on the JSON-object like on a Python dictionary.
We iterate through the ``quotes`` and print out the ``quote["text"]``. We iterate through the ``quotes`` and print out the ``quote["text"]``.
If the handy ``has_next`` element is ``true`` (try loading If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number and ``yield`` a new request, inserting the incremented page-number
into our ``url``. into our ``url``.
.. _requests-from-curl: .. _requests-from-curl:
@ -298,7 +298,7 @@ Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_. you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider. to identifying the correct request and replicating it in your spider.

6
docs/topics/exporters.rst
View File

@ -57,7 +57,7 @@ value of one of their fields::
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
year = adapter['year'] year = adapter['year']
if year not in self.year_to_exporter: if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb') f = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(f) exporter = XmlItemExporter(f)
exporter.start_exporting() exporter.start_exporting()
self.year_to_exporter[year] = exporter self.year_to_exporter[year] = exporter
@ -98,7 +98,7 @@ Example::
import scrapy import scrapy
def serialize_price(value): def serialize_price(value):
return '$ %s' % str(value) return f'$ {str(value)}'
class Product(scrapy.Item): class Product(scrapy.Item):
name = scrapy.Field() name = scrapy.Field()
@ -122,7 +122,7 @@ Example::
def serialize_field(self, field, name, value): def serialize_field(self, field, name, value):
if field == 'price': if field == 'price':
return '$ %s' % str(value) return f'$ {str(value)}'
return super(Product, self).serialize_field(field, name, value) return super(Product, self).serialize_field(field, name, value)
.. _topics-exporters-reference: .. _topics-exporters-reference:

6
docs/topics/item-pipeline.rst
View File

@ -96,7 +96,7 @@ contain a price::
adapter['price'] = adapter['price'] * self.vat_factor adapter['price'] = adapter['price'] * self.vat_factor
return item return item
else: else:
raise DropItem("Missing price in %s" % item) raise DropItem(f"Missing price in {item}")
Write items to a JSON file Write items to a JSON file
@ -211,7 +211,7 @@ item.
# Save screenshot to file, filename will be hash of url. # Save screenshot to file, filename will be hash of url.
url = adapter["url"] url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest() url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash) filename = f"{url_hash}.png"
with open(filename, "wb") as f: with open(filename, "wb") as f:
f.write(response.body) f.write(response.body)
@ -240,7 +240,7 @@ returns multiples items with the same id::
def process_item(self, item, spider): def process_item(self, item, spider):
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen: if adapter['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %r" % item) raise DropItem(f"Duplicate item found: {item!r}")
else: else:
self.ids_seen.add(adapter['id']) self.ids_seen.add(adapter['id'])
return item return item

2
docs/topics/leaks.rst
View File

@ -102,7 +102,7 @@ A real example
Let's see a concrete example of a hypothetical case of memory leaks. Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one:: Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
callback=self.parse, cb_kwargs={'referer': response}) callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively That line is passing a response reference inside a request which effectively

7
docs/topics/selectors.rst
View File

@ -328,8 +328,9 @@ too. Here's an example:
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links): >>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) ... href_xpath = link.xpath('@href').get()
... print('Link number %d points to url %r and image %r' % args) ... img_xpath = link.xpath('img/@src').get()
... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}')
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
@ -822,7 +823,7 @@ with groups of itemscopes and corresponding itemprops::
... props = scope.xpath(''' ... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop, ... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''') ... .//*[@itemscope]/*/@itemprop)''')
... print(" properties: %s" % (props.getall())) ... print(f" properties: {props.getall()}")
... print("") ... print("")
current scope: ['http://schema.org/Product'] current scope: ['http://schema.org/Product']

2
docs/topics/settings.rst
View File

@ -110,7 +110,7 @@ In a spider, the settings are available through ``self.settings``::
start_urls = ['http://example.com'] start_urls = ['http://example.com']
def parse(self, response): def parse(self, response):
print("Existing settings: %s" % self.settings.attributes.keys()) print(f"Existing settings: {self.settings.attributes.keys()}")
.. note:: .. note::
The ``settings`` attribute is set in the base Spider class after the spider The ``settings`` attribute is set in the base Spider class after the spider

4
docs/topics/spiders.rst
View File

@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods::
def __init__(self, category=None, *args, **kwargs): def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs) super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = ['http://www.example.com/categories/%s' % category] self.start_urls = [f'http://www.example.com/categories/{category}']
# ... # ...
The default `__init__` method will take any spider arguments The default `__init__` method will take any spider arguments
@ -292,7 +292,7 @@ The above example can also be written as follows::
name = 'myspider' name = 'myspider'
def start_requests(self): def start_requests(self):
yield scrapy.Request('http://www.example.com/categories/%s' % self.category) yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
Keep in mind that spider arguments are only strings. Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own. The spider will not do any parsing on its own.

2
extras/qps-bench-server.py
View File

@ -37,7 +37,7 @@ class Root(Resource):
if now - self.lastmark >= 3: if now - self.lastmark >= 3:
self.lastmark = now self.lastmark = now
qps = len(self.tail) / sum(self.tail) qps = len(self.tail) / sum(self.tail)
print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps)) print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
if 'latency' in request.args: if 'latency' in request.args:
latency = float(request.args['latency'][0]) latency = float(request.args['latency'][0])

4
extras/qpsclient.py
View File

@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self): def start_requests(self):
url = self.benchurl url = self.benchurl
if self.latency is not None: if self.latency is not None:
url += '?latency={0}'.format(self.latency) url += f'?latency={self.latency}'
slots = int(self.slots) slots = int(self.slots)
if slots > 1: if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
else: else:
urls = [url] urls = [url]

16
scrapy/cmdline.py
View File

@ -44,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
if inspect.isclass(obj): if inspect.isclass(obj):
cmds[entry_point.name] = obj() cmds[entry_point.name] = obj()
else: else:
raise Exception("Invalid entry point %s" % entry_point.name) raise Exception(f"Invalid entry point {entry_point.name}")
return cmds return cmds
@ -67,11 +67,11 @@ def _pop_command_name(argv):
def _print_header(settings, inproject): def _print_header(settings, inproject):
version = scrapy.__version__
if inproject: if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__, print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
settings['BOT_NAME']))
else: else:
print("Scrapy %s - no active project\n" % scrapy.__version__) print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject): def _print_commands(settings, inproject):
@ -81,7 +81,7 @@ def _print_commands(settings, inproject):
print("Available commands:") print("Available commands:")
cmds = _get_commands_dict(settings, inproject) cmds = _get_commands_dict(settings, inproject)
for cmdname, cmdclass in sorted(cmds.items()): for cmdname, cmdclass in sorted(cmds.items()):
print(" %-13s %s" % (cmdname, cmdclass.short_desc())) print(f" {cmdname:<13} {cmdclass.short_desc()}")
if not inproject: if not inproject:
print() print()
print(" [ more ] More commands available when run from project directory") print(" [ more ] More commands available when run from project directory")
@ -91,7 +91,7 @@ def _print_commands(settings, inproject):
def _print_unknown_command(settings, cmdname, inproject): def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject) _print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname) print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands') print('Use "scrapy" to see available commands')
@ -133,7 +133,7 @@ def execute(argv=None, settings=None):
sys.exit(2) sys.exit(2)
cmd = cmds[cmdname] cmd = cmds[cmdname]
parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax()) parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc() parser.description = cmd.long_desc()
settings.setdict(cmd.default_settings, priority='command') settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings cmd.settings = settings
@ -155,7 +155,7 @@ def _run_command(cmd, args, opts):
def _run_command_profiled(cmd, args, opts): def _run_command_profiled(cmd, args, opts):
if opts.profile: if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals() loc = locals()
p = cProfile.Profile() p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc) p.runctx('cmd.run(args, opts)', globals(), loc)

2
scrapy/commands/__init__.py
View File

@ -61,7 +61,7 @@ class ScrapyCommand:
group.add_option("--logfile", metavar="FILE", group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used") help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL']) help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true", group.add_option("--nolog", action="store_true",
help="disable logging completely") help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None, group.add_option("--profile", metavar="FILE", default=None,

2
scrapy/commands/bench.py
View File

@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self): def start_requests(self):
qargs = {'total': self.total, 'show': self.show} qargs = {'total': self.total, 'show': self.show}
url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1)) url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
return [scrapy.Request(url, dont_filter=True)] return [scrapy.Request(url, dont_filter=True)]
def parse(self, response): def parse(self, response):

10
scrapy/commands/check.py
View File

@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult):
plural = "s" if run != 1 else "" plural = "s" if run != 1 else ""
writeln(self.separator2) writeln(self.separator2)
writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start)) writeln(f"Ran {run} contract{plural} in {stop - start:.3f}")
writeln() writeln()
infos = [] infos = []
@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult):
write("FAILED") write("FAILED")
failed, errored = map(len, (self.failures, self.errors)) failed, errored = map(len, (self.failures, self.errors))
if failed: if failed:
infos.append("failures=%d" % failed) infos.append(f"failures={failed}")
if errored: if errored:
infos.append("errors=%d" % errored) infos.append(f"errors={errored}")
else: else:
write("OK") write("OK")
if infos: if infos:
writeln(" (%s)" % (", ".join(infos),)) writeln(f" ({', '.join(infos)})")
else: else:
write("\n") write("\n")
@ -85,7 +85,7 @@ class Command(ScrapyCommand):
continue continue
print(spider) print(spider)
for method in sorted(methods): for method in sorted(methods):
print(' * %s' % method) print(f' * {method}')
else: else:
start = time.time() start = time.time()
self.crawler_process.start() self.crawler_process.start()

4
scrapy/commands/edit.py
View File

@ -32,8 +32,8 @@ class Command(ScrapyCommand):
try: try:
spidercls = self.crawler_process.spider_loader.load(args[0]) spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError: except KeyError:
return self._err("Spider not found: %s" % args[0]) return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__ sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py') sfile = sfile.replace('.pyc', '.py')
self.exitcode = os.system('%s "%s"' % (editor, sfile)) self.exitcode = os.system(f'{editor} "{sfile}"')

27
scrapy/commands/genspider.py
View File

@ -73,17 +73,18 @@ class Command(ScrapyCommand):
if template_file: if template_file:
self._genspider(module, name, domain, opts.template, template_file) self._genspider(module, name, domain, opts.template, template_file)
if opts.edit: if opts.edit:
self.exitcode = os.system('scrapy edit "%s"' % name) self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file): def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template""" """Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
tvars = { tvars = {
'project_name': self.settings.get('BOT_NAME'), 'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')), 'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module, 'module': module,
'name': name, 'name': name,
'domain': domain, 'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_')) 'classname': f'{capitalized_module}Spider'
} }
if self.settings.get('NEWSPIDER_MODULE'): if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -91,32 +92,32 @@ class Command(ScrapyCommand):
else: else:
spiders_module = None spiders_module = None
spiders_dir = "." spiders_dir = "."
spider_file = "%s.py" % join(spiders_dir, module) spider_file = f"{join(spiders_dir, module)}.py"
shutil.copyfile(template_file, spider_file) shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars) render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r " print(f"Created spider {name!r} using template {template_name!r} ",
% (name, template_name), end=('' if spiders_module else '\n')) end=('' if spiders_module else '\n'))
if spiders_module: if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module)) print("in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template): def _find_template(self, template):
template_file = join(self.templates_dir, '%s.tmpl' % template) template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file): if exists(template_file):
return template_file return template_file
print("Unable to find template: %s\n" % template) print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.') print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self): def _list_templates(self):
print("Available templates:") print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)): for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'): if filename.endswith('.tmpl'):
print(" %s" % splitext(filename)[0]) print(f" {splitext(filename)[0]}")
def _spider_exists(self, name): def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'): if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists # if run as a standalone command and file with same filename already exists
if exists(name + ".py"): if exists(name + ".py"):
print("%s already exists" % (abspath(name + ".py"))) print(f"{abspath(name + '.py')} already exists")
return True return True
return False return False
@ -126,8 +127,8 @@ class Command(ScrapyCommand):
pass pass
else: else:
# if spider with same name exists # if spider with same name exists
print("Spider %r already exists in module:" % name) print(f"Spider {name!r} already exists in module:")
print(" %s" % spidercls.__module__) print(f" {spidercls.__module__}")
return True return True
# a file with the same name exists in the target directory # a file with the same name exists in the target directory
@ -135,7 +136,7 @@ class Command(ScrapyCommand):
spiders_dir = dirname(spiders_module.__file__) spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir) spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")): if exists(join(spiders_dir_abs, name + ".py")):
print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
return True return True
return False return False

4
scrapy/commands/parse.py
View File

@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand):
if opts.verbose: if opts.verbose:
for level in range(1, self.max_level + 1): for level in range(1, self.max_level + 1):
print('\n>>> DEPTH LEVEL: %s <<<' % level) print(f'\n>>> DEPTH LEVEL: {level} <<<')
if not opts.noitems: if not opts.noitems:
self.print_items(level, colour) self.print_items(level, colour)
if not opts.nolinks: if not opts.nolinks:
self.print_requests(level, colour) self.print_requests(level, colour)
else: else:
print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level) print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
if not opts.noitems: if not opts.noitems:
self.print_items(colour=colour) self.print_items(colour=colour)
if not opts.nolinks: if not opts.nolinks:

8
scrapy/commands/runspider.py
View File

@ -12,7 +12,7 @@ def _import_file(filepath):
dirname, file = os.path.split(abspath) dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file) fname, fext = os.path.splitext(file)
if fext != '.py': if fext != '.py':
raise ValueError("Not a Python source file: %s" % abspath) raise ValueError(f"Not a Python source file: {abspath}")
if dirname: if dirname:
sys.path = [dirname] + sys.path sys.path = [dirname] + sys.path
try: try:
@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand):
raise UsageError() raise UsageError()
filename = args[0] filename = args[0]
if not os.path.exists(filename): if not os.path.exists(filename):
raise UsageError("File not found: %s\n" % filename) raise UsageError(f"File not found: {filename}\n")
try: try:
module = _import_file(filename) module = _import_file(filename)
except (ImportError, ValueError) as e: except (ImportError, ValueError) as e:
raise UsageError("Unable to load %r: %s\n" % (filename, e)) raise UsageError(f"Unable to load {filename!r}: {e}\n")
spclasses = list(iter_spider_classes(module)) spclasses = list(iter_spider_classes(module))
if not spclasses: if not spclasses:
raise UsageError("No spider found in file: %s\n" % filename) raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop() spidercls = spclasses.pop()
self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.crawl(spidercls, **opts.spargs)

12
scrapy/commands/startproject.py
View File

@ -52,7 +52,7 @@ class Command(ScrapyCommand):
print('Error: Project names must begin with a letter and contain' print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores') ' only\nletters, numbers and underscores')
elif _module_exists(project_name): elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name) print(f'Error: Module {project_name!r} already exists')
else: else:
return True return True
return False return False
@ -100,7 +100,7 @@ class Command(ScrapyCommand):
if exists(join(project_dir, 'scrapy.cfg')): if exists(join(project_dir, 'scrapy.cfg')):
self.exitcode = 1 self.exitcode = 1
print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
return return
if not self._is_valid_name(project_name): if not self._is_valid_name(project_name):
@ -113,11 +113,11 @@ class Command(ScrapyCommand):
path = join(*paths) path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print("New Scrapy project '%s', using template directory '%s', " print(f"New Scrapy project '{project_name}', using template directory "
"created in:" % (project_name, self.templates_dir)) f"'{self.templates_dir}', created in:")
print(" %s\n" % abspath(project_dir)) print(f" {abspath(project_dir)}\n")
print("You can start your first spider with:") print("You can start your first spider with:")
print(" cd %s" % project_dir) print(f" cd {project_dir}")
print(" scrapy genspider example example.com") print(" scrapy genspider example example.com")
@property @property

4
scrapy/commands/version.py
View File

@ -23,8 +23,8 @@ class Command(ScrapyCommand):
if opts.verbose: if opts.verbose:
versions = scrapy_components_versions() versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions) width = max(len(n) for (n, _) in versions)
patt = "%-{}s : %s".format(width) patt = f"%-{width}s : %s"
for name, version in versions: for name, version in versions:
print(patt % (name, version)) print(patt % (name, version))
else: else:
print("Scrapy %s" % scrapy.__version__) print(f"Scrapy {scrapy.__version__}")

8
scrapy/contracts/__init__.py
View File

@ -112,8 +112,8 @@ class Contract:
request_cls = None request_cls = None
def __init__(self, method, *args): def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name) self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args self.args = args
def add_pre_hook(self, request, results): def add_pre_hook(self, request, results):
@ -172,8 +172,8 @@ def _create_testcase(method, desc):
class ContractTestCase(TestCase): class ContractTestCase(TestCase):
def __str__(_self): def __str__(_self):
return "[%s] %s (%s)" % (spider, method.__name__, desc) return f"[{spider}] {method.__name__} ({desc})"
name = '%s_%s' % (spider, method.__name__) name = f'{spider}_{method.__name__}'
setattr(ContractTestCase, name, lambda x: x) setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name) return ContractTestCase(name)

12
scrapy/contracts/default.py
View File

@ -60,8 +60,7 @@ class ReturnsContract(Contract):
if len(self.args) not in [1, 2, 3]: if len(self.args) not in [1, 2, 3]:
raise ValueError( raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i" f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
% len(self.args)
) )
self.obj_name = self.args[0] or None self.obj_name = self.args[0] or None
self.obj_type_verifier = self.object_type_verifiers[self.obj_name] self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
@ -88,10 +87,9 @@ class ReturnsContract(Contract):
if self.min_bound == self.max_bound: if self.min_bound == self.max_bound:
expected = self.min_bound expected = self.min_bound
else: else:
expected = '%s..%s' % (self.min_bound, self.max_bound) expected = f'{self.min_bound}..{self.max_bound}'
raise ContractFail("Returned %s %s, expected %s" % raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
(occurrences, self.obj_name, expected))
class ScrapesContract(Contract): class ScrapesContract(Contract):
@ -106,5 +104,5 @@ class ScrapesContract(Contract):
if is_item(x): if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)] missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing: if missing:
missing_str = ", ".join(missing) missing_fields = ", ".join(missing)
raise ContractFail("Missing fields: %s" % missing_str) raise ContractFail(f"Missing fields: {missing_fields}")

16
scrapy/core/downloader/__init__.py
View File

@ -41,17 +41,17 @@ class Slot:
def __repr__(self): def __repr__(self):
cls_name = self.__class__.__name__ cls_name = self.__class__.__name__
return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( return (f"{cls_name}(concurrency={self.concurrency!r}, "
cls_name, self.concurrency, self.delay, self.randomize_delay) f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r}")
def __str__(self): def __str__(self):
return ( return (
"<downloader.Slot concurrency=%r delay=%0.2f randomize_delay=%r " f"<downloader.Slot concurrency={self.concurrency!r} "
"len(active)=%d len(queue)=%d len(transferring)=%d lastseen=%s>" % ( f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
self.concurrency, self.delay, self.randomize_delay, f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
len(self.active), len(self.queue), len(self.transferring), f"len(transferring)={len(self.transferring)} "
datetime.fromtimestamp(self.lastseen).isoformat() f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
)
) )

3
scrapy/core/downloader/handlers/__init__.py
View File

@ -71,8 +71,7 @@ class DownloadHandlers:
scheme = urlparse_cached(request).scheme scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme) handler = self._get_handler(scheme)
if not handler: if not handler:
raise NotSupported("Unsupported URL scheme '%s': %s" % raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
(scheme, self._notconfigured[scheme]))
return handler.download_request(request, spider) return handler.download_request(request, spider)
@defer.inlineCallbacks @defer.inlineCallbacks

17
scrapy/core/downloader/handlers/http11.py
View File

@ -60,11 +60,11 @@ class HTTP11DownloadHandler:
settings=settings, settings=settings,
crawler=crawler, crawler=crawler,
) )
msg = """ msg = f"""
'%s' does not accept `method` argument (type OpenSSL.SSL method,\ '{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \
Please upgrade your context factory class to handle them or ignore them.""" % ( `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) Please upgrade your context factory class to handle them or ignore them."""
warnings.warn(msg) warnings.warn(msg)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
else: else:
extra = rcvd_bytes[:32] extra = rcvd_bytes[:32]
self._tunnelReadyDeferred.errback( self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( TunnelError('Could not open CONNECT tunnel with proxy '
self._host, self._port, extra))) f'{self._host}:{self._port} [{extra!r}]')
)
def connectFailed(self, reason): def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred.""" """Propagates the errback to the appropriate deferred."""
@ -371,7 +372,7 @@ class ScrapyAgent:
if self._txresponse: if self._txresponse:
self._txresponse._transport.stopProducing() self._txresponse._transport.stopProducing()
raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time): def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time request.meta['download_latency'] = time() - start_time

6
scrapy/core/downloader/handlers/s3.py
View File

@ -56,7 +56,7 @@ class S3DownloadHandler:
import botocore.credentials import botocore.credentials
kw.pop('anon', None) kw.pop('anon', None)
if kw: if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw) raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon: if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials( self._signer = SignerCls(botocore.credentials.Credentials(
@ -85,14 +85,14 @@ class S3DownloadHandler:
scheme = 'https' if request.meta.get('is_secure') else 'http' scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon: if self.anon:
request = request.replace(url=url) request = request.replace(url=url)
elif self._signer is not None: elif self._signer is not None:
import botocore.awsrequest import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest( awsrequest = botocore.awsrequest.AWSRequest(
method=request.method, method=request.method,
url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path), url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
headers=request.headers.to_unicode_dict(), headers=request.headers.to_unicode_dict(),
data=request.body) data=request.body)
self._signer.add_auth(awsrequest) self._signer.add_auth(awsrequest)

15
scrapy/core/downloader/middleware.py
View File

@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, spider=spider)) response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)): if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput( raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s" f"Middleware {method.__self__.__class__.__name__}"
% (method.__self__.__class__.__name__, response.__class__.__name__) ".process_request must return None, Response or "
f"Request, got {response.__class__.__name__}"
) )
if response: if response:
return response return response
@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)): if not isinstance(response, (Response, Request)):
raise _InvalidOutput( raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s" f"Middleware {method.__self__.__class__.__name__}"
% (method.__self__.__class__.__name__, type(response)) ".process_response must return Response or Request, "
f"got {type(response)}"
) )
if isinstance(response, Request): if isinstance(response, Request):
return response return response
@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)): if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput( raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s" f"Middleware {method.__self__.__class__.__name__}"
% (method.__self__.__class__.__name__, type(response)) ".process_exception must return None, Response or "
f"Request, got {type(response)}"
) )
if response: if response:
return response return response

6
scrapy/core/downloader/webclient.py
View File

@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing() self.transport.stopProducing()
self.factory.noPage( self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds." defer.TimeoutError(f"Getting {self.factory.url} took longer "
% (self.factory.url, self.factory.timeout))) f"than {self.factory.timeout} seconds."))
# This class used to inherit from Twisted’s # This class used to inherit from Twisted’s
@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
self.headers['Content-Length'] = 0 self.headers['Content-Length'] = 0
def __repr__(self): def __repr__(self):
return "<%s: %s>" % (self.__class__.__name__, self.url) return f"<{self.__class__.__name__}: {self.url}>"
def _cancelTimeout(self, result, timeoutCall): def _cancelTimeout(self, result, timeoutCall):
if timeoutCall.active(): if timeoutCall.active():

12
scrapy/core/engine.py
View File

@ -171,8 +171,8 @@ class ExecutionEngine:
def _handle_downloader_output(self, response, request, spider): def _handle_downloader_output(self, response, request, spider):
if not isinstance(response, (Request, Response, Failure)): if not isinstance(response, (Request, Response, Failure)):
raise TypeError( raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r" "Incorrect type: expected Request, Response or Failure, got "
% (type(response), response) f"{type(response)}: {response!r}"
) )
# downloader middleware can return requests (for example, redirects) # downloader middleware can return requests (for example, redirects)
if isinstance(response, Request): if isinstance(response, Request):
@ -214,7 +214,7 @@ class ExecutionEngine:
def crawl(self, request, spider): def crawl(self, request, spider):
if spider not in self.open_spiders: if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}")
self.schedule(request, spider) self.schedule(request, spider)
self.slot.nextcall.schedule() self.slot.nextcall.schedule()
@ -239,8 +239,8 @@ class ExecutionEngine:
def _on_success(response): def _on_success(response):
if not isinstance(response, (Response, Request)): if not isinstance(response, (Response, Request)):
raise TypeError( raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r" "Incorrect type: expected Response or Request, got "
% (type(response), response) f"{type(response)}: {response!r}"
) )
if isinstance(response, Response): if isinstance(response, Response):
if response.request is None: if response.request is None:
@ -268,7 +268,7 @@ class ExecutionEngine:
@defer.inlineCallbacks @defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True): def open_spider(self, spider, start_requests=(), close_if_idle=True):
if not self.has_capacity(): if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name) raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={'spider': spider}) logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider) nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler) scheduler = self.scheduler_cls.from_crawler(self.crawler)

4
scrapy/core/scraper.py
View File

@ -125,7 +125,7 @@ class Scraper:
Handle the downloaded response or failure through the spider callback/errback Handle the downloaded response or failure through the spider callback/errback
""" """
if not isinstance(result, (Response, Failure)): if not isinstance(result, (Response, Failure)):
raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")
dfd = self._scrape2(result, request, spider) # returns spider's processed output dfd = self._scrape2(result, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, result, spider) dfd.addErrback(self.handle_spider_error, request, result, spider)
dfd.addCallback(self.handle_spider_output, request, result, spider) dfd.addCallback(self.handle_spider_output, request, result, spider)
@ -173,7 +173,7 @@ class Scraper:
spider=spider spider=spider
) )
self.crawler.stats.inc_value( self.crawler.stats.inc_value(
"spider_exceptions/%s" % _failure.value.__class__.__name__, f"spider_exceptions/{_failure.value.__class__.__name__}",
spider=spider spider=spider
) )

20
scrapy/core/spidermw.py
View File

@ -19,10 +19,7 @@ def _isiterable(possible_iterator):
def _fname(f): def _fname(f):
return "{}.{}".format( return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
f.__self__.__class__.__name__,
f.__func__.__name__
)
class SpiderMiddlewareManager(MiddlewareManager): class SpiderMiddlewareManager(MiddlewareManager):
@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
try: try:
result = method(response=response, spider=spider) result = method(response=response, spider=spider)
if result is not None: if result is not None:
msg = "Middleware {} must return None or raise an exception, got {}" msg = (f"Middleware {_fname(method)} must return None "
raise _InvalidOutput(msg.format(_fname(method), type(result))) f"or raise an exception, got {type(result)}")
raise _InvalidOutput(msg)
except _InvalidOutput: except _InvalidOutput:
raise raise
except Exception: except Exception:
@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None: elif result is None:
continue continue
else: else:
msg = "Middleware {} must return None or an iterable, got {}" msg = (f"Middleware {_fname(method)} must return None "
raise _InvalidOutput(msg.format(_fname(method), type(result))) f"or an iterable, got {type(result)}")
raise _InvalidOutput(msg)
return _failure return _failure
def process_spider_output(result, start_index=0): def process_spider_output(result, start_index=0):
@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result): if _isiterable(result):
result = _evaluate_iterable(result, method_index + 1, recovered) result = _evaluate_iterable(result, method_index + 1, recovered)
else: else:
msg = "Middleware {} must return an iterable, got {}" msg = (f"Middleware {_fname(method)} must return an "
raise _InvalidOutput(msg.format(_fname(method), type(result))) f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
return MutableChain(result, recovered) return MutableChain(result, recovered)

12
scrapy/downloadermiddlewares/cookies.py
View File

@ -54,8 +54,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace') cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')] for c in request.headers.getlist('Cookie')]
if cl: if cl:
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
msg = "Sending cookies to: {}\n{}".format(request, cookies) msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={'spider': spider}) logger.debug(msg, extra={'spider': spider})
def _debug_set_cookie(self, response, spider): def _debug_set_cookie(self, response, spider):
@ -63,8 +63,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace') cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')] for c in response.headers.getlist('Set-Cookie')]
if cl: if cl:
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
msg = "Received cookies from: {}\n{}".format(response, cookies) msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={'spider': spider}) logger.debug(msg, extra={'spider': spider})
def _format_cookie(self, cookie, request): def _format_cookie(self, cookie, request):
@ -90,9 +90,9 @@ class CookiesMiddleware:
request, cookie) request, cookie)
decoded[key] = cookie[key].decode("latin1", errors="replace") decoded[key] = cookie[key].decode("latin1", errors="replace")
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
for key, value in decoded.items(): # path, domain for key, value in decoded.items(): # path, domain
cookie_str += "; {}={}".format(key.capitalize(), value) cookie_str += f"; {key.capitalize()}={value}"
return cookie_str return cookie_str
def _get_request_cookies(self, jar, request): def _get_request_cookies(self, jar, request):

2
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -24,7 +24,7 @@ class HttpProxyMiddleware:
def _basic_auth_header(self, username, password): def _basic_auth_header(self, username, password):
user_pass = to_bytes( user_pass = to_bytes(
'%s:%s' % (unquote(username), unquote(password)), f'{unquote(username)}:{unquote(password)}',
encoding=self.auth_encoding) encoding=self.auth_encoding)
return base64.b64encode(user_pass) return base64.b64encode(user_pass)

2
scrapy/downloadermiddlewares/retry.py
View File

@ -88,7 +88,7 @@ class RetryMiddleware:
reason = global_object_name(reason.__class__) reason = global_object_name(reason.__class__)
stats.inc_value('retry/count') stats.inc_value('retry/count')
stats.inc_value('retry/reason_count/%s' % reason) stats.inc_value(f'retry/reason_count/{reason}')
return retryreq return retryreq
else: else:
stats.inc_value('retry/max_reached') stats.inc_value('retry/max_reached')

6
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -61,7 +61,7 @@ class RobotsTxtMiddleware:
if netloc not in self._parsers: if netloc not in self._parsers:
self._parsers[netloc] = Deferred() self._parsers[netloc] = Deferred()
robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"
robotsreq = Request( robotsreq = Request(
robotsurl, robotsurl,
priority=self.DOWNLOAD_PRIORITY, priority=self.DOWNLOAD_PRIORITY,
@ -94,7 +94,7 @@ class RobotsTxtMiddleware:
def _parse_robots(self, response, netloc, spider): def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count') self.crawler.stats.inc_value('robotstxt/response_count')
self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status)) self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}')
rp = self._parserimpl.from_crawler(self.crawler, response.body) rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc] rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp self._parsers[netloc] = rp
@ -102,7 +102,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, failure, netloc): def _robots_error(self, failure, netloc):
if failure.type is not IgnoreRequest: if failure.type is not IgnoreRequest:
key = 'robotstxt/exception_count/{}'.format(failure.type) key = f'robotstxt/exception_count/{failure.type}'
self.crawler.stats.inc_value(key) self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc] rp_dfd = self._parsers[netloc]
self._parsers[netloc] = None self._parsers[netloc] = None

6
scrapy/downloadermiddlewares/stats.py
View File

@ -17,13 +17,13 @@ class DownloaderStats:
def process_request(self, request, spider): def process_request(self, request, spider):
self.stats.inc_value('downloader/request_count', spider=spider) self.stats.inc_value('downloader/request_count', spider=spider)
self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
reqlen = len(request_httprepr(request)) reqlen = len(request_httprepr(request))
self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
def process_response(self, request, response, spider): def process_response(self, request, response, spider):
self.stats.inc_value('downloader/response_count', spider=spider) self.stats.inc_value('downloader/response_count', spider=spider)
self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
reslen = len(response_httprepr(response)) reslen = len(response_httprepr(response))
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
return response return response
@ -31,4 +31,4 @@ class DownloaderStats:
def process_exception(self, request, exception, spider): def process_exception(self, request, exception, spider):
ex_class = global_object_name(exception.__class__) ex_class = global_object_name(exception.__class__)
self.stats.inc_value('downloader/exception_count', spider=spider) self.stats.inc_value('downloader/exception_count', spider=spider)
self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)

2
scrapy/exporters.py
View File

@ -39,7 +39,7 @@ class BaseItemExporter:
self.export_empty_fields = options.pop('export_empty_fields', False) self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent = options.pop('indent', None) self.indent = options.pop('indent', None)
if not dont_fail and options: if not dont_fail and options:
raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
def export_item(self, item): def export_item(self, item):
raise NotImplementedError raise NotImplementedError

2
scrapy/extensions/corestats.py
View File

@ -43,4 +43,4 @@ class CoreStats:
def item_dropped(self, item, spider, exception): def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__ reason = exception.__class__.__name__
self.stats.inc_value('item_dropped_count', spider=spider) self.stats.inc_value('item_dropped_count', spider=spider)
self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)

2
scrapy/extensions/debug.py
View File

@ -48,7 +48,7 @@ class StackTraceDump:
for id_, frame in sys._current_frames().items(): for id_, frame in sys._current_frames().items():
name = id2name.get(id_, '') name = id2name.get(id_, '')
dump = ''.join(traceback.format_stack(frame)) dump = ''.join(traceback.format_stack(frame))
dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump) dumps += f"# Thread: {name}({id_})\n{dump}\n"
return dumps return dumps

10
scrapy/extensions/httpcache.py
View File

@ -223,7 +223,7 @@ class DbmCacheStorage:
self.db = None self.db = None
def open_spider(self, spider): def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c') self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
@ -251,13 +251,13 @@ class DbmCacheStorage:
'headers': dict(response.headers), 'headers': dict(response.headers),
'body': response.body, 'body': response.body,
} }
self.db['%s_data' % key] = pickle.dumps(data, protocol=4) self.db[f'{key}_data'] = pickle.dumps(data, protocol=4)
self.db['%s_time' % key] = str(time()) self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request): def _read_data(self, spider, request):
key = self._request_key(request) key = self._request_key(request)
db = self.db db = self.db
tkey = '%s_time' % key tkey = f'{key}_time'
if tkey not in db: if tkey not in db:
return # not found return # not found
@ -265,7 +265,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts): if 0 < self.expiration_secs < time() - float(ts):
return # expired return # expired
return pickle.loads(db['%s_data' % key]) return pickle.loads(db[f'{key}_data'])
def _request_key(self, request): def _request_key(self, request):
return request_fingerprint(request) return request_fingerprint(request)

2
scrapy/extensions/memdebug.py
View File

@ -30,4 +30,4 @@ class MemoryDebugger:
for cls, wdict in live_refs.items(): for cls, wdict in live_refs.items():
if not wdict: if not wdict:
continue continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)

14
scrapy/extensions/memusage.py
View File

@ -82,8 +82,8 @@ class MemoryUsage:
{'memusage': mem}, extra={'crawler': self.crawler}) {'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails: if self.notify_mails:
subj = ( subj = (
"%s terminated: memory usage exceeded %dM at %s" f"{self.crawler.settings['BOT_NAME']} terminated: "
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) f"memory usage exceeded {mem}M at {socket.gethostname()}"
) )
self._send_report(self.notify_mails, subj) self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/limit_notified', 1) self.crawler.stats.set_value('memusage/limit_notified', 1)
@ -105,8 +105,8 @@ class MemoryUsage:
{'memusage': mem}, extra={'crawler': self.crawler}) {'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails: if self.notify_mails:
subj = ( subj = (
"%s warning: memory usage reached %dM at %s" f"{self.crawler.settings['BOT_NAME']} warning: "
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) f"memory usage reached {mem}M at {socket.gethostname()}"
) )
self._send_report(self.notify_mails, subj) self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/warning_notified', 1) self.crawler.stats.set_value('memusage/warning_notified', 1)
@ -115,9 +115,9 @@ class MemoryUsage:
def _send_report(self, rcpts, subject): def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info""" """send notification mail with some additional useful info"""
stats = self.crawler.stats stats = self.crawler.stats
s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n" s += "\r\n"

10
scrapy/extensions/statsmailer.py
View File

@ -24,11 +24,11 @@ class StatsMailer:
o = cls(crawler.stats, recipients, mail) o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o return o
def spider_closed(self, spider): def spider_closed(self, spider):
spider_stats = self.stats.get_stats(spider) spider_stats = self.stats.get_stats(spider)
body = "Global stats\n\n" body = "Global stats\n\n"
body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats())
body += "\n\n%s stats\n\n" % spider.name body += f"\n\n{spider.name} stats\n\n"
body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats)
return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body)

2
scrapy/http/common.py
View File

@ -1,6 +1,6 @@
def obsolete_setter(setter, attrname): def obsolete_setter(setter, attrname):
def newsetter(self, value): def newsetter(self, value):
c = self.__class__.__name__ c = self.__class__.__name__
msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
raise AttributeError(msg) raise AttributeError(msg)
return newsetter return newsetter

2
scrapy/http/headers.py
View File

@ -33,7 +33,7 @@ class Headers(CaselessDict):
elif isinstance(x, int): elif isinstance(x, int):
return str(x).encode(self.encoding) return str(x).encode(self.encoding)
else: else:
raise TypeError('Unsupported value type: {}'.format(type(x))) raise TypeError(f'Unsupported value type: {type(x)}')
def __getitem__(self, key): def __getitem__(self, key):
try: try:

12
scrapy/http/request/__init__.py
View File

@ -25,13 +25,13 @@ class Request(object_ref):
self._set_url(url) self._set_url(url)
self._set_body(body) self._set_body(body)
if not isinstance(priority, int): if not isinstance(priority, int):
raise TypeError("Request priority not an integer: %r" % priority) raise TypeError(f"Request priority not an integer: {priority!r}")
self.priority = priority self.priority = priority
if callback is not None and not callable(callback): if callback is not None and not callable(callback):
raise TypeError('callback must be a callable, got %s' % type(callback).__name__) raise TypeError(f'callback must be a callable, got {type(callback).__name__}')
if errback is not None and not callable(errback): if errback is not None and not callable(errback):
raise TypeError('errback must be a callable, got %s' % type(errback).__name__) raise TypeError(f'errback must be a callable, got {type(errback).__name__}')
self.callback = callback self.callback = callback
self.errback = errback self.errback = errback
@ -60,13 +60,13 @@ class Request(object_ref):
def _set_url(self, url): def _set_url(self, url):
if not isinstance(url, str): if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}')
s = safe_url_string(url, self.encoding) s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s) self._url = escape_ajax(s)
if ('://' not in self._url) and (not self._url.startswith('data:')): if ('://' not in self._url) and (not self._url.startswith('data:')):
raise ValueError('Missing scheme in request url: %s' % self._url) raise ValueError(f'Missing scheme in request url: {self._url}')
url = property(_get_url, obsolete_setter(_set_url, 'url')) url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -86,7 +86,7 @@ class Request(object_ref):
return self._encoding return self._encoding
def __str__(self): def __str__(self):
return "<%s %s>" % (self.method, self.url) return f"<{self.method} {self.url}>"
__repr__ = __str__ __repr__ = __str__

19
scrapy/http/request/form.py
View File

@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath):
base_url=get_base_url(response)) base_url=get_base_url(response))
forms = root.xpath('//form') forms = root.xpath('//form')
if not forms: if not forms:
raise ValueError("No <form> element found in %s" % response) raise ValueError(f"No <form> element found in {response}")
if formname is not None: if formname is not None:
f = root.xpath('//form[@name="%s"]' % formname) f = root.xpath(f'//form[@name="{formname}"]')
if f: if f:
return f[0] return f[0]
if formid is not None: if formid is not None:
f = root.xpath('//form[@id="%s"]' % formid) f = root.xpath(f'//form[@id="{formid}"]')
if f: if f:
return f[0] return f[0]
@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
el = el.getparent() el = el.getparent()
if el is None: if el is None:
break break
raise ValueError('No <form> element found with %s' % formxpath) raise ValueError(f'No <form> element found with {formxpath}')
# If we get here, it means that either formname was None # If we get here, it means that either formname was None
# or invalid # or invalid
@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
try: try:
form = forms[formnumber] form = forms[formnumber]
except IndexError: except IndexError:
raise IndexError("Form number %d not found in %s" % raise IndexError(f"Form number {formnumber} not found in {response}")
(formnumber, response))
else: else:
return form return form
@ -205,12 +204,12 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other # We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such # arguments, because they can be used as such
xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata)
el = form.xpath(xpath) el = form.xpath(xpath)
if len(el) == 1: if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '') return (el[0].get('name'), el[0].get('value') or '')
elif len(el) > 1: elif len(el) > 1:
raise ValueError("Multiple elements found (%r) matching the criteria " raise ValueError(f"Multiple elements found ({el!r}) matching the "
"in clickdata: %r" % (el, clickdata)) f"criteria in clickdata: {clickdata!r}")
else: else:
raise ValueError('No clickable element matching clickdata: %r' % (clickdata,)) raise ValueError(f'No clickable element matching clickdata: {clickdata!r}')

6
scrapy/http/response/__init__.py
View File

@ -55,8 +55,8 @@ class Response(object_ref):
if isinstance(url, str): if isinstance(url, str):
self._url = url self._url = url
else: else:
raise TypeError('%s url must be str, got %s:' % raise TypeError(f'{type(self).__name__} url must be str, '
(type(self).__name__, type(url).__name__)) f'got {type(url).__name__}')
url = property(_get_url, obsolete_setter(_set_url, 'url')) url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -77,7 +77,7 @@ class Response(object_ref):
body = property(_get_body, obsolete_setter(_set_body, 'body')) body = property(_get_body, obsolete_setter(_set_body, 'body'))
def __str__(self): def __str__(self):
return "<%d %s>" % (self.status, self.url) return f"<{self.status} {self.url}>"
__repr__ = __str__ __repr__ = __str__

15
scrapy/http/response/text.py
View File

@ -47,8 +47,8 @@ class TextResponse(Response):
self._body = b'' # used by encoding detection self._body = b'' # used by encoding detection
if isinstance(body, str): if isinstance(body, str):
if self._encoding is None: if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' % raise TypeError('Cannot convert unicode body - '
type(self).__name__) f'{type(self).__name__} has no encoding')
self._body = body.encode(self._encoding) self._body = body.encode(self._encoding)
else: else:
super()._set_body(body) super()._set_body(body)
@ -92,7 +92,7 @@ class TextResponse(Response):
# _body_inferred_encoding is called # _body_inferred_encoding is called
benc = self.encoding benc = self.encoding
if self._cached_ubody is None: if self._cached_ubody is None:
charset = 'charset=%s' % benc charset = f'charset={benc}'
self._cached_ubody = html_to_unicode(charset, self.body)[1] self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody return self._cached_ubody
@ -255,12 +255,11 @@ def _url_from_selector(sel):
# e.g. ::attr(href) result # e.g. ::attr(href) result
return strip_html5_whitespace(sel.root) return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'): if not hasattr(sel.root, 'tag'):
raise _InvalidSelector("Unsupported selector: %s" % sel) raise _InvalidSelector(f"Unsupported selector: {sel}")
if sel.root.tag not in ('a', 'link'): if sel.root.tag not in ('a', 'link'):
raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" % raise _InvalidSelector("Only <a> and <link> elements are supported; "
sel.root.tag) f"got <{sel.root.tag}>")
href = sel.root.get('href') href = sel.root.get('href')
if href is None: if href is None:
raise _InvalidSelector("<%s> element has no href attribute: %s" % raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}")
(sel.root.tag, sel))
return strip_html5_whitespace(href) return strip_html5_whitespace(href)

6
scrapy/item.py
View File

@ -96,19 +96,19 @@ class DictItem(MutableMapping, BaseItem):
if key in self.fields: if key in self.fields:
self._values[key] = value self._values[key] = value
else: else:
raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) raise KeyError(f"{self.__class__.__name__} does not support field: {key}")
def __delitem__(self, key): def __delitem__(self, key):
del self._values[key] del self._values[key]
def __getattr__(self, name): def __getattr__(self, name):
if name in self.fields: if name in self.fields:
raise AttributeError("Use item[%r] to get field value" % name) raise AttributeError(f"Use item[{name!r}] to get field value")
raise AttributeError(name) raise AttributeError(name)
def __setattr__(self, name, value): def __setattr__(self, name, value):
if not name.startswith('_'): if not name.startswith('_'):
raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value")
super().__setattr__(name, value) super().__setattr__(name, value)
def __len__(self): def __len__(self):

6
scrapy/link.py
View File

@ -14,7 +14,7 @@ class Link:
def __init__(self, url, text='', fragment='', nofollow=False): def __init__(self, url, text='', fragment='', nofollow=False):
if not isinstance(url, str): if not isinstance(url, str):
got = url.__class__.__name__ got = url.__class__.__name__
raise TypeError("Link urls must be str objects, got %s" % got) raise TypeError(f"Link urls must be str objects, got {got}")
self.url = url self.url = url
self.text = text self.text = text
self.fragment = fragment self.fragment = fragment
@ -33,6 +33,6 @@ class Link:
def __repr__(self): def __repr__(self):
return ( return (
'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' f'Link(url={self.url!r}, text={self.text!r}, '
% (self.url, self.text, self.fragment, self.nofollow) f'fragment={self.fragment!r}, nofollow={self.nofollow!r})'
) )

4
scrapy/logformatter.py
View File

@ -54,8 +54,8 @@ class LogFormatter:
def crawled(self, request, response, spider): def crawled(self, request, response, spider):
"""Logs a message when the crawler finds a webpage.""" """Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else '' request_flags = f' {str(request.flags)}' if request.flags else ''
response_flags = ' %s' % str(response.flags) if response.flags else '' response_flags = f' {str(response.flags)}' if response.flags else ''
return { return {
'level': logging.DEBUG, 'level': logging.DEBUG,
'msg': CRAWLEDMSG, 'msg': CRAWLEDMSG,

23
scrapy/pipelines/files.py
View File

@ -108,7 +108,7 @@ class S3FilesStore:
from boto.s3.connection import S3Connection from boto.s3.connection import S3Connection
self.S3Connection = S3Connection self.S3Connection = S3Connection
if not uri.startswith("s3://"): if not uri.startswith("s3://"):
raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split('/', 1) self.bucket, self.prefix = uri[5:].split('/', 1)
def stat_file(self, path, info): def stat_file(self, path, info):
@ -133,7 +133,7 @@ class S3FilesStore:
return c.get_bucket(self.bucket, validate=False) return c.get_bucket(self.bucket, validate=False)
def _get_boto_key(self, path): def _get_boto_key(self, path):
key_name = '%s%s' % (self.prefix, path) key_name = f'{self.prefix}{path}'
if self.is_botocore: if self.is_botocore:
return threads.deferToThread( return threads.deferToThread(
self.s3_client.head_object, self.s3_client.head_object,
@ -145,7 +145,7 @@ class S3FilesStore:
def persist_file(self, path, buf, info, meta=None, headers=None): def persist_file(self, path, buf, info, meta=None, headers=None):
"""Upload file to S3 storage""" """Upload file to S3 storage"""
key_name = '%s%s' % (self.prefix, path) key_name = f'{self.prefix}{path}'
buf.seek(0) buf.seek(0)
if self.is_botocore: if self.is_botocore:
extra = self._headers_to_botocore_kwargs(self.HEADERS) extra = self._headers_to_botocore_kwargs(self.HEADERS)
@ -208,8 +208,7 @@ class S3FilesStore:
try: try:
kwarg = mapping[key] kwarg = mapping[key]
except KeyError: except KeyError:
raise TypeError( raise TypeError(f'Header "{key}" is not supported by botocore')
'Header "%s" is not supported by botocore' % key)
else: else:
extra[kwarg] = value extra[kwarg] = value
return extra return extra
@ -283,7 +282,7 @@ class FTPFilesStore:
def __init__(self, uri): def __init__(self, uri):
if not uri.startswith("ftp://"): if not uri.startswith("ftp://"):
raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")
u = urlparse(uri) u = urlparse(uri)
self.port = u.port self.port = u.port
self.host = u.hostname self.host = u.hostname
@ -293,7 +292,7 @@ class FTPFilesStore:
self.basedir = u.path.rstrip('/') self.basedir = u.path.rstrip('/')
def persist_file(self, path, buf, info, meta=None, headers=None): def persist_file(self, path, buf, info, meta=None, headers=None):
path = '%s/%s' % (self.basedir, path) path = f'{self.basedir}/{path}'
return threads.deferToThread( return threads.deferToThread(
ftp_store_file, path=path, file=buf, ftp_store_file, path=path, file=buf,
host=self.host, port=self.port, username=self.username, host=self.host, port=self.port, username=self.username,
@ -308,10 +307,10 @@ class FTPFilesStore:
ftp.login(self.username, self.password) ftp.login(self.username, self.password)
if self.USE_ACTIVE_MODE: if self.USE_ACTIVE_MODE:
ftp.set_pasv(False) ftp.set_pasv(False)
file_path = "%s/%s" % (self.basedir, path) file_path = f"{self.basedir}/{path}"
last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip()) last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
m = hashlib.md5() m = hashlib.md5()
ftp.retrbinary('RETR %s' % file_path, m.update) ftp.retrbinary(f'RETR {file_path}', m.update)
return {'last_modified': last_modified, 'checksum': m.hexdigest()} return {'last_modified': last_modified, 'checksum': m.hexdigest()}
# The file doesn't exist # The file doesn't exist
except Exception: except Exception:
@ -515,7 +514,7 @@ class FilesPipeline(MediaPipeline):
def inc_stats(self, spider, status): def inc_stats(self, spider, status):
spider.crawler.stats.inc_value('file_count', spider=spider) spider.crawler.stats.inc_value('file_count', spider=spider)
spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider)
# Overridable Interface # Overridable Interface
def get_media_requests(self, item, info): def get_media_requests(self, item, info):
@ -545,4 +544,4 @@ class FilesPipeline(MediaPipeline):
media_type = mimetypes.guess_type(request.url)[0] media_type = mimetypes.guess_type(request.url)[0]
if media_type: if media_type:
media_ext = mimetypes.guess_extension(media_type) media_ext = mimetypes.guess_extension(media_type)
return 'full/%s%s' % (media_guid, media_ext) return f'full/{media_guid}{media_ext}'

9
scrapy/pipelines/images.py
View File

@ -125,8 +125,9 @@ class ImagesPipeline(FilesPipeline):
width, height = orig_image.size width, height = orig_image.size
if width < self.min_width or height < self.min_height: if width < self.min_width or height < self.min_height:
raise ImageException("Image too small (%dx%d < %dx%d)" % raise ImageException("Image too small "
(width, height, self.min_width, self.min_height)) f"({width}x{height} < "
f"{self.min_width}x{self.min_height})")
image, buf = self.convert_image(orig_image) image, buf = self.convert_image(orig_image)
yield path, image, buf yield path, image, buf
@ -168,8 +169,8 @@ class ImagesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None): def file_path(self, request, response=None, info=None, *, item=None):
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'full/%s.jpg' % (image_guid) return f'full/{image_guid}.jpg'
def thumb_path(self, request, thumb_id, response=None, info=None): def thumb_path(self, request, thumb_id, response=None, info=None):
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) return f'thumbs/{thumb_id}/{thumb_guid}.jpg'

7
scrapy/pipelines/media.py
View File

@ -61,7 +61,7 @@ class MediaPipeline:
'MYPIPE_IMAGES' 'MYPIPE_IMAGES'
""" """
class_name = self.__class__.__name__ class_name = self.__class__.__name__
formatted_key = "{}_{}".format(class_name.upper(), key) formatted_key = f"{class_name.upper()}_{key}"
if ( if (
not base_class_name not base_class_name
or class_name == base_class_name or class_name == base_class_name
@ -151,9 +151,8 @@ class MediaPipeline:
if 'item' not in sig.parameters: if 'item' not in sig.parameters:
old_params = str(sig)[1:-1] old_params = str(sig)[1:-1]
new_params = old_params + ", *, item=None" new_params = old_params + ", *, item=None"
warn('%s(self, %s) is deprecated, ' warn(f'{func.__name__}(self, {old_params}) is deprecated, '
'please use %s(self, %s)' f'please use {func.__name__}(self, {new_params})',
% (func.__name__, old_params, func.__name__, new_params),
ScrapyDeprecationWarning, stacklevel=2) ScrapyDeprecationWarning, stacklevel=2)
self._expects_item[func.__name__] = False self._expects_item[func.__name__] = False

9
scrapy/pqueues.py
View File

@ -141,17 +141,16 @@ class DownloaderAwarePriorityQueue:
def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP')
% (self.__class__,))
if slot_startprios and not isinstance(slot_startprios, dict): if slot_startprios and not isinstance(slot_startprios, dict):
raise ValueError("DownloaderAwarePriorityQueue accepts " raise ValueError("DownloaderAwarePriorityQueue accepts "
"``slot_startprios`` as a dict; %r instance " "``slot_startprios`` as a dict; "
f"{slot_startprios.__class__!r} instance "
"is passed. Most likely, it means the state is" "is passed. Most likely, it means the state is"
"created by an incompatible priority queue. " "created by an incompatible priority queue. "
"Only a crawl started with the same priority " "Only a crawl started with the same priority "
"queue class can be resumed." % "queue class can be resumed.")
slot_startprios.__class__)
self._downloader_interface = DownloaderInterface(crawler) self._downloader_interface = DownloaderInterface(crawler)
self.downstream_queue_cls = downstream_queue_cls self.downstream_queue_cls = downstream_queue_cls

2
scrapy/responsetypes.py
View File

@ -45,7 +45,7 @@ class ResponseTypes:
elif mimetype in self.classes: elif mimetype in self.classes:
return self.classes[mimetype] return self.classes[mimetype]
else: else:
basetype = "%s/*" % mimetype.split('/')[0] basetype = f"{mimetype.split('/')[0]}/*"
return self.classes.get(basetype, Response) return self.classes.get(basetype, Response)
def from_content_type(self, content_type, content_encoding=None): def from_content_type(self, content_type, content_encoding=None):

4
scrapy/selector/unified.py
View File

@ -66,8 +66,8 @@ class Selector(_ParselSelector, object_ref):
def __init__(self, response=None, text=None, type=None, root=None, **kwargs): def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
if response is not None and text is not None: if response is not None and text is not None:
raise ValueError('%s.__init__() received both response and text' raise ValueError(f'{self.__class__.__name__}.__init__() received '
% self.__class__.__name__) 'both response and text')
st = _st(response, type or self._default_type) st = _st(response, type or self._default_type)

2
scrapy/settings/__init__.py
View File

@ -52,7 +52,7 @@ class SettingsAttribute:
self.priority = priority self.priority = priority
def __str__(self): def __str__(self):
return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self) return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
__repr__ = __str__ __repr__ = __str__

2
scrapy/settings/default_settings.py
View File

@ -287,7 +287,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
URLLENGTH_LIMIT = 2083 URLLENGTH_LIMIT = 2083
USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)'
TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_ENABLED = 1
TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_PORT = [6023, 6073]

4
scrapy/shell.py
View File

@ -140,7 +140,7 @@ class Shell:
b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)")
for k, v in sorted(self.vars.items()): for k, v in sorted(self.vars.items()):
if self._is_relevant(v): if self._is_relevant(v):
b.append(" %-10s %s" % (k, v)) b.append(f" {k:<10} {v}")
b.append("Useful shortcuts:") b.append("Useful shortcuts:")
if self.inthread: if self.inthread:
b.append(" fetch(url[, redirect=True]) " b.append(" fetch(url[, redirect=True]) "
@ -150,7 +150,7 @@ class Shell:
b.append(" shelp() Shell help (print this help)") b.append(" shelp() Shell help (print this help)")
b.append(" view(response) View response in a browser") b.append(" view(response) View response in a browser")
return "\n".join("[s] %s" % line for line in b) return "\n".join(f"[s] {line}" for line in b)
def _is_relevant(self, value): def _is_relevant(self, value):
return isinstance(value, self.relevant_classes) or is_item(value) return isinstance(value, self.relevant_classes) or is_item(value)

13
scrapy/spiderloader.py
View File

@ -27,7 +27,7 @@ class SpiderLoader:
dupes = [] dupes = []
for name, locations in self._found.items(): for name, locations in self._found.items():
dupes.extend([ dupes.extend([
" {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) f" {cls} named {name!r} (in {mod})"
for mod, cls in locations for mod, cls in locations
if len(locations) > 1 if len(locations) > 1
]) ])
@ -36,7 +36,7 @@ class SpiderLoader:
dupes_string = "\n\n".join(dupes) dupes_string = "\n\n".join(dupes)
warnings.warn( warnings.warn(
"There are several spiders with the same name:\n\n" "There are several spiders with the same name:\n\n"
"{}\n\n This can cause unexpected behavior.".format(dupes_string), f"{dupes_string}\n\n This can cause unexpected behavior.",
category=UserWarning, category=UserWarning,
) )
@ -53,10 +53,9 @@ class SpiderLoader:
except ImportError: except ImportError:
if self.warn_only: if self.warn_only:
warnings.warn( warnings.warn(
"\n{tb}Could not load spiders from module '{modname}'. " f"\n{traceback.format_exc()}Could not load spiders "
"See above traceback for details.".format( f"from module '{name}'. "
modname=name, tb=traceback.format_exc() "See above traceback for details.",
),
category=RuntimeWarning, category=RuntimeWarning,
) )
else: else:
@ -75,7 +74,7 @@ class SpiderLoader:
try: try:
return self._spiders[spider_name] return self._spiders[spider_name]
except KeyError: except KeyError:
raise KeyError("Spider not found: {}".format(spider_name)) raise KeyError(f"Spider not found: {spider_name}")
def find_by_request(self, request): def find_by_request(self, request):
""" """

2
scrapy/spidermiddlewares/depth.py
View File

@ -43,7 +43,7 @@ class DepthMiddleware:
return False return False
else: else:
if self.verbose_stats: if self.verbose_stats:
self.stats.inc_value('request_depth_count/%s' % depth, self.stats.inc_value(f'request_depth_count/{depth}',
spider=spider) spider=spider)
self.stats.max_value('request_depth_max', depth, self.stats.max_value('request_depth_max', depth,
spider=spider) spider=spider)

2
scrapy/spidermiddlewares/httperror.py
View File

@ -48,7 +48,7 @@ class HttpErrorMiddleware:
if isinstance(exception, HttpError): if isinstance(exception, HttpError):
spider.crawler.stats.inc_value('httperror/response_ignored_count') spider.crawler.stats.inc_value('httperror/response_ignored_count')
spider.crawler.stats.inc_value( spider.crawler.stats.inc_value(
'httperror/response_ignored_status_count/%s' % response.status f'httperror/response_ignored_status_count/{response.status}'
) )
logger.info( logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed", "Ignoring response %(response)r: HTTP status code is not handled or not allowed",

6
scrapy/spidermiddlewares/offsite.py
View File

@ -61,15 +61,15 @@ class OffsiteMiddleware:
continue continue
elif url_pattern.match(domain): elif url_pattern.match(domain):
message = ("allowed_domains accepts only domains, not URLs. " message = ("allowed_domains accepts only domains, not URLs. "
"Ignoring URL entry %s in allowed_domains." % domain) f"Ignoring URL entry {domain} in allowed_domains.")
warnings.warn(message, URLWarning) warnings.warn(message, URLWarning)
elif port_pattern.search(domain): elif port_pattern.search(domain):
message = ("allowed_domains accepts only domains without ports. " message = ("allowed_domains accepts only domains without ports. "
"Ignoring entry %s in allowed_domains." % domain) f"Ignoring entry {domain} in allowed_domains.")
warnings.warn(message, PortWarning) warnings.warn(message, PortWarning)
else: else:
domains.append(re.escape(domain)) domains.append(re.escape(domain))
regex = r'^(.*\.)?(%s)$' % '|'.join(domains) regex = fr'^(.*\.)?({"|".join(domains)})$'
return re.compile(regex) return re.compile(regex)
def spider_opened(self, spider): def spider_opened(self, spider):

2
scrapy/spidermiddlewares/referer.py
View File

@ -278,7 +278,7 @@ def _load_policy_class(policy, warning_only=False):
try: try:
return _policy_classes[policy.lower()] return _policy_classes[policy.lower()]
except KeyError: except KeyError:
msg = "Could not load referrer policy %r" % policy msg = f"Could not load referrer policy {policy!r}"
if not warning_only: if not warning_only:
raise RuntimeError(msg) raise RuntimeError(msg)
else: else:

11
scrapy/spiders/__init__.py
View File

@ -25,7 +25,7 @@ class Spider(object_ref):
if name is not None: if name is not None:
self.name = name self.name = name
elif not getattr(self, 'name', None): elif not getattr(self, 'name', None):
raise ValueError("%s must have a name" % type(self).__name__) raise ValueError(f"{type(self).__name__} must have a name")
self.__dict__.update(kwargs) self.__dict__.update(kwargs)
if not hasattr(self, 'start_urls'): if not hasattr(self, 'start_urls'):
self.start_urls = [] self.start_urls = []
@ -66,9 +66,8 @@ class Spider(object_ref):
warnings.warn( warnings.warn(
"Spider.make_requests_from_url method is deprecated; it " "Spider.make_requests_from_url method is deprecated; it "
"won't be called in future Scrapy releases. Please " "won't be called in future Scrapy releases. Please "
"override Spider.start_requests method instead (see %s.%s)." % ( "override Spider.start_requests method instead "
cls.__module__, cls.__name__ f"(see {cls.__module__}.{cls.__name__}).",
),
) )
for url in self.start_urls: for url in self.start_urls:
yield self.make_requests_from_url(url) yield self.make_requests_from_url(url)
@ -90,7 +89,7 @@ class Spider(object_ref):
return self.parse(response, **kwargs) return self.parse(response, **kwargs)
def parse(self, response, **kwargs): def parse(self, response, **kwargs):
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined')
@classmethod @classmethod
def update_settings(cls, settings): def update_settings(cls, settings):
@ -107,7 +106,7 @@ class Spider(object_ref):
return closed(reason) return closed(reason)
def __str__(self): def __str__(self):
return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>"
__repr__ = __str__ __repr__ = __str__

4
scrapy/spiders/feed.py
View File

@ -71,11 +71,11 @@ class XMLFeedSpider(Spider):
elif self.iterator == 'xml': elif self.iterator == 'xml':
selector = Selector(response, type='xml') selector = Selector(response, type='xml')
self._register_namespaces(selector) self._register_namespaces(selector)
nodes = selector.xpath('//%s' % self.itertag) nodes = selector.xpath(f'//{self.itertag}')
elif self.iterator == 'html': elif self.iterator == 'html':
selector = Selector(response, type='html') selector = Selector(response, type='html')
self._register_namespaces(selector) self._register_namespaces(selector)
nodes = selector.xpath('//%s' % self.itertag) nodes = selector.xpath(f'//{self.itertag}')
else: else:
raise NotSupported('Unsupported node iterator') raise NotSupported('Unsupported node iterator')

6
scrapy/utils/benchserver.py
View File

@ -21,8 +21,8 @@ class Root(Resource):
for nl in nlist: for nl in nlist:
args['n'] = nl args['n'] = nl
argstr = urlencode(args, doseq=True) argstr = urlencode(args, doseq=True)
request.write("<a href='/follow?{0}'>follow {1}</a><br>" request.write(f"<a href='/follow?{argstr}'>follow {nl}</a><br>"
.format(argstr, nl).encode('utf8')) .encode('utf8'))
request.write(b"</body></html>") request.write(b"</body></html>")
return b'' return b''
@ -39,6 +39,6 @@ if __name__ == '__main__':
def _print_listening(): def _print_listening():
httpHost = httpPort.getHost() httpHost = httpPort.getHost()
print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port)) print(f"Bench server at http://{httpHost.host}:{httpHost.port}")
reactor.callWhenRunning(_print_listening) reactor.callWhenRunning(_print_listening)
reactor.run() reactor.run()

22
scrapy/utils/conf.py
View File

@ -17,8 +17,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
def _check_components(complist): def _check_components(complist):
if len({convert(c) for c in complist}) != len(complist): if len({convert(c) for c in complist}) != len(complist):
raise ValueError('Some paths in {!r} convert to the same object, ' raise ValueError('Some paths in {complist!r} convert to the same object, '
'please update your settings'.format(complist)) 'please update your settings')
def _map_keys(compdict): def _map_keys(compdict):
if isinstance(compdict, BaseSettings): if isinstance(compdict, BaseSettings):
@ -26,9 +26,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
for k, v in compdict.items(): for k, v in compdict.items():
prio = compdict.getpriority(k) prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio: if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same ' raise ValueError(f'Some paths in {list(compdict.keys())!r} '
'convert to the same '
'object, please update your settings' 'object, please update your settings'
''.format(list(compdict.keys()))) )
else: else:
compbs.set(convert(k), v, priority=prio) compbs.set(convert(k), v, priority=prio)
return compbs return compbs
@ -40,8 +41,9 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
"""Fail if a value in the components dict is not a real number or None.""" """Fail if a value in the components dict is not a real number or None."""
for name, value in compdict.items(): for name, value in compdict.items():
if value is not None and not isinstance(value, numbers.Real): if value is not None and not isinstance(value, numbers.Real):
raise ValueError('Invalid value {} for component {}, please provide ' raise ValueError(f'Invalid value {value} for component {name}, '
'a real number or None instead'.format(value, name)) 'please provide a real number or None instead'
)
# BEGIN Backward compatibility for old (base, custom) call signature # BEGIN Backward compatibility for old (base, custom) call signature
if isinstance(custom, (list, tuple)): if isinstance(custom, (list, tuple)):
@ -141,12 +143,10 @@ def feed_process_params_from_cli(settings, output, output_format=None,
def check_valid_format(output_format): def check_valid_format(output_format):
if output_format not in valid_output_formats: if output_format not in valid_output_formats:
raise UsageError( raise UsageError(
"Unrecognized output format '%s'. Set a supported one (%s) " f"Unrecognized output format '{output_format}'. "
f"Set a supported one ({tuple(valid_output_formats)}) "
"after a colon at the end of the output URI (i.e. -o/-O " "after a colon at the end of the output URI (i.e. -o/-O "
"<URI>:<FORMAT>) or as a file extension." % ( "<URI>:<FORMAT>) or as a file extension."
output_format,
tuple(valid_output_formats),
)
) )
overwrite = False overwrite = False

4
scrapy/utils/curl.py
View File

@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header
class CurlParser(argparse.ArgumentParser): class CurlParser(argparse.ArgumentParser):
def error(self, message): def error(self, message):
error_msg = 'There was an error parsing the curl command: {}'.format(message) error_msg = f'There was an error parsing the curl command: {message}'
raise ValueError(error_msg) raise ValueError(error_msg)
@ -52,7 +52,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
if argv: if argv:
msg = 'Unrecognized options: {}'.format(', '.join(argv)) msg = f'Unrecognized options: {", ".join(argv)}'
if ignore_unknown_options: if ignore_unknown_options:
warnings.warn(msg) warnings.warn(msg)
else: else:

4
scrapy/utils/decorators.py
View File

@ -14,9 +14,9 @@ def deprecated(use_instead=None):
def deco(func): def deco(func):
@wraps(func) @wraps(func)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
message = "Call to deprecated function %s." % func.__name__ message = f"Call to deprecated function {func.__name__}."
if use_instead: if use_instead:
message += " Use %s instead." % use_instead message += f" Use {use_instead} instead."
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
return func(*args, **kwargs) return func(*args, **kwargs)
return wrapped return wrapped

11
scrapy/utils/deprecate.py
View File

@ -8,9 +8,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
def attribute(obj, oldattr, newattr, version='0.12'): def attribute(obj, oldattr, newattr, version='0.12'):
cname = obj.__class__.__name__ cname = obj.__class__.__name__
warnings.warn( warnings.warn(
"%s.%s attribute is deprecated and will be no longer supported " f"{cname}.{oldattr} attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead" f"in Scrapy {version}, use {cname}.{newattr} attribute instead",
% (cname, oldattr, version, cname, newattr),
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
stacklevel=3) stacklevel=3)
@ -116,7 +115,7 @@ def create_deprecated_class(
# deprecated class is in jinja2 template). __module__ attribute is not # deprecated class is in jinja2 template). __module__ attribute is not
# important enough to raise an exception as users may be unable # important enough to raise an exception as users may be unable
# to fix inspect.stack() errors. # to fix inspect.stack() errors.
warnings.warn("Error detecting parent module: %r" % e) warnings.warn(f"Error detecting parent module: {e!r}")
return deprecated_cls return deprecated_cls
@ -124,7 +123,7 @@ def create_deprecated_class(
def _clspath(cls, forced=None): def _clspath(cls, forced=None):
if forced is not None: if forced is not None:
return forced return forced
return '{}.{}'.format(cls.__module__, cls.__name__) return f'{cls.__module__}.{cls.__name__}'
DEPRECATION_RULES = [ DEPRECATION_RULES = [
@ -137,7 +136,7 @@ def update_classpath(path):
for prefix, replacement in DEPRECATION_RULES: for prefix, replacement in DEPRECATION_RULES:
if path.startswith(prefix): if path.startswith(prefix):
new_path = path.replace(prefix, replacement, 1) new_path = path.replace(prefix, replacement, 1)
warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead",
ScrapyDeprecationWarning) ScrapyDeprecationWarning)
return new_path return new_path
return path return path

4
scrapy/utils/engine.py
View File

@ -29,7 +29,7 @@ def get_engine_status(engine):
try: try:
checks += [(test, eval(test))] checks += [(test, eval(test))]
except Exception as e: except Exception as e:
checks += [(test, "%s (exception)" % type(e).__name__)] checks += [(test, f"{type(e).__name__} (exception)")]
return checks return checks
@ -38,7 +38,7 @@ def format_engine_status(engine=None):
checks = get_engine_status(engine) checks = get_engine_status(engine)
s = "Execution engine status\n\n" s = "Execution engine status\n\n"
for test, result in checks: for test, result in checks:
s += "%-47s : %s\n" % (test, result) s += f"{test:<47} : {result}\n"
s += "\n" s += "\n"
return s return s

2
scrapy/utils/ftp.py
View File

@ -33,5 +33,5 @@ def ftp_store_file(
dirname, filename = posixpath.split(path) dirname, filename = posixpath.split(path)
ftp_makedirs_cwd(ftp, dirname) ftp_makedirs_cwd(ftp, dirname)
command = 'STOR' if overwrite else 'APPE' command = 'STOR' if overwrite else 'APPE'
ftp.storbinary('%s %s' % (command, filename), file) ftp.storbinary(f'{command} {filename}', file)
file.close() file.close()

13
scrapy/utils/iterators.py
View File

@ -22,8 +22,8 @@ def xmliter(obj, nodename):
""" """
nodename_patt = re.escape(nodename) nodename_patt = re.escape(nodename)
HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S)
HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S)
text = _body_or_str(obj) text = _body_or_str(obj)
header_start = re.search(HEADER_START_RE, text) header_start = re.search(HEADER_START_RE, text)
@ -31,7 +31,7 @@ def xmliter(obj, nodename):
header_end = re_rsearch(HEADER_END_RE, text) header_end = re_rsearch(HEADER_END_RE, text)
header_end = text[header_end[1]:].strip() if header_end else '' header_end = text[header_end[1]:].strip() if header_end else ''
r = re.compile(r'<%(np)s[\s>].*?</%(np)s>' % {'np': nodename_patt}, re.DOTALL) r = re.compile(fr'<{nodename_patt}[\s>].*?</{nodename_patt}>', re.DOTALL)
for match in r.finditer(text): for match in r.finditer(text):
nodetext = header_start + match.group() + header_end nodetext = header_start + match.group() + header_end
yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
@ -40,9 +40,9 @@ def xmliter(obj, nodename):
def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
from lxml import etree from lxml import etree
reader = _StreamReader(obj) reader = _StreamReader(obj)
tag = '{%s}%s' % (namespace, nodename) if namespace else nodename tag = f'{{{namespace}}}{nodename}'if namespace else nodename
iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename)
for _, node in iterable: for _, node in iterable:
nodetext = etree.tostring(node, encoding='unicode') nodetext = etree.tostring(node, encoding='unicode')
node.clear() node.clear()
@ -131,8 +131,7 @@ def _body_or_str(obj, unicode=True):
if not isinstance(obj, expected_types): if not isinstance(obj, expected_types):
expected_types_str = " or ".join(t.__name__ for t in expected_types) expected_types_str = " or ".join(t.__name__ for t in expected_types)
raise TypeError( raise TypeError(
"Object %r must be %s, not %s" f"Object {obj!r} must be {expected_types_str}, not {type(obj).__name__}"
% (obj, expected_types_str, type(obj).__name__)
) )
if isinstance(obj, Response): if isinstance(obj, Response):
if not unicode: if not unicode:

4
scrapy/utils/log.py
View File

@ -143,7 +143,7 @@ def log_scrapy_info(settings):
logger.info("Scrapy %(version)s started (bot: %(bot)s)", logger.info("Scrapy %(version)s started (bot: %(bot)s)",
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
versions = [ versions = [
"%s %s" % (name, version) f"{name} {version}"
for name, version in scrapy_components_versions() for name, version in scrapy_components_versions()
if name != "Scrapy" if name != "Scrapy"
] ]
@ -187,7 +187,7 @@ class LogCounterHandler(logging.Handler):
self.crawler = crawler self.crawler = crawler
def emit(self, record): def emit(self, record):
sname = 'log_count/{}'.format(record.levelname) sname = f'log_count/{record.levelname}'
self.crawler.stats.inc_value(sname) self.crawler.stats.inc_value(sname)

15
scrapy/utils/misc.py
View File

@ -46,7 +46,7 @@ def load_object(path):
try: try:
dot = path.rindex('.') dot = path.rindex('.')
except ValueError: except ValueError:
raise ValueError("Error loading object '%s': not a full path" % path) raise ValueError(f"Error loading object '{path}': not a full path")
module, name = path[:dot], path[dot + 1:] module, name = path[:dot], path[dot + 1:]
mod = import_module(module) mod = import_module(module)
@ -54,7 +54,7 @@ def load_object(path):
try: try:
obj = getattr(mod, name) obj = getattr(mod, name)
except AttributeError: except AttributeError:
raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name)) raise NameError(f"Module '{module}' doesn't define any object named '{name}'")
return obj return obj
@ -163,7 +163,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
instance = objcls(*args, **kwargs) instance = objcls(*args, **kwargs)
method_name = '__new__' method_name = '__new__'
if instance is None: if instance is None:
raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return instance return instance
@ -234,9 +234,10 @@ def warn_on_generator_with_return_value(spider, callable):
""" """
if is_generator_with_return_value(callable): if is_generator_with_return_value(callable):
warnings.warn( warnings.warn(
'The "{}.{}" method is a generator and includes a "return" statement with a ' f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
'value different than None. This could lead to unexpected behaviour. Please see ' 'a generator and includes a "return" statement with a value '
'different than None. This could lead to unexpected behaviour. Please see '
'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
'for details about the semantics of the "return" statement within generators' 'for details about the semantics of the "return" statement within generators',
.format(spider.__class__.__name__, callable.__name__), stacklevel=2, stacklevel=2,
) )

4
scrapy/utils/project.py
View File

@ -20,7 +20,7 @@ def inside_project():
try: try:
import_module(scrapy_module) import_module(scrapy_module)
except ImportError as exc: except ImportError as exc:
warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}")
else: else:
return True return True
return bool(closest_scrapy_cfg()) return bool(closest_scrapy_cfg())
@ -90,7 +90,7 @@ def get_project_settings():
warnings.warn( warnings.warn(
'Use of environment variables prefixed with SCRAPY_ to override ' 'Use of environment variables prefixed with SCRAPY_ to override '
'settings is deprecated. The following environment variables are ' 'settings is deprecated. The following environment variables are '
'currently defined: {}'.format(setting_envvar_list), f'currently defined: {setting_envvar_list}',
ScrapyDeprecationWarning ScrapyDeprecationWarning
) )
settings.setdict(scrapy_envvars, priority='project') settings.setdict(scrapy_envvars, priority='project')

12
scrapy/utils/python.py
View File

@ -91,7 +91,7 @@ def to_unicode(text, encoding=None, errors='strict'):
return text return text
if not isinstance(text, (bytes, str)): if not isinstance(text, (bytes, str)):
raise TypeError('to_unicode must receive a bytes or str ' raise TypeError('to_unicode must receive a bytes or str '
'object, got %s' % type(text).__name__) f'object, got {type(text).__name__}')
if encoding is None: if encoding is None:
encoding = 'utf-8' encoding = 'utf-8'
return text.decode(encoding, errors) return text.decode(encoding, errors)
@ -104,7 +104,7 @@ def to_bytes(text, encoding=None, errors='strict'):
return text return text
if not isinstance(text, str): if not isinstance(text, str):
raise TypeError('to_bytes must receive a str or bytes ' raise TypeError('to_bytes must receive a str or bytes '
'object, got %s' % type(text).__name__) f'object, got {type(text).__name__}')
if encoding is None: if encoding is None:
encoding = 'utf-8' encoding = 'utf-8'
return text.encode(encoding, errors) return text.encode(encoding, errors)
@ -174,7 +174,7 @@ def binary_is_text(data):
does not contain unprintable control characters. does not contain unprintable control characters.
""" """
if not isinstance(data, bytes): if not isinstance(data, bytes):
raise TypeError("data must be bytes, got '%s'" % type(data).__name__) raise TypeError(f"data must be bytes, got '{type(data).__name__}'")
return all(c not in _BINARYCHARS for c in data) return all(c not in _BINARYCHARS for c in data)
@ -217,7 +217,7 @@ def get_func_args(func, stripself=False):
else: else:
return get_func_args(func.__call__, True) return get_func_args(func.__call__, True)
else: else:
raise TypeError('%s is not callable' % type(func)) raise TypeError(f'{type(func)} is not callable')
if stripself: if stripself:
func_args.pop(0) func_args.pop(0)
return func_args return func_args
@ -250,7 +250,7 @@ def get_spec(func):
elif hasattr(func, '__call__'): elif hasattr(func, '__call__'):
spec = _getargspec_py23(func.__call__) spec = _getargspec_py23(func.__call__)
else: else:
raise TypeError('%s is not callable' % type(func)) raise TypeError(f'{type(func)} is not callable')
defaults = spec.defaults or [] defaults = spec.defaults or []
@ -322,7 +322,7 @@ def global_object_name(obj):
>>> global_object_name(Request) >>> global_object_name(Request)
'scrapy.http.request.Request' 'scrapy.http.request.Request'
""" """
return "%s.%s" % (obj.__module__, obj.__name__) return f"{obj.__module__}.{obj.__name__}"
if hasattr(sys, "pypy_version_info"): if hasattr(sys, "pypy_version_info"):

8
scrapy/utils/reactor.py
View File

@ -10,7 +10,7 @@ def listen_tcp(portrange, host, factory):
"""Like reactor.listenTCP but tries different ports in a range.""" """Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor from twisted.internet import reactor
if len(portrange) > 2: if len(portrange) > 2:
raise ValueError("invalid portrange: %s" % portrange) raise ValueError(f"invalid portrange: {portrange}")
if not portrange: if not portrange:
return reactor.listenTCP(0, factory, interface=host) return reactor.listenTCP(0, factory, interface=host)
if not hasattr(portrange, '__iter__'): if not hasattr(portrange, '__iter__'):
@ -78,9 +78,9 @@ def verify_installed_reactor(reactor_path):
from twisted.internet import reactor from twisted.internet import reactor
reactor_class = load_object(reactor_path) reactor_class = load_object(reactor_path)
if not isinstance(reactor, reactor_class): if not isinstance(reactor, reactor_class):
msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( msg = ("The installed reactor "
reactor.__module__, reactor.__class__.__name__, reactor_path f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
) f"match the requested one ({reactor_path})")
raise Exception(msg) raise Exception(msg)

4
scrapy/utils/reqser.py
View File

@ -84,7 +84,7 @@ def _find_method(obj, func):
# https://docs.python.org/3/reference/datamodel.html # https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__: if obj_func.__func__ is func.__func__:
return name return name
raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) raise ValueError(f"Function {func} is not an instance method in: {obj}")
def _get_method(obj, name): def _get_method(obj, name):
@ -92,4 +92,4 @@ def _get_method(obj, name):
try: try:
return getattr(obj, name) return getattr(obj, name)
except AttributeError: except AttributeError:
raise ValueError("Method %r not found in: %s" % (name, obj)) raise ValueError(f"Method {name!r} not found in: {obj}")

10
scrapy/utils/response.py
View File

@ -39,7 +39,7 @@ def response_status_message(status):
"""Return status code plus status text descriptive message """Return status code plus status text descriptive message
""" """
message = http.RESPONSES.get(int(status), "Unknown Status") message = http.RESPONSES.get(int(status), "Unknown Status")
return '%s %s' % (status, to_unicode(message)) return f'{status} {to_unicode(message)}'
def response_httprepr(response): def response_httprepr(response):
@ -69,15 +69,15 @@ def open_in_browser(response, _openfunc=webbrowser.open):
body = response.body body = response.body
if isinstance(response, HtmlResponse): if isinstance(response, HtmlResponse):
if b'<base' not in body: if b'<base' not in body:
repl = '<head><base href="%s">' % response.url repl = f'<head><base href="{response.url}">'
body = body.replace(b'<head>', to_bytes(repl)) body = body.replace(b'<head>', to_bytes(repl))
ext = '.html' ext = '.html'
elif isinstance(response, TextResponse): elif isinstance(response, TextResponse):
ext = '.txt' ext = '.txt'
else: else:
raise TypeError("Unsupported response type: %s" % raise TypeError("Unsupported response type: "
response.__class__.__name__) f"{response.__class__.__name__}")
fd, fname = tempfile.mkstemp(ext) fd, fname = tempfile.mkstemp(ext)
os.write(fd, body) os.write(fd, body)
os.close(fd) os.close(fd)
return _openfunc("file://%s" % fname) return _openfunc(f"file://{fname}")

6
scrapy/utils/serialize.py
View File

@ -17,7 +17,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
if isinstance(o, set): if isinstance(o, set):
return list(o) return list(o)
elif isinstance(o, datetime.datetime): elif isinstance(o, datetime.datetime):
return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}")
elif isinstance(o, datetime.date): elif isinstance(o, datetime.date):
return o.strftime(self.DATE_FORMAT) return o.strftime(self.DATE_FORMAT)
elif isinstance(o, datetime.time): elif isinstance(o, datetime.time):
@ -29,9 +29,9 @@ class ScrapyJSONEncoder(json.JSONEncoder):
elif is_item(o): elif is_item(o):
return ItemAdapter(o).asdict() return ItemAdapter(o).asdict()
elif isinstance(o, Request): elif isinstance(o, Request):
return "<%s %s %s>" % (type(o).__name__, o.method, o.url) return f"<{type(o).__name__} {o.method} {o.url}>"
elif isinstance(o, Response): elif isinstance(o, Response):
return "<%s %s %s>" % (type(o).__name__, o.status, o.url) return f"<{type(o).__name__} {o.status} {o.url}>"
else: else:
return super().default(o) return super().default(o)

4
scrapy/utils/ssl.py
View File

@ -50,7 +50,7 @@ def get_temp_key_info(ssl_object):
key_info.append(ffi_buf_to_string(cname)) key_info.append(ffi_buf_to_string(cname))
else: else:
key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits')
return ', '.join(key_info) return ', '.join(key_info)
@ -58,4 +58,4 @@ def get_openssl_version():
system_openssl = OpenSSL.SSL.SSLeay_version( system_openssl = OpenSSL.SSL.SSLeay_version(
OpenSSL.SSL.SSLEAY_VERSION OpenSSL.SSL.SSLEAY_VERSION
).decode('ascii', errors='replace') ).decode('ascii', errors='replace')
return '{} ({})'.format(OpenSSL.version.__version__, system_openssl) return f'{OpenSSL.version.__version__} ({system_openssl})'

2
scrapy/utils/test.py
View File

@ -79,7 +79,7 @@ def get_ftp_content_and_delete(
def buffer_data(data): def buffer_data(data):
ftp_data.append(data) ftp_data.append(data)
ftp.retrbinary('RETR %s' % path, buffer_data) ftp.retrbinary(f'RETR {path}', buffer_data)
dirname, filename = split(path) dirname, filename = split(path)
ftp.cwd(dirname) ftp.cwd(dirname)
ftp.delete(filename) ftp.delete(filename)

6
scrapy/utils/testproc.py
View File

@ -23,10 +23,10 @@ class ProcessTest:
def _process_finished(self, pp, cmd, check_code): def _process_finished(self, pp, cmd, check_code):
if pp.exitcode and check_code: if pp.exitcode and check_code:
msg = "process %s exit with code %d" % (cmd, pp.exitcode) msg = f"process {cmd} exit with code {pp.exitcode}"
msg += "\n>>> stdout <<<\n%s" % pp.out msg += f"\n>>> stdout <<<\n{pp.out}"
msg += "\n" msg += "\n"
msg += "\n>>> stderr <<<\n%s" % pp.err msg += f"\n>>> stderr <<<\n{pp.err}"
raise RuntimeError(msg) raise RuntimeError(msg)
return pp.exitcode, pp.out, pp.err return pp.exitcode, pp.out, pp.err

4
scrapy/utils/testsite.py
View File

@ -9,7 +9,7 @@ class SiteTest:
from twisted.internet import reactor from twisted.internet import reactor
super().setUp() super().setUp()
self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
self.baseurl = "http://localhost:%d/" % self.site.getHost().port self.baseurl = f"http://localhost:{self.site.getHost().port}/"
def tearDown(self): def tearDown(self):
super().tearDown() super().tearDown()
@ -40,5 +40,5 @@ def test_site():
if __name__ == '__main__': if __name__ == '__main__':
from twisted.internet import reactor from twisted.internet import reactor
port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
print("http://localhost:%d/" % port.getHost().port) print(f"http://localhost:{port.getHost().port}/")
reactor.run() reactor.run()

4
scrapy/utils/trackref.py
View File

@ -41,9 +41,7 @@ def format_live_refs(ignore=NoneType):
if issubclass(cls, ignore): if issubclass(cls, ignore):
continue continue
oldest = min(wdict.values()) oldest = min(wdict.values())
s += "%-30s %6d oldest: %ds ago\n" % ( s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n"
cls.__name__, len(wdict), now - oldest
)
return s return s

4
scrapy/utils/url.py
View File

@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains):
if not host: if not host:
return False return False
domains = [d.lower() for d in domains] domains = [d.lower() for d in domains]
return any((host == d) or (host.endswith('.%s' % d)) for d in domains) return any((host == d) or (host.endswith(f'.{d}')) for d in domains)
def url_is_from_spider(url, spider): def url_is_from_spider(url, spider):
@ -153,7 +153,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=
if (parsed_url.scheme, parsed_url.port) in (('http', 80), if (parsed_url.scheme, parsed_url.port) in (('http', 80),
('https', 443), ('https', 443),
('ftp', 21)): ('ftp', 21)):
netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') netloc = netloc.replace(f':{parsed_url.port}', '')
return urlunparse(( return urlunparse((
parsed_url.scheme, parsed_url.scheme,
netloc, netloc,

2
sep/sep-002.rst
View File

@ -30,7 +30,7 @@ Proposed Implementation
if hasattr(value, '__iter__'): # str/unicode not allowed if hasattr(value, '__iter__'): # str/unicode not allowed
return [self._field.to_python(v) for v in value] return [self._field.to_python(v) for v in value]
else: else:
raise TypeError("Expected iterable, got %s" % type(value).__name__) raise TypeError(f"Expected iterable, got {type(value).__name__}")
def get_default(self): def get_default(self):
# must return a new copy to avoid unexpected behaviors with mutable defaults # must return a new copy to avoid unexpected behaviors with mutable defaults

4
sep/sep-004.rst
View File

@ -11,7 +11,7 @@ SEP-004: Library API
==================== ====================
.. note:: the library API has been implemented, but slightly different from .. note:: the library API has been implemented, but slightly different from
proposed in this SEP. You can run a Scrapy crawler inside a Twisted proposed in this SEP. You can run a Scrapy crawler inside a Twisted
reactor, but not outside it. reactor, but not outside it.
Introduction Introduction
============ ============
@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script:
cr = Crawler(start_urls, callback=parse_start_page) cr = Crawler(start_urls, callback=parse_start_page)
cr.run() # blocking call - this populates scraped_items cr.run() # blocking call - this populates scraped_items
print "%d items scraped" % len(scraped_items) print(f"{len(scraped_items)} items scraped")
# ... do something more interesting with scraped_items ... # ... do something more interesting with scraped_items ...
The behaviour of the Scrapy crawler would be controller by the Scrapy settings, The behaviour of the Scrapy crawler would be controller by the Scrapy settings,

29
sep/sep-014.rst
View File

@ -21,7 +21,7 @@ Current flaws and inconsistencies
2. Link extractors are inflexible and hard to maintain, link 2. Link extractors are inflexible and hard to maintain, link
processing/filtering is tightly coupled. (e.g. canonicalize) processing/filtering is tightly coupled. (e.g. canonicalize)
3. Isn't possible to crawl an url directly from command line because the Spider 3. Isn't possible to crawl an url directly from command line because the Spider
does not know which callback use. does not know which callback use.
These flaws will be corrected by the changes proposed in this SEP. These flaws will be corrected by the changes proposed in this SEP.
@ -55,7 +55,7 @@ Request Extractors
Request Extractors takes response object and determines which requests follow. Request Extractors takes response object and determines which requests follow.
This is an enhancement to ``LinkExtractors`` which returns urls (links), This is an enhancement to ``LinkExtractors`` which returns urls (links),
Request Extractors return Request objects. Request Extractors return Request objects.
Request Processors Request Processors
------------------ ------------------
@ -142,7 +142,7 @@ Custom Processor and External Callback
# Callback defined out of spider # Callback defined out of spider
def my_external_callback(response): def my_external_callback(response):
# process item # process item
pass pass
class SampleSpider(CrawlSpider): class SampleSpider(CrawlSpider):
@ -233,7 +233,7 @@ Request/Response Matchers
def matches_request(self, request): def matches_request(self, request):
"""Returns True if Request's url matches initial url""" """Returns True if Request's url matches initial url"""
return self.matches_url(request.url) return self.matches_url(request.url)
def matches_response(self, response): def matches_response(self, response):
"""REturns True if Response's url matches initial url""" """REturns True if Response's url matches initial url"""
@ -305,14 +305,14 @@ Request Extractor
for req in self.requests: for req in self.requests:
req.meta.setdefault('link_text', '') req.meta.setdefault('link_text', '')
req.meta['link_text'] = str_to_unicode(req.meta['link_text'], req.meta['link_text'] = str_to_unicode(req.meta['link_text'],
encoding) encoding)
def reset(self): def reset(self):
"""Reset state""" """Reset state"""
FixedSGMLParser.reset(self) FixedSGMLParser.reset(self)
self.requests = [] self.requests = []
self.base_url = None self.base_url = None
def unknown_starttag(self, tag, attrs): def unknown_starttag(self, tag, attrs):
"""Process unknown start tag""" """Process unknown start tag"""
if 'base' tag: if 'base' tag:
@ -376,7 +376,7 @@ Request Processor
#!python #!python
# #
# Request Processors # Request Processors
# Processors receive list of requests and return list of requests # Processors receive list of requests and return list of requests
# #
"""Request Processors""" """Request Processors"""
@ -390,7 +390,7 @@ Request Processor
# replace in-place # replace in-place
req.url = canonicalize_url(req.url) req.url = canonicalize_url(req.url)
yield req yield req
class Unique(object): class Unique(object):
"""Filter duplicate Requests""" """Filter duplicate Requests"""
@ -455,9 +455,9 @@ Request Processor
"""Initialize allow/deny attributes""" """Initialize allow/deny attributes"""
_re_type = type(re.compile('', 0)) _re_type = type(re.compile('', 0))
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(allow)] for x in arg_to_iter(allow)]
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(deny)] for x in arg_to_iter(deny)]
def __call__(self, requests): def __call__(self, requests):
@ -524,7 +524,7 @@ Rules Manager
# #
# Handles rules matcher/callbacks # Handles rules matcher/callbacks
# Resolve rule for given response # Resolve rule for given response
# #
class RulesManager(object): class RulesManager(object):
"""Rules Manager""" """Rules Manager"""
def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): def __init__(self, rules, spider, default_matcher=UrlRegexMatcher):
@ -542,8 +542,8 @@ Rules Manager
# instance default matcher # instance default matcher
matcher = default_matcher(rule.matcher) matcher = default_matcher(rule.matcher)
else: else:
raise ValueError('Not valid matcher given %r in %r' \ raise ValueError('Not valid matcher given '
% (rule.matcher, rule)) f'{rule.matcher!r} in {rule!r}')
# prepare callback # prepare callback
if callable(rule.callback): if callable(rule.callback):
@ -553,8 +553,7 @@ Rules Manager
callback = getattr(spider, rule.callback) callback = getattr(spider, rule.callback)
if not callable(callback): if not callable(callback):
raise AttributeError('Invalid callback %r can not be resolved' \ raise AttributeError(f'Invalid callback {callback!r} can not be resolved')
% callback)
else: else:
callback = None callback = None

22
sep/sep-018.rst
View File

@ -171,7 +171,7 @@ the same spider:
#!python #!python
class MySpider(BaseSpider): class MySpider(BaseSpider):
middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(),
ItemIdSetter(), OffsiteMiddleware()] ItemIdSetter(), OffsiteMiddleware()]
allowed_domains = ['example.com', 'sub.example.com'] allowed_domains = ['example.com', 'sub.example.com']
@ -196,7 +196,7 @@ the same spider:
# extract item from response # extract item from response
return item return item
The Spider Middleware that implements spider code The Spider Middleware that implements spider code
================================================= =================================================
There's gonna be one middleware that will take care of calling the proper There's gonna be one middleware that will take care of calling the proper
@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers:
class CanonializeUrl(object): class CanonializeUrl(object):
def process_request(self, request, response, spider): def process_request(self, request, response, spider):
curl = canonicalize_url(request.url, curl = canonicalize_url(request.url,
rules=spider.canonicalization_rules) rules=spider.canonicalization_rules)
return request.replace(url=curl) return request.replace(url=curl)
@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers:
class MySpider(BaseSpider): class MySpider(BaseSpider):
middlewares = [CanonicalizeUrl()] middlewares = [CanonicalizeUrl()]
canonicalization_rules = ['sort-query-args', canonicalization_rules = ['sort-query-args',
'normalize-percent-encoding', ...] 'normalize-percent-encoding', ...]
# ... # ...
@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt:
if netloc in info.pending: if netloc in info.pending:
res = None res = None
else: else:
robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) robotsurl = f"{url.scheme}://{netloc}/robots.txt"
meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]}
res = Request(robotsurl, callback=self.parse_robots, res = Request(robotsurl, callback=self.parse_robots,
meta=meta, priority=self.REQUEST_PRIORITY) meta=meta, priority=self.REQUEST_PRIORITY)
@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API:
if host and host not in info.hosts_seen: if host and host not in info.hosts_seen:
spider.log("Filtered offsite request to %r: %s" % (host, request)) spider.log("Filtered offsite request to %r: %s" % (host, request))
info.hosts_seen.add(host) info.hosts_seen.add(host)
def should_follow(self, request, spider): def should_follow(self, request, spider):
info = self.spiders[spider] info = self.spiders[spider]
# hostname can be None for wrong urls (like javascript links) # hostname can be None for wrong urls (like javascript links)
@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API:
def get_host_regex(self, spider): def get_host_regex(self, spider):
"""Override this method to implement a different offsite policy""" """Override this method to implement a different offsite policy"""
domains = [d.replace('.', r'\.') for d in spider.allowed_domains] domains = [d.replace('.', r'\.') for d in spider.allowed_domains]
regex = r'^(.*\.)?(%s)$' % '|'.join(domains) regex = fr'^(.*\.)?({"|".join(domains)})$'
return re.compile(regex) return re.compile(regex)
def spider_opened(self, spider): def spider_opened(self, spider):
@ -570,7 +570,7 @@ A middleware to filter out requests already seen:
self.dupefilter = load_object(clspath)() self.dupefilter = load_object(clspath)()
dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
def enqueue_request(self, spider, request): def enqueue_request(self, spider, request):
seen = self.dupefilter.request_seen(spider, request) seen = self.dupefilter.request_seen(spider, request)
if not seen or request.dont_filter: if not seen or request.dont_filter:
@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley
for name in parslet.keys(): for name in parslet.keys():
self.fields[name] = Field() self.fields[name] = Field()
super(ParsleyItem, self).__init__(*a, **kw) super(ParsleyItem, self).__init__(*a, **kw)
self.item_class = ParsleyItem self.item_class = ParsleyItem
self.parsley = PyParsley(parslet, output='python') self.parsley = PyParsley(parslet, output='python')
def process_response(self, response, request, spider): def process_response(self, response, request, spider):
return self.item_class(self.parsly.parse(string=response.body)) return self.item_class(self.parsly.parse(string=response.body))
@ -627,7 +627,7 @@ Resolved:
not the original one (think of redirections), but it does carry the ``meta`` not the original one (think of redirections), but it does carry the ``meta``
of the original one. The original one may not be available anymore (in of the original one. The original one may not be available anymore (in
memory) if we're using a persistent scheduler., but in that case it would be memory) if we're using a persistent scheduler., but in that case it would be
the deserialized request from the persistent scheduler queue. the deserialized request from the persistent scheduler queue.
- No - this would make implementation more complex and we're not sure it's - No - this would make implementation more complex and we're not sure it's
really needed really needed

2
tests/CrawlerRunner/ip_address.py
View File

@ -38,7 +38,7 @@ class LocalhostSpider(Spider):
if __name__ == "__main__": if __name__ == "__main__":
with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
port = urlparse(mock_http_server.http_address).port port = urlparse(mock_http_server.http_address).port
url = "http://not.a.real.domain:{port}/echo".format(port=port) url = f"http://not.a.real.domain:{port}/echo"
servers = [(mock_dns_server.host, mock_dns_server.port)] servers = [(mock_dns_server.host, mock_dns_server.port)]
reactor.installResolver(createResolver(servers=servers)) reactor.installResolver(createResolver(servers=servers))

10
tests/mockserver.py
View File

@ -73,7 +73,7 @@ class Follow(LeafResource):
for nl in nlist: for nl in nlist:
args[b"n"] = [to_bytes(str(nl))] args[b"n"] = [to_bytes(str(nl))]
argstr = urlencode(args, doseq=True) argstr = urlencode(args, doseq=True)
s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl) s += f"<a href='/follow?{argstr}'>follow {nl}</a><br>"
s += """</body>""" s += """</body>"""
request.write(to_bytes(s)) request.write(to_bytes(s))
request.finish() request.finish()
@ -91,7 +91,7 @@ class Delay(LeafResource):
return NOT_DONE_YET return NOT_DONE_YET
def _delayedRender(self, request, n): def _delayedRender(self, request, n):
request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))
request.finish() request.finish()
@ -310,8 +310,8 @@ if __name__ == "__main__":
def print_listening(): def print_listening():
httpHost = httpPort.getHost() httpHost = httpPort.getHost()
httpsHost = httpsPort.getHost() httpsHost = httpsPort.getHost()
httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) httpAddress = f'http://{httpHost.host}:{httpHost.port}'
httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) httpsAddress = f'https://{httpsHost.host}:{httpsHost.port}'
print(httpAddress) print(httpAddress)
print(httpsAddress) print(httpsAddress)
@ -323,7 +323,7 @@ if __name__ == "__main__":
def print_listening(): def print_listening():
host = listener.getHost() host = listener.getHost()
print("%s:%s" % (host.host, host.port)) print(f"{host.host}:{host.port}")
reactor.callWhenRunning(print_listening) reactor.callWhenRunning(print_listening)
reactor.run() reactor.run()

2
tests/py36/_test_crawl.py
View File

@ -33,7 +33,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
depth = 2 depth = 2
def _get_req(self, index, cb=None): def _get_req(self, index, cb=None):
return Request(self.mockserver.url("/status?n=200&request=%d" % index), return Request(self.mockserver.url(f"/status?n=200&request={index}"),
meta={'index': index}, meta={'index': index},
dont_filter=True, dont_filter=True,
callback=cb) callback=cb)

8
tests/spiders.py
View File

@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider):
self.urls_visited = [] self.urls_visited = []
self.times = [] self.times = []
qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}")
self.start_urls = [url] self.start_urls = [url]
def parse(self, response): def parse(self, response):
@ -67,7 +67,7 @@ class DelaySpider(MetaSpider):
def start_requests(self): def start_requests(self):
self.t1 = time.time() self.t1 = time.time()
url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback) yield Request(url, callback=self.parse, errback=self.errback)
def parse(self, response): def parse(self, response):
@ -192,7 +192,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
for s in range(100): for s in range(100):
qargs = {'total': 10, 'seed': s} qargs = {'total': 10, 'seed': s}
url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}")
yield Request(url, meta={'seed': s}) yield Request(url, meta={'seed': s})
if self.fail_yielding: if self.fail_yielding:
2 / 0 2 / 0
@ -239,7 +239,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):
def start_requests(self): def start_requests(self):
for i in range(0, self.distinct_urls): for i in range(0, self.distinct_urls):
for j in range(0, self.dupe_factor): for j in range(0, self.dupe_factor):
url = self.mockserver.url("/echo?headers=1&body=test%d" % i) url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
yield Request(url, dont_filter=self.dont_filter) yield Request(url, dont_filter=self.dont_filter)
def __init__(self, url="http://localhost:8998", *args, **kwargs): def __init__(self, url="http://localhost:8998", *args, **kwargs):

2
tests/test_cmdline/extensions.py
View File

@ -4,7 +4,7 @@
class TestExtension: class TestExtension:
def __init__(self, settings): def __init__(self, settings):
settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) settings.set('TEST1', f"{settings['TEST1']} + started")
@classmethod @classmethod
def from_crawler(cls, crawler): def from_crawler(cls, crawler):

Some files were not shown because too many files have changed in this diff Show More