diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html
new file mode 100644
index 000000000..71aff8847
--- /dev/null
+++ b/docs/_tests/quotes.html
@@ -0,0 +1,281 @@
+
+
+
+
+ Quotes to Scrape
+
+
+
+
+
+
+
+
+
+
+
+
+
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is our choices, Harry, that show what we truly are, far more than our abilities.”
+
by J.K. Rowling
+ (about)
+
+
+
+
+
+
“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”
+
by Jane Austen
+ (about)
+
+
+
+
+
+
“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”
+
by Marilyn Monroe
+ (about)
+
+
+
+
+
+
“Try not to become a man of success. Rather become a man of value.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is better to be hated for what you are than to be loved for what you are not.”
+
by André Gide
+ (about)
+
+
+
+
+
+
“I have not failed. I've just found 10,000 ways that won't work.”
+
by Thomas A. Edison
+ (about)
+
+
+
+
+
+
“A woman is like a tea bag; you never know how strong it is until it's in hot water.”
+
by Eleanor Roosevelt
+ (about)
+
+
+
+
+
+
“A day without sunshine is like, you know, night.”
+
by Steve Martin
+ (about)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index d1d4ebf5d..b7d740c19 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things:
.. _topics-inspector:
Inspecting a website
-===================================
+====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
@@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can
expand and collapse a tag by clicking on the arrow in front of it or by double
clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
-copy XPaths to selected elements. Let's try it out: Right-click on the ``span``
-tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so::
+copy XPaths to selected elements. Let's try it out.
+
+First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
+
+.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
- (...)
- >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
- ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
+
+Then, back to your web browser, right-click on the ``span`` tag, select
+``Copy > XPath`` and paste it in the Scrapy shell like so:
+
+.. invisible-code-block: python
+
+ response = load_response('http://quotes.toscrape.com/', 'quotes.html')
+
+>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@@ -112,13 +122,13 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
-the `has-class-extension`_::
+the `has-class-extension`_:
- >>> response.xpath('//span[has-class("text")]/text()').getall()
- ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
- '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
- '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
- (...)]
+>>> response.xpath('//span[has-class("text")]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
+'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
+'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
+...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
-we'll check another quite useful command from the Scrapy shell::
+we'll check another quite useful command from the Scrapy shell:
+
+.. skip: next
+
+.. code-block:: none
$ scrapy shell "quotes.toscrape.com/scroll"
(...)
diff --git a/pytest.ini b/pytest.ini
index f088e10ef..7bb759c28 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,7 +8,6 @@ addopts =
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
- --ignore=docs/topics/developer-tools.rst
--ignore=docs/topics/dynamic-content.rst
--ignore=docs/topics/items.rst
--ignore=docs/topics/leaks.rst
@@ -85,8 +84,8 @@ flake8-ignore =
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
- scrapy/http/response/__init__.py E501 E128 W293 W291
- scrapy/http/response/text.py E501 W293 E128 E124
+ scrapy/http/response/__init__.py E501 E128
+ scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E501 E402
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
@@ -127,9 +126,9 @@ flake8-ignore =
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/log.py E128 W503
- scrapy/utils/markup.py F403 W292
+ scrapy/utils/markup.py F403
scrapy/utils/misc.py E501 E226
- scrapy/utils/multipart.py F403 W292
+ scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E226
@@ -144,7 +143,6 @@ flake8-ignore =
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
- scrapy/_monkeypatches.py W293
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
@@ -153,7 +151,7 @@ flake8-ignore =
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
- scrapy/logformatter.py E501 W293
+ scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
@@ -174,7 +172,7 @@ flake8-ignore =
tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
- tests/test_contracts.py E501 E128 W293
+ tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py F841 E501 E305
@@ -189,17 +187,17 @@ flake8-ignore =
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
- tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126
+ tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
- tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
+ tests/test_dupefilters.py E221 E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
- tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123
+ tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E301 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
@@ -209,20 +207,20 @@ flake8-ignore =
tests/test_mail.py E128 E501 E305
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
- tests/test_pipeline_files.py E501 W293 E303 E272 E226
+ tests/test_pipeline_files.py E501 E303 E272 E226
tests/test_pipeline_images.py F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E305
- tests/test_robotstxt_interface.py E501 W291 E501
+ tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
- tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
- tests/test_spidermiddleware_output_chain.py E501 W293 E226
+ tests/test_spidermiddleware_offsite.py E501 E128 E111
+ tests/test_spidermiddleware_output_chain.py E501 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_conf.py E501 E303 E128
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 64e9c6c20..e79ce9acc 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -113,8 +113,8 @@ class Response(object_ref):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
-
- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
+
+ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 1079fd6e8..4f9afde87 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -125,7 +125,7 @@ class TextResponse(Response):
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
-
+
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
@@ -133,7 +133,7 @@ class TextResponse(Response):
``response.xpath('//img/@src')[0]``.
* a Selector for ```` or `` `` element, e.g.
``response.css('a.my_link')[0]``.
-
+
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 5189d7cfa..4e5963e99 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'"
class LogFormatter(object):
"""Class for generating log messages for different actions.
-
+
All methods must return a dictionary listing the parameters ``level``, ``msg``
and ``args`` which are going to be used for constructing the log message when
calling ``logging.log``.
@@ -48,7 +48,7 @@ class LogFormatter(object):
}
}
"""
-
+
def crawled(self, request, response, spider):
"""Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else ''
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index 2455fcc16..9728c542a 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -11,4 +11,4 @@ from w3lib.html import * # noqa: F401
warnings.warn("Module `scrapy.utils.markup` is deprecated. "
"Please import from `w3lib.html` instead.",
- ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file
+ ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
index e81f63152..5dcf791b8 100644
--- a/scrapy/utils/multipart.py
+++ b/scrapy/utils/multipart.py
@@ -12,4 +12,4 @@ from w3lib.form import * # noqa: F401
warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
"If you're using `encode_multipart` function, please use "
"`urllib3.filepost.encode_multipart_formdata` instead",
- ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file
+ ScrapyDeprecationWarning, stacklevel=2)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 582e3d052..11d41c1fe 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -252,7 +252,7 @@ class ContractsManagerTest(unittest.TestCase):
self.assertEqual(len(contracts), 3)
self.assertEqual(frozenset(type(x) for x in contracts),
frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
-
+
contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
self.assertEqual(len(contracts), 3)
self.assertEqual(frozenset(type(x) for x in contracts),
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 76f87458b..f433fcea6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -30,25 +30,44 @@ class CrawlTestCase(TestCase):
self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url
@defer.inlineCallbacks
- def test_delay(self):
- # short to long delays
- yield self._test_delay(0.2, False)
- yield self._test_delay(1, False)
- # randoms
- yield self._test_delay(0.2, True)
- yield self._test_delay(1, True)
+ def test_fixed_delay(self):
+ yield self._test_delay(total=3, delay=0.1)
@defer.inlineCallbacks
- def _test_delay(self, delay, randomize):
- settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
+ def test_randomized_delay(self):
+ yield self._test_delay(total=3, delay=0.1, randomize=True)
+
+ @defer.inlineCallbacks
+ def _test_delay(self, total, delay, randomize=False):
+ crawl_kwargs = dict(
+ maxlatency=delay * 2,
+ mockserver=self.mockserver,
+ total=total,
+ )
+ tolerance = (1 - (0.6 if randomize else 0.2))
+
+ settings = {"DOWNLOAD_DELAY": delay,
+ 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
- yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver)
- t = crawler.spider.times
- totaltime = t[-1] - t[0]
- avgd = totaltime / (len(t) - 1)
- tolerance = 0.6 if randomize else 0.2
- self.assertTrue(avgd > delay * (1 - tolerance),
- "download delay too small: %s" % avgd)
+ yield crawler.crawl(**crawl_kwargs)
+ times = crawler.spider.times
+ total_time = times[-1] - times[0]
+ average = total_time / (len(times) - 1)
+ self.assertTrue(average > delay * tolerance,
+ "download delay too small: %s" % average)
+
+ # Ensure that the same test parameters would cause a failure if no
+ # download delay is set. Otherwise, it means we are using a combination
+ # of ``total`` and ``delay`` values that are too small for the test
+ # code above to have any meaning.
+ settings["DOWNLOAD_DELAY"] = 0
+ crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+ yield crawler.crawl(**crawl_kwargs)
+ times = crawler.spider.times
+ total_time = times[-1] - times[0]
+ average = total_time / (len(times) - 1)
+ self.assertFalse(average > delay / tolerance,
+ "test total or delay values are too small")
@defer.inlineCallbacks
def test_timeout_success(self):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index e09d66086..9c989977e 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -124,7 +124,7 @@ class MaxRetryTimesTest(unittest.TestCase):
# SETTINGS: meta(max_retry_times) = 0
meta_max_retry_times = 0
-
+
req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
@@ -137,7 +137,7 @@ class MaxRetryTimesTest(unittest.TestCase):
self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
def test_with_metakey_greater(self):
-
+
# SETINGS: RETRY_TIMES < meta(max_retry_times)
self.mw.max_retry_times = 2
meta_max_retry_times = 3
@@ -149,7 +149,7 @@ class MaxRetryTimesTest(unittest.TestCase):
self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
def test_with_metakey_lesser(self):
-
+
# SETINGS: RETRY_TIMES > meta(max_retry_times)
self.mw.max_retry_times = 5
meta_max_retry_times = 4
@@ -172,7 +172,7 @@ class MaxRetryTimesTest(unittest.TestCase):
self._test_retry(req, DNSLookupError('foo'), 0)
def _test_retry(self, req, exception, max_retry_times):
-
+
for i in range(0, max_retry_times):
req = self.mw.process_exception(req, exception, self.spider)
assert isinstance(req, Request)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index e4b0bdf83..0546558bc 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -142,12 +142,12 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html')
-
+
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
- l.check_present(('scrapy.dupefilters', 'DEBUG',
+ l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: '
' - no more duplicates will be shown'
' (see DUPEFILTER_DEBUG to show all duplicates)')))
@@ -169,7 +169,7 @@ class RFPDupeFilterTest(unittest.TestCase):
r2 = Request('http://scrapytest.org/index.html',
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
)
-
+
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 52f2b554e..141141671 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -58,7 +58,6 @@ class FilesPipelineTestCase(unittest.TestCase):
self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
//+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
-
def test_fs_store(self):
assert isinstance(self.pipeline.store, FSFilesStore)
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 27d79437b..24aaaf7ec 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -44,7 +44,7 @@ class BaseRobotParserTest:
def test_allowed_wildcards(self):
robotstxt_robotstxt_body = """User-agent: first
- Disallow: /disallowed/*/end$
+ Disallow: /disallowed/*/end$
User-agent: second
Allow: /*allowed
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 992e60be2..7511aa568 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -73,7 +73,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
-
+
def test_get_host_regex(self):
self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']
with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 5b7b5e7aa..739cf1c2d 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -156,7 +156,7 @@ class GeneratorFailMiddleware:
r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
yield r
raise LookupError()
-
+
def process_spider_exception(self, response, exception, spider):
method = '{}.process_spider_exception'.format(self.__class__.__name__)
spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
@@ -264,7 +264,7 @@ class TestSpiderMiddleware(TestCase):
@classmethod
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
-
+
@defer.inlineCallbacks
def crawl_log(self, spider):
crawler = get_crawler(spider)
@@ -308,7 +308,7 @@ class TestSpiderMiddleware(TestCase):
self.assertIn("{'from': 'errback'}", str(log1))
self.assertNotIn("{'from': 'callback'}", str(log1))
self.assertIn("'item_scraped_count': 1", str(log1))
-
+
@defer.inlineCallbacks
def test_generator_callback(self):
"""
@@ -319,7 +319,7 @@ class TestSpiderMiddleware(TestCase):
log2 = yield self.crawl_log(GeneratorCallbackSpider)
self.assertIn("Middleware: ImportError exception caught", str(log2))
self.assertIn("'item_scraped_count': 2", str(log2))
-
+
@defer.inlineCallbacks
def test_not_a_generator_callback(self):
"""