mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'scrapy/master' into async-seeds
This commit is contained in:
Adrián Chaves
commit
3d93441ea6
|
|
@ -14,7 +14,7 @@ from it.
|
|||
|
||||
If you fail to do that, and you can nonetheless access the desired data through
|
||||
the :ref:`DOM <topics-livedom>` from your web browser, see
|
||||
:ref:`topics-javascript-rendering`.
|
||||
:ref:`topics-headless-browsing`.
|
||||
|
||||
.. _topics-finding-data-source:
|
||||
|
||||
|
|
@ -97,7 +97,7 @@ it <topics-handling-response-formats>`.
|
|||
You can reproduce any request with Scrapy. However, some times reproducing all
|
||||
necessary requests may not seem efficient in developer time. If that is your
|
||||
case, and crawling speed is not a major concern for you, you can alternatively
|
||||
consider :ref:`JavaScript pre-rendering <topics-javascript-rendering>`.
|
||||
consider :ref:`using a headless browser <topics-headless-browsing>`.
|
||||
|
||||
If you get the expected response `sometimes`, but not always, the issue is
|
||||
probably not your request, but the target server. The target server might be
|
||||
|
|
@ -220,9 +220,9 @@ data from it:
|
|||
>>> selector.css('var[name="data"]').get()
|
||||
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
|
||||
|
||||
.. _topics-javascript-rendering:
|
||||
.. _topics-headless-browsing:
|
||||
|
||||
Pre-rendering JavaScript
|
||||
Using a headless browser
|
||||
========================
|
||||
|
||||
On webpages that fetch data from additional requests, reproducing those
|
||||
|
|
@ -232,29 +232,10 @@ network transfer.
|
|||
|
||||
However, sometimes it can be really hard to reproduce certain requests. Or you
|
||||
may need something that no request can give you, such as a screenshot of a
|
||||
webpage as seen in a web browser.
|
||||
webpage as seen in a web browser. In this case using a `headless browser`_ will
|
||||
help.
|
||||
|
||||
In these cases use the Splash_ JavaScript-rendering service, along with
|
||||
`scrapy-splash`_ for seamless integration.
|
||||
|
||||
Splash returns as HTML the :ref:`DOM <topics-livedom>` of a webpage, so that
|
||||
you can parse it with :ref:`selectors <topics-selectors>`. It provides great
|
||||
flexibility through configuration_ or scripting_.
|
||||
|
||||
If you need something beyond what Splash offers, such as interacting with the
|
||||
DOM on-the-fly from Python code instead of using a previously-written script,
|
||||
or handling multiple web browser windows, you might need to
|
||||
:ref:`use a headless browser <topics-headless-browsing>` instead.
|
||||
|
||||
.. _configuration: https://splash.readthedocs.io/en/stable/api.html
|
||||
.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
|
||||
|
||||
.. _topics-headless-browsing:
|
||||
|
||||
Using a headless browser
|
||||
========================
|
||||
|
||||
A `headless browser`_ is a special web browser that provides an API for
|
||||
A headless browser is a special web browser that provides an API for
|
||||
automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
|
||||
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
|
||||
|
||||
|
|
@ -287,7 +268,6 @@ We recommend using `scrapy-playwright`_ for a better integration.
|
|||
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
|
||||
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
|
||||
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
|
||||
.. _Splash: https://github.com/scrapinghub/splash
|
||||
.. _chompjs: https://github.com/Nykakin/chompjs
|
||||
.. _curl: https://curl.se/
|
||||
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
|
||||
|
|
@ -297,7 +277,6 @@ We recommend using `scrapy-playwright`_ for a better integration.
|
|||
.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
|
||||
.. _pytesseract: https://github.com/madmaze/pytesseract
|
||||
.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
.. _tabula-py: https://github.com/chezou/tabula-py
|
||||
.. _wget: https://www.gnu.org/software/wget/
|
||||
.. _wgrep: https://github.com/stav/wgrep
|
||||
|
|
|
|||
|
|
@ -1,750 +0,0 @@
|
|||
"""
|
||||
These tests are kept as references from the ones that were ported to a itemloaders library.
|
||||
Once we remove the references from scrapy, we can remove these tests.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
from itemloaders.processors import (
|
||||
Compose,
|
||||
Identity,
|
||||
Join,
|
||||
MapCompose,
|
||||
SelectJmes,
|
||||
TakeFirst,
|
||||
)
|
||||
|
||||
from scrapy.item import Field, Item
|
||||
from scrapy.loader import ItemLoader
|
||||
|
||||
|
||||
# test items
|
||||
class NameItem(Item):
|
||||
name = Field()
|
||||
|
||||
|
||||
class SummaryItem(NameItem):
|
||||
url = Field()
|
||||
summary = Field()
|
||||
|
||||
|
||||
# test item loaders
|
||||
class NameItemLoader(ItemLoader):
|
||||
default_item_class = SummaryItem
|
||||
|
||||
|
||||
class ProcessorItemLoader(NameItemLoader):
|
||||
name_in = MapCompose(lambda v: v.title())
|
||||
|
||||
|
||||
class DefaultedItemLoader(NameItemLoader):
|
||||
default_input_processor = MapCompose(lambda v: v[:-1])
|
||||
|
||||
|
||||
# test processors
|
||||
def processor_with_args(value, other=None, loader_context=None):
|
||||
if "key" in loader_context:
|
||||
return loader_context["key"]
|
||||
return value
|
||||
|
||||
|
||||
class BasicItemLoaderTest(unittest.TestCase):
|
||||
def test_load_item_using_default_loader(self):
|
||||
i = SummaryItem()
|
||||
i["summary"] = "lala"
|
||||
il = ItemLoader(item=i)
|
||||
il.add_value("name", "marta")
|
||||
item = il.load_item()
|
||||
assert item is i
|
||||
self.assertEqual(item["summary"], ["lala"])
|
||||
self.assertEqual(item["name"], ["marta"])
|
||||
|
||||
def test_load_item_using_custom_loader(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
item = il.load_item()
|
||||
self.assertEqual(item["name"], ["Marta"])
|
||||
|
||||
def test_load_item_ignore_none_field_values(self):
|
||||
def validate_sku(value):
|
||||
# Let's assume a SKU is only digits.
|
||||
return value if value.isdigit() else None
|
||||
|
||||
class MyLoader(ItemLoader):
|
||||
name_out = Compose(lambda vs: vs[0]) # take first which allows empty values
|
||||
price_out = Compose(TakeFirst(), float)
|
||||
sku_out = Compose(TakeFirst(), validate_sku)
|
||||
|
||||
valid_fragment = "SKU: 1234"
|
||||
invalid_fragment = "SKU: not available"
|
||||
sku_re = "SKU: (.+)"
|
||||
|
||||
il = MyLoader(item={})
|
||||
# Should not return "sku: None".
|
||||
il.add_value("sku", [invalid_fragment], re=sku_re)
|
||||
# Should not ignore empty values.
|
||||
il.add_value("name", "")
|
||||
il.add_value("price", ["0"])
|
||||
self.assertEqual(
|
||||
il.load_item(),
|
||||
{
|
||||
"name": "",
|
||||
"price": 0.0,
|
||||
},
|
||||
)
|
||||
|
||||
il.replace_value("sku", [valid_fragment], re=sku_re)
|
||||
self.assertEqual(il.load_item()["sku"], "1234")
|
||||
|
||||
def test_self_referencing_loader(self):
|
||||
class MyLoader(ItemLoader):
|
||||
url_out = TakeFirst()
|
||||
|
||||
def img_url_out(self, values):
|
||||
return (self.get_output_value("url") or "") + values[0]
|
||||
|
||||
il = MyLoader(item={})
|
||||
il.add_value("url", "http://example.com/")
|
||||
il.add_value("img_url", "1234.png")
|
||||
self.assertEqual(
|
||||
il.load_item(),
|
||||
{
|
||||
"url": "http://example.com/",
|
||||
"img_url": "http://example.com/1234.png",
|
||||
},
|
||||
)
|
||||
|
||||
il = MyLoader(item={})
|
||||
il.add_value("img_url", "1234.png")
|
||||
self.assertEqual(
|
||||
il.load_item(),
|
||||
{
|
||||
"img_url": "1234.png",
|
||||
},
|
||||
)
|
||||
|
||||
def test_add_value(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_collected_values("name"), ["Marta"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Marta"])
|
||||
il.add_value("name", "pepe")
|
||||
self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"])
|
||||
|
||||
# test add object value
|
||||
il.add_value("summary", {"key": 1})
|
||||
self.assertEqual(il.get_collected_values("summary"), [{"key": 1}])
|
||||
|
||||
il.add_value(None, "Jim", lambda x: {"name": x})
|
||||
self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"])
|
||||
|
||||
def test_add_zero(self):
|
||||
il = NameItemLoader()
|
||||
il.add_value("name", 0)
|
||||
self.assertEqual(il.get_collected_values("name"), [0])
|
||||
|
||||
def test_replace_value(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.replace_value("name", "marta")
|
||||
self.assertEqual(il.get_collected_values("name"), ["Marta"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Marta"])
|
||||
il.replace_value("name", "pepe")
|
||||
self.assertEqual(il.get_collected_values("name"), ["Pepe"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Pepe"])
|
||||
|
||||
il.replace_value(None, "Jim", lambda x: {"name": x})
|
||||
self.assertEqual(il.get_collected_values("name"), ["Jim"])
|
||||
|
||||
def test_get_value(self):
|
||||
il = NameItemLoader()
|
||||
self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper))
|
||||
self.assertEqual(
|
||||
["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$")
|
||||
)
|
||||
self.assertEqual(
|
||||
"foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$")
|
||||
)
|
||||
|
||||
il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$")
|
||||
self.assertEqual(["foo"], il.get_collected_values("name"))
|
||||
il.replace_value("name", "name:bar", re="name:(.*)$")
|
||||
self.assertEqual(["bar"], il.get_collected_values("name"))
|
||||
|
||||
def test_iter_on_input_processor_input(self):
|
||||
class NameFirstItemLoader(NameItemLoader):
|
||||
name_in = TakeFirst()
|
||||
|
||||
il = NameFirstItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_collected_values("name"), ["marta"])
|
||||
il = NameFirstItemLoader()
|
||||
il.add_value("name", ["marta", "jose"])
|
||||
self.assertEqual(il.get_collected_values("name"), ["marta"])
|
||||
|
||||
il = NameFirstItemLoader()
|
||||
il.replace_value("name", "marta")
|
||||
self.assertEqual(il.get_collected_values("name"), ["marta"])
|
||||
il = NameFirstItemLoader()
|
||||
il.replace_value("name", ["marta", "jose"])
|
||||
self.assertEqual(il.get_collected_values("name"), ["marta"])
|
||||
|
||||
il = NameFirstItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
il.add_value("name", ["jose", "pedro"])
|
||||
self.assertEqual(il.get_collected_values("name"), ["marta", "jose"])
|
||||
|
||||
def test_map_compose_filter(self):
|
||||
def filter_world(x):
|
||||
return None if x == "world" else x
|
||||
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
self.assertEqual(
|
||||
proc(["hello", "world", "this", "is", "scrapy"]),
|
||||
["HELLO", "THIS", "IS", "SCRAPY"],
|
||||
)
|
||||
|
||||
def test_map_compose_filter_multil(self):
|
||||
class TestItemLoader(NameItemLoader):
|
||||
name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["Mart"])
|
||||
item = il.load_item()
|
||||
self.assertEqual(item["name"], ["Mart"])
|
||||
|
||||
def test_default_input_processor(self):
|
||||
il = DefaultedItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["mart"])
|
||||
|
||||
def test_inherited_default_input_processor(self):
|
||||
class InheritDefaultedItemLoader(DefaultedItemLoader):
|
||||
pass
|
||||
|
||||
il = InheritDefaultedItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["mart"])
|
||||
|
||||
def test_input_processor_inheritance(self):
|
||||
class ChildItemLoader(ProcessorItemLoader):
|
||||
url_in = MapCompose(lambda v: v.lower())
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value("url", "HTTP://scrapy.ORG")
|
||||
self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"])
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["Marta"])
|
||||
|
||||
class ChildChildItemLoader(ChildItemLoader):
|
||||
url_in = MapCompose(lambda v: v.upper())
|
||||
summary_in = MapCompose(lambda v: v)
|
||||
|
||||
il = ChildChildItemLoader()
|
||||
il.add_value("url", "http://scrapy.org")
|
||||
self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"])
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["Marta"])
|
||||
|
||||
def test_empty_map_compose(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_in = MapCompose()
|
||||
|
||||
il = IdentityDefaultedItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["marta"])
|
||||
|
||||
def test_identity_input_processor(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_in = Identity()
|
||||
|
||||
il = IdentityDefaultedItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["marta"])
|
||||
|
||||
def test_extend_custom_input_processors(self):
|
||||
class ChildItemLoader(ProcessorItemLoader):
|
||||
name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase)
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["mARTA"])
|
||||
|
||||
def test_extend_default_input_processors(self):
|
||||
class ChildDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_in = MapCompose(
|
||||
DefaultedItemLoader.default_input_processor, str.swapcase
|
||||
)
|
||||
|
||||
il = ChildDefaultedItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
self.assertEqual(il.get_output_value("name"), ["MART"])
|
||||
|
||||
def test_output_processor_using_function(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
|
||||
|
||||
class TakeFirstItemLoader(ProcessorItemLoader):
|
||||
name_out = " ".join
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), "Mar Ta")
|
||||
|
||||
def test_output_processor_error(self):
|
||||
class TestItemLoader(ItemLoader):
|
||||
default_item_class = SummaryItem
|
||||
name_out = MapCompose(float)
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value("name", ["$10"])
|
||||
try:
|
||||
float("$10")
|
||||
except Exception as e:
|
||||
expected_exc_str = str(e)
|
||||
|
||||
exc = None
|
||||
try:
|
||||
il.load_item()
|
||||
except Exception as e:
|
||||
exc = e
|
||||
assert isinstance(exc, ValueError)
|
||||
s = str(exc)
|
||||
assert "name" in s, s
|
||||
assert "$10" in s, s
|
||||
assert "ValueError" in s, s
|
||||
assert expected_exc_str in s, s
|
||||
|
||||
def test_output_processor_using_classes(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
|
||||
|
||||
class TakeFirstItemLoader(ProcessorItemLoader):
|
||||
name_out = Join()
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), "Mar Ta")
|
||||
|
||||
class TakeFirstItemLoader2(ProcessorItemLoader):
|
||||
name_out = Join("<br>")
|
||||
|
||||
il = TakeFirstItemLoader2()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")
|
||||
|
||||
def test_default_output_processor(self):
|
||||
il = ProcessorItemLoader()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
|
||||
|
||||
class LalaItemLoader(ProcessorItemLoader):
|
||||
default_output_processor = Identity()
|
||||
|
||||
il = LalaItemLoader()
|
||||
il.add_value("name", ["mar", "ta"])
|
||||
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
|
||||
|
||||
def test_loader_context_on_declaration(self):
|
||||
class ChildItemLoader(ProcessorItemLoader):
|
||||
url_in = MapCompose(processor_with_args, key="val")
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value("url", "text")
|
||||
self.assertEqual(il.get_output_value("url"), ["val"])
|
||||
il.replace_value("url", "text2")
|
||||
self.assertEqual(il.get_output_value("url"), ["val"])
|
||||
|
||||
def test_loader_context_on_instantiation(self):
|
||||
class ChildItemLoader(ProcessorItemLoader):
|
||||
url_in = MapCompose(processor_with_args)
|
||||
|
||||
il = ChildItemLoader(key="val")
|
||||
il.add_value("url", "text")
|
||||
self.assertEqual(il.get_output_value("url"), ["val"])
|
||||
il.replace_value("url", "text2")
|
||||
self.assertEqual(il.get_output_value("url"), ["val"])
|
||||
|
||||
def test_loader_context_on_assign(self):
|
||||
class ChildItemLoader(ProcessorItemLoader):
|
||||
url_in = MapCompose(processor_with_args)
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.context["key"] = "val"
|
||||
il.add_value("url", "text")
|
||||
self.assertEqual(il.get_output_value("url"), ["val"])
|
||||
il.replace_value("url", "text2")
|
||||
self.assertEqual(il.get_output_value("url"), ["val"])
|
||||
|
||||
def test_item_passed_to_input_processor_functions(self):
|
||||
def processor(value, loader_context):
|
||||
return loader_context["item"]["name"]
|
||||
|
||||
class ChildItemLoader(ProcessorItemLoader):
|
||||
url_in = MapCompose(processor)
|
||||
|
||||
it = SummaryItem(name="marta")
|
||||
il = ChildItemLoader(item=it)
|
||||
il.add_value("url", "text")
|
||||
self.assertEqual(il.get_output_value("url"), ["marta"])
|
||||
il.replace_value("url", "text2")
|
||||
self.assertEqual(il.get_output_value("url"), ["marta"])
|
||||
|
||||
def test_compose_processor(self):
|
||||
class TestItemLoader(NameItemLoader):
|
||||
name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value("name", ["marta", "other"])
|
||||
self.assertEqual(il.get_output_value("name"), "Mart")
|
||||
item = il.load_item()
|
||||
self.assertEqual(item["name"], "Mart")
|
||||
|
||||
def test_partial_processor(self):
|
||||
def join(values, sep=None, loader_context=None, ignored=None):
|
||||
if sep is not None:
|
||||
return sep.join(values)
|
||||
if loader_context and "sep" in loader_context:
|
||||
return loader_context["sep"].join(values)
|
||||
return "".join(values)
|
||||
|
||||
class TestItemLoader(NameItemLoader):
|
||||
name_out = Compose(partial(join, sep="+"))
|
||||
url_out = Compose(partial(join, loader_context={"sep": "."}))
|
||||
summary_out = Compose(partial(join, ignored="foo"))
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value("name", ["rabbit", "hole"])
|
||||
il.add_value("url", ["rabbit", "hole"])
|
||||
il.add_value("summary", ["rabbit", "hole"])
|
||||
item = il.load_item()
|
||||
self.assertEqual(item["name"], "rabbit+hole")
|
||||
self.assertEqual(item["url"], "rabbit.hole")
|
||||
self.assertEqual(item["summary"], "rabbithole")
|
||||
|
||||
def test_error_input_processor(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
||||
class TestItemLoader(ItemLoader):
|
||||
default_item_class = TestItem
|
||||
name_in = MapCompose(float)
|
||||
|
||||
il = TestItemLoader()
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error with input processor MapCompose: .* "
|
||||
"error='ValueError: Error in MapCompose .* "
|
||||
"error='ValueError: could not convert",
|
||||
):
|
||||
il.add_value("name", ["marta", "other"])
|
||||
|
||||
def test_error_output_processor(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
||||
class TestItemLoader(ItemLoader):
|
||||
default_item_class = TestItem
|
||||
name_out = Compose(Join(), float)
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value("name", "marta")
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error with output processor: .* "
|
||||
"error='ValueError: Error in Compose .* "
|
||||
"error='ValueError: could not convert",
|
||||
):
|
||||
il.load_item()
|
||||
|
||||
def test_error_processor_as_argument(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
||||
class TestItemLoader(ItemLoader):
|
||||
default_item_class = TestItem
|
||||
|
||||
il = TestItemLoader()
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=r"Error with processor Compose .* "
|
||||
r"error='ValueError: Error in Compose .* "
|
||||
r"error='TypeError: float\(\) argument",
|
||||
):
|
||||
il.add_value("name", ["marta", "other"], Compose(float))
|
||||
|
||||
|
||||
class InitializationFromDictTest(unittest.TestCase):
|
||||
item_class = dict
|
||||
|
||||
def test_keep_single_value(self):
|
||||
"""Loaded item should contain values from the initial item"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {"name": ["foo"]})
|
||||
|
||||
def test_keep_list(self):
|
||||
"""Loaded item should contain values from the initial item"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]})
|
||||
|
||||
def test_add_value_singlevalue_singlevalue(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", "bar")
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]})
|
||||
|
||||
def test_add_value_singlevalue_list(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", ["item", "loader"])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]})
|
||||
|
||||
def test_add_value_list_singlevalue(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", "qwerty")
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]})
|
||||
|
||||
def test_add_value_list_list(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value("name", ["item", "loader"])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]})
|
||||
|
||||
def test_get_output_value_singlevalue(self):
|
||||
"""Getting output value must not remove value from item"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il.get_output_value("name"), ["foo"])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(loaded_item, {"name": ["foo"]})
|
||||
|
||||
def test_get_output_value_list(self):
|
||||
"""Getting output value must not remove value from item"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(loaded_item, {"name": ["foo", "bar"]})
|
||||
|
||||
def test_values_single(self):
|
||||
"""Values from initial item must be added to loader._values"""
|
||||
input_item = self.item_class(name="foo")
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il._values.get("name"), ["foo"])
|
||||
|
||||
def test_values_list(self):
|
||||
"""Values from initial item must be added to loader._values"""
|
||||
input_item = self.item_class(name=["foo", "bar"])
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il._values.get("name"), ["foo", "bar"])
|
||||
|
||||
|
||||
class BaseNoInputReprocessingLoader(ItemLoader):
|
||||
title_in = MapCompose(str.upper)
|
||||
title_out = TakeFirst()
|
||||
|
||||
|
||||
class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
|
||||
default_item_class = dict
|
||||
|
||||
|
||||
class NoInputReprocessingFromDictTest(unittest.TestCase):
|
||||
"""
|
||||
Loaders initialized from loaded items must not reprocess fields (dict instances)
|
||||
"""
|
||||
|
||||
def test_avoid_reprocessing_with_initial_values_single(self):
|
||||
il = NoInputReprocessingDictLoader(item={"title": "foo"})
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "foo"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"}
|
||||
)
|
||||
|
||||
def test_avoid_reprocessing_with_initial_values_list(self):
|
||||
il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]})
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "foo"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"}
|
||||
)
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_single(self):
|
||||
il = NoInputReprocessingDictLoader()
|
||||
il.add_value("title", "foo")
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "FOO"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"}
|
||||
)
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_list(self):
|
||||
il = NoInputReprocessingDictLoader()
|
||||
il.add_value("title", ["foo", "bar"])
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {"title": "FOO"})
|
||||
self.assertEqual(
|
||||
NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"}
|
||||
)
|
||||
|
||||
|
||||
class TestOutputProcessorDict(unittest.TestCase):
|
||||
def test_output_processor(self):
|
||||
class TempDict(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(self, *args, **kwargs)
|
||||
self.setdefault("temp", 0.3)
|
||||
|
||||
class TempLoader(ItemLoader):
|
||||
default_item_class = TempDict
|
||||
default_input_processor = Identity()
|
||||
default_output_processor = Compose(TakeFirst())
|
||||
|
||||
loader = TempLoader()
|
||||
item = loader.load_item()
|
||||
self.assertIsInstance(item, TempDict)
|
||||
self.assertEqual(dict(item), {"temp": 0.3})
|
||||
|
||||
|
||||
class ProcessorsTest(unittest.TestCase):
|
||||
def test_take_first(self):
|
||||
proc = TakeFirst()
|
||||
self.assertEqual(proc([None, "", "hello", "world"]), "hello")
|
||||
self.assertEqual(proc([None, "", 0, "hello", "world"]), 0)
|
||||
|
||||
def test_identity(self):
|
||||
proc = Identity()
|
||||
self.assertEqual(
|
||||
proc([None, "", "hello", "world"]), [None, "", "hello", "world"]
|
||||
)
|
||||
|
||||
def test_join(self):
|
||||
proc = Join()
|
||||
with pytest.raises(TypeError):
|
||||
proc([None, "", "hello", "world"])
|
||||
self.assertEqual(proc(["", "hello", "world"]), " hello world")
|
||||
self.assertEqual(proc(["hello", "world"]), "hello world")
|
||||
self.assertIsInstance(proc(["hello", "world"]), str)
|
||||
|
||||
def test_compose(self):
|
||||
proc = Compose(lambda v: v[0], str.upper)
|
||||
self.assertEqual(proc(["hello", "world"]), "HELLO")
|
||||
proc = Compose(str.upper)
|
||||
self.assertEqual(proc(None), None)
|
||||
proc = Compose(str.upper, stop_on_none=False)
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
|
||||
):
|
||||
proc(None)
|
||||
proc = Compose(str.upper, lambda x: x + 1)
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error in Compose with .* error='TypeError: (can only|unsupported operand)",
|
||||
):
|
||||
proc("hello")
|
||||
|
||||
def test_mapcompose(self):
|
||||
def filter_world(x):
|
||||
return None if x == "world" else x
|
||||
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
self.assertEqual(
|
||||
proc(["hello", "world", "this", "is", "scrapy"]),
|
||||
["HELLO", "THIS", "IS", "SCRAPY"],
|
||||
)
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
self.assertEqual(proc(None), [])
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
|
||||
):
|
||||
proc([1])
|
||||
proc = MapCompose(filter_world, lambda x: x + 1)
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)",
|
||||
):
|
||||
proc("hello")
|
||||
|
||||
|
||||
class SelectJmesTestCase(unittest.TestCase):
|
||||
test_list_equals = {
|
||||
"simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"),
|
||||
"invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None),
|
||||
"top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}),
|
||||
"double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"),
|
||||
"dict": (
|
||||
"foo.bar[*].name",
|
||||
{"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
|
||||
["one", "two"],
|
||||
),
|
||||
"list": ("[1]", [1, 2], 2),
|
||||
}
|
||||
|
||||
def test_output(self):
|
||||
for k, v in self.test_list_equals.items():
|
||||
expr, test_list, expected = v
|
||||
test = SelectJmes(expr)(test_list)
|
||||
self.assertEqual(
|
||||
test, expected, msg=f'test "{k}" got {test} expected {expected}'
|
||||
)
|
||||
|
||||
|
||||
# Functions as processors
|
||||
|
||||
|
||||
def function_processor_strip(iterable):
|
||||
return [x.strip() for x in iterable]
|
||||
|
||||
|
||||
def function_processor_upper(iterable):
|
||||
return [x.upper() for x in iterable]
|
||||
|
||||
|
||||
class FunctionProcessorItem(Item):
|
||||
foo = Field(
|
||||
input_processor=function_processor_strip,
|
||||
output_processor=function_processor_upper,
|
||||
)
|
||||
|
||||
|
||||
class FunctionProcessorDictLoader(ItemLoader):
|
||||
default_item_class = dict
|
||||
foo_in = function_processor_strip
|
||||
foo_out = function_processor_upper
|
||||
|
||||
|
||||
class FunctionProcessorTestCase(unittest.TestCase):
|
||||
def test_processor_defined_in_item_loader(self):
|
||||
lo = FunctionProcessorDictLoader()
|
||||
lo.add_value("foo", " bar ")
|
||||
lo.add_value("foo", [" asdf ", " qwerty "])
|
||||
self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
|
||||
|
|
@ -30,7 +30,7 @@ from scrapy.utils.test import get_crawler
|
|||
from tests import get_testdata, tests_datadir
|
||||
|
||||
|
||||
class SpiderTest(unittest.TestCase):
|
||||
class TestSpider(unittest.TestCase):
|
||||
spider_class = Spider
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -41,13 +41,13 @@ class SpiderTest(unittest.TestCase):
|
|||
|
||||
def test_base_spider(self):
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(spider.name, "example.com")
|
||||
self.assertEqual(spider.start_urls, [])
|
||||
assert spider.name == "example.com"
|
||||
assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison
|
||||
|
||||
def test_spider_args(self):
|
||||
"""``__init__`` method arguments are assigned to spider attributes"""
|
||||
spider = self.spider_class("example.com", foo="bar")
|
||||
self.assertEqual(spider.foo, "bar")
|
||||
assert spider.foo == "bar"
|
||||
|
||||
def test_spider_without_name(self):
|
||||
"""``__init__`` method arguments are assigned to spider attributes"""
|
||||
|
|
@ -60,10 +60,10 @@ class SpiderTest(unittest.TestCase):
|
|||
def test_from_crawler_crawler_and_settings_population(self):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertTrue(hasattr(spider, "crawler"))
|
||||
self.assertIs(spider.crawler, crawler)
|
||||
self.assertTrue(hasattr(spider, "settings"))
|
||||
self.assertIs(spider.settings, crawler.settings)
|
||||
assert hasattr(spider, "crawler")
|
||||
assert spider.crawler is crawler
|
||||
assert hasattr(spider, "settings")
|
||||
assert spider.settings is crawler.settings
|
||||
|
||||
def test_from_crawler_init_call(self):
|
||||
with mock.patch.object(
|
||||
|
|
@ -85,7 +85,7 @@ class SpiderTest(unittest.TestCase):
|
|||
crawler.signals.send_catch_log(
|
||||
signal=signals.spider_closed, spider=spider, reason=None
|
||||
)
|
||||
self.assertTrue(spider.closed_called)
|
||||
assert spider.closed_called
|
||||
|
||||
def test_update_settings(self):
|
||||
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
|
||||
|
|
@ -94,9 +94,9 @@ class SpiderTest(unittest.TestCase):
|
|||
settings = Settings(project_settings, priority="project")
|
||||
|
||||
self.spider_class.update_settings(settings)
|
||||
self.assertEqual(settings.get("TEST1"), "spider")
|
||||
self.assertEqual(settings.get("TEST2"), "spider")
|
||||
self.assertEqual(settings.get("TEST3"), "project")
|
||||
assert settings.get("TEST1") == "spider"
|
||||
assert settings.get("TEST2") == "spider"
|
||||
assert settings.get("TEST3") == "project"
|
||||
|
||||
@inlineCallbacks
|
||||
def test_settings_in_from_crawler(self):
|
||||
|
|
@ -114,11 +114,11 @@ class SpiderTest(unittest.TestCase):
|
|||
return spider
|
||||
|
||||
crawler = Crawler(TestSpider, project_settings)
|
||||
self.assertEqual(crawler.settings.get("TEST1"), "spider")
|
||||
self.assertEqual(crawler.settings.get("TEST2"), "spider")
|
||||
self.assertEqual(crawler.settings.get("TEST3"), "project")
|
||||
assert crawler.settings.get("TEST1") == "spider"
|
||||
assert crawler.settings.get("TEST2") == "spider"
|
||||
assert crawler.settings.get("TEST3") == "project"
|
||||
yield crawler.crawl()
|
||||
self.assertEqual(crawler.settings.get("TEST1"), "spider_instance")
|
||||
assert crawler.settings.get("TEST1") == "spider_instance"
|
||||
|
||||
def test_logger(self):
|
||||
spider = self.spider_class("example.com")
|
||||
|
|
@ -127,8 +127,8 @@ class SpiderTest(unittest.TestCase):
|
|||
lc.check(("example.com", "INFO", "test log msg"))
|
||||
|
||||
record = lc.records[0]
|
||||
self.assertIn("spider", record.__dict__)
|
||||
self.assertIs(record.spider, spider)
|
||||
assert "spider" in record.__dict__
|
||||
assert record.spider is spider
|
||||
|
||||
def test_log(self):
|
||||
spider = self.spider_class("example.com")
|
||||
|
|
@ -137,11 +137,11 @@ class SpiderTest(unittest.TestCase):
|
|||
mock_logger.log.assert_called_once_with("INFO", "test log msg")
|
||||
|
||||
|
||||
class InitSpiderTest(SpiderTest):
|
||||
class TestInitSpider(TestSpider):
|
||||
spider_class = InitSpider
|
||||
|
||||
|
||||
class XMLFeedSpiderTest(SpiderTest):
|
||||
class TestXMLFeedSpider(TestSpider):
|
||||
spider_class = XMLFeedSpider
|
||||
|
||||
def test_register_namespace(self):
|
||||
|
|
@ -173,28 +173,24 @@ class XMLFeedSpiderTest(SpiderTest):
|
|||
for iterator in ("iternodes", "xml"):
|
||||
spider = _XMLSpider("example", iterator=iterator)
|
||||
output = list(spider._parse(response))
|
||||
self.assertEqual(len(output), 2, iterator)
|
||||
self.assertEqual(
|
||||
output,
|
||||
[
|
||||
{
|
||||
"loc": ["http://www.example.com/Special-Offers.html"],
|
||||
"updated": ["2009-08-16"],
|
||||
"custom": ["fuu"],
|
||||
"other": ["bar"],
|
||||
},
|
||||
{
|
||||
"loc": [],
|
||||
"updated": ["2009-08-16"],
|
||||
"other": ["foo"],
|
||||
"custom": [],
|
||||
},
|
||||
],
|
||||
iterator,
|
||||
)
|
||||
assert len(output) == 2, iterator
|
||||
assert output == [
|
||||
{
|
||||
"loc": ["http://www.example.com/Special-Offers.html"],
|
||||
"updated": ["2009-08-16"],
|
||||
"custom": ["fuu"],
|
||||
"other": ["bar"],
|
||||
},
|
||||
{
|
||||
"loc": [],
|
||||
"updated": ["2009-08-16"],
|
||||
"other": ["foo"],
|
||||
"custom": [],
|
||||
},
|
||||
], iterator
|
||||
|
||||
|
||||
class CSVFeedSpiderTest(SpiderTest):
|
||||
class TestCSVFeedSpider(TestSpider):
|
||||
spider_class = CSVFeedSpider
|
||||
|
||||
def test_parse_rows(self):
|
||||
|
|
@ -215,7 +211,7 @@ class CSVFeedSpiderTest(SpiderTest):
|
|||
assert len(rows) == 4
|
||||
|
||||
|
||||
class CrawlSpiderTest(SpiderTest):
|
||||
class TestCrawlSpider(TestSpider):
|
||||
test_body = b"""<html><head><title>Page title<title>
|
||||
<body>
|
||||
<p><a href="item/12.html">Item 12</a></p>
|
||||
|
|
@ -240,16 +236,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_links(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -266,16 +259,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_links_filter(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -295,15 +285,12 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 2)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 2
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
]
|
||||
|
||||
def test_process_links_generator(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -320,16 +307,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_request(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -348,16 +332,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.com/somepage/item/12.html",
|
||||
"http://example.com/about.html",
|
||||
"http://example.com/nofollow.html",
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.com/somepage/item/12.html",
|
||||
"http://example.com/about.html",
|
||||
"http://example.com/nofollow.html",
|
||||
]
|
||||
|
||||
def test_process_request_with_response(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -379,20 +360,18 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[r.meta["response_class"] for r in output],
|
||||
["HtmlResponse", "HtmlResponse", "HtmlResponse"],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
assert [r.meta["response_class"] for r in output] == [
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
]
|
||||
|
||||
def test_process_request_instance_method(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -409,16 +388,13 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
|
||||
],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
|
||||
safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
|
||||
]
|
||||
|
||||
def test_process_request_instance_method_with_response(self):
|
||||
response = HtmlResponse(
|
||||
|
|
@ -441,32 +417,30 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(isinstance(r, Request) for r in output))
|
||||
self.assertEqual(
|
||||
[r.url for r in output],
|
||||
[
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[r.meta["response_class"] for r in output],
|
||||
["HtmlResponse", "HtmlResponse", "HtmlResponse"],
|
||||
)
|
||||
assert len(output) == 3
|
||||
assert all(isinstance(r, Request) for r in output)
|
||||
assert [r.url for r in output] == [
|
||||
"http://example.org/somepage/item/12.html",
|
||||
"http://example.org/about.html",
|
||||
"http://example.org/nofollow.html",
|
||||
]
|
||||
assert [r.meta["response_class"] for r in output] == [
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
"HtmlResponse",
|
||||
]
|
||||
|
||||
def test_follow_links_attribute_population(self):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertTrue(hasattr(spider, "_follow_links"))
|
||||
self.assertTrue(spider._follow_links)
|
||||
assert hasattr(spider, "_follow_links")
|
||||
assert spider._follow_links
|
||||
|
||||
settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertTrue(hasattr(spider, "_follow_links"))
|
||||
self.assertFalse(spider._follow_links)
|
||||
assert hasattr(spider, "_follow_links")
|
||||
assert not spider._follow_links
|
||||
|
||||
@inlineCallbacks
|
||||
def test_start_url(self):
|
||||
|
|
@ -486,7 +460,7 @@ class CrawlSpiderTest(SpiderTest):
|
|||
)
|
||||
|
||||
|
||||
class SitemapSpiderTest(SpiderTest):
|
||||
class TestSitemapSpider(TestSpider):
|
||||
spider_class = SitemapSpider
|
||||
|
||||
BODY = b"SITEMAP"
|
||||
|
|
@ -499,7 +473,7 @@ class SitemapSpiderTest(SpiderTest):
|
|||
def assertSitemapBody(self, response, body):
|
||||
crawler = get_crawler()
|
||||
spider = self.spider_class.from_crawler(crawler, "example.com")
|
||||
self.assertEqual(spider._get_sitemap_body(response), body)
|
||||
assert spider._get_sitemap_body(response) == body
|
||||
|
||||
def test_get_sitemap_body(self):
|
||||
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
|
||||
|
|
@ -546,15 +520,12 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://example.com/sitemap.xml",
|
||||
"http://example.com/sitemap-product-index.xml",
|
||||
"http://example.com/sitemap-uppercase.xml",
|
||||
"http://www.example.com/sitemap-relative-url.xml",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://example.com/sitemap.xml",
|
||||
"http://example.com/sitemap-product-index.xml",
|
||||
"http://example.com/sitemap-uppercase.xml",
|
||||
"http://www.example.com/sitemap-relative-url.xml",
|
||||
]
|
||||
|
||||
def test_alternate_url_locs(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -573,21 +544,17 @@ Sitemap: /sitemap-relative-url.xml
|
|||
</urlset>"""
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/english/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/"
|
||||
]
|
||||
|
||||
spider.sitemap_alternate_links = True
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://www.example.com/english/",
|
||||
"http://www.example.com/deutsch/",
|
||||
"http://www.example.com/schweiz-deutsch/",
|
||||
"http://www.example.com/italiano/",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/",
|
||||
"http://www.example.com/deutsch/",
|
||||
"http://www.example.com/schweiz-deutsch/",
|
||||
"http://www.example.com/italiano/",
|
||||
]
|
||||
|
||||
def test_sitemap_filter(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -614,16 +581,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/english/", "http://www.example.com/portuguese/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/",
|
||||
"http://www.example.com/portuguese/",
|
||||
]
|
||||
|
||||
spider = FilteredSitemapSpider("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/english/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/"
|
||||
]
|
||||
|
||||
def test_sitemap_filter_with_alternate_links(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -652,19 +618,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://www.example.com/english/article_1/",
|
||||
"http://www.example.com/english/article_2/",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/english/article_1/",
|
||||
"http://www.example.com/english/article_2/",
|
||||
]
|
||||
|
||||
spider = FilteredSitemapSpider("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/deutsch/article_1/"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/deutsch/article_1/"
|
||||
]
|
||||
|
||||
def test_sitemapindex_filter(self):
|
||||
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -692,19 +654,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
[
|
||||
"http://www.example.com/sitemap1.xml",
|
||||
"http://www.example.com/sitemap2.xml",
|
||||
],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/sitemap1.xml",
|
||||
"http://www.example.com/sitemap2.xml",
|
||||
]
|
||||
|
||||
spider = FilteredSitemapSpider("example.com")
|
||||
self.assertEqual(
|
||||
[req.url for req in spider._parse_sitemap(r)],
|
||||
["http://www.example.com/sitemap2.xml"],
|
||||
)
|
||||
assert [req.url for req in spider._parse_sitemap(r)] == [
|
||||
"http://www.example.com/sitemap2.xml"
|
||||
]
|
||||
|
||||
def test_compression_bomb_setting(self):
|
||||
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
|
||||
|
|
@ -714,7 +672,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
body = body_path.read_bytes()
|
||||
request = Request(url="https://example.com")
|
||||
response = Response(url="https://example.com", body=body, request=request)
|
||||
self.assertIsNone(spider._get_sitemap_body(response))
|
||||
assert spider._get_sitemap_body(response) is None
|
||||
|
||||
def test_compression_bomb_spider_attr(self):
|
||||
class DownloadMaxSizeSpider(self.spider_class):
|
||||
|
|
@ -726,7 +684,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
body = body_path.read_bytes()
|
||||
request = Request(url="https://example.com")
|
||||
response = Response(url="https://example.com", body=body, request=request)
|
||||
self.assertIsNone(spider._get_sitemap_body(response))
|
||||
assert spider._get_sitemap_body(response) is None
|
||||
|
||||
def test_compression_bomb_request_meta(self):
|
||||
crawler = get_crawler()
|
||||
|
|
@ -737,7 +695,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
url="https://example.com", meta={"download_maxsize": 10_000_000}
|
||||
)
|
||||
response = Response(url="https://example.com", body=body, request=request)
|
||||
self.assertIsNone(spider._get_sitemap_body(response))
|
||||
assert spider._get_sitemap_body(response) is None
|
||||
|
||||
def test_download_warnsize_setting(self):
|
||||
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
|
||||
|
|
@ -817,13 +775,13 @@ Sitemap: /sitemap-relative-url.xml
|
|||
)
|
||||
|
||||
|
||||
class DeprecationTest(unittest.TestCase):
|
||||
class TestDeprecation:
|
||||
def test_crawl_spider(self):
|
||||
assert issubclass(CrawlSpider, Spider)
|
||||
assert isinstance(CrawlSpider(name="foo"), Spider)
|
||||
|
||||
|
||||
class NoParseMethodSpiderTest(unittest.TestCase):
|
||||
class TestNoParseMethodSpider:
|
||||
spider_class = Spider
|
||||
|
||||
def test_undefined_parse_method(self):
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from tempfile import mkdtemp
|
|||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from twisted.trial import unittest
|
||||
from zope.interface.verify import verifyObject
|
||||
|
||||
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
|
||||
|
|
@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path):
|
|||
shutil.copytree(source, target)
|
||||
|
||||
|
||||
class SpiderLoaderTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestSpiderLoader:
|
||||
def setup_method(self):
|
||||
orig_spiders_dir = module_dir / "test_spiders"
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
|
||||
|
|
@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
|
||||
self.spider_loader = SpiderLoader.from_settings(settings)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
del self.spider_loader
|
||||
del sys.modules["test_spiders_xxx"]
|
||||
sys.path.remove(str(self.tmpdir))
|
||||
|
|
@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
verifyObject(ISpiderLoader, self.spider_loader)
|
||||
|
||||
def test_list(self):
|
||||
self.assertEqual(
|
||||
set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}
|
||||
)
|
||||
assert set(self.spider_loader.list()) == {
|
||||
"spider1",
|
||||
"spider2",
|
||||
"spider3",
|
||||
"spider4",
|
||||
}
|
||||
|
||||
def test_load(self):
|
||||
spider1 = self.spider_loader.load("spider1")
|
||||
self.assertEqual(spider1.__name__, "Spider1")
|
||||
assert spider1.__name__ == "Spider1"
|
||||
|
||||
def test_find_by_request(self):
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),
|
||||
["spider1"],
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),
|
||||
["spider2"],
|
||||
)
|
||||
self.assertEqual(
|
||||
set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),
|
||||
{"spider1", "spider2"},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://spider3.com")), []
|
||||
)
|
||||
self.assertEqual(
|
||||
self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),
|
||||
["spider3"],
|
||||
assert self.spider_loader.find_by_request(
|
||||
Request("http://scrapy1.org/test")
|
||||
) == ["spider1"]
|
||||
assert self.spider_loader.find_by_request(
|
||||
Request("http://scrapy2.org/test")
|
||||
) == ["spider2"]
|
||||
assert set(
|
||||
self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))
|
||||
) == {"spider1", "spider2"}
|
||||
assert (
|
||||
self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))
|
||||
== []
|
||||
)
|
||||
assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []
|
||||
assert self.spider_loader.find_by_request(
|
||||
Request("http://spider3.com/onlythis")
|
||||
) == ["spider3"]
|
||||
|
||||
def test_load_spider_module(self):
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
|
|
@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}})
|
||||
|
||||
crawler = runner.create_crawler("spider_from_addon")
|
||||
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
|
||||
self.assertEqual(crawler.spidercls.name, "spider_from_addon")
|
||||
self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1)
|
||||
assert issubclass(crawler.spidercls, scrapy.Spider)
|
||||
assert crawler.spidercls.name == "spider_from_addon"
|
||||
assert len(crawler.settings["SPIDER_MODULES"]) == 1
|
||||
|
||||
def test_crawler_runner_loading(self):
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
|
|
@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
runner.create_crawler("spider2")
|
||||
|
||||
crawler = runner.create_crawler("spider1")
|
||||
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
|
||||
self.assertEqual(crawler.spidercls.name, "spider1")
|
||||
assert issubclass(crawler.spidercls, scrapy.Spider)
|
||||
assert crawler.spidercls.name == "spider1"
|
||||
|
||||
def test_bad_spider_modules_exception(self):
|
||||
module = "tests.test_spiderloader.test_spiders.doesnotexist"
|
||||
|
|
@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
# at least until all six versions we can import (including botocore.vendored.six)
|
||||
# are updated to 1.16.0+
|
||||
w.pop(0)
|
||||
self.assertIn("Could not load spiders from module", str(w[0].message))
|
||||
assert "Could not load spiders from module" in str(w[0].message)
|
||||
|
||||
spiders = spider_loader.list()
|
||||
self.assertEqual(spiders, [])
|
||||
assert not spiders
|
||||
|
||||
def test_syntax_error_exception(self):
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
|
|
@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
# at least until all six versions we can import (including botocore.vendored.six)
|
||||
# are updated to 1.16.0+
|
||||
w.pop(0)
|
||||
self.assertIn("Could not load spiders from module", str(w[0].message))
|
||||
assert "Could not load spiders from module" in str(w[0].message)
|
||||
|
||||
spiders = spider_loader.list()
|
||||
self.assertEqual(spiders, [])
|
||||
assert not spiders
|
||||
|
||||
|
||||
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestDuplicateSpiderNameLoader:
|
||||
def setup_method(self):
|
||||
orig_spiders_dir = module_dir / "test_spiders"
|
||||
self.tmpdir = Path(mkdtemp())
|
||||
self.spiders_dir = self.tmpdir / "test_spiders_xxx"
|
||||
|
|
@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
|||
sys.path.append(str(self.tmpdir))
|
||||
self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
del sys.modules["test_spiders_xxx"]
|
||||
sys.path.remove(str(self.tmpdir))
|
||||
|
||||
|
|
@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
|||
with warnings.catch_warnings(record=True) as w:
|
||||
spider_loader = SpiderLoader.from_settings(self.settings)
|
||||
|
||||
self.assertEqual(len(w), 1)
|
||||
assert len(w) == 1
|
||||
msg = str(w[0].message)
|
||||
self.assertIn("several spiders with the same name", msg)
|
||||
self.assertIn("'spider3'", msg)
|
||||
self.assertTrue(msg.count("'spider3'") == 2)
|
||||
assert "several spiders with the same name" in msg
|
||||
assert "'spider3'" in msg
|
||||
assert msg.count("'spider3'") == 2
|
||||
|
||||
self.assertNotIn("'spider1'", msg)
|
||||
self.assertNotIn("'spider2'", msg)
|
||||
self.assertNotIn("'spider4'", msg)
|
||||
assert "'spider1'" not in msg
|
||||
assert "'spider2'" not in msg
|
||||
assert "'spider4'" not in msg
|
||||
|
||||
spiders = set(spider_loader.list())
|
||||
self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
|
||||
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
|
||||
|
||||
def test_multiple_dupename_warning(self):
|
||||
# copy 2 spider modules so as to have duplicate spider name
|
||||
|
|
@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
|
|||
with warnings.catch_warnings(record=True) as w:
|
||||
spider_loader = SpiderLoader.from_settings(self.settings)
|
||||
|
||||
self.assertEqual(len(w), 1)
|
||||
assert len(w) == 1
|
||||
msg = str(w[0].message)
|
||||
self.assertIn("several spiders with the same name", msg)
|
||||
self.assertIn("'spider1'", msg)
|
||||
self.assertTrue(msg.count("'spider1'") == 2)
|
||||
assert "several spiders with the same name" in msg
|
||||
assert "'spider1'" in msg
|
||||
assert msg.count("'spider1'") == 2
|
||||
|
||||
self.assertIn("'spider2'", msg)
|
||||
self.assertTrue(msg.count("'spider2'") == 2)
|
||||
assert "'spider2'" in msg
|
||||
assert msg.count("'spider2'") == 2
|
||||
|
||||
self.assertNotIn("'spider3'", msg)
|
||||
self.assertNotIn("'spider4'", msg)
|
||||
assert "'spider3'" not in msg
|
||||
assert "'spider4'" not in msg
|
||||
|
||||
spiders = set(spider_loader.list())
|
||||
self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
|
||||
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class SpiderMiddlewareTestCase(TestCase):
|
||||
class TestSpiderMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
self.request = Request("http://example.com/index.html")
|
||||
self.response = Response(self.request.url, request=self.request)
|
||||
|
|
@ -43,7 +43,7 @@ class SpiderMiddlewareTestCase(TestCase):
|
|||
return results[0]
|
||||
|
||||
|
||||
class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
|
||||
"""Invalid return value for process_spider_input method"""
|
||||
|
||||
def test_invalid_process_spider_input(self):
|
||||
|
|
@ -53,11 +53,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
|
|||
|
||||
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, _InvalidOutput)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, _InvalidOutput)
|
||||
|
||||
|
||||
class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
|
||||
"""Invalid return value for process_spider_output method"""
|
||||
|
||||
def test_invalid_process_spider_output(self):
|
||||
|
|
@ -67,11 +67,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
|
|||
|
||||
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, _InvalidOutput)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, _InvalidOutput)
|
||||
|
||||
|
||||
class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
|
||||
"""Invalid return value for process_spider_exception method"""
|
||||
|
||||
def test_invalid_process_spider_exception(self):
|
||||
|
|
@ -86,11 +86,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
|
|||
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
|
||||
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, _InvalidOutput)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, _InvalidOutput)
|
||||
|
||||
|
||||
class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
|
||||
"""Re raise the exception by returning None"""
|
||||
|
||||
def test_process_spider_exception_return_none(self):
|
||||
|
|
@ -105,11 +105,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
|
|||
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
|
||||
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
|
||||
result = self._scrape_response()
|
||||
self.assertIsInstance(result, Failure)
|
||||
self.assertIsInstance(result.value, ZeroDivisionError)
|
||||
assert isinstance(result, Failure)
|
||||
assert isinstance(result.value, ZeroDivisionError)
|
||||
|
||||
|
||||
class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
||||
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
|
||||
"""Helpers for testing sync, async and mixed middlewares.
|
||||
|
||||
Should work for process_spider_output and, when it's supported, process_test_yield_seeds.
|
||||
|
|
@ -150,14 +150,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
|||
result = yield self._get_middleware_result(
|
||||
*mw_classes, start_index=start_index
|
||||
)
|
||||
self.assertIsInstance(result, Iterable)
|
||||
assert isinstance(result, Iterable)
|
||||
result_list = list(result)
|
||||
self.assertEqual(len(result_list), self.RESULT_COUNT)
|
||||
self.assertIsInstance(result_list[0], self.ITEM_TYPE)
|
||||
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
|
||||
self.assertEqual(
|
||||
"doesn't support asynchronous spider output" in str(log),
|
||||
ProcessSpiderOutputSimpleMiddleware in mw_classes,
|
||||
assert len(result_list) == self.RESULT_COUNT
|
||||
assert isinstance(result_list[0], self.ITEM_TYPE)
|
||||
assert ("downgraded to a non-async" in str(log)) == downgrade
|
||||
assert ("doesn't support asynchronous spider output" in str(log)) == (
|
||||
ProcessSpiderOutputSimpleMiddleware in mw_classes
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -168,11 +167,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
|||
result = yield self._get_middleware_result(
|
||||
*mw_classes, start_index=start_index
|
||||
)
|
||||
self.assertIsInstance(result, AsyncIterator)
|
||||
assert isinstance(result, AsyncIterator)
|
||||
result_list = yield deferred_from_coro(collect_asyncgen(result))
|
||||
self.assertEqual(len(result_list), self.RESULT_COUNT)
|
||||
self.assertIsInstance(result_list[0], self.ITEM_TYPE)
|
||||
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
|
||||
assert len(result_list) == self.RESULT_COUNT
|
||||
assert isinstance(result_list[0], self.ITEM_TYPE)
|
||||
assert ("downgraded to a non-async" in str(log)) == downgrade
|
||||
|
||||
|
||||
class ProcessSpiderOutputSimpleMiddleware:
|
||||
|
|
@ -214,7 +213,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware:
|
|||
yield {"foo": 3}
|
||||
|
||||
|
||||
class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
|
||||
"""process_spider_output tests for simple callbacks"""
|
||||
|
||||
ITEM_TYPE = dict
|
||||
|
|
@ -259,7 +258,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
|
|||
return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
|
||||
|
||||
|
||||
class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
|
||||
class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
|
||||
"""process_spider_output tests for async generator callbacks"""
|
||||
|
||||
async def _scrape_func(self, *args, **kwargs):
|
||||
|
|
@ -299,7 +298,7 @@ class ProcessSpiderOutputCoroutineMiddleware:
|
|||
return result
|
||||
|
||||
|
||||
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
|
||||
@defer.inlineCallbacks
|
||||
def test_non_iterable(self):
|
||||
with pytest.raises(
|
||||
|
|
@ -326,8 +325,8 @@ class ProcessYieldSeedsSimpleMiddleware:
|
|||
yield from test_yield_seeds
|
||||
|
||||
|
||||
class ProcessYieldSeedsSimple(BaseAsyncSpiderMiddlewareTestCase):
|
||||
"""process_test_yield_seeds tests for simple test_yield_seeds"""
|
||||
class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware):
|
||||
"""process_seeds tests for simple yield_seeds"""
|
||||
|
||||
ITEM_TYPE = (Request, dict)
|
||||
MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware
|
||||
|
|
@ -382,67 +381,65 @@ class UniversalMiddlewareBothAsync:
|
|||
yield
|
||||
|
||||
|
||||
class UniversalMiddlewareManagerTest(TestCase):
|
||||
def setUp(self):
|
||||
class TestUniversalMiddlewareManager:
|
||||
def setup_method(self):
|
||||
self.mwman = SpiderMiddlewareManager()
|
||||
|
||||
def test_simple_mw(self):
|
||||
mw = ProcessSpiderOutputSimpleMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
assert (
|
||||
self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
|
||||
)
|
||||
|
||||
def test_async_mw(self):
|
||||
mw = ProcessSpiderOutputAsyncGenMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
assert (
|
||||
self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
|
||||
)
|
||||
|
||||
def test_universal_mw(self):
|
||||
mw = ProcessSpiderOutputUniversalMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0],
|
||||
(mw.process_spider_output, mw.process_spider_output_async),
|
||||
assert self.mwman.methods["process_spider_output"][0] == (
|
||||
mw.process_spider_output,
|
||||
mw.process_spider_output_async,
|
||||
)
|
||||
|
||||
def test_universal_mw_no_sync(self):
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(UniversalMiddlewareNoSync())
|
||||
self.assertIn(
|
||||
assert (
|
||||
"UniversalMiddlewareNoSync has process_spider_output_async"
|
||||
" without process_spider_output",
|
||||
str(log),
|
||||
" without process_spider_output" in str(log)
|
||||
)
|
||||
self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
|
||||
assert self.mwman.methods["process_spider_output"][0] is None
|
||||
|
||||
def test_universal_mw_both_sync(self):
|
||||
mw = UniversalMiddlewareBothSync()
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertIn(
|
||||
assert (
|
||||
"UniversalMiddlewareBothSync.process_spider_output_async "
|
||||
"is not an async generator function",
|
||||
str(log),
|
||||
"is not an async generator function" in str(log)
|
||||
)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
assert (
|
||||
self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
|
||||
)
|
||||
|
||||
def test_universal_mw_both_async(self):
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(UniversalMiddlewareBothAsync())
|
||||
self.assertIn(
|
||||
assert (
|
||||
"UniversalMiddlewareBothAsync.process_spider_output "
|
||||
"is an async generator function while process_spider_output_async exists",
|
||||
str(log),
|
||||
"is an async generator function while process_spider_output_async exists"
|
||||
in str(log)
|
||||
)
|
||||
self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
|
||||
assert self.mwman.methods["process_spider_output"][0] is None
|
||||
|
||||
|
||||
class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
|
||||
ITEM_TYPE = dict
|
||||
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
|
||||
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
|
||||
|
|
@ -483,7 +480,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
|
|||
return self._test_simple_base(self.MW_UNIVERSAL)
|
||||
|
||||
|
||||
class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
|
||||
class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
|
||||
async def _scrape_func(self, *args, **kwargs):
|
||||
for item in super()._scrape_func():
|
||||
yield item
|
||||
|
|
@ -512,7 +509,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
|
|||
return self._test_asyncgen_base(self.MW_UNIVERSAL)
|
||||
|
||||
|
||||
class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
|
||||
class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
|
||||
ITEM_TYPE = dict
|
||||
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
|
||||
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spidermiddlewares.depth import DepthMiddleware
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestDepthMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestDepthMiddleware:
|
||||
def setup_method(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = crawler._create_spider("scrapytest.org")
|
||||
|
||||
|
|
@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase):
|
|||
result = [Request("http://scrapytest.org")]
|
||||
|
||||
out = list(self.mw.process_spider_output(resp, result, self.spider))
|
||||
self.assertEqual(out, result)
|
||||
assert out == result
|
||||
|
||||
rdc = self.stats.get_value("request_depth_count/1", spider=self.spider)
|
||||
self.assertEqual(rdc, 1)
|
||||
assert rdc == 1
|
||||
|
||||
req.meta["depth"] = 1
|
||||
|
||||
out2 = list(self.mw.process_spider_output(resp, result, self.spider))
|
||||
self.assertEqual(out2, [])
|
||||
assert not out2
|
||||
|
||||
rdm = self.stats.get_value("request_depth_max", spider=self.spider)
|
||||
self.assertEqual(rdm, 1)
|
||||
assert rdm == 1
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
self.stats.close_spider(self.spider, "")
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
import logging
|
||||
from unittest import TestCase
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.trial.unittest import TestCase as TrialTestCase
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -59,8 +58,8 @@ def _responses(request, status_codes):
|
|||
return responses
|
||||
|
||||
|
||||
class TestHttpErrorMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestHttpErrorMiddleware:
|
||||
def setup_method(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = Spider.from_crawler(crawler, name="foo")
|
||||
self.mw = HttpErrorMiddleware(Settings({}))
|
||||
|
|
@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase):
|
|||
self.res200, self.res404 = _responses(self.req, [200, 404])
|
||||
|
||||
def test_process_spider_input(self):
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
|
||||
assert self.mw.process_spider_input(self.res200, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(self.res404, self.spider)
|
||||
|
||||
def test_process_spider_exception(self):
|
||||
self.assertEqual(
|
||||
[],
|
||||
assert (
|
||||
self.mw.process_spider_exception(
|
||||
self.res404, HttpError(self.res404), self.spider
|
||||
),
|
||||
)
|
||||
== []
|
||||
)
|
||||
self.assertIsNone(
|
||||
assert (
|
||||
self.mw.process_spider_exception(self.res404, Exception(), self.spider)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_handle_httpstatus_list(self):
|
||||
|
|
@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase):
|
|||
res.request = Request(
|
||||
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
|
||||
)
|
||||
self.assertIsNone(self.mw.process_spider_input(res, self.spider))
|
||||
assert self.mw.process_spider_input(res, self.spider) is None
|
||||
|
||||
self.spider.handle_httpstatus_list = [404]
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
|
||||
assert self.mw.process_spider_input(self.res404, self.spider) is None
|
||||
|
||||
|
||||
class TestHttpErrorMiddlewareSettings(TestCase):
|
||||
class TestHttpErrorMiddlewareSettings:
|
||||
"""Similar test, but with settings"""
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
|
||||
self.req = Request("http://scrapytest.org")
|
||||
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
|
||||
|
||||
def test_process_spider_input(self):
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
|
||||
assert self.mw.process_spider_input(self.res200, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(self.res404, self.spider)
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
|
||||
assert self.mw.process_spider_input(self.res402, self.spider) is None
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request(
|
||||
|
|
@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase):
|
|||
res402 = self.res402.copy()
|
||||
res402.request = request
|
||||
|
||||
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
|
||||
assert self.mw.process_spider_input(res404, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(res402, self.spider)
|
||||
|
||||
def test_spider_override_settings(self):
|
||||
self.spider.handle_httpstatus_list = [404]
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
|
||||
assert self.mw.process_spider_input(self.res404, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(self.res402, self.spider)
|
||||
|
||||
|
||||
class TestHttpErrorMiddlewareHandleAll(TestCase):
|
||||
def setUp(self):
|
||||
class TestHttpErrorMiddlewareHandleAll:
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
|
||||
self.req = Request("http://scrapytest.org")
|
||||
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
|
||||
|
||||
def test_process_spider_input(self):
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
|
||||
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
|
||||
assert self.mw.process_spider_input(self.res200, self.spider) is None
|
||||
assert self.mw.process_spider_input(self.res404, self.spider) is None
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request(
|
||||
|
|
@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
res402 = self.res402.copy()
|
||||
res402.request = request
|
||||
|
||||
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
|
||||
assert self.mw.process_spider_input(res404, self.spider) is None
|
||||
with pytest.raises(HttpError):
|
||||
self.mw.process_spider_input(res402, self.spider)
|
||||
|
||||
|
|
@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
|
||||
with pytest.raises(HttpError):
|
||||
mw.process_spider_input(res404, self.spider)
|
||||
self.assertIsNone(mw.process_spider_input(res402, self.spider))
|
||||
assert mw.process_spider_input(res402, self.spider) is None
|
||||
|
||||
|
||||
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
||||
class TestHttpErrorMiddlewareIntegrational(TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
crawler = get_crawler(_HttpErrorSpider)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert not crawler.spider.skipped, crawler.spider.skipped
|
||||
self.assertEqual(crawler.spider.parsed, {"200"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
|
||||
assert crawler.spider.parsed == {"200"}
|
||||
assert crawler.spider.failed == {"404", "402", "500"}
|
||||
|
||||
get_value = crawler.stats.get_value
|
||||
self.assertEqual(get_value("httperror/response_ignored_count"), 3)
|
||||
self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1)
|
||||
self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1)
|
||||
self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1)
|
||||
assert get_value("httperror/response_ignored_count") == 3
|
||||
assert get_value("httperror/response_ignored_status_count/404") == 1
|
||||
assert get_value("httperror/response_ignored_status_count/402") == 1
|
||||
assert get_value("httperror/response_ignored_status_count/500") == 1
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_logging(self):
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
|
||||
self.assertEqual(crawler.spider.parsed, {"200", "402"})
|
||||
self.assertEqual(crawler.spider.skipped, {"402"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "500"})
|
||||
assert crawler.spider.parsed == {"200", "402"}
|
||||
assert crawler.spider.skipped == {"402"}
|
||||
assert crawler.spider.failed == {"404", "500"}
|
||||
|
||||
self.assertIn("Ignoring response <404", str(log))
|
||||
self.assertIn("Ignoring response <500", str(log))
|
||||
self.assertNotIn("Ignoring response <200", str(log))
|
||||
self.assertNotIn("Ignoring response <402", str(log))
|
||||
assert "Ignoring response <404" in str(log)
|
||||
assert "Ignoring response <500" in str(log)
|
||||
assert "Ignoring response <200" not in str(log)
|
||||
assert "Ignoring response <402" not in str(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_logging_level(self):
|
||||
|
|
@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.INFO) as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.parsed, {"200"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
|
||||
assert crawler.spider.parsed == {"200"}
|
||||
assert crawler.spider.failed == {"404", "402", "500"}
|
||||
|
||||
self.assertIn("Ignoring response <402", str(log))
|
||||
self.assertIn("Ignoring response <404", str(log))
|
||||
self.assertIn("Ignoring response <500", str(log))
|
||||
self.assertNotIn("Ignoring response <200", str(log))
|
||||
assert "Ignoring response <402" in str(log)
|
||||
assert "Ignoring response <404" in str(log)
|
||||
assert "Ignoring response <500" in str(log)
|
||||
assert "Ignoring response <200" not in str(log)
|
||||
|
||||
# with level WARNING, we shouldn't capture anything from HttpError
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.WARNING) as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.parsed, {"200"})
|
||||
self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
|
||||
assert crawler.spider.parsed == {"200"}
|
||||
assert crawler.spider.failed == {"404", "402", "500"}
|
||||
|
||||
self.assertNotIn("Ignoring response <402", str(log))
|
||||
self.assertNotIn("Ignoring response <404", str(log))
|
||||
self.assertNotIn("Ignoring response <500", str(log))
|
||||
self.assertNotIn("Ignoring response <200", str(log))
|
||||
assert "Ignoring response <402" not in str(log)
|
||||
assert "Ignoring response <404" not in str(log)
|
||||
assert "Ignoring response <500" not in str(log)
|
||||
assert "Ignoring response <200" not in str(log)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import warnings
|
||||
from unittest import TestCase
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -8,8 +7,8 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestOffsiteMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestOffsiteMiddleware:
|
||||
def setup_method(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = crawler._create_spider(**self._get_spiderargs())
|
||||
self.mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
|
|
@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase):
|
|||
reqs = onsite_reqs + offsite_reqs
|
||||
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEqual(out, onsite_reqs)
|
||||
assert out == onsite_reqs
|
||||
|
||||
|
||||
class TestOffsiteMiddleware2(TestOffsiteMiddleware):
|
||||
|
|
@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
|
|||
res = Response("http://scrapytest.org")
|
||||
reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEqual(out, reqs)
|
||||
assert out == reqs
|
||||
|
||||
|
||||
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
|
||||
|
|
@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
|
|||
res = Response("http://scrapytest.org")
|
||||
reqs = [Request("http://scrapytest.org/1")]
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEqual(out, reqs)
|
||||
assert out == reqs
|
||||
|
||||
|
||||
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
|
||||
|
|
|
|||
|
|
@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase):
|
|||
was enqueued from the recovery middleware)
|
||||
"""
|
||||
log = yield self.crawl_log(RecoverySpider)
|
||||
self.assertIn("Middleware: TabError exception caught", str(log))
|
||||
self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
|
||||
self.assertIn("'item_scraped_count': 3", str(log))
|
||||
assert "Middleware: TabError exception caught" in str(log)
|
||||
assert str(log).count("Middleware: TabError exception caught") == 1
|
||||
assert "'item_scraped_count': 3" in str(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_recovery_asyncgen(self):
|
||||
|
|
@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase):
|
|||
Same as test_recovery but with an async callback.
|
||||
"""
|
||||
log = yield self.crawl_log(RecoveryAsyncGenSpider)
|
||||
self.assertIn("Middleware: TabError exception caught", str(log))
|
||||
self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
|
||||
self.assertIn("'item_scraped_count': 3", str(log))
|
||||
assert "Middleware: TabError exception caught" in str(log)
|
||||
assert str(log).count("Middleware: TabError exception caught") == 1
|
||||
assert "'item_scraped_count': 3" in str(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_process_spider_input_without_errback(self):
|
||||
|
|
@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
process_spider_exception chain from the start if the Request has no errback
|
||||
"""
|
||||
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
|
||||
self.assertIn("Middleware: will raise IndexError", str(log1))
|
||||
self.assertIn("Middleware: IndexError exception caught", str(log1))
|
||||
assert "Middleware: will raise IndexError" in str(log1)
|
||||
assert "Middleware: IndexError exception caught" in str(log1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_process_spider_input_with_errback(self):
|
||||
|
|
@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase):
|
|||
process_spider_exception chain if the Request has an errback
|
||||
"""
|
||||
log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)
|
||||
self.assertNotIn("Middleware: IndexError exception caught", str(log1))
|
||||
self.assertIn("Middleware: will raise IndexError", str(log1))
|
||||
self.assertIn("Got a Failure on the Request errback", str(log1))
|
||||
self.assertIn("{'from': 'errback'}", str(log1))
|
||||
self.assertNotIn("{'from': 'callback'}", str(log1))
|
||||
self.assertIn("'item_scraped_count': 1", str(log1))
|
||||
assert "Middleware: IndexError exception caught" not in str(log1)
|
||||
assert "Middleware: will raise IndexError" in str(log1)
|
||||
assert "Got a Failure on the Request errback" in str(log1)
|
||||
assert "{'from': 'errback'}" in str(log1)
|
||||
assert "{'from': 'callback'}" not in str(log1)
|
||||
assert "'item_scraped_count': 1" in str(log1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_generator_callback(self):
|
||||
|
|
@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
exception is raised should be processed normally.
|
||||
"""
|
||||
log2 = yield self.crawl_log(GeneratorCallbackSpider)
|
||||
self.assertIn("Middleware: ImportError exception caught", str(log2))
|
||||
self.assertIn("'item_scraped_count': 2", str(log2))
|
||||
assert "Middleware: ImportError exception caught" in str(log2)
|
||||
assert "'item_scraped_count': 2" in str(log2)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_async_generator_callback(self):
|
||||
|
|
@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
Same as test_generator_callback but with an async callback.
|
||||
"""
|
||||
log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
|
||||
self.assertIn("Middleware: ImportError exception caught", str(log2))
|
||||
self.assertIn("'item_scraped_count': 2", str(log2))
|
||||
assert "Middleware: ImportError exception caught" in str(log2)
|
||||
assert "'item_scraped_count': 2" in str(log2)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_generator_callback_right_after_callback(self):
|
||||
|
|
@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
even if the middleware is placed right after the spider
|
||||
"""
|
||||
log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
|
||||
self.assertIn("Middleware: ImportError exception caught", str(log21))
|
||||
self.assertIn("'item_scraped_count': 2", str(log21))
|
||||
assert "Middleware: ImportError exception caught" in str(log21)
|
||||
assert "'item_scraped_count': 2" in str(log21)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_not_a_generator_callback(self):
|
||||
|
|
@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
be caught by the process_spider_exception chain. No items should be processed.
|
||||
"""
|
||||
log3 = yield self.crawl_log(NotGeneratorCallbackSpider)
|
||||
self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))
|
||||
self.assertNotIn("item_scraped_count", str(log3))
|
||||
assert "Middleware: ZeroDivisionError exception caught" in str(log3)
|
||||
assert "item_scraped_count" not in str(log3)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_not_a_generator_callback_right_after_callback(self):
|
||||
|
|
@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase):
|
|||
log31 = yield self.crawl_log(
|
||||
NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
|
||||
)
|
||||
self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))
|
||||
self.assertNotIn("item_scraped_count", str(log31))
|
||||
assert "Middleware: ZeroDivisionError exception caught" in str(log31)
|
||||
assert "item_scraped_count" not in str(log31)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_generator_output_chain(self):
|
||||
|
|
@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase):
|
|||
process_spider_exception chain)
|
||||
"""
|
||||
log4 = yield self.crawl_log(GeneratorOutputChainSpider)
|
||||
self.assertIn("'item_scraped_count': 2", str(log4))
|
||||
self.assertIn(
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert "'item_scraped_count': 2" in str(log4)
|
||||
assert (
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
|
||||
in str(log4)
|
||||
)
|
||||
self.assertIn(
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
|
||||
in str(log4)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorFailMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert (
|
||||
"GeneratorFailMiddleware.process_spider_exception: LookupError caught"
|
||||
not in str(log4)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
|
||||
str(log4),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
|
||||
not in str(log4)
|
||||
)
|
||||
item_from_callback = {
|
||||
"processed": [
|
||||
|
|
@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase):
|
|||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
|
||||
]
|
||||
}
|
||||
self.assertIn(str(item_from_callback), str(log4))
|
||||
self.assertIn(str(item_recovered), str(log4))
|
||||
self.assertNotIn("parse-second-item", str(log4))
|
||||
assert str(item_from_callback) in str(log4)
|
||||
assert str(item_recovered) in str(log4)
|
||||
assert "parse-second-item" not in str(log4)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_not_a_generator_output_chain(self):
|
||||
|
|
@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase):
|
|||
from the spider callback are lost)
|
||||
"""
|
||||
log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
|
||||
self.assertIn("'item_scraped_count': 1", str(log5))
|
||||
self.assertIn(
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert "'item_scraped_count': 1" in str(log5)
|
||||
assert (
|
||||
"GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught"
|
||||
in str(log5)
|
||||
)
|
||||
self.assertIn(
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught"
|
||||
in str(log5)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorFailMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert (
|
||||
"GeneratorFailMiddleware.process_spider_exception: ReferenceError caught"
|
||||
not in str(log5)
|
||||
)
|
||||
self.assertNotIn(
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
|
||||
str(log5),
|
||||
assert (
|
||||
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught"
|
||||
not in str(log5)
|
||||
)
|
||||
item_recovered = {
|
||||
"processed": [
|
||||
|
|
@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase):
|
|||
"NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
|
||||
]
|
||||
}
|
||||
self.assertIn(str(item_recovered), str(log5))
|
||||
self.assertNotIn("parse-first-item", str(log5))
|
||||
self.assertNotIn("parse-second-item", str(log5))
|
||||
assert str(item_recovered) in str(log5)
|
||||
assert "parse-first-item" not in str(log5)
|
||||
assert "parse-second-item" not in str(log5)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from __future__ import annotations
|
|||
|
||||
import warnings
|
||||
from typing import Any
|
||||
from unittest import TestCase
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
|
|
@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import (
|
|||
from scrapy.spiders import Spider
|
||||
|
||||
|
||||
class TestRefererMiddleware(TestCase):
|
||||
class TestRefererMiddleware:
|
||||
req_meta: dict[str, Any] = {}
|
||||
resp_headers: dict[str, str] = {}
|
||||
settings: dict[str, Any] = {}
|
||||
|
|
@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase):
|
|||
("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
settings = Settings(self.settings)
|
||||
self.mw = RefererMiddleware(settings)
|
||||
|
|
@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase):
|
|||
response = self.get_response(origin)
|
||||
request = self.get_request(target)
|
||||
out = list(self.mw.process_spider_output(response, [request], self.spider))
|
||||
self.assertEqual(out[0].headers.get("Referer"), referrer)
|
||||
assert out[0].headers.get("Referer") == referrer
|
||||
|
||||
|
||||
class MixinDefault:
|
||||
|
|
@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
|
|||
req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaSettingFallback(TestCase):
|
||||
class TestRequestMetaSettingFallback:
|
||||
params = [
|
||||
(
|
||||
# When an unknown policy is referenced in Request.meta
|
||||
|
|
@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase):
|
|||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
policy = mw.policy(response, request)
|
||||
self.assertIsInstance(policy, policy_class)
|
||||
assert isinstance(policy, policy_class)
|
||||
|
||||
if check_warning:
|
||||
self.assertEqual(len(w), 1)
|
||||
self.assertEqual(w[0].category, RuntimeWarning, w[0].message)
|
||||
assert len(w) == 1
|
||||
assert w[0].category is RuntimeWarning, w[0].message
|
||||
|
||||
|
||||
class TestSettingsPolicyByName(TestCase):
|
||||
class TestSettingsPolicyByName:
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
|
|
@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
]:
|
||||
settings = Settings({"REFERRER_POLICY": s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
assert mw.default_policy == p
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
|
|
@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
]:
|
||||
settings = Settings({"REFERRER_POLICY": s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
assert mw.default_policy == p
|
||||
|
||||
def test_invalid_name(self):
|
||||
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
|
||||
|
|
@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
}
|
||||
)
|
||||
mw1 = RefererMiddleware(settings1)
|
||||
self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)
|
||||
assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy
|
||||
|
||||
# test parsing with space(s) after the comma
|
||||
settings2 = Settings(
|
||||
|
|
@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase):
|
|||
}
|
||||
)
|
||||
mw2 = RefererMiddleware(settings2)
|
||||
self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)
|
||||
assert mw2.default_policy == UnsafeUrlPolicy
|
||||
|
||||
def test_multiple_policy_tokens_all_invalid(self):
|
||||
settings = Settings(
|
||||
|
|
@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.spider = Spider("foo")
|
||||
settings = Settings(self.settings)
|
||||
self.referrermw = RefererMiddleware(settings)
|
||||
|
|
@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
out = list(
|
||||
self.referrermw.process_spider_output(response, [request], self.spider)
|
||||
)
|
||||
self.assertEqual(out[0].headers.get("Referer"), init_referrer)
|
||||
assert out[0].headers.get("Referer") == init_referrer
|
||||
|
||||
for status, url in redirections:
|
||||
response = Response(
|
||||
|
|
@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
|
|||
self.referrermw.request_scheduled(request, self.spider)
|
||||
|
||||
assert isinstance(request, Request)
|
||||
self.assertEqual(request.headers.get("Referer"), final_referrer)
|
||||
assert request.headers.get("Referer") == final_referrer
|
||||
|
||||
|
||||
class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from testfixtures import LogCapture
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -8,8 +6,8 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestUrlLengthMiddleware(TestCase):
|
||||
def setUp(self):
|
||||
class TestUrlLengthMiddleware:
|
||||
def setup_method(self):
|
||||
self.maxlength = 25
|
||||
crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength})
|
||||
self.spider = crawler._create_spider("foo")
|
||||
|
|
@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase):
|
|||
)
|
||||
|
||||
def test_middleware_works(self):
|
||||
self.assertEqual(self.process_spider_output(), [self.short_url_req])
|
||||
assert self.process_spider_output() == [self.short_url_req]
|
||||
|
||||
def test_logging(self):
|
||||
with LogCapture() as log:
|
||||
|
|
@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase):
|
|||
ric = self.stats.get_value(
|
||||
"urllength/request_ignored_count", spider=self.spider
|
||||
)
|
||||
self.assertEqual(ric, 1)
|
||||
assert ric == 1
|
||||
|
||||
self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log))
|
||||
assert f"Ignoring link (url length > {self.maxlength})" in str(log)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from datetime import datetime, timezone
|
|||
from tempfile import mkdtemp
|
||||
|
||||
import pytest
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.extensions.spiderstate import SpiderState
|
||||
|
|
@ -11,7 +10,7 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class SpiderStateTest(unittest.TestCase):
|
||||
class TestSpiderState:
|
||||
def test_store_load(self):
|
||||
jobdir = mkdtemp()
|
||||
try:
|
||||
|
|
@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase):
|
|||
spider2 = Spider(name="default")
|
||||
ss2 = SpiderState(jobdir)
|
||||
ss2.spider_opened(spider2)
|
||||
self.assertEqual(spider.state, {"one": 1, "dt": dt})
|
||||
assert spider.state == {"one": 1, "dt": dt}
|
||||
ss2.spider_closed(spider2)
|
||||
finally:
|
||||
shutil.rmtree(jobdir)
|
||||
|
|
@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase):
|
|||
spider = Spider(name="default")
|
||||
ss = SpiderState()
|
||||
ss.spider_opened(spider)
|
||||
self.assertEqual(spider.state, {})
|
||||
assert spider.state == {}
|
||||
ss.spider_closed(spider)
|
||||
|
||||
def test_not_configured(self):
|
||||
|
|
|
|||
|
|
@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin:
|
|||
q.push("a")
|
||||
q.push(123)
|
||||
q.push({"a": "dict"})
|
||||
self.assertEqual(q.pop(), "a")
|
||||
self.assertEqual(q.pop(), 123)
|
||||
self.assertEqual(q.pop(), {"a": "dict"})
|
||||
assert q.pop() == "a"
|
||||
assert q.pop() == 123
|
||||
assert q.pop() == {"a": "dict"}
|
||||
|
||||
test_nonserializable_object = nonserializable_object_test
|
||||
|
||||
|
|
@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
q.push(i)
|
||||
i2 = q.pop()
|
||||
assert isinstance(i2, MyItem)
|
||||
self.assertEqual(i, i2)
|
||||
assert i == i2
|
||||
|
||||
def test_serialize_loader(self):
|
||||
q = self.queue()
|
||||
|
|
@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
loader2 = q.pop()
|
||||
assert isinstance(loader2, MyLoader)
|
||||
assert loader2.default_item_class is MyItem
|
||||
self.assertEqual(loader2.name_out("x"), "xx")
|
||||
assert loader2.name_out("x") == "xx"
|
||||
|
||||
def test_serialize_request_recursive(self):
|
||||
q = self.queue()
|
||||
|
|
@ -110,23 +110,29 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
q.push(r)
|
||||
r2 = q.pop()
|
||||
assert isinstance(r2, Request)
|
||||
self.assertEqual(r.url, r2.url)
|
||||
assert r.url == r2.url
|
||||
assert r2.meta["request"] is r2
|
||||
|
||||
def test_non_pickable_object(self):
|
||||
q = self.queue()
|
||||
try:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",
|
||||
) as exc_info:
|
||||
q.push(lambda x: x)
|
||||
except ValueError as exc:
|
||||
if hasattr(sys, "pypy_version_info"):
|
||||
self.assertIsInstance(exc.__context__, pickle.PicklingError)
|
||||
else:
|
||||
self.assertIsInstance(exc.__context__, AttributeError)
|
||||
if hasattr(sys, "pypy_version_info"):
|
||||
assert isinstance(exc_info.value.__context__, pickle.PicklingError)
|
||||
else:
|
||||
assert isinstance(exc_info.value.__context__, AttributeError)
|
||||
sel = Selector(text="<html><body><p>some text</p></body></html>")
|
||||
try:
|
||||
with pytest.raises(
|
||||
ValueError, match="can't pickle Selector objects"
|
||||
) as exc_info:
|
||||
q.push(sel)
|
||||
except ValueError as exc:
|
||||
self.assertIsInstance(exc.__context__, TypeError)
|
||||
assert isinstance(exc_info.value.__context__, TypeError)
|
||||
# This seems to help with https://github.com/scrapy/queuelib/issues/70.
|
||||
# It will need to remain under a queuelib version check after that bug is fixed.
|
||||
del exc_info
|
||||
|
||||
|
||||
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
|
||||
|
|
@ -151,9 +157,9 @@ class LifoDiskQueueTestMixin:
|
|||
q.push("a")
|
||||
q.push(123)
|
||||
q.push({"a": "dict"})
|
||||
self.assertEqual(q.pop(), {"a": "dict"})
|
||||
self.assertEqual(q.pop(), 123)
|
||||
self.assertEqual(q.pop(), "a")
|
||||
assert q.pop() == {"a": "dict"}
|
||||
assert q.pop() == 123
|
||||
assert q.pop() == "a"
|
||||
|
||||
test_nonserializable_object = nonserializable_object_test
|
||||
|
||||
|
|
@ -173,7 +179,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
|
|||
q.push(i)
|
||||
i2 = q.pop()
|
||||
assert isinstance(i2, MyItem)
|
||||
self.assertEqual(i, i2)
|
||||
assert i == i2
|
||||
|
||||
def test_serialize_loader(self):
|
||||
q = self.queue()
|
||||
|
|
@ -182,7 +188,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
|
|||
loader2 = q.pop()
|
||||
assert isinstance(loader2, MyLoader)
|
||||
assert loader2.default_item_class is MyItem
|
||||
self.assertEqual(loader2.name_out("x"), "xx")
|
||||
assert loader2.name_out("x") == "xx"
|
||||
|
||||
def test_serialize_request_recursive(self):
|
||||
q = self.queue()
|
||||
|
|
@ -191,5 +197,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
|
|||
q.push(r)
|
||||
r2 = q.pop()
|
||||
assert isinstance(r2, Request)
|
||||
self.assertEqual(r.url, r2.url)
|
||||
assert r.url == r2.url
|
||||
assert r2.meta["request"] is r2
|
||||
|
|
|
|||
|
|
@ -22,14 +22,14 @@ from scrapy.squeues import (
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class BaseQueueTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestBaseQueue:
|
||||
def setup_method(self):
|
||||
self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-")
|
||||
self.qpath = self.tempfilename()
|
||||
self.qdir = tempfile.mkdtemp()
|
||||
self.crawler = get_crawler(Spider)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
shutil.rmtree(self.tmpdir)
|
||||
|
||||
def tempfilename(self):
|
||||
|
|
@ -48,36 +48,36 @@ class RequestQueueTestMixin:
|
|||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
req = Request("http://www.example.com")
|
||||
q.push(req)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.peek().url, req.url)
|
||||
self.assertEqual(q.pop().url, req.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 1
|
||||
assert q.peek().url == req.url
|
||||
assert q.pop().url == req.url
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
def test_one_element_without_peek(self):
|
||||
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
req = Request("http://www.example.com")
|
||||
q.push(req)
|
||||
self.assertEqual(len(q), 1)
|
||||
assert len(q) == 1
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match="The underlying queue class does not implement 'peek'",
|
||||
):
|
||||
q.peek()
|
||||
self.assertEqual(q.pop().url, req.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert q.pop().url == req.url
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
|
||||
|
|
@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin):
|
|||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
q.push(req1)
|
||||
q.push(req2)
|
||||
q.push(req3)
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.peek().url, req1.url)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.peek().url, req2.url)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.peek().url, req3.url)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.peek().url == req1.url
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 2
|
||||
assert q.peek().url == req2.url
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.peek().url == req3.url
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
def test_fifo_without_peek(self):
|
||||
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
|
|
@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin):
|
|||
match="The underlying queue class does not implement 'peek'",
|
||||
):
|
||||
q.peek()
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 2
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
|
||||
|
|
@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin):
|
|||
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
q.push(req1)
|
||||
q.push(req2)
|
||||
q.push(req3)
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.peek().url, req3.url)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.peek().url, req2.url)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.peek().url, req1.url)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.peek())
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.peek().url == req3.url
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 2
|
||||
assert q.peek().url == req2.url
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.peek().url == req1.url
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 0
|
||||
assert q.peek() is None
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
def test_lifo_without_peek(self):
|
||||
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
|
||||
raise unittest.SkipTest("The queuelib queues do not define peek")
|
||||
q = self.queue()
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
req1 = Request("http://www.example.com/1")
|
||||
req2 = Request("http://www.example.com/2")
|
||||
req3 = Request("http://www.example.com/3")
|
||||
|
|
@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin):
|
|||
match="The underlying queue class does not implement 'peek'",
|
||||
):
|
||||
q.peek()
|
||||
self.assertEqual(len(q), 3)
|
||||
self.assertEqual(q.pop().url, req3.url)
|
||||
self.assertEqual(len(q), 2)
|
||||
self.assertEqual(q.pop().url, req2.url)
|
||||
self.assertEqual(len(q), 1)
|
||||
self.assertEqual(q.pop().url, req1.url)
|
||||
self.assertEqual(len(q), 0)
|
||||
self.assertIsNone(q.pop())
|
||||
assert len(q) == 3
|
||||
assert q.pop().url == req3.url
|
||||
assert len(q) == 2
|
||||
assert q.pop().url == req2.url
|
||||
assert len(q) == 1
|
||||
assert q.pop().url == req1.url
|
||||
assert len(q) == 0
|
||||
assert q.pop() is None
|
||||
q.close()
|
||||
|
||||
|
||||
class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
|
||||
class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo")
|
||||
|
||||
|
||||
class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
|
||||
class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo")
|
||||
|
||||
|
||||
class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
|
||||
class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return MarshalFifoDiskQueue.from_crawler(
|
||||
crawler=self.crawler, key="marshal/fifo"
|
||||
)
|
||||
|
||||
|
||||
class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
|
||||
class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return MarshalLifoDiskQueue.from_crawler(
|
||||
crawler=self.crawler, key="marshal/lifo"
|
||||
)
|
||||
|
||||
|
||||
class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
|
||||
class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return FifoMemoryQueue.from_crawler(crawler=self.crawler)
|
||||
|
||||
|
||||
class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
|
||||
class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue):
|
||||
def queue(self):
|
||||
return LifoMemoryQueue.from_crawler(crawler=self.crawler)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class CoreStatsExtensionTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestCoreStatsExtension:
|
||||
def setup_method(self):
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.spider = self.crawler._create_spider("foo")
|
||||
|
||||
|
|
@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase):
|
|||
ext.response_received(self.spider)
|
||||
ext.item_dropped({}, self.spider, ZeroDivisionError())
|
||||
ext.spider_closed(self.spider, "finished")
|
||||
self.assertEqual(
|
||||
ext.stats._stats,
|
||||
{
|
||||
"start_time": fixed_datetime,
|
||||
"finish_time": fixed_datetime,
|
||||
"item_scraped_count": 1,
|
||||
"response_received_count": 1,
|
||||
"item_dropped_count": 1,
|
||||
"item_dropped_reasons_count/ZeroDivisionError": 1,
|
||||
"finish_reason": "finished",
|
||||
"elapsed_time_seconds": 0.0,
|
||||
},
|
||||
)
|
||||
assert ext.stats._stats == {
|
||||
"start_time": fixed_datetime,
|
||||
"finish_time": fixed_datetime,
|
||||
"item_scraped_count": 1,
|
||||
"response_received_count": 1,
|
||||
"item_dropped_count": 1,
|
||||
"item_dropped_reasons_count/ZeroDivisionError": 1,
|
||||
"finish_reason": "finished",
|
||||
"elapsed_time_seconds": 0.0,
|
||||
}
|
||||
|
||||
def test_core_stats_dummy_stats_collector(self):
|
||||
self.crawler.stats = DummyStatsCollector(self.crawler)
|
||||
|
|
@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase):
|
|||
ext.response_received(self.spider)
|
||||
ext.item_dropped({}, self.spider, ZeroDivisionError())
|
||||
ext.spider_closed(self.spider, "finished")
|
||||
self.assertEqual(ext.stats._stats, {})
|
||||
assert ext.stats._stats == {}
|
||||
|
||||
|
||||
class StatsCollectorTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestStatsCollector:
|
||||
def setup_method(self):
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.spider = self.crawler._create_spider("foo")
|
||||
|
||||
def test_collector(self):
|
||||
stats = StatsCollector(self.crawler)
|
||||
self.assertEqual(stats.get_stats(), {})
|
||||
self.assertEqual(stats.get_value("anything"), None)
|
||||
self.assertEqual(stats.get_value("anything", "default"), "default")
|
||||
assert stats.get_stats() == {}
|
||||
assert stats.get_value("anything") is None
|
||||
assert stats.get_value("anything", "default") == "default"
|
||||
stats.set_value("test", "value")
|
||||
self.assertEqual(stats.get_stats(), {"test": "value"})
|
||||
assert stats.get_stats() == {"test": "value"}
|
||||
stats.set_value("test2", 23)
|
||||
self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23})
|
||||
self.assertEqual(stats.get_value("test2"), 23)
|
||||
assert stats.get_stats() == {"test": "value", "test2": 23}
|
||||
assert stats.get_value("test2") == 23
|
||||
stats.inc_value("test2")
|
||||
self.assertEqual(stats.get_value("test2"), 24)
|
||||
assert stats.get_value("test2") == 24
|
||||
stats.inc_value("test2", 6)
|
||||
self.assertEqual(stats.get_value("test2"), 30)
|
||||
assert stats.get_value("test2") == 30
|
||||
stats.max_value("test2", 6)
|
||||
self.assertEqual(stats.get_value("test2"), 30)
|
||||
assert stats.get_value("test2") == 30
|
||||
stats.max_value("test2", 40)
|
||||
self.assertEqual(stats.get_value("test2"), 40)
|
||||
assert stats.get_value("test2") == 40
|
||||
stats.max_value("test3", 1)
|
||||
self.assertEqual(stats.get_value("test3"), 1)
|
||||
assert stats.get_value("test3") == 1
|
||||
stats.min_value("test2", 60)
|
||||
self.assertEqual(stats.get_value("test2"), 40)
|
||||
assert stats.get_value("test2") == 40
|
||||
stats.min_value("test2", 35)
|
||||
self.assertEqual(stats.get_value("test2"), 35)
|
||||
assert stats.get_value("test2") == 35
|
||||
stats.min_value("test4", 7)
|
||||
self.assertEqual(stats.get_value("test4"), 7)
|
||||
assert stats.get_value("test4") == 7
|
||||
|
||||
def test_dummy_collector(self):
|
||||
stats = DummyStatsCollector(self.crawler)
|
||||
self.assertEqual(stats.get_stats(), {})
|
||||
self.assertEqual(stats.get_value("anything"), None)
|
||||
self.assertEqual(stats.get_value("anything", "default"), "default")
|
||||
assert stats.get_stats() == {}
|
||||
assert stats.get_value("anything") is None
|
||||
assert stats.get_value("anything", "default") == "default"
|
||||
stats.set_value("test", "value")
|
||||
stats.inc_value("v1")
|
||||
stats.max_value("v2", 100)
|
||||
stats.min_value("v3", 100)
|
||||
stats.open_spider("a")
|
||||
stats.set_value("test", "value", spider=self.spider)
|
||||
self.assertEqual(stats.get_stats(), {})
|
||||
self.assertEqual(stats.get_stats("a"), {})
|
||||
assert stats.get_stats() == {}
|
||||
assert stats.get_stats("a") == {}
|
||||
|
|
|
|||
Loading…
Reference in New Issue