diff --git a/conftest.py b/conftest.py index ed7d14166..f952127b9 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,8 @@ def _py_files(folder): collect_ignore = [ + # may need extra deps + "docs/_ext", # not a test, but looks like a test "scrapy/utils/testproc.py", "scrapy/utils/testsite.py", @@ -46,12 +48,6 @@ if not H2_ENABLED: ) -@pytest.fixture -def chdir(tmpdir): - """Change to pytest-provided temporary directory""" - tmpdir.chdir() - - def pytest_addoption(parser): parser.addoption( "--reactor", diff --git a/docs/news.rst b/docs/news.rst index 36d229760..05ad611ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4454,6 +4454,8 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. skip: start + * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: @@ -4494,6 +4496,8 @@ Backward-incompatible changes (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`, :issue:`4036`) +.. skip: end + See also :ref:`1.8-deprecation-removals` below. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 65270433f..6c57a88f1 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -111,6 +111,8 @@ you may use `curl2scrapy `_. Handling different response formats =================================== +.. skip: start + Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: @@ -157,11 +159,15 @@ data from it depends on the type of response: Otherwise, you might need to convert the SVG code into a raster image, and :ref:`handle that raster image `. +.. skip: end + .. _topics-parsing-javascript: Parsing JavaScript code ======================= +.. skip: start + If the desired data is hardcoded in JavaScript, you first need to get the JavaScript code: @@ -220,6 +226,8 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' +.. skip: end + .. _topics-headless-browsing: Using a headless browser @@ -242,6 +250,7 @@ it is possible to integrate ``asyncio``-based libraries which handle headless br One such library is `playwright-python`_ (an official Python port of `playwright`_). The following is a simple snippet to illustrate its usage within a Scrapy spider: +.. skip: next .. code-block:: python import scrapy diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 0365c95b3..3588d033e 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -214,6 +214,8 @@ the :attr:`~scrapy.Item.fields` attribute. Working with Item objects ------------------------- +.. skip: start + Here are some examples of common tasks performed with items, using the ``Product`` item :ref:`declared above `. You will notice the API is very similar to the :class:`dict` API. @@ -375,6 +377,8 @@ appending more values, or changing existing values, like this: That adds (or replaces) the ``serializer`` metadata key for the ``name`` field, keeping all the previously existing metadata values. +.. skip: end + .. _supporting-item-types: diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index cd8914644..bbe1f3dd4 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -60,6 +60,8 @@ in control. Debugging memory leaks with ``trackref`` ======================================== +.. skip: start + :mod:`trackref` is a module provided by Scrapy to debug the most common cases of memory leaks. It basically tracks the references to all live Request, Response, Item, Spider and Selector objects. @@ -203,6 +205,8 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. ``None`` if none is found. Use :func:`print_live_refs` first to get a list of all tracked live objects per class name. +.. skip: end + .. _topics-leaks-muppy: Debugging memory leaks with muppy @@ -226,6 +230,7 @@ If you use ``pip``, you can install muppy with the following command:: Here's an example to view all Python objects available in the heap using muppy: +.. skip: start .. code-block:: pycon >>> from pympler import muppy @@ -253,6 +258,8 @@ the heap using muppy: `, using the :ref:`Product item ` declared in the :ref:`Items chapter `: +.. skip: next .. code-block:: python from scrapy.loader import ItemLoader @@ -130,6 +131,7 @@ assigned to the item. Let's see an example to illustrate how the input and output processors are called for a particular field (the same applies for any other field): +.. skip: next .. code-block:: python l = ItemLoader(Product(), some_selector) @@ -250,6 +252,7 @@ metadata. Here is an example: ) +.. skip: start .. code-block:: pycon >>> from scrapy.loader import ItemLoader @@ -259,6 +262,8 @@ metadata. Here is an example: >>> il.load_item() {'name': 'Welcome to my website', 'price': '1000'} +.. skip: end + The precedence order, for both input and output processors, is as follows: 1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most @@ -294,6 +299,8 @@ the Item Loader that it's able to receive an Item Loader context, so the Item Loader passes the currently active context when calling it, and the processor function (``parse_length`` in this case) can thus use them. +.. skip: start + There are several ways to modify Item Loader context values: 1. By modifying the currently active Item Loader context @@ -320,6 +327,8 @@ There are several ways to modify Item Loader context values: class ProductLoader(ItemLoader): length_out = MapCompose(parse_length, unit="cm") +.. skip: end + ItemLoader objects ================== @@ -350,6 +359,7 @@ that you wish to extract. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -364,6 +374,7 @@ the footer selector. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -401,6 +412,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``): +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose @@ -418,6 +430,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it: +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index dbef07b73..40a85201a 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -308,6 +308,7 @@ Examples: * ``*::text`` selects all descendant text nodes of the current selector context: +..skip: next .. code-block:: pycon >>> response.css("#images *::text").getall() @@ -878,7 +879,7 @@ Example selecting links in list item with a "class" attribute ending with a digi >>> sel = Selector(text=doc, type="html") >>> sel.xpath("//li//@href").getall() ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() + >>> sel.xpath(r'//li[re:test(@class, "item-\d$")]//@href').getall() ['link1.html', 'link2.html', 'link4.html', 'link5.html'] .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4898843e4..85a08cebd 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -142,6 +142,8 @@ Those objects are: Example of shell session ======================== +.. skip: start + Here's an example of a typical shell session where we start by scraping the https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/ page. Finally, we modify the (Reddit) request method to POST and re-fetch it @@ -232,6 +234,8 @@ After that, we can start playing with the objects: 'X-Ua-Compatible': ['IE=edge'], 'X-Xss-Protection': ['1; mode=block']} +.. skip: end + .. _topics-shell-inspect-response: @@ -268,6 +272,8 @@ Here's an example of how you would call it from your spider: # Rest of parsing code. +.. skip: start + When you run the spider, you will get something similar to this:: 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) @@ -301,6 +307,8 @@ crawling:: 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... +.. skip: end + Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is blocked by the shell. However, after you leave the shell, the spider will continue crawling where it stopped, as shown above. diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 9572a3785..e34999b58 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -42,6 +42,8 @@ attribute. Here is an example of an extension that access stats: def from_crawler(cls, crawler): return cls(crawler.stats) +.. skip: start + Set stat value: .. code-block:: python @@ -80,6 +82,8 @@ Get all stats: >>> stats.get_stats() {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} +.. skip: end + Available Stats Collectors ========================== diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 3e9bbe56e..ae9cb634c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -97,6 +97,8 @@ convenience: Telnet console usage examples ============================= +.. skip: start + Here are some example tasks you can do with the telnet console: View engine status @@ -146,6 +148,8 @@ To stop:: >>> engine.stop() Connection closed by foreign host. +.. skip: end + Telnet Console signals ====================== diff --git a/pyproject.toml b/pyproject.toml index 0742991db..bc809a7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,23 +224,7 @@ disable = [ [tool.pytest.ini_options] xfail_strict = true -usefixtures = "chdir" python_files = ["test_*.py", "test_*/__init__.py"] -addopts = [ - "--assert=plain", - "--ignore=docs/_ext", - "--ignore=docs/conf.py", - "--ignore=docs/news.rst", - "--ignore=docs/topics/dynamic-content.rst", - "--ignore=docs/topics/items.rst", - "--ignore=docs/topics/leaks.rst", - "--ignore=docs/topics/loaders.rst", - "--ignore=docs/topics/selectors.rst", - "--ignore=docs/topics/shell.rst", - "--ignore=docs/topics/stats.rst", - "--ignore=docs/topics/telnetconsole.rst", - "--ignore=docs/utils", -] markers = [ "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 01797fd20..f8f3eb22a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import csv import gzip import json import lzma +import os import random import shutil import string @@ -107,8 +108,13 @@ class TestFileFeedStorage: self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self, tmp_path): - path = (tmp_path / "foo" / "bar").relative_to(Path.cwd()) - self._assert_stores(FileFeedStorage(str(path)), path) + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + path = Path("foo", "bar") + self._assert_stores(FileFeedStorage(str(path)), path) + finally: + os.chdir(old_cwd) def test_interface(self, tmp_path): path = tmp_path / "file.txt" @@ -236,24 +242,22 @@ class TestBlockingFeedStorage: def test_default_temp_dir(self): b = BlockingFeedStorage() - tmp = b.open(self.get_test_spider()) - tmp_path = Path(tmp.name).parent - assert str(tmp_path) == tempfile.gettempdir() + storage_file = b.open(self.get_test_spider()) + storage_dir = Path(storage_file.name).parent + assert str(storage_dir) == tempfile.gettempdir() - def test_temp_file(self): + def test_temp_file(self, tmp_path): b = BlockingFeedStorage() - tests_path = Path(__file__).resolve().parent - spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) - tmp = b.open(spider) - tmp_path = Path(tmp.name).parent - assert tmp_path == tests_path + spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + storage_file = b.open(spider) + storage_dir = Path(storage_file.name).parent + assert storage_dir == tmp_path - def test_invalid_folder(self): + def test_invalid_folder(self, tmp_path): b = BlockingFeedStorage() - tests_path = Path(__file__).resolve().parent - invalid_path = tests_path / "invalid_path" + invalid_path = tmp_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) with pytest.raises(OSError, match="Not a Directory:"): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b4eae108f..808fde23d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -524,21 +524,20 @@ class TestFilesPipelineCustomSettings: expected_value = settings.get(settings_attr) assert getattr(pipeline_cls, pipe_inst_attr) == expected_value - def test_file_pipeline_using_pathlike_objects(self): + def test_file_pipeline_using_pathlike_objects(self, tmp_path): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): return Path("subdir") / Path(request.url).name pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler( - get_crawler(None, {"FILES_STORE": Path("./Temp")}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) request = Request("http://example.com/image01.jpg") assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self): - path = Path("./FileDir") - fs_store = FSFilesStore(path) - assert fs_store.basedir == str(path) + def test_files_store_constructor_with_pathlike_object(self, tmp_path): + fs_store = FSFilesStore(tmp_path) + assert fs_store.basedir == str(tmp_path) @pytest.mark.requires_botocore diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 68bd6df68..8353ad73c 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -2,8 +2,7 @@ Queues that handle requests """ -import shutil -import tempfile +from pathlib import Path import pytest import queuelib @@ -23,30 +22,17 @@ from scrapy.utils.test import get_crawler class TestBaseQueue: def setup_method(self): - self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") - self.qpath = self.tempfilename() - self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def teardown_method(self): - shutil.rmtree(self.tmpdir) - - def tempfilename(self): - with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: - return nf.name - - def mkdtemp(self): - return tempfile.mkdtemp(dir=self.tmpdir) - class RequestQueueTestMixin: - def queue(self): + def queue(self, base_path: Path): raise NotImplementedError - def test_one_element_with_peek(self): + def test_one_element_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -60,10 +46,10 @@ class RequestQueueTestMixin: assert q.pop() is None q.close() - def test_one_element_without_peek(self): + def test_one_element_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req = Request("http://www.example.com") @@ -81,10 +67,10 @@ class RequestQueueTestMixin: class FifoQueueMixin(RequestQueueTestMixin): - def test_fifo_with_peek(self): + def test_fifo_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -108,10 +94,10 @@ class FifoQueueMixin(RequestQueueTestMixin): assert q.pop() is None q.close() - def test_fifo_without_peek(self): + def test_fifo_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - pytest.skip("The queuelib queues do not define peek") - q = self.queue() + pytest.skip("The queuelib queues define peek") + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req1 = Request("http://www.example.com/1") @@ -137,10 +123,10 @@ class FifoQueueMixin(RequestQueueTestMixin): class LifoQueueMixin(RequestQueueTestMixin): - def test_lifo_with_peek(self): + def test_lifo_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -164,10 +150,10 @@ class LifoQueueMixin(RequestQueueTestMixin): assert q.pop() is None q.close() - def test_lifo_without_peek(self): + def test_lifo_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - pytest.skip("The queuelib queues do not define peek") - q = self.queue() + pytest.skip("The queuelib queues define peek") + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req1 = Request("http://www.example.com/1") @@ -193,34 +179,38 @@ class LifoQueueMixin(RequestQueueTestMixin): class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): - return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + def queue(self, base_path): + return PickleFifoDiskQueue.from_crawler( + crawler=self.crawler, key=str(base_path / "pickle" / "fifo") + ) class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): - return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + def queue(self, base_path): + return PickleLifoDiskQueue.from_crawler( + crawler=self.crawler, key=str(base_path / "pickle" / "lifo") + ) class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return MarshalFifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/fifo" + crawler=self.crawler, key=str(base_path / "marshal" / "fifo") ) class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return MarshalLifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/lifo" + crawler=self.crawler, key=str(base_path / "marshal" / "lifo") ) class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return FifoMemoryQueue.from_crawler(crawler=self.crawler) class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return LifoMemoryQueue.from_crawler(crawler=self.crawler)