diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 70f20d4ed..9337375bb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -120,3 +120,23 @@ get the failures pretty printed:: for header in self.args: if header not in response.headers: raise ContractFail('X-CustomHeader not present') + + +Detecting check runs +==================== + +When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is +set to the ``true`` string. You can use `os.environ`_ to perform any change to +your spiders or your settings when ``scrapy check`` is used:: + + import os + import scrapy + + class ExampleSpider(scrapy.Spider): + name = 'example' + + def __init__(self): + if os.environ.get('SCRAPY_CHECK'): + pass # Do some scraper adjustments when a check is running + +.. _os.environ: https://docs.python.org/3/library/os.html#os.environ diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b8a9ef989..ab73e85e7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, set_environ from scrapy.utils.conf import build_component_list @@ -68,16 +68,17 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - for spidername in args or spider_loader.list(): - spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + with set_environ(SCRAPY_CHECK='true'): + for spidername in args or spider_loader.list(): + spidercls = spider_loader.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) - tested_methods = conman.tested_methods_from_spidercls(spidercls) - if opts.list: - for method in tested_methods: - contract_reqs[spidercls.name].append(method) - elif tested_methods: - self.crawler_process.crawl(spidercls) + tested_methods = conman.tested_methods_from_spidercls(spidercls) + if opts.list: + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ddaa7f7bf..f638adb25 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,6 +1,8 @@ """Helper functions which don't fit anywhere else""" +import os import re import hashlib +from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -142,3 +144,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return objcls.from_settings(settings, *args, **kwargs) else: return objcls(*args, **kwargs) + + +@contextmanager +def set_environ(**kwargs): + """Temporarily set environment variables inside the context manager and + fully restore previous environment afterwards + """ + + original_env = {k: os.environ.get(k) for k in kwargs} + os.environ.update(kwargs) + try: + yield + finally: + for k, v in original_env.items(): + if v is None: + del os.environ[k] + else: + os.environ[k] = v diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index fcb7772ab..e109d5343 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,12 +3,13 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules from tests import mock __doctests__ = ['scrapy.utils.misc'] + class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): @@ -130,5 +131,18 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(ValueError): create_instance(m, None, None) + def test_set_environ(self): + assert os.environ.get('some_test_environ') is None + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') is None + + os.environ['some_test_environ'] = 'test' + assert os.environ.get('some_test_environ') == 'test' + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') == 'test' + + if __name__ == "__main__": unittest.main()