From d27c2c68ba4f201c18255a6ec8a735f140b13773 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 12:56:50 +0200 Subject: [PATCH 01/10] Wrap scrapy check in environment --- scrapy/commands/check.py | 21 +++++++++++---------- scrapy/utils/misc.py | 27 +++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b8a9ef989..b29587fa7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, set_environ from scrapy.utils.conf import build_component_list @@ -68,16 +68,17 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - for spidername in args or spider_loader.list(): - spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + with set_environ(SCRAPY_CHECK=True): + for spidername in args or spider_loader.list(): + spidercls = spider_loader.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) - tested_methods = conman.tested_methods_from_spidercls(spidercls) - if opts.list: - for method in tested_methods: - contract_reqs[spidercls.name].append(method) - elif tested_methods: - self.crawler_process.crawl(spidercls) + tested_methods = conman.tested_methods_from_spidercls(spidercls) + if opts.list: + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ddaa7f7bf..7a2cd18ea 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,6 +1,8 @@ """Helper functions which don't fit anywhere else""" +import os import re import hashlib +from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -10,7 +12,6 @@ from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem - _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes @@ -40,7 +41,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot+1:] + module, name = path[:dot], path[dot + 1:] mod = import_module(module) try: @@ -85,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): @@ -142,3 +143,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return objcls.from_settings(settings, *args, **kwargs) else: return objcls(*args, **kwargs) + + +@contextmanager +def set_environ(**kwargs): + """Temporarily set environment variables inside the context manager and + fully restore previous environment afterwards + """ + + original_env = {k: os.environ.get(k) for k in kwargs} + os.environ.update(kwargs) + try: + yield + finally: + for k, v in original_env: + if v is None: + del os.environ[k] + else: + os.environ[k] = v From 50730ed2280dec6384986d34999cd277d7568ff9 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 13:01:01 +0200 Subject: [PATCH 02/10] Try it with a string --- scrapy/commands/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b29587fa7..ab73e85e7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -68,7 +68,7 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - with set_environ(SCRAPY_CHECK=True): + with set_environ(SCRAPY_CHECK='true'): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) spidercls.start_requests = lambda s: conman.from_spider(s, result) From 07adca34e1378b11dff9e3f11d3760c54f5fa1ef Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 13:01:46 +0200 Subject: [PATCH 03/10] Fix 'Too many values to unpack' --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7a2cd18ea..cdd5a11c9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -156,7 +156,7 @@ def set_environ(**kwargs): try: yield finally: - for k, v in original_env: + for k, v in original_env.items(): if v is None: del os.environ[k] else: From 29739989478bb1b10467b036df91326071b1ffbc Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 14:50:02 +0200 Subject: [PATCH 04/10] Add set_environ test --- tests/test_utils_misc/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index fcb7772ab..738120a0b 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,7 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ from tests import mock @@ -130,5 +130,12 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(ValueError): create_instance(m, None, None) + def test_set_environ(self): + assert os.environ.get('some_test_environ') is None + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') is None + + if __name__ == "__main__": unittest.main() From 6d52708579be05c29c58d6ccc63486f761466d18 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 15:19:23 +0200 Subject: [PATCH 05/10] Add reset case --- tests/test_utils_misc/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 738120a0b..eeb995e48 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -136,6 +136,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') is None + os.environ['some_test_environ'] = 'test' + assert os.environ.get('some_test_environ') == 'test' + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') == 'test' + if __name__ == "__main__": unittest.main() From 935387aaea2f0bdc28504b40dfc9ccbbb437bec4 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 22:10:23 +0200 Subject: [PATCH 06/10] Revert some non-changes --- scrapy/utils/misc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index cdd5a11c9..f51012e3d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -41,7 +41,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot + 1:] + module, name = path[:dot], path[dot+1:] mod = import_module(module) try: @@ -86,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): From 7809c0b14e3ad62aea8e62c7309997ecb64fbbf1 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Sat, 20 Apr 2019 09:25:01 +0200 Subject: [PATCH 07/10] Revert another non-change comment --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f51012e3d..b2164d4a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -88,7 +88,7 @@ def extract_regex(regex, text, encoding='utf-8'): try: strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): From 8bd207a2f639216eb51f61ae312dfd22f4b39781 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Sun, 28 Apr 2019 21:47:47 +0200 Subject: [PATCH 08/10] Add documentation --- docs/topics/contracts.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 70f20d4ed..3aa32cba8 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -120,3 +120,22 @@ get the failures pretty printed:: for header in self.args: if header not in response.headers: raise ContractFail('X-CustomHeader not present') + + +Detecting check run +=================== +It is not encouraged to created different behaviour when running test. +However, sometimes it is useful to know when a spider is started by scrapy check. +It can for example be needed to enforce less settings to be set, or to disable some +uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment +variable is set. This can be retrieved via ``os.environ``:: + + import os + import scrapy + + class ExampleSpider(scrapy.Spider): + name = 'example' + + def __init__(self): + if os.environ.get('SCRAPY_CHECK'): + # Do some scraper adjustments when check is running \ No newline at end of file From f6485e669772a940c3c319c71dbcca7bd747d57a Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Fri, 3 May 2019 13:53:45 +0200 Subject: [PATCH 09/10] Restore alphabetic order and two lines between import and code --- scrapy/utils/misc.py | 1 + tests/test_utils_misc/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b2164d4a8..f638adb25 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -12,6 +12,7 @@ from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem + _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index eeb995e48..e109d5343 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,12 +3,13 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules from tests import mock __doctests__ = ['scrapy.utils.misc'] + class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): From bc1a92921364de8f4616feb0ea7dcb7b6d42d2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 3 May 2019 14:42:12 +0200 Subject: [PATCH 10/10] Improve the documentation about detecting check runs --- docs/topics/contracts.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 3aa32cba8..9337375bb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -122,13 +122,12 @@ get the failures pretty printed:: raise ContractFail('X-CustomHeader not present') -Detecting check run -=================== -It is not encouraged to created different behaviour when running test. -However, sometimes it is useful to know when a spider is started by scrapy check. -It can for example be needed to enforce less settings to be set, or to disable some -uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment -variable is set. This can be retrieved via ``os.environ``:: +Detecting check runs +==================== + +When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is +set to the ``true`` string. You can use `os.environ`_ to perform any change to +your spiders or your settings when ``scrapy check`` is used:: import os import scrapy @@ -138,4 +137,6 @@ variable is set. This can be retrieved via ``os.environ``:: def __init__(self): if os.environ.get('SCRAPY_CHECK'): - # Do some scraper adjustments when check is running \ No newline at end of file + pass # Do some scraper adjustments when a check is running + +.. _os.environ: https://docs.python.org/3/library/os.html#os.environ