From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001 From: faizan2700 Date: Sun, 2 Feb 2020 16:54:22 +0530 Subject: [PATCH 1/6] SCRAPY_CHECK will be set while running contact --- scrapy/commands/check.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 9d4437a47..09a76ca7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -78,19 +78,19 @@ class Command(ScrapyCommand): elif tested_methods: self.crawler_process.crawl(spidercls) - # start checks - if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(' * %s' % method) - else: - start = time.time() - self.crawler_process.start() - stop = time.time() + # start checks + if opts.list: + for spider, methods in sorted(contract_reqs.items()): + if not methods and not opts.verbose: + continue + print(spider) + for method in sorted(methods): + print(' * %s' % method) + else: + start = time.time() + self.crawler_process.start() + stop = time.time() - result.printErrors() - result.printSummary(start, stop) - self.exitcode = int(not result.wasSuccessful()) + result.printErrors() + result.printSummary(start, stop) + self.exitcode = int(not result.wasSuccessful()) From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Tue, 7 Jul 2020 15:23:29 +0530 Subject: [PATCH 2/6] Added basic `scrapy check` tests --- tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_command_check.py diff --git a/tests/test_command_check.py b/tests/test_command_check.py new file mode 100644 index 000000000..52005a4c5 --- /dev/null +++ b/tests/test_command_check.py @@ -0,0 +1,96 @@ +from os.path import join, abspath + +from tests.test_commands import CommandTest + + +class CheckCommandTest(CommandTest): + + command = 'check' + + def setUp(self): + super(CheckCommandTest, self).setUp() + self.spider_name = 'check_spider' + self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + + def _write_contract(self, contracts, parse_def): + with open(self.spider, 'w') as file: + file.write(f""" +import scrapy + +class CheckSpider(scrapy.Spider): + name = '{self.spider_name}' + start_urls = ['http://example.com'] + + def parse(self, response, **cb_kwargs): + \"\"\" + @url http://www.amazon.com/s?field-keywords=selfish+gene + {contracts} + \"\"\" + {parse_def} + """) + + def _test_contract(self, contracts='', parse_def='pass'): + self._write_contract(contracts, parse_def) + p, out, err = self.proc('check') + self.assertIn('OK', err) + self.assertEqual(p.returncode, 0) + + def test_check_returns_requests_contract(self): + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(contracts, parse_def) + + def test_check_returns_items_contract(self): + contracts = """ + @returns items 1 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_cb_kwargs_contract(self): + contracts = """ + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_check_scrapes_contract(self): + contracts = """ + @scrapes key1 key2 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_all_default_contracts(self): + contracts = """ + @returns items 1 + @returns requests 1 + @scrapes key1 key2 + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + yield scrapy.Request(url='http://next-url.com') + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_SCRAPY_CHECK_set(self): + parse_def = """ + import os + if not os.environ.get('SCRAPY_CHECK'): + raise Exception('SCRAPY_CHECK not set') + """ + self._test_contract(parse_def=parse_def) From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Tue, 7 Jul 2020 15:24:33 +0530 Subject: [PATCH 3/6] Ignore flake8 E501 for `scrapy check` tests` --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index bae68cd3a..97320a008 100644 --- a/pytest.ini +++ b/pytest.ini @@ -173,6 +173,7 @@ flake8-ignore = tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 + tests/test_command_check.py E501 tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 8 Jul 2020 17:13:57 +0530 Subject: [PATCH 4/6] Convert f-string to .format() --- tests/test_command_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 52005a4c5..72acd817c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(f""" + file.write(""" import scrapy class CheckSpider(scrapy.Spider): - name = '{self.spider_name}' + name = '{0}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://www.amazon.com/s?field-keywords=selfish+gene - {contracts} + {1} \"\"\" - {parse_def} - """) + {2} + """.format(self.spider_name, contracts, parse_def)) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 8 Jul 2020 19:48:42 +0530 Subject: [PATCH 5/6] Update url contract value --- tests/test_command_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 72acd817c..f27f526a3 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider): def parse(self, response, **cb_kwargs): \"\"\" - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://example.com {1} \"\"\" {2} @@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider): def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) p, out, err = self.proc('check') + self.assertNotIn('F', out) self.assertIn('OK', err) self.assertEqual(p.returncode, 0) From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Fri, 10 Jul 2020 18:22:43 +0530 Subject: [PATCH 6/6] Update pytest.ini --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 92c5bcb75..ca8191f42 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,4 +39,5 @@ flake8-ignore = scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 \ No newline at end of file + tests/test_loader.py E741 +