Add a test for a CrawlerProcess script. (#4218)

* Add a test for a CrawlerProcess script.

* Add tests/CrawlerProcess to collect_ignore.

* Remove an extra line.

* Fix/improve conftest.py.

conftest.py

@@ -1,12 +1,19 @@
+from pathlib import Path
+
 import pytest
 
 
+def _py_files(folder):
+    return (str(p) for p in Path(folder).rglob('*.py'))
+
+
 collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
+    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+    *_py_files("tests/CrawlerProcess")
 ]
 
-
 for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':

tests/CrawlerProcess/simple.py

@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()

tests/test_crawler.py

@@ -1,4 +1,7 @@
 import logging
+import os
+import subprocess
+import sys
 import warnings
 
 from twisted.internet import defer
@@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         yield runner.crawl(NoRequestsSpider)
 
         self.assertEqual(runner.bootstrap_failed, True)
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+    def run_script(self, script_name):
+        script_path = os.path.join(self.script_dir, script_name)
+        args = (sys.executable, script_path)
+        p = subprocess.Popen(args, env=get_testenv(),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        return stderr.decode('utf-8')
+
+    def test_simple(self):
+        log = self.run_script('simple.py')
+        self.assertIn('Spider closed (finished)', log)