allowing to run .pyw files (#4646)

* allow .pyw in scrapy/commands/runspider.py

* aesthetics

* added tests for '.pyw'

* created class for testing .pyw files

* name=None parameter in get_log

* small fix

* .pyw tests for non-windows

* used @skipIf for tests

* two more tests skipped

scrapy/commands/runspider.py

@@ -11,7 +11,7 @@ def _import_file(filepath):
     abspath = os.path.abspath(filepath)
     dirname, file = os.path.split(abspath)
     fname, fext = os.path.splitext(file)
-    if fext != '.py':
+    if fext not in ('.py', '.pyw'):
         raise ValueError(f"Not a Python source file: {abspath}")
     if dirname:
         sys.path = [dirname] + sys.path

tests/test_commands.py

@@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest):
 
 class RunSpiderCommandTest(CommandTest):
 
+    spider_filename = 'myspider.py'
+
     debug_log_spider = """
 import scrapy
 
@@ -507,11 +509,23 @@ class MySpider(scrapy.Spider):
         return []
 """
 
+    badspider = """
+import scrapy
+
+class BadSpider(scrapy.Spider):
+    name = "bad"
+    def start_requests(self):
+        raise Exception("oops!")
+        """
+
     @contextmanager
-    def _create_file(self, content, name):
+    def _create_file(self, content, name=None):
         tmpdir = self.mktemp()
         os.mkdir(tmpdir)
-        fname = abspath(join(tmpdir, name))
+        if name:
+            fname = abspath(join(tmpdir, name))
+        else:
+            fname = abspath(join(tmpdir, self.spider_filename))
         with open(fname, 'w') as f:
             f.write(content)
         try:
@@ -519,12 +533,12 @@ class MySpider(scrapy.Spider):
         finally:
             rmtree(tmpdir)
 
-    def runspider(self, code, name='myspider.py', args=()):
+    def runspider(self, code, name=None, args=()):
         with self._create_file(code, name) as fname:
             return self.proc('runspider', fname, *args)
 
-    def get_log(self, code, name='myspider.py', args=()):
-        p, stdout, stderr = self.runspider(code, name=name, args=args)
+    def get_log(self, code, name=None, args=()):
+        p, stdout, stderr = self.runspider(code, name, args=args)
         return stderr
 
     def test_runspider(self):
@@ -556,7 +570,7 @@ class MySpider(scrapy.Spider):
         # which is intended,
         # but this should not be because of DNS lookup error
         # assumption: localhost will resolve in all cases (true?)
-        log = self.get_log("""
+        dnscache_spider = """
 import scrapy
 
 class MySpider(scrapy.Spider):
@@ -565,23 +579,20 @@ class MySpider(scrapy.Spider):
 
     def parse(self, response):
         return {'test': 'value'}
-""",
-                           args=('-s', 'DNSCACHE_ENABLED=False'))
-        print(log)
+"""
+        log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
         self.assertNotIn("DNSLookupError", log)
         self.assertIn("INFO: Spider opened", log)
 
     def test_runspider_log_short_names(self):
         log1 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=1'))
-        print(log1)
         self.assertIn("[myspider] DEBUG: It Works!", log1)
         self.assertIn("[scrapy]", log1)
         self.assertNotIn("[scrapy.core.engine]", log1)
 
         log2 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=0'))
-        print(log2)
         self.assertIn("[myspider] DEBUG: It Works!", log2)
         self.assertNotIn("[scrapy]", log2)
         self.assertIn("[scrapy.core.engine]", log2)
@@ -599,15 +610,7 @@ class MySpider(scrapy.Spider):
         self.assertIn('Unable to load', log)
 
     def test_start_requests_errors(self):
-        log = self.get_log("""
-import scrapy
-
-class BadSpider(scrapy.Spider):
-    name = "bad"
-    def start_requests(self):
-        raise Exception("oops!")
-        """, name="badspider.py")
-        print(log)
+        log = self.get_log(self.badspider, name='badspider.py')
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
@@ -696,6 +699,54 @@ class MySpider(scrapy.Spider):
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
 
+class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
+
+    spider_filename = 'myspider.pyw'
+
+    def setUp(self):
+        super(WindowsRunSpiderCommandTest, self).setUp()
+
+    def test_start_requests_errors(self):
+        log = self.get_log(self.badspider, name='badspider.pyw')
+        self.assertIn("start_requests", log)
+        self.assertIn("badspider.pyw", log)
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_run_good_spider(self):
+        super().test_run_good_spider()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider(self):
+        super().test_runspider()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_dnscache_disabled(self):
+        super().test_runspider_dnscache_disabled()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_log_level(self):
+        super().test_runspider_log_level()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_log_short_names(self):
+        super().test_runspider_log_short_names()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_no_spider_found(self):
+        super().test_runspider_no_spider_found()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_output(self):
+        super().test_output()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_overwrite_output(self):
+        super().test_overwrite_output()
+
+    def test_runspider_unable_to_load(self):
+        raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
+
+
 class BenchCommandTest(CommandTest):
 
     def test_run(self):