Create a logger for every Spider and adapt Spider.log to log through it

scrapy/spider.py

@@ -3,9 +3,9 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+import logging
 import warnings
 
-from scrapy import log
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
@@ -31,11 +31,19 @@ class Spider(object_ref):
         if not hasattr(self, 'start_urls'):
             self.start_urls = []
 
-    def log(self, message, level=log.DEBUG, **kw):
-        """Log the given messages at the given log level. Always use this
-        method to send log messages from your spider
+    @property
+    def logger(self):
+        logger = logging.getLogger(self.name)
+        return logging.LoggerAdapter(logger, {'spider': self})
+
+    def log(self, message, level=logging.DEBUG, **kw):
+        """Log the given message at the given log level
+
+        This helper wraps a log call to the logger within the spider, but you
+        can use it directly (e.g. Spider.logger.info('msg')) or use any other
+        Python logger too.
         """
-        log.msg(message, spider=self, level=level, **kw)
+        self.logger.log(level, message, **kw)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):

tests/spiders.py

@@ -74,7 +74,7 @@ class SimpleSpider(MetaSpider):
         self.start_urls = [url]
 
     def parse(self, response):
-        self.log("Got response %d" % response.status)
+        self.logger.info("Got response %d" % response.status)
 
 
 class ItemSpider(FollowAllSpider):

tests/test_commands.py

@@ -142,7 +142,7 @@ class MySpider(scrapy.Spider):
     name = 'myspider'
 
     def start_requests(self):
-        self.log("It Works!")
+        self.logger.debug("It Works!")
         return []
 """)
         p = self.proc('runspider', fname)
@@ -197,7 +197,7 @@ class MySpider(scrapy.Spider):
 
     def parse(self, response):
         if getattr(self, 'test_arg', None):
-            self.log('It Works!')
+            self.logger.debug('It Works!')
         return [scrapy.Item(), dict(foo='bar')]
 """.format(self.spider_name))
 

tests/test_spider.py

@@ -2,6 +2,8 @@ import gzip
 import inspect
 import warnings
 from io import BytesIO
+
+from testfixtures import LogCapture
 from twisted.trial import unittest
 
 from scrapy import signals
@@ -102,6 +104,23 @@ class SpiderTest(unittest.TestCase):
         self.assertEqual(settings.get('TEST2'), 'spider')
         self.assertEqual(settings.get('TEST3'), 'project')
 
+    def test_logger(self):
+        spider = self.spider_class('example.com')
+        with LogCapture() as l:
+            spider.logger.info('test log msg')
+        l.check(('example.com', 'INFO', 'test log msg'))
+
+        record = l.records[0]
+        self.assertIn('spider', record.__dict__)
+        self.assertIs(record.spider, spider)
+
+    def test_log(self):
+        spider = self.spider_class('example.com')
+        with mock.patch('scrapy.spider.Spider.logger') as mock_logger:
+            spider.log('test log msg', 'INFO')
+        mock_logger.log.assert_called_once_with('INFO', 'test log msg')
+
+
 class InitSpiderTest(SpiderTest):
 
     spider_class = InitSpider