minor (and inoffensive) code improvements and fixes found while documenting

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40700

scrapy/trunk/scrapy/conf/default_settings.py

@@ -82,7 +82,6 @@ EXTENSIONS = [
     'scrapy.xpath.extension.ResponseLibxml2',
     'scrapy.management.web.WebConsole',
     'scrapy.management.telnet.TelnetConsole',
-    'scrapy.contrib.webconsole.schedstats.SchedulerStats',
     'scrapy.contrib.webconsole.livestats.LiveStats',
     'scrapy.contrib.webconsole.spiderctl.Spiderctl',
     'scrapy.contrib.webconsole.enginestatus.EngineStatus',

scrapy/trunk/scrapy/contrib/memdebug.py

@@ -1,3 +1,9 @@
+"""
+MemoryDebugger extension
+
+See documentation in docs/ref/extensions.rst
+"""
+
 import pprint
 import gc
 import socket

scrapy/trunk/scrapy/contrib/memusage.py

@@ -1,3 +1,9 @@
+"""
+MemoryUsage extension
+
+See documentation in docs/ref/extensions.rst
+"""
+
 import sys
 import os
 import pprint

scrapy/trunk/scrapy/contrib/response/soup.py

@@ -1,12 +1,7 @@
 """
-The ResponseSoup extension causes the Response objects to grow a new method
-("getsoup") which returns a (cached) BeautifulSoup object of its body, and a
-"soup" attribute with the same effect. The soup argument is provided for
-convenience, but you cannot pass any BeautifulSoup constructor arguments (which
-you can do with the getsoup() method).
+ResponseSoup extension 
 
-For more information about BeautifulSoup see:
-http://www.crummy.com/software/BeautifulSoup/documentation.html
+See documentation in docs/ref/extensions.rst
 """ 
 
 from BeautifulSoup import BeautifulSoup
@@ -20,6 +15,7 @@ class ResponseSoup(object):
         setattr(Response, 'soup', property(getsoup))
 
 def getsoup(response, **kwargs):
+    # TODO: use different cache buckets depending on constructor parameters
     if not hasattr(response, '_soup'):
         body = response.body.to_string() if response.body is not None else ""
         setattr(response, '_soup', BeautifulSoup(body, **kwargs))

scrapy/trunk/scrapy/contrib/webconsole/livestats.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from pydispatch import dispatcher
 from scrapy.core import signals
 from scrapy.core.engine import scrapyengine
-from scrapy.management.web import banner
+from scrapy.management.web import banner, webconsole_discover_module
 
 class SpiderStats(object):
     def __init__(self):
@@ -25,24 +25,23 @@ class LiveStats(object):
         dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
         dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded)
 
-        from scrapy.management.web import webconsole_discover_module
         dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
 
     def domain_open(self, domain, spider):
         pstats = SpiderStats()
         self.domains[spider.domain_name] = pstats
-        pstats.started = datetime.now()
+        pstats.started = datetime.now().replace(microsecond=0)
         pstats.finished = None
 
     def domain_closed(self, domain, spider):
-        self.domains[spider.domain_name].finished = datetime.now()
+        self.domains[spider.domain_name].finished = datetime.now().replace(microsecond=0)
 
     def item_scraped(self, item, spider):
         self.domains[spider.domain_name].scraped += 1
 
     def response_downloaded(self, response, spider):
-        #sometimes we download responses without opening/closing domains,
-        #for example from scrapy shell
+        # sometimes we download responses without opening/closing domains,
+        # for example from scrapy shell
         if self.domains.get(spider.domain_name):
             self.domains[spider.domain_name].crawled += 1
             

scrapy/trunk/scrapy/contrib/webconsole/schedstats.py

@@ -1,20 +1,25 @@
 """
 Scheduler information module for Scrapy webconsole
+
+FIXME: this webconsole extension needs to be fixed after we removed
+PriorityStack/Queue from the scheduler and replace them by bisect or something
+like that.
 """
+
 from pydispatch import dispatcher
-from scrapy.core import signals
 from scrapy.core.engine import scrapyengine
-from scrapy.spider import spiders
-from scrapy.management.web import banner
+from scrapy.management.web import banner, webconsole_discover_module
 
 class SchedulerStats(object):
     webconsole_id = 'scheduler'
     webconsole_name = 'Scheduler queue'
 
     def __init__(self):
-        from scrapy.management.web import webconsole_discover_module
         dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
 
+    def webconsole_discover_module(self):
+        return self
+
     def webconsole_render(self, wc_request):
         s = banner(self)
         s += "<ul>\n"
@@ -36,6 +41,3 @@ class SchedulerStats(object):
         s += "</html>\n"
 
         return s
-
-    def webconsole_discover_module(self):
-        return self

scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py

@@ -139,6 +139,7 @@ class Spiderctl(object):
 
     def webconsole_control(self, wc_request):
         args = wc_request.args
+        enabled_domains = spiders.asdict(include_disabled=False).keys()
         s = "<hr />\n"
         if "reloadspiders" in args:
             scrapymanager.reload_spiders()
@@ -164,7 +165,7 @@ class Spiderctl(object):
             scheduled = []
             to_remove = set([d.strip() for d in args["bulk_remove_domains"][0].split("\n")])
             removed = set([d for d in scrapyengine.scheduler.pending_domains if d in to_remove])
-            scrapyengine.scheduler.pending_domains = [d for d in execengine.scheduler.pending_domains if d not in removed]
+            scrapyengine.scheduler.pending_domains = [d for d in scrapyengine.scheduler.pending_domains if d not in removed]
             if removed:
                 s += "<p>"
                 s += "Removed: <ul><li>%s</li></ul>" % "</li><li>".join(removed)

scrapy/trunk/scrapy/management/web.py

@@ -14,6 +14,18 @@ webconsole_discover_module = object()
 
 urlpath_re = re.compile(r"^/(\w+)/")
 
+def error404(module):
+    return """
+<html>
+  <head><title>404 Not Found</title></head>
+  <body>
+    <h1>Not found</h1>
+    <p>Web console module not found: <b>%s</b></p>
+    <p><a href="/">Back to main menu</a></p>
+  </body>
+</html>
+""" % module
+
 def banner(module=None):
     s  = "<html>\n"
     s += "<head><title>Scrapy</title></head>\n"
@@ -21,7 +33,7 @@ def banner(module=None):
     s += "<h1><a href='/'>Scrapy web console</a></h1>\n"
     now = datetime.now()
     uptime = now - scrapyengine.start_time
-    s += "<p>Bot: <b>%s</b> | Host: <b>%s</b> | Uptime: <b>%s</b> | Time: <b>%s</b> </p>\n" % (settings['BOT_NAME'], socket.gethostname(), str(uptime), str(now))
+    s += "<p>Bot: <b>%s</b> | Host: <b>%s</b> | Uptime: <b>%s</b> | Time: <b>%s</b> </p>\n" % (settings['BOT_NAME'], socket.gethostname(), str(uptime), str(now.replace(microsecond=0)))
     if module:
         s += "<h2><a href='/%s/'>%s</a></h2>\n" % (module.webconsole_id, module.webconsole_name)
     return s
@@ -40,7 +52,12 @@ class WebConsoleResource(resource.Resource):
     def render_GET(self, request):
         m = urlpath_re.search(request.path)
         if m:
-            return self.modules[m.group(1)].webconsole_render(request)
+            module = m.group(1)
+            if module in self.modules:
+                return self.modules[m.group(1)].webconsole_render(request)
+            else:
+                request.setResponseCode(404)
+                return error404(module)
         else:
             return self.module_list()