cleaned up scrapy.utils.db module

scrapy/conf/default_settings.py

@@ -119,6 +119,7 @@ MEMUSAGE_REPORT = False
 MEMUSAGE_WARNING_MB = 0
 
 MYSQL_CONNECTION_SETTINGS = {}
+MYSQL_CONNECTION_PING_PERIOD = 0
 
 NEWSPIDER_MODULE = ''
 

scrapy/utils/db.py

@@ -1,21 +1,18 @@
-"""
-Function for dealing with databases
-"""
+"""Functions for dealing with databases"""
+
 import re
+
+import MySQLdb
+
+from scrapy.core.engine import scrapyengine
 from scrapy.conf import settings
 from scrapy import log
 
-class URIValidationError(Exception):
-    pass
+mysql_uri_re = r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/:]+)(:(?P<port>\d+))?/(?P<db>.*)$"
 
-def parse_uri(db_uri):
-    """
-    Parse mysql URI and return settings dict
-    """
-    if not (isinstance(db_uri, basestring) and db_uri.startswith('mysql://')):
-        raise URIValidationError("Incorrect MySQL URI: %s" % db_uri)
-
-    m = re.search(r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/:]+)(:(?P<port>\d+))?/(?P<db>.*)$", db_uri)
+def parse_uri(mysql_uri):
+    """Parse mysql URI and return a dict with its parameters"""
+    m = re.search(mysql_uri_re, mysql_uri)
     if m:
         d = m.groupdict()
         if d['passwd'] is None:
@@ -27,33 +24,26 @@ def parse_uri(db_uri):
         return d
 
 def mysql_connect(db_uri_or_dict, **kwargs):
-    """
-    Connects to a MySQL DB given a mysql URI
-    """
-    import MySQLdb
-    from scrapy.core.engine import scrapyengine
-
+    """Connects to a MySQL DB given a mysql URI"""
     if isinstance(db_uri_or_dict, dict):
         d = db_uri_or_dict
     else:
         d = parse_uri(db_uri_or_dict)
+    if not d:
+        return
+    d.update(settings.get("MYSQL_CONNECTION_SETTINGS", {}))
+    d.update(kwargs)
+    log.msg("Connecting to MySQL: db=%r, host=%r, user=%r" % (d['db'], \
+        d['host'], d['user']), level=log.DEBUG)
+    conn = MySQLdb.connect(**d)
 
-    if d:
-        d['charset'] = 'utf8'
-        d.update(settings.get("MYSQL_CONNECTION_SETTINGS", {}))
-        d.update(kwargs)
+    # connection keep-alive
+    def conn_ping():
+        log.msg("Pinging connection to %s/%s" % (d['host'], d['db']), \
+            level=log.DEBUG)
+        conn.ping()
+    ping_period = settings.getint("MYSQL_CONNECTION_PING_PERIOD")
+    if ping_period:
+        scrapyengine.addtask(conn_ping, ping_period)
 
-        dcopy = d.copy()
-        if dcopy.get("passwd"):
-            dcopy["passwd"] = "********"
-        log.msg("Connecting db with settings %s" % dcopy )
-
-        conn = MySQLdb.connect(**d)
-
-        #this is to maintain active the connection
-        def _ping():
-            log.msg("Pinging connection to %s/%s" % (d.get('host'), d.get('db')) )
-            conn.ping()
-        scrapyengine.addtask(_ping, settings.getint("MYSQL_CONNECTION_PING_PERIOD", 600))
-
-        return conn
+    return conn