diff --git a/scrapy/contrib_exp/history/__init__.py b/scrapy/contrib_exp/history/__init__.py deleted file mode 100644 index 81eb201d6..000000000 --- a/scrapy/contrib_exp/history/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.contrib.history.history import ItemHistory -from scrapy.contrib.history.scheduler import RulesScheduler -from scrapy.contrib.history.store import SQLHistoryStore diff --git a/scrapy/contrib_exp/history/history.py b/scrapy/contrib_exp/history/history.py deleted file mode 100644 index 6767ef260..000000000 --- a/scrapy/contrib_exp/history/history.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -History management -""" -import re -from datetime import datetime - -import MySQLdb - -class History(object): - """Base class for tracking different kinds of histories""" - - def __init__(self, db_uri): - self.db_uri = db_uri - self._mysql_conn = None - - def connect(self): - """ - Connect to PDB and open mysql connect to PRODUCT_DB - """ - m = re.search(r"mysql:\/\/(?P[^:]+)(:(?P[^@]+))?@(?P[^/]+)/(?P.*)$", self.db_uri) - if m: - d = m.groupdict() - if d['passwd'] is None: - del(d['passwd']) - - d['charset'] = "utf8" - self._mysql_conn = MySQLdb.connect(**d) - - def get_mysql_conn(self): - if self._mysql_conn is None: - self.connect() - return self._mysql_conn - mysql_conn = property(get_mysql_conn) - - -class ItemHistory(History): - """ - Class instance registers item guids, and versions with check_item method. - It also gives access to stored ItemTicket and ItemVersion objects. - """ - NEW = 0 - UPDATE = 1 - DUPLICATE = 2 - - def check_item(self, domain, item): - """ - Store item's guid and version to the database along - with date and time of last occurence. - Return: - * ItemHistory.NEW - if guid hasn't been met - * ItemHistory.UPDATE - if history already contains guid, but versions doesn't match - * ItemHistory.DUPLICATE - both guid and version are not new - """ - version = item.version - - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - - def add_version(version): - insert = "INSERT INTO version (guid, version, seen) VALUES (%s,%s,%s)" - c.execute(insert, (item.guid, version, datetime.now())) - - select = "SELECT * FROM ticket WHERE guid=%s" - c.execute(select, item.guid) - r = c.fetchone() - if r: - select = "SELECT * FROM version WHERE version=%s" - if c.execute(select, version): - update = "UPDATE version SET seen=%s WHERE version=%s" - c.execute(update, (datetime.now(), version)) - self.mysql_conn.commit() - return ItemHistory.DUPLICATE - else: - add_version(version) - self.mysql_conn.commit() - return ItemHistory.UPDATE - else: - insert = "INSERT INTO ticket (guid, domain, url, url_hash) VALUES (%s,%s,%s,%s)" - c.execute(insert, (item.guid, domain, item.url, hash(item.url))) - add_version(version) - self.mysql_conn.commit() - return ItemHistory.NEW - - def get_ticket(self, guid): - """ - Return ItemTicket object for guid. - ItemVersion objects can be accessed via 'versions' list. - """ - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM ticket WHERE guid=%s" - c.execute(select, guid) - ticket = c.fetchone() - if not ticket: - raise Exception("Item ticket with guid = '%s' not found" % guid) - ticket['versions'] = [] - select = "SELECT * FROM version WHERE guid=%s" - c.execute(select, guid) - for version in c.fetchall(): - ticket['versions'].append(version) - return ticket - - def delete_ticket(self, guid): - """Delete item ticket and associated versions from DB""" - c = self.mysql_conn.cursor() - delete = "DELETE FROM ticket WHERE guid=%s" - c.execute(delete, guid) - self.mysql_conn.commit() - - -class URLHistory(History): - """ - Access URL status and history for the scraping engine - - This is degsigned to have an instance per domain where typically - a call will be made to get_url_status, followed by either - update_checked or record_version. - """ - - def get_url_status(self, urlkey): - """ - Get the url status (url, last_version, last_checked), - or None if the url data has not been seen before - """ - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM url_status WHERE url_hash=%s" - c.execute(select, urlkey) - r = c.fetchone() - return (r['url'], r['last_version'], r['last_checked']) if r else None - - def record_version(self, urlkey, url, parent_key, version, postdata_hash=None): - """ - Record a version of a page and update the last checked time. - - If the same version (or None) is passed, the last checked time is still updated. - """ - now = datetime.now() - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM url_status WHERE url_hash=%s" - c.execute(select, urlkey) - r = c.fetchone() - - if not r: - insert = "INSERT INTO url_status (url_hash, url, parent_hash, last_version, last_checked) VALUES (%s,%s,%s,%s,%s)" - c.execute(insert, (urlkey, url, parent_key, version, now)) - else: - update = "UPDATE url_status SET last_version=%s, last_checked=%s WHERE url_hash=%s" - c.execute(update, (version, now, urlkey)) - self.mysql_conn.commit() - - last_version = r['last_version'] if r else None - if version and version != last_version: - if not c.execute("SELECT url_hash FROM url_history WHERE version=%s", version): - insert = "INSERT INTO url_history (url_hash, version, postdata_hash, created) VALUES (%s,%s,%s,%s)" - c.execute(insert, (urlkey, version, postdata_hash, now)) - self.mysql_conn.commit() - - def get_version_info(self, version): - """Simple accessor method""" - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM url_history WHERE version=%s" - c.execute(select, version) - r = c.fetchone() - return (r['url_hash'], r['created']) if r else None diff --git a/scrapy/contrib_exp/history/memorystore.py b/scrapy/contrib_exp/history/memorystore.py deleted file mode 100644 index b7eddfcf4..000000000 --- a/scrapy/contrib_exp/history/memorystore.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -The datastore module contains implementations of data storage engines for the -crawling process. These are used to track metrics on each site and on each -page visited. -""" -from datetime import datetime - -class MemoryStore(object) : - """Simple implementation of a data store. This is useful if no persistent - history is required (e.g. unit testing or development) and provides a - simpler reference implementation. - """ - def __init__(self) : - """The store will just be a dict with an entry for each site, that - entry will contain dicts and lists of data. - """ - self._store = {} - - def open(self, site): - self._store[site] = {} - - def close_site(self, site): - del self._store[site] - - def store(self, site, key, url, parent=None, version=None, post_version=None): - checked = datetime.now() - self._store[key] = (version, checked) - - def status(self, site, key): - """Get the version and last checked time for a key. - - this will be changed later to support checking (lazily) the last modified - time and perhaps other statistics needed by the scheduling algorithms - """ - return self._store.get(key) diff --git a/scrapy/contrib_exp/history/middleware.py b/scrapy/contrib_exp/history/middleware.py deleted file mode 100644 index 32dde7913..000000000 --- a/scrapy/contrib_exp/history/middleware.py +++ /dev/null @@ -1,92 +0,0 @@ -import hashlib -from datetime import datetime - -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.utils.misc import load_object -from scrapy.core import signals -from scrapy import log -from scrapy.core.exceptions import NotConfigured, IgnoreRequest -from scrapy.conf import settings - -class HistoryMiddleware(object): - # How often we should re-check links we know about - MIN_CHECK_DAYS = 4 - # How often we should process pages that have not changed (need to include depth) - MIN_PROCESS_UNCHANGED_DAYS = 12 - - MEMORYSTORE = 'scrapy.contrib_exp.history.memorystore.MemoryStore' - - def __init__(self): - historycls = load_object(self.MEMORYSTORE) - if not historycls: - raise NotConfigured - self.historydata = historycls() - dispatcher.connect(self.open_domain, signal=signals.domain_opened) - dispatcher.connect(self.close_domain, signal=signals.domain_closed) - - def process_request(self, request, spider): - key = urlkey(request.url) - status = self.historydata.status(domain, key) - if status: - _url, version, last_checked = status - d = datetime.now() - last_checked - if d.days < self.MIN_CHECK_DAYS: - raise IgnoreRequest("Not scraping %s (scraped %s ago)" % (request.url, d)) - request.meta['history_response_version'] = version - - def process_response(self, request, response, spider): - version = request.meta.get('history_response_version') - if version == self.get_version(response): - del request.content['history_response_version'] - hist = self.historydata.version_info(domain, version) - if hist: - versionkey, created = hist - # if versionkey != urlkey(url) this means - # the same content is available on a different url - delta = datetime.now() - created - if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS: - message = "skipping %s: unchanged for %s" % (response.url, delta) - raise IgnoreRequest(message) - self.record_visit(domain, request, response) - return response - - def process_exception(self, request, exception, spider): - self.record_visit(spider.domain_name, request, None) - - def open_domain(self, domain): - self.historydata.open(domain) - - def close_domain(self, domain): - self.historydata.close_site(domain) - - def record_visit(self, domain, request, response): - """record the fact that the url has been visited""" - url = request.url - post_version = hash(request.body) - key = urlkey(url) - if response: - redirect_url = response.url - parentkey = urlkey(response.request.headers.get('referer')) if response.request else None - version = self.get_version(response) - else: - redirect_url, parentkey, version = url, None, None - self.historydata.store(domain, key, url, parentkey, version, post_version) - - def get_version(self, response): - key = hashlib.sha1(response.body).hexdigest() - -def urlkey(url): - """Generate a 'key' for a given url - - >>> urlkey("http://www.example.com/") - '89e6a0649e06d83370cdf2cbfb05f363934a8d0c' - >>> urlkey("http://www.example.com/") == urlkey("http://www.example.com/?") - True - """ - from scrapy.utils.c14n import canonicalize - return hash(canonicalize(url)) - - -def hash(value): - return hashlib.sha1(value).hexdigest() if value else None diff --git a/scrapy/contrib_exp/history/scheduler.py b/scrapy/contrib_exp/history/scheduler.py deleted file mode 100644 index 56a858349..000000000 --- a/scrapy/contrib_exp/history/scheduler.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -WARNING: This Scheduler code is obsolete and needs to be rewritten -""" - -import hashlib -from datetime import datetime - -from twisted.internet import defer - -from scrapy import log -from scrapy.core.scheduler import Scheduler -from scrapy.core.exceptions import IgnoreRequest -from scrapy.utils.request import request_fingerprint - -class RulesScheduler(Scheduler): - """Scheduler that uses rules to determine if we should follow links - - TODO: - * take into account where in chain of links we are (less depth should - be crawled more often) - * Be more strict about scraping product pages that rarely lead to new - versions of products. The same applies to pages with links. Particularly - useful for filtering out when there are many urls for the same product. - (but be careful to also filter out pages that almost always lead to new - output). - """ - - # if these parameters change, then update bin/unavailable.py - - # How often we should re-check links we know about - MIN_CHECK_DAYS = 4 - - # How often we should process pages that have not changed (need to include depth) - MIN_PROCESS_UNCHANGED_DAYS = 12 - - def enqueue_request(self, domain, request): - """Add a page to be scraped for a domain that is currently being scraped. - - The url will only be added if we have not checked it already within - a specified time period. - """ - requestid = request_fingerprint(request) - added = self.groupfilter.add(domain, requestid) - - if request.dont_filter or added: - key = urlkey(request.url) # we can not use fingerprint unless lost crawled history - status = self.historydata.status(domain, key) - now = datetime.now() - version = None - if status: - _url, version, last_checked = status - d = now - last_checked - if d.days < self.MIN_CHECK_DAYS: - log.msg("Not scraping %s (scraped %s ago)" % (request.url, d), level=log.DEBUG) - return - # put the version in the pending pages to avoid querying DB again - record = (request, version, now) - self.pending_requests[domain].push(record, request.priority) - - def next_request(self, domain): - """Get the next page from the superclass. This will add a callback - to prevent processing the page unless its content has been - changed. - - In the event that it a page is not processed, the record_visit method - is called to update the last_checked time. - """ - pending_list = self.pending_requests.get(domain) - if not pending_list : - return None - request, version, timestamp = pending_list.get_nowait()[1] - post_version = hash(request.body) - - def callback(pagedata): - """process other callback if we pass the checks""" - - if version == self.get_version(pagedata): - hist = self.historydata.version_info(domain, version) - if hist: - versionkey, created = hist - # if versionkey != urlkey(url) this means - # the same content is available on a different url - delta = timestamp - created - if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS: - message = "skipping %s: unchanged for %s" % (pagedata.url, delta) - raise IgnoreRequest(message) - self.record_visit(domain, request.url, pagedata.url, - pagedata.parent, self.get(pagedata), - post_version) - return pagedata - - def errback(error) : - self.record_visit(domain, request.url, request.url, None, None, - post_version) - return error - - d = defer.Deferred() - d.addCallbacks(callback, errback) - # prepend_callback Request method was removed (it never worked properly anyways) - #request.prepend_callback(d) - - return request - - def get_version(self, response): - key = hashlib.sha1(response.body).hexdigest() diff --git a/scrapy/contrib_exp/history/sqlstore.py b/scrapy/contrib_exp/history/sqlstore.py deleted file mode 100644 index bd9d01681..000000000 --- a/scrapy/contrib_exp/history/sqlstore.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -SQLHistoryStore - -Persistent history storage using relational database -""" -from scrapy.contrib.history.history import URLHistory -from scrapy import log -from scrapy.conf import settings - -class SQLHistoryStore(object) : - """Implementation of a data store that stores information in a relation - database - - This maintains a URLHistory object per site. That means each domain - has it's own session and is isolated from the others. - """ - - def __init__(self): - self._store = {} - self._dbinfo = settings['SCRAPING_DB'] - self._debug = settings['DEBUG_SQL_HISTORY_STORE'] - - def open(self, site): - self._store[site] = URLHistory(self._dbinfo) - - def close_site(self, site): - self._store[site].close() - del self._store[site] - - def store(self, site, key, url, parent=None, version=None, post_version=None): - history = self._store[site] - if self._debug: - log.msg("record_version(key=%s, url=%s, parent=%s, version=%s, post_version=%s)" % - (key, url, parent, version, post_version), domain=site, level=log.DEBUG) - history.record_version(key, url, parent, version, post_version) - - def has_site(self, site): - return site in self._store - - def status(self, site, key): - if site in self._store: - history = self._store[site] - return history.get_url_status(key) - - def version_info(self, site, version): - history = self._store[site] - return history.get_version_info(version)