mirror of https://github.com/scrapy/scrapy.git
removed unused module: scrapy.contrib_exp.history
This commit is contained in:
Pablo Hoffman
parent
b65c87cb67
commit
b763350340
|
|
@ -1,3 +0,0 @@
|
|||
from scrapy.contrib.history.history import ItemHistory
|
||||
from scrapy.contrib.history.scheduler import RulesScheduler
|
||||
from scrapy.contrib.history.store import SQLHistoryStore
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
"""
|
||||
History management
|
||||
"""
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import MySQLdb
|
||||
|
||||
class History(object):
|
||||
"""Base class for tracking different kinds of histories"""
|
||||
|
||||
def __init__(self, db_uri):
|
||||
self.db_uri = db_uri
|
||||
self._mysql_conn = None
|
||||
|
||||
def connect(self):
|
||||
"""
|
||||
Connect to PDB and open mysql connect to PRODUCT_DB
|
||||
"""
|
||||
m = re.search(r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/]+)/(?P<db>.*)$", self.db_uri)
|
||||
if m:
|
||||
d = m.groupdict()
|
||||
if d['passwd'] is None:
|
||||
del(d['passwd'])
|
||||
|
||||
d['charset'] = "utf8"
|
||||
self._mysql_conn = MySQLdb.connect(**d)
|
||||
|
||||
def get_mysql_conn(self):
|
||||
if self._mysql_conn is None:
|
||||
self.connect()
|
||||
return self._mysql_conn
|
||||
mysql_conn = property(get_mysql_conn)
|
||||
|
||||
|
||||
class ItemHistory(History):
|
||||
"""
|
||||
Class instance registers item guids, and versions with check_item method.
|
||||
It also gives access to stored ItemTicket and ItemVersion objects.
|
||||
"""
|
||||
NEW = 0
|
||||
UPDATE = 1
|
||||
DUPLICATE = 2
|
||||
|
||||
def check_item(self, domain, item):
|
||||
"""
|
||||
Store item's guid and version to the database along
|
||||
with date and time of last occurence.
|
||||
Return:
|
||||
* ItemHistory.NEW - if guid hasn't been met
|
||||
* ItemHistory.UPDATE - if history already contains guid, but versions doesn't match
|
||||
* ItemHistory.DUPLICATE - both guid and version are not new
|
||||
"""
|
||||
version = item.version
|
||||
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
|
||||
def add_version(version):
|
||||
insert = "INSERT INTO version (guid, version, seen) VALUES (%s,%s,%s)"
|
||||
c.execute(insert, (item.guid, version, datetime.now()))
|
||||
|
||||
select = "SELECT * FROM ticket WHERE guid=%s"
|
||||
c.execute(select, item.guid)
|
||||
r = c.fetchone()
|
||||
if r:
|
||||
select = "SELECT * FROM version WHERE version=%s"
|
||||
if c.execute(select, version):
|
||||
update = "UPDATE version SET seen=%s WHERE version=%s"
|
||||
c.execute(update, (datetime.now(), version))
|
||||
self.mysql_conn.commit()
|
||||
return ItemHistory.DUPLICATE
|
||||
else:
|
||||
add_version(version)
|
||||
self.mysql_conn.commit()
|
||||
return ItemHistory.UPDATE
|
||||
else:
|
||||
insert = "INSERT INTO ticket (guid, domain, url, url_hash) VALUES (%s,%s,%s,%s)"
|
||||
c.execute(insert, (item.guid, domain, item.url, hash(item.url)))
|
||||
add_version(version)
|
||||
self.mysql_conn.commit()
|
||||
return ItemHistory.NEW
|
||||
|
||||
def get_ticket(self, guid):
|
||||
"""
|
||||
Return ItemTicket object for guid.
|
||||
ItemVersion objects can be accessed via 'versions' list.
|
||||
"""
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM ticket WHERE guid=%s"
|
||||
c.execute(select, guid)
|
||||
ticket = c.fetchone()
|
||||
if not ticket:
|
||||
raise Exception("Item ticket with guid = '%s' not found" % guid)
|
||||
ticket['versions'] = []
|
||||
select = "SELECT * FROM version WHERE guid=%s"
|
||||
c.execute(select, guid)
|
||||
for version in c.fetchall():
|
||||
ticket['versions'].append(version)
|
||||
return ticket
|
||||
|
||||
def delete_ticket(self, guid):
|
||||
"""Delete item ticket and associated versions from DB"""
|
||||
c = self.mysql_conn.cursor()
|
||||
delete = "DELETE FROM ticket WHERE guid=%s"
|
||||
c.execute(delete, guid)
|
||||
self.mysql_conn.commit()
|
||||
|
||||
|
||||
class URLHistory(History):
|
||||
"""
|
||||
Access URL status and history for the scraping engine
|
||||
|
||||
This is degsigned to have an instance per domain where typically
|
||||
a call will be made to get_url_status, followed by either
|
||||
update_checked or record_version.
|
||||
"""
|
||||
|
||||
def get_url_status(self, urlkey):
|
||||
"""
|
||||
Get the url status (url, last_version, last_checked),
|
||||
or None if the url data has not been seen before
|
||||
"""
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM url_status WHERE url_hash=%s"
|
||||
c.execute(select, urlkey)
|
||||
r = c.fetchone()
|
||||
return (r['url'], r['last_version'], r['last_checked']) if r else None
|
||||
|
||||
def record_version(self, urlkey, url, parent_key, version, postdata_hash=None):
|
||||
"""
|
||||
Record a version of a page and update the last checked time.
|
||||
|
||||
If the same version (or None) is passed, the last checked time is still updated.
|
||||
"""
|
||||
now = datetime.now()
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM url_status WHERE url_hash=%s"
|
||||
c.execute(select, urlkey)
|
||||
r = c.fetchone()
|
||||
|
||||
if not r:
|
||||
insert = "INSERT INTO url_status (url_hash, url, parent_hash, last_version, last_checked) VALUES (%s,%s,%s,%s,%s)"
|
||||
c.execute(insert, (urlkey, url, parent_key, version, now))
|
||||
else:
|
||||
update = "UPDATE url_status SET last_version=%s, last_checked=%s WHERE url_hash=%s"
|
||||
c.execute(update, (version, now, urlkey))
|
||||
self.mysql_conn.commit()
|
||||
|
||||
last_version = r['last_version'] if r else None
|
||||
if version and version != last_version:
|
||||
if not c.execute("SELECT url_hash FROM url_history WHERE version=%s", version):
|
||||
insert = "INSERT INTO url_history (url_hash, version, postdata_hash, created) VALUES (%s,%s,%s,%s)"
|
||||
c.execute(insert, (urlkey, version, postdata_hash, now))
|
||||
self.mysql_conn.commit()
|
||||
|
||||
def get_version_info(self, version):
|
||||
"""Simple accessor method"""
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM url_history WHERE version=%s"
|
||||
c.execute(select, version)
|
||||
r = c.fetchone()
|
||||
return (r['url_hash'], r['created']) if r else None
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
"""
|
||||
The datastore module contains implementations of data storage engines for the
|
||||
crawling process. These are used to track metrics on each site and on each
|
||||
page visited.
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
class MemoryStore(object) :
|
||||
"""Simple implementation of a data store. This is useful if no persistent
|
||||
history is required (e.g. unit testing or development) and provides a
|
||||
simpler reference implementation.
|
||||
"""
|
||||
def __init__(self) :
|
||||
"""The store will just be a dict with an entry for each site, that
|
||||
entry will contain dicts and lists of data.
|
||||
"""
|
||||
self._store = {}
|
||||
|
||||
def open(self, site):
|
||||
self._store[site] = {}
|
||||
|
||||
def close_site(self, site):
|
||||
del self._store[site]
|
||||
|
||||
def store(self, site, key, url, parent=None, version=None, post_version=None):
|
||||
checked = datetime.now()
|
||||
self._store[key] = (version, checked)
|
||||
|
||||
def status(self, site, key):
|
||||
"""Get the version and last checked time for a key.
|
||||
|
||||
this will be changed later to support checking (lazily) the last modified
|
||||
time and perhaps other statistics needed by the scheduling algorithms
|
||||
"""
|
||||
return self._store.get(key)
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
import hashlib
|
||||
from datetime import datetime
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.conf import settings
|
||||
|
||||
class HistoryMiddleware(object):
|
||||
# How often we should re-check links we know about
|
||||
MIN_CHECK_DAYS = 4
|
||||
# How often we should process pages that have not changed (need to include depth)
|
||||
MIN_PROCESS_UNCHANGED_DAYS = 12
|
||||
|
||||
MEMORYSTORE = 'scrapy.contrib_exp.history.memorystore.MemoryStore'
|
||||
|
||||
def __init__(self):
|
||||
historycls = load_object(self.MEMORYSTORE)
|
||||
if not historycls:
|
||||
raise NotConfigured
|
||||
self.historydata = historycls()
|
||||
dispatcher.connect(self.open_domain, signal=signals.domain_opened)
|
||||
dispatcher.connect(self.close_domain, signal=signals.domain_closed)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
key = urlkey(request.url)
|
||||
status = self.historydata.status(domain, key)
|
||||
if status:
|
||||
_url, version, last_checked = status
|
||||
d = datetime.now() - last_checked
|
||||
if d.days < self.MIN_CHECK_DAYS:
|
||||
raise IgnoreRequest("Not scraping %s (scraped %s ago)" % (request.url, d))
|
||||
request.meta['history_response_version'] = version
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
version = request.meta.get('history_response_version')
|
||||
if version == self.get_version(response):
|
||||
del request.content['history_response_version']
|
||||
hist = self.historydata.version_info(domain, version)
|
||||
if hist:
|
||||
versionkey, created = hist
|
||||
# if versionkey != urlkey(url) this means
|
||||
# the same content is available on a different url
|
||||
delta = datetime.now() - created
|
||||
if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS:
|
||||
message = "skipping %s: unchanged for %s" % (response.url, delta)
|
||||
raise IgnoreRequest(message)
|
||||
self.record_visit(domain, request, response)
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
self.record_visit(spider.domain_name, request, None)
|
||||
|
||||
def open_domain(self, domain):
|
||||
self.historydata.open(domain)
|
||||
|
||||
def close_domain(self, domain):
|
||||
self.historydata.close_site(domain)
|
||||
|
||||
def record_visit(self, domain, request, response):
|
||||
"""record the fact that the url has been visited"""
|
||||
url = request.url
|
||||
post_version = hash(request.body)
|
||||
key = urlkey(url)
|
||||
if response:
|
||||
redirect_url = response.url
|
||||
parentkey = urlkey(response.request.headers.get('referer')) if response.request else None
|
||||
version = self.get_version(response)
|
||||
else:
|
||||
redirect_url, parentkey, version = url, None, None
|
||||
self.historydata.store(domain, key, url, parentkey, version, post_version)
|
||||
|
||||
def get_version(self, response):
|
||||
key = hashlib.sha1(response.body).hexdigest()
|
||||
|
||||
def urlkey(url):
|
||||
"""Generate a 'key' for a given url
|
||||
|
||||
>>> urlkey("http://www.example.com/")
|
||||
'89e6a0649e06d83370cdf2cbfb05f363934a8d0c'
|
||||
>>> urlkey("http://www.example.com/") == urlkey("http://www.example.com/?")
|
||||
True
|
||||
"""
|
||||
from scrapy.utils.c14n import canonicalize
|
||||
return hash(canonicalize(url))
|
||||
|
||||
|
||||
def hash(value):
|
||||
return hashlib.sha1(value).hexdigest() if value else None
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
"""
|
||||
WARNING: This Scheduler code is obsolete and needs to be rewritten
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.core.scheduler import Scheduler
|
||||
from scrapy.core.exceptions import IgnoreRequest
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
class RulesScheduler(Scheduler):
|
||||
"""Scheduler that uses rules to determine if we should follow links
|
||||
|
||||
TODO:
|
||||
* take into account where in chain of links we are (less depth should
|
||||
be crawled more often)
|
||||
* Be more strict about scraping product pages that rarely lead to new
|
||||
versions of products. The same applies to pages with links. Particularly
|
||||
useful for filtering out when there are many urls for the same product.
|
||||
(but be careful to also filter out pages that almost always lead to new
|
||||
output).
|
||||
"""
|
||||
|
||||
# if these parameters change, then update bin/unavailable.py
|
||||
|
||||
# How often we should re-check links we know about
|
||||
MIN_CHECK_DAYS = 4
|
||||
|
||||
# How often we should process pages that have not changed (need to include depth)
|
||||
MIN_PROCESS_UNCHANGED_DAYS = 12
|
||||
|
||||
def enqueue_request(self, domain, request):
|
||||
"""Add a page to be scraped for a domain that is currently being scraped.
|
||||
|
||||
The url will only be added if we have not checked it already within
|
||||
a specified time period.
|
||||
"""
|
||||
requestid = request_fingerprint(request)
|
||||
added = self.groupfilter.add(domain, requestid)
|
||||
|
||||
if request.dont_filter or added:
|
||||
key = urlkey(request.url) # we can not use fingerprint unless lost crawled history
|
||||
status = self.historydata.status(domain, key)
|
||||
now = datetime.now()
|
||||
version = None
|
||||
if status:
|
||||
_url, version, last_checked = status
|
||||
d = now - last_checked
|
||||
if d.days < self.MIN_CHECK_DAYS:
|
||||
log.msg("Not scraping %s (scraped %s ago)" % (request.url, d), level=log.DEBUG)
|
||||
return
|
||||
# put the version in the pending pages to avoid querying DB again
|
||||
record = (request, version, now)
|
||||
self.pending_requests[domain].push(record, request.priority)
|
||||
|
||||
def next_request(self, domain):
|
||||
"""Get the next page from the superclass. This will add a callback
|
||||
to prevent processing the page unless its content has been
|
||||
changed.
|
||||
|
||||
In the event that it a page is not processed, the record_visit method
|
||||
is called to update the last_checked time.
|
||||
"""
|
||||
pending_list = self.pending_requests.get(domain)
|
||||
if not pending_list :
|
||||
return None
|
||||
request, version, timestamp = pending_list.get_nowait()[1]
|
||||
post_version = hash(request.body)
|
||||
|
||||
def callback(pagedata):
|
||||
"""process other callback if we pass the checks"""
|
||||
|
||||
if version == self.get_version(pagedata):
|
||||
hist = self.historydata.version_info(domain, version)
|
||||
if hist:
|
||||
versionkey, created = hist
|
||||
# if versionkey != urlkey(url) this means
|
||||
# the same content is available on a different url
|
||||
delta = timestamp - created
|
||||
if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS:
|
||||
message = "skipping %s: unchanged for %s" % (pagedata.url, delta)
|
||||
raise IgnoreRequest(message)
|
||||
self.record_visit(domain, request.url, pagedata.url,
|
||||
pagedata.parent, self.get(pagedata),
|
||||
post_version)
|
||||
return pagedata
|
||||
|
||||
def errback(error) :
|
||||
self.record_visit(domain, request.url, request.url, None, None,
|
||||
post_version)
|
||||
return error
|
||||
|
||||
d = defer.Deferred()
|
||||
d.addCallbacks(callback, errback)
|
||||
# prepend_callback Request method was removed (it never worked properly anyways)
|
||||
#request.prepend_callback(d)
|
||||
|
||||
return request
|
||||
|
||||
def get_version(self, response):
|
||||
key = hashlib.sha1(response.body).hexdigest()
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
"""
|
||||
SQLHistoryStore
|
||||
|
||||
Persistent history storage using relational database
|
||||
"""
|
||||
from scrapy.contrib.history.history import URLHistory
|
||||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
|
||||
class SQLHistoryStore(object) :
|
||||
"""Implementation of a data store that stores information in a relation
|
||||
database
|
||||
|
||||
This maintains a URLHistory object per site. That means each domain
|
||||
has it's own session and is isolated from the others.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._store = {}
|
||||
self._dbinfo = settings['SCRAPING_DB']
|
||||
self._debug = settings['DEBUG_SQL_HISTORY_STORE']
|
||||
|
||||
def open(self, site):
|
||||
self._store[site] = URLHistory(self._dbinfo)
|
||||
|
||||
def close_site(self, site):
|
||||
self._store[site].close()
|
||||
del self._store[site]
|
||||
|
||||
def store(self, site, key, url, parent=None, version=None, post_version=None):
|
||||
history = self._store[site]
|
||||
if self._debug:
|
||||
log.msg("record_version(key=%s, url=%s, parent=%s, version=%s, post_version=%s)" %
|
||||
(key, url, parent, version, post_version), domain=site, level=log.DEBUG)
|
||||
history.record_version(key, url, parent, version, post_version)
|
||||
|
||||
def has_site(self, site):
|
||||
return site in self._store
|
||||
|
||||
def status(self, site, key):
|
||||
if site in self._store:
|
||||
history = self._store[site]
|
||||
return history.get_url_status(key)
|
||||
|
||||
def version_info(self, site, version):
|
||||
history = self._store[site]
|
||||
return history.get_version_info(version)
|
||||
Loading…
Reference in New Issue