mirror of https://github.com/scrapy/scrapy.git
guid: cleanup guid attribute references including removal of replays and deprecated commands
--HG-- rename : scrapy/trunk/scrapy/contrib/history/__init__.py => scrapy/trunk/scrapy/contrib_exp/history/__init__.py rename : scrapy/trunk/scrapy/contrib/history/history.py => scrapy/trunk/scrapy/contrib_exp/history/history.py rename : scrapy/trunk/scrapy/contrib/history/middleware.py => scrapy/trunk/scrapy/contrib_exp/history/middleware.py rename : scrapy/trunk/scrapy/contrib/history/scheduler.py => scrapy/trunk/scrapy/contrib_exp/history/scheduler.py rename : scrapy/trunk/scrapy/contrib/history/store.py => scrapy/trunk/scrapy/contrib_exp/history/store.py rename : scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py => scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40881
This commit is contained in:
Daniel Grana
parent
81ce9bd458
commit
7fc8d590f3
|
|
@ -1,25 +0,0 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.fetcher import fetch
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
def get_item_attr(pagedata, attr="guid"):
|
||||
spider = spiders.fromurl(pagedata.url)
|
||||
items = spider.parse(pagedata)
|
||||
attrs = [getattr(i, attr) for i in items if isinstance(i, ScrapedItem)]
|
||||
return attrs
|
||||
|
||||
def get_attr(url, attr="guid"):
|
||||
pagedatas = fetch([url])
|
||||
if pagedatas:
|
||||
return get_item_attr(pagedatas[0], attr)
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<url> <attribute>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Print an attribute from the item scraped in the given URL"
|
||||
|
||||
def run(self, args, opts):
|
||||
print get_attr(args[0], args[1])
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
import os
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.replay import Replay
|
||||
from scrapy.utils import display
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command import cmdline
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <replay_file> [action]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Replay a session previously recorded with crawl --record"
|
||||
|
||||
def help(self):
|
||||
s = "Replay a session previously recorded with crawl --record\n"
|
||||
s += "\n"
|
||||
s += "Available actions:\n"
|
||||
s += " crawl: just replay the crawl (default if action omitted)\n"
|
||||
s += " diff: replay the crawl and show differences in items scraped/passed\n"
|
||||
s += " update: replay the crawl and update both scraped and passed items\n"
|
||||
s += " showitems: show items stored\n"
|
||||
s += " showpages: show all responses downloaded (not only HTML pages)\n"
|
||||
return s
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show verbose output (full items/responses)")
|
||||
parser.add_option("-t", "--item-type", dest="itype", help="item type (scraped, passed). default: scraped", metavar="TYPE")
|
||||
parser.add_option("--output", dest="outfile", help="write output to FILE. if omitted uses stdout", metavar="FILE")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true", help="disable colorized output (for console only)")
|
||||
parser.add_option("-i", "--ignore", dest="ignores", action="append", help="item attribute to ignore. can be passed multiple times", metavar="ATTR")
|
||||
parser.add_option("--target", dest="targets", action="append", help="crawl TARGET instead of recorded urls/domains. can be passed multiple times")
|
||||
# adding option to update
|
||||
parser.add_option("--pages", dest="pages", action="store_true", help="update all the pages in the replay file, recording it again.")
|
||||
parser.add_option("-q", "--quiet", dest="quiet", action="store_true", help="no verbose mode in option diff.")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
if args:
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
self.opts = opts
|
||||
self.action = args[1] if len(args) > 1 else 'crawl'
|
||||
mode = 'update' if self.action == 'update' else 'play'
|
||||
usedir = args and os.path.isdir(args[0])
|
||||
self.replay = Replay(args[0], mode=mode, usedir=usedir)
|
||||
if self.action not in ['crawl', 'diff', 'update']:
|
||||
settings.overrides['LOG_ENABLED'] = False
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A <replay_dir> is required"
|
||||
return
|
||||
|
||||
display.nocolour = opts.nocolour
|
||||
|
||||
if opts.itype == 'passed':
|
||||
self.before_db = self.replay.passed_old
|
||||
self.now_db = self.replay.passed_new
|
||||
else: # default is 'scraped'
|
||||
opts.itype = 'scraped'
|
||||
self.before_db = self.replay.scraped_old
|
||||
self.now_db = self.replay.scraped_new
|
||||
|
||||
actionfunc = getattr(self, 'action_%s' % self.action, None)
|
||||
if actionfunc:
|
||||
rep = actionfunc(opts)
|
||||
if rep:
|
||||
if opts.outfile:
|
||||
f = open(opts.outfile, "w")
|
||||
f.write(rep)
|
||||
f.close()
|
||||
else:
|
||||
print rep,
|
||||
self.replay.cleanup()
|
||||
else:
|
||||
print "Unknown replay action: %s" % self.action
|
||||
|
||||
def action_crawl(self, opts):
|
||||
self.replay.play(args=opts.targets)
|
||||
|
||||
def action_update(self, opts):
|
||||
self.replay.update(args=opts.targets, opts=opts.__dict__)
|
||||
if (opts.pages):
|
||||
args = ['scrapy-crawl', 'crawl']
|
||||
args.extend(self.replay.options['args'])
|
||||
for k in self.replay.options['opts']:
|
||||
if self.replay.options['opts'][k]:
|
||||
args.append("--%s" % k)
|
||||
if self.replay.options['opts'][k] != True:
|
||||
args.append(self.replay.options['opts'][k])
|
||||
cmdline.execute_with_args(args)
|
||||
|
||||
def action_showitems(self, opts):
|
||||
s = ""
|
||||
s += self._format_items(self.before_db.values())
|
||||
s += ">>> Total: %d items %s\n" % (len(self.before_db), opts.itype)
|
||||
return s
|
||||
|
||||
def action_showpages(self, opts):
|
||||
s = ""
|
||||
if self.opts.verbose:
|
||||
for r in self.replay.responses_old.values():
|
||||
s += ">>> %s\n" % str(r)
|
||||
s += display.pformat(r)
|
||||
else:
|
||||
s += "\n".join([str(r) for r in self.replay.responses_old.values()]) + "\n"
|
||||
s += ">>> Total: %d responses received\n" % len(self.replay.responses_old)
|
||||
return s
|
||||
|
||||
def action_diff(self, opts):
|
||||
self.action_crawl(opts)
|
||||
|
||||
guids_before = set(self.before_db.keys())
|
||||
guids_now = set(map(str, self.now_db.keys()))
|
||||
|
||||
guids_new = guids_now - guids_before
|
||||
guids_missing = guids_before - guids_now
|
||||
guids_both = guids_now & guids_before
|
||||
|
||||
changed_items, chreport = self._report_differences(self.before_db, self.now_db, guids_both)
|
||||
|
||||
ok_items = len(guids_both) - changed_items
|
||||
new_items = len(guids_new)
|
||||
missing_items = len(guids_missing)
|
||||
|
||||
if (new_items - missing_items - changed_items) == 0 and opts.quiet:
|
||||
s = ""
|
||||
else:
|
||||
s = "CRAWLING DIFFERENCES REPORT\n\n"
|
||||
|
||||
s += "Total items : %d\n" % (len(guids_both) + new_items + missing_items)
|
||||
s += " Items OK : %d\n" % ok_items
|
||||
s += " New items : %d\n" % new_items
|
||||
s += " Missing items : %d\n" % missing_items
|
||||
s += " Changed items : %d\n" % changed_items
|
||||
|
||||
s += "\n"
|
||||
s += "- NEW ITEMS (%d) -----------------------------------------\n" % new_items
|
||||
s += self._format_items([self.now_db[g] for g in guids_new])
|
||||
|
||||
s += "\n"
|
||||
s += "- MISSING ITEMS (%d) -------------------------------------\n" % missing_items
|
||||
s += self._format_items([self.before_db[g] for g in guids_missing])
|
||||
|
||||
s += "\n"
|
||||
s += "- CHANGED ITEMS (%d) -------------------------------------\n" % changed_items
|
||||
|
||||
s += chreport
|
||||
|
||||
return s
|
||||
|
||||
def _report_differences(self, old_items, new_items, guids):
|
||||
items_old = [old_items[g] for g in guids]
|
||||
items_new = [new_items[g] for g in guids]
|
||||
|
||||
c = 0
|
||||
s = ""
|
||||
for old, new in zip(items_old, items_new):
|
||||
d = self._item_diff(old, new)
|
||||
if d:
|
||||
c += 1
|
||||
s += d
|
||||
return c, s
|
||||
|
||||
def _item_diff(self, old, new):
|
||||
delta = new - old
|
||||
|
||||
s = ""
|
||||
if delta.diff:
|
||||
s += ">>> Item guid=%s name=%s\n" % (old.guid, old.name)
|
||||
s += display.pformat(delta.diff) + "\n"
|
||||
return s
|
||||
|
||||
def _format_items(self, items):
|
||||
if self.opts.verbose:
|
||||
s = display.pformat(items)
|
||||
else:
|
||||
s = ""
|
||||
for i in items:
|
||||
s += "%s\n" % str(i)
|
||||
s += " <%s>\n" % i.url
|
||||
return s
|
||||
|
||||
|
|
@ -1,207 +0,0 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import os
|
||||
import cPickle as pickle
|
||||
import shutil
|
||||
import tempfile
|
||||
import tarfile
|
||||
import copy
|
||||
import hashlib
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
class Replay(object):
|
||||
"""
|
||||
This is the Replay extension, used to replay a crawling and see if the
|
||||
items scraped are the same.
|
||||
"""
|
||||
|
||||
def __init__(self, repfile, mode='play', usedir=False):
|
||||
"""
|
||||
Available modes: record, play, update
|
||||
|
||||
repfile can be either a dir (is usedir=True) or a tar.gz file (if
|
||||
usedir=False)
|
||||
"""
|
||||
|
||||
# XXX: this is ugly, and should be removed. but how?
|
||||
cachemw = 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware'
|
||||
if not cachemw in settings['DOWNLOADER_MIDDLEWARES']:
|
||||
raise NotConfigured("Cache middleware must be enabled to use Replay")
|
||||
|
||||
self.recording = mode == 'record'
|
||||
self.updating = mode == 'update'
|
||||
self.playing = mode == 'play'
|
||||
|
||||
self.options = {}
|
||||
self.scraped_old = {}
|
||||
self.scraped_new = {}
|
||||
self.passed_old = {}
|
||||
self.passed_new = {}
|
||||
self.responses_old = {}
|
||||
self.responses_new = {}
|
||||
|
||||
#self._opendb(repfile, usedir)
|
||||
self._load(repfile, usedir)
|
||||
|
||||
settings.overrides['CACHE2_DIR'] = self.cache2dir
|
||||
settings.overrides['CACHE2_EXPIRATION_SECS'] = -1
|
||||
settings.overrides['CACHE2_IGNORE_MISSING'] = self.playing or self.updating
|
||||
settings.overrides['CACHE2_SECTORIZE'] = False
|
||||
|
||||
dispatcher.connect(self.engine_started, signal=signals.engine_started)
|
||||
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
|
||||
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
|
||||
dispatcher.connect(self.item_passed, signal=signals.item_passed)
|
||||
dispatcher.connect(self.response_received, signal=signals.response_received)
|
||||
|
||||
|
||||
def play(self, args=None, opts=None):
|
||||
if self.recording:
|
||||
raise ValueError("Replay.play() not available in record mode")
|
||||
|
||||
args = args or self.options['args']
|
||||
opts = opts or self.options['opts']
|
||||
scrapymanager.runonce(*args, **opts)
|
||||
|
||||
def record(self, args=None, opts=None):
|
||||
self.recording = True
|
||||
self.options.clear()
|
||||
self.options['args'] = args or []
|
||||
self.options['opts'] = opts or {}
|
||||
|
||||
if os.path.exists(self.cache2dir):
|
||||
shutil.rmtree(self.cache2dir)
|
||||
else:
|
||||
os.mkdir(self.cache2dir)
|
||||
|
||||
def update(self, args=None, opts=None):
|
||||
self.updating = True
|
||||
if (opts['pages']):
|
||||
args = self.options['args']
|
||||
opts = self.options['opts']
|
||||
settings.overrides['CACHE2_EXPIRATION_SECS'] = 0
|
||||
settings.overrides['CACHE2_IGNORE_MISSING'] = False
|
||||
else:
|
||||
self.play(args, opts)
|
||||
|
||||
def engine_started(self):
|
||||
log.msg("Replay: recording session in %s" % self.repfile)
|
||||
|
||||
def engine_stopped(self):
|
||||
if self.recording or self.updating:
|
||||
log.msg("Replay: recorded in %s: %d/%d scraped/passed items, %d received responses" % \
|
||||
(self.repfile, len(self.scraped_old), len(self.passed_old), len(self.responses_old)))
|
||||
self._save()
|
||||
self.cleanup()
|
||||
|
||||
def clean_private(self, src):
|
||||
if isinstance(src, ScrapedItem):
|
||||
target = src.__dict__
|
||||
elif isinstance(src, (list, tuple)):
|
||||
return type(src)(self.clean_private(elem) for elem in src)
|
||||
elif isinstance(src, dict):
|
||||
target = src
|
||||
else:
|
||||
return src
|
||||
for key, val in target.items():
|
||||
if key[0] == '_' and key[1] != '_':
|
||||
del target[key]
|
||||
else:
|
||||
target[key] = self.clean_private(val)
|
||||
return src
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
item = self.clean_private(copy.deepcopy(item))
|
||||
if self.recording or self.updating:
|
||||
self.scraped_old[str(item.guid)] = item
|
||||
else:
|
||||
self.scraped_new[str(item.guid)] = item
|
||||
|
||||
def item_passed(self, item, spider):
|
||||
item = self.clean_private(copy.deepcopy(item))
|
||||
if self.recording or self.updating:
|
||||
self.passed_old[str(item.guid)] = item
|
||||
else:
|
||||
self.passed_new[str(item.guid)] = item
|
||||
|
||||
def response_received(self, response, spider):
|
||||
key = hashlib.sha1(response.body).hexdigest()
|
||||
if (self.recording or self.updating) and key:
|
||||
self.responses_old[key] = response.copy()
|
||||
elif key:
|
||||
self.responses_new[key] = response.copy()
|
||||
|
||||
def _load(self, repfile, usedir):
|
||||
if usedir:
|
||||
if os.path.isdir(repfile):
|
||||
replay_dir = repfile
|
||||
elif self.recording:
|
||||
os.makedirs(repfile)
|
||||
replay_dir = repfile
|
||||
else:
|
||||
raise IOError("No such dir: " % repfile)
|
||||
elif os.path.exists(repfile) and (self.playing or self.updating):
|
||||
if tarfile.is_tarfile(repfile):
|
||||
replay_dir = tempfile.mkdtemp(prefix="replay-")
|
||||
tar = tarfile.open(repfile)
|
||||
tar.extractall(replay_dir)
|
||||
tar.close()
|
||||
else:
|
||||
raise IOError("Wrong tarfile: %s" % repfile)
|
||||
elif self.recording:
|
||||
replay_dir = tempfile.mkdtemp(prefix="replay-")
|
||||
else:
|
||||
raise IOError("No such file: %s" % repfile)
|
||||
|
||||
self.cache2dir = os.path.join(replay_dir, "httpcache")
|
||||
self.replay_dir = replay_dir
|
||||
self.repfile = repfile
|
||||
self.usedir = usedir
|
||||
|
||||
self.options_path = os.path.join(replay_dir, "options.db")
|
||||
self.scraped_path = os.path.join(replay_dir, "items_scraped.db")
|
||||
self.passed_path = os.path.join(replay_dir, "items_passed.db")
|
||||
self.responses_path = os.path.join(replay_dir, "responses.db")
|
||||
|
||||
if self.updating:
|
||||
with open(self.options_path, 'r') as f:
|
||||
self.options = pickle.load(f)
|
||||
if self.playing:
|
||||
with open(self.options_path, 'r') as f:
|
||||
self.options = pickle.load(f)
|
||||
with open(self.responses_path, 'r') as f:
|
||||
self.responses_old = pickle.load(f)
|
||||
with open(self.scraped_path, 'r') as f:
|
||||
self.scraped_old = pickle.load(f)
|
||||
with open(self.passed_path, 'r') as f:
|
||||
self.passed_old = pickle.load(f)
|
||||
|
||||
def _save(self):
|
||||
if self.recording or self.updating:
|
||||
with open(self.options_path, 'w') as f:
|
||||
pickle.dump(self.options, f)
|
||||
with open(self.responses_path, 'w') as f:
|
||||
pickle.dump(self.responses_old, f)
|
||||
with open(self.scraped_path, 'w') as f:
|
||||
pickle.dump(self.scraped_old, f)
|
||||
with open(self.passed_path, 'w') as f:
|
||||
pickle.dump(self.passed_old, f)
|
||||
|
||||
if not self.usedir:
|
||||
if self.recording or self.updating:
|
||||
tar = tarfile.open(self.repfile, "w:gz")
|
||||
for name in os.listdir(self.replay_dir):
|
||||
tar.add(os.path.join(self.replay_dir, name), name)
|
||||
tar.close()
|
||||
|
||||
def cleanup(self):
|
||||
if not self.usedir and os.path.exists(self.replay_dir):
|
||||
shutil.rmtree(self.replay_dir)
|
||||
Loading…
Reference in New Issue