From 7fc8d590f3dc091ae3e6cdfec18145af4e110b3c Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 19 Feb 2009 14:22:23 +0000 Subject: [PATCH] guid: cleanup guid attribute references including removal of replays and deprecated commands --HG-- rename : scrapy/trunk/scrapy/contrib/history/__init__.py => scrapy/trunk/scrapy/contrib_exp/history/__init__.py rename : scrapy/trunk/scrapy/contrib/history/history.py => scrapy/trunk/scrapy/contrib_exp/history/history.py rename : scrapy/trunk/scrapy/contrib/history/middleware.py => scrapy/trunk/scrapy/contrib_exp/history/middleware.py rename : scrapy/trunk/scrapy/contrib/history/scheduler.py => scrapy/trunk/scrapy/contrib_exp/history/scheduler.py rename : scrapy/trunk/scrapy/contrib/history/store.py => scrapy/trunk/scrapy/contrib_exp/history/store.py rename : scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py => scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40881 --- .../trunk/scrapy/command/commands/getattr.py | 25 --- .../trunk/scrapy/command/commands/replay.py | 184 ---------------- .../history/__init__.py | 0 .../history/history.py | 0 .../history/middleware.py | 0 .../history/scheduler.py | 0 .../{contrib => contrib_exp}/history/store.py | 0 .../pipeline/shoveitem.py | 0 scrapy/trunk/scrapy/replay/__init__.py | 207 ------------------ 9 files changed, 416 deletions(-) delete mode 100644 scrapy/trunk/scrapy/command/commands/getattr.py delete mode 100644 scrapy/trunk/scrapy/command/commands/replay.py rename scrapy/trunk/scrapy/{contrib => contrib_exp}/history/__init__.py (100%) rename scrapy/trunk/scrapy/{contrib => contrib_exp}/history/history.py (100%) rename scrapy/trunk/scrapy/{contrib => contrib_exp}/history/middleware.py (100%) rename scrapy/trunk/scrapy/{contrib => contrib_exp}/history/scheduler.py (100%) rename scrapy/trunk/scrapy/{contrib => contrib_exp}/history/store.py (100%) rename scrapy/trunk/scrapy/{contrib => contrib_exp}/pipeline/shoveitem.py (100%) delete mode 100644 scrapy/trunk/scrapy/replay/__init__.py diff --git a/scrapy/trunk/scrapy/command/commands/getattr.py b/scrapy/trunk/scrapy/command/commands/getattr.py deleted file mode 100644 index ac639bf3d..000000000 --- a/scrapy/trunk/scrapy/command/commands/getattr.py +++ /dev/null @@ -1,25 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.fetcher import fetch -from scrapy.spider import spiders -from scrapy.item import ScrapedItem - -def get_item_attr(pagedata, attr="guid"): - spider = spiders.fromurl(pagedata.url) - items = spider.parse(pagedata) - attrs = [getattr(i, attr) for i in items if isinstance(i, ScrapedItem)] - return attrs - -def get_attr(url, attr="guid"): - pagedatas = fetch([url]) - if pagedatas: - return get_item_attr(pagedatas[0], attr) - -class Command(ScrapyCommand): - def syntax(self): - return " " - - def short_desc(self): - return "Print an attribute from the item scraped in the given URL" - - def run(self, args, opts): - print get_attr(args[0], args[1]) diff --git a/scrapy/trunk/scrapy/command/commands/replay.py b/scrapy/trunk/scrapy/command/commands/replay.py deleted file mode 100644 index 590f976a9..000000000 --- a/scrapy/trunk/scrapy/command/commands/replay.py +++ /dev/null @@ -1,184 +0,0 @@ -import os - -from scrapy.command import ScrapyCommand -from scrapy.replay import Replay -from scrapy.utils import display -from scrapy.conf import settings -from scrapy.command import cmdline - -class Command(ScrapyCommand): - def syntax(self): - return "[options] [action]" - - def short_desc(self): - return "Replay a session previously recorded with crawl --record" - - def help(self): - s = "Replay a session previously recorded with crawl --record\n" - s += "\n" - s += "Available actions:\n" - s += " crawl: just replay the crawl (default if action omitted)\n" - s += " diff: replay the crawl and show differences in items scraped/passed\n" - s += " update: replay the crawl and update both scraped and passed items\n" - s += " showitems: show items stored\n" - s += " showpages: show all responses downloaded (not only HTML pages)\n" - return s - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show verbose output (full items/responses)") - parser.add_option("-t", "--item-type", dest="itype", help="item type (scraped, passed). default: scraped", metavar="TYPE") - parser.add_option("--output", dest="outfile", help="write output to FILE. if omitted uses stdout", metavar="FILE") - parser.add_option("--nocolour", dest="nocolour", action="store_true", help="disable colorized output (for console only)") - parser.add_option("-i", "--ignore", dest="ignores", action="append", help="item attribute to ignore. can be passed multiple times", metavar="ATTR") - parser.add_option("--target", dest="targets", action="append", help="crawl TARGET instead of recorded urls/domains. can be passed multiple times") - # adding option to update - parser.add_option("--pages", dest="pages", action="store_true", help="update all the pages in the replay file, recording it again.") - parser.add_option("-q", "--quiet", dest="quiet", action="store_true", help="no verbose mode in option diff.") - - def process_options(self, args, opts): - if args: - ScrapyCommand.process_options(self, args, opts) - self.opts = opts - self.action = args[1] if len(args) > 1 else 'crawl' - mode = 'update' if self.action == 'update' else 'play' - usedir = args and os.path.isdir(args[0]) - self.replay = Replay(args[0], mode=mode, usedir=usedir) - if self.action not in ['crawl', 'diff', 'update']: - settings.overrides['LOG_ENABLED'] = False - - def run(self, args, opts): - if not args: - print "A is required" - return - - display.nocolour = opts.nocolour - - if opts.itype == 'passed': - self.before_db = self.replay.passed_old - self.now_db = self.replay.passed_new - else: # default is 'scraped' - opts.itype = 'scraped' - self.before_db = self.replay.scraped_old - self.now_db = self.replay.scraped_new - - actionfunc = getattr(self, 'action_%s' % self.action, None) - if actionfunc: - rep = actionfunc(opts) - if rep: - if opts.outfile: - f = open(opts.outfile, "w") - f.write(rep) - f.close() - else: - print rep, - self.replay.cleanup() - else: - print "Unknown replay action: %s" % self.action - - def action_crawl(self, opts): - self.replay.play(args=opts.targets) - - def action_update(self, opts): - self.replay.update(args=opts.targets, opts=opts.__dict__) - if (opts.pages): - args = ['scrapy-crawl', 'crawl'] - args.extend(self.replay.options['args']) - for k in self.replay.options['opts']: - if self.replay.options['opts'][k]: - args.append("--%s" % k) - if self.replay.options['opts'][k] != True: - args.append(self.replay.options['opts'][k]) - cmdline.execute_with_args(args) - - def action_showitems(self, opts): - s = "" - s += self._format_items(self.before_db.values()) - s += ">>> Total: %d items %s\n" % (len(self.before_db), opts.itype) - return s - - def action_showpages(self, opts): - s = "" - if self.opts.verbose: - for r in self.replay.responses_old.values(): - s += ">>> %s\n" % str(r) - s += display.pformat(r) - else: - s += "\n".join([str(r) for r in self.replay.responses_old.values()]) + "\n" - s += ">>> Total: %d responses received\n" % len(self.replay.responses_old) - return s - - def action_diff(self, opts): - self.action_crawl(opts) - - guids_before = set(self.before_db.keys()) - guids_now = set(map(str, self.now_db.keys())) - - guids_new = guids_now - guids_before - guids_missing = guids_before - guids_now - guids_both = guids_now & guids_before - - changed_items, chreport = self._report_differences(self.before_db, self.now_db, guids_both) - - ok_items = len(guids_both) - changed_items - new_items = len(guids_new) - missing_items = len(guids_missing) - - if (new_items - missing_items - changed_items) == 0 and opts.quiet: - s = "" - else: - s = "CRAWLING DIFFERENCES REPORT\n\n" - - s += "Total items : %d\n" % (len(guids_both) + new_items + missing_items) - s += " Items OK : %d\n" % ok_items - s += " New items : %d\n" % new_items - s += " Missing items : %d\n" % missing_items - s += " Changed items : %d\n" % changed_items - - s += "\n" - s += "- NEW ITEMS (%d) -----------------------------------------\n" % new_items - s += self._format_items([self.now_db[g] for g in guids_new]) - - s += "\n" - s += "- MISSING ITEMS (%d) -------------------------------------\n" % missing_items - s += self._format_items([self.before_db[g] for g in guids_missing]) - - s += "\n" - s += "- CHANGED ITEMS (%d) -------------------------------------\n" % changed_items - - s += chreport - - return s - - def _report_differences(self, old_items, new_items, guids): - items_old = [old_items[g] for g in guids] - items_new = [new_items[g] for g in guids] - - c = 0 - s = "" - for old, new in zip(items_old, items_new): - d = self._item_diff(old, new) - if d: - c += 1 - s += d - return c, s - - def _item_diff(self, old, new): - delta = new - old - - s = "" - if delta.diff: - s += ">>> Item guid=%s name=%s\n" % (old.guid, old.name) - s += display.pformat(delta.diff) + "\n" - return s - - def _format_items(self, items): - if self.opts.verbose: - s = display.pformat(items) - else: - s = "" - for i in items: - s += "%s\n" % str(i) - s += " <%s>\n" % i.url - return s - diff --git a/scrapy/trunk/scrapy/contrib/history/__init__.py b/scrapy/trunk/scrapy/contrib_exp/history/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/history/__init__.py rename to scrapy/trunk/scrapy/contrib_exp/history/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/history/history.py b/scrapy/trunk/scrapy/contrib_exp/history/history.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/history/history.py rename to scrapy/trunk/scrapy/contrib_exp/history/history.py diff --git a/scrapy/trunk/scrapy/contrib/history/middleware.py b/scrapy/trunk/scrapy/contrib_exp/history/middleware.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/history/middleware.py rename to scrapy/trunk/scrapy/contrib_exp/history/middleware.py diff --git a/scrapy/trunk/scrapy/contrib/history/scheduler.py b/scrapy/trunk/scrapy/contrib_exp/history/scheduler.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/history/scheduler.py rename to scrapy/trunk/scrapy/contrib_exp/history/scheduler.py diff --git a/scrapy/trunk/scrapy/contrib/history/store.py b/scrapy/trunk/scrapy/contrib_exp/history/store.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/history/store.py rename to scrapy/trunk/scrapy/contrib_exp/history/store.py diff --git a/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py b/scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py rename to scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py diff --git a/scrapy/trunk/scrapy/replay/__init__.py b/scrapy/trunk/scrapy/replay/__init__.py deleted file mode 100644 index 7e9cd2525..000000000 --- a/scrapy/trunk/scrapy/replay/__init__.py +++ /dev/null @@ -1,207 +0,0 @@ -from __future__ import with_statement - -import os -import cPickle as pickle -import shutil -import tempfile -import tarfile -import copy -import hashlib - -from pydispatch import dispatcher - -from scrapy import log -from scrapy.core import signals -from scrapy.core.manager import scrapymanager -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings -from scrapy.item import ScrapedItem - -class Replay(object): - """ - This is the Replay extension, used to replay a crawling and see if the - items scraped are the same. - """ - - def __init__(self, repfile, mode='play', usedir=False): - """ - Available modes: record, play, update - - repfile can be either a dir (is usedir=True) or a tar.gz file (if - usedir=False) - """ - - # XXX: this is ugly, and should be removed. but how? - cachemw = 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware' - if not cachemw in settings['DOWNLOADER_MIDDLEWARES']: - raise NotConfigured("Cache middleware must be enabled to use Replay") - - self.recording = mode == 'record' - self.updating = mode == 'update' - self.playing = mode == 'play' - - self.options = {} - self.scraped_old = {} - self.scraped_new = {} - self.passed_old = {} - self.passed_new = {} - self.responses_old = {} - self.responses_new = {} - - #self._opendb(repfile, usedir) - self._load(repfile, usedir) - - settings.overrides['CACHE2_DIR'] = self.cache2dir - settings.overrides['CACHE2_EXPIRATION_SECS'] = -1 - settings.overrides['CACHE2_IGNORE_MISSING'] = self.playing or self.updating - settings.overrides['CACHE2_SECTORIZE'] = False - - dispatcher.connect(self.engine_started, signal=signals.engine_started) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.item_passed, signal=signals.item_passed) - dispatcher.connect(self.response_received, signal=signals.response_received) - - - def play(self, args=None, opts=None): - if self.recording: - raise ValueError("Replay.play() not available in record mode") - - args = args or self.options['args'] - opts = opts or self.options['opts'] - scrapymanager.runonce(*args, **opts) - - def record(self, args=None, opts=None): - self.recording = True - self.options.clear() - self.options['args'] = args or [] - self.options['opts'] = opts or {} - - if os.path.exists(self.cache2dir): - shutil.rmtree(self.cache2dir) - else: - os.mkdir(self.cache2dir) - - def update(self, args=None, opts=None): - self.updating = True - if (opts['pages']): - args = self.options['args'] - opts = self.options['opts'] - settings.overrides['CACHE2_EXPIRATION_SECS'] = 0 - settings.overrides['CACHE2_IGNORE_MISSING'] = False - else: - self.play(args, opts) - - def engine_started(self): - log.msg("Replay: recording session in %s" % self.repfile) - - def engine_stopped(self): - if self.recording or self.updating: - log.msg("Replay: recorded in %s: %d/%d scraped/passed items, %d received responses" % \ - (self.repfile, len(self.scraped_old), len(self.passed_old), len(self.responses_old))) - self._save() - self.cleanup() - - def clean_private(self, src): - if isinstance(src, ScrapedItem): - target = src.__dict__ - elif isinstance(src, (list, tuple)): - return type(src)(self.clean_private(elem) for elem in src) - elif isinstance(src, dict): - target = src - else: - return src - for key, val in target.items(): - if key[0] == '_' and key[1] != '_': - del target[key] - else: - target[key] = self.clean_private(val) - return src - - def item_scraped(self, item, spider): - item = self.clean_private(copy.deepcopy(item)) - if self.recording or self.updating: - self.scraped_old[str(item.guid)] = item - else: - self.scraped_new[str(item.guid)] = item - - def item_passed(self, item, spider): - item = self.clean_private(copy.deepcopy(item)) - if self.recording or self.updating: - self.passed_old[str(item.guid)] = item - else: - self.passed_new[str(item.guid)] = item - - def response_received(self, response, spider): - key = hashlib.sha1(response.body).hexdigest() - if (self.recording or self.updating) and key: - self.responses_old[key] = response.copy() - elif key: - self.responses_new[key] = response.copy() - - def _load(self, repfile, usedir): - if usedir: - if os.path.isdir(repfile): - replay_dir = repfile - elif self.recording: - os.makedirs(repfile) - replay_dir = repfile - else: - raise IOError("No such dir: " % repfile) - elif os.path.exists(repfile) and (self.playing or self.updating): - if tarfile.is_tarfile(repfile): - replay_dir = tempfile.mkdtemp(prefix="replay-") - tar = tarfile.open(repfile) - tar.extractall(replay_dir) - tar.close() - else: - raise IOError("Wrong tarfile: %s" % repfile) - elif self.recording: - replay_dir = tempfile.mkdtemp(prefix="replay-") - else: - raise IOError("No such file: %s" % repfile) - - self.cache2dir = os.path.join(replay_dir, "httpcache") - self.replay_dir = replay_dir - self.repfile = repfile - self.usedir = usedir - - self.options_path = os.path.join(replay_dir, "options.db") - self.scraped_path = os.path.join(replay_dir, "items_scraped.db") - self.passed_path = os.path.join(replay_dir, "items_passed.db") - self.responses_path = os.path.join(replay_dir, "responses.db") - - if self.updating: - with open(self.options_path, 'r') as f: - self.options = pickle.load(f) - if self.playing: - with open(self.options_path, 'r') as f: - self.options = pickle.load(f) - with open(self.responses_path, 'r') as f: - self.responses_old = pickle.load(f) - with open(self.scraped_path, 'r') as f: - self.scraped_old = pickle.load(f) - with open(self.passed_path, 'r') as f: - self.passed_old = pickle.load(f) - - def _save(self): - if self.recording or self.updating: - with open(self.options_path, 'w') as f: - pickle.dump(self.options, f) - with open(self.responses_path, 'w') as f: - pickle.dump(self.responses_old, f) - with open(self.scraped_path, 'w') as f: - pickle.dump(self.scraped_old, f) - with open(self.passed_path, 'w') as f: - pickle.dump(self.passed_old, f) - - if not self.usedir: - if self.recording or self.updating: - tar = tarfile.open(self.repfile, "w:gz") - for name in os.listdir(self.replay_dir): - tar.add(os.path.join(self.replay_dir, name), name) - tar.close() - - def cleanup(self): - if not self.usedir and os.path.exists(self.replay_dir): - shutil.rmtree(self.replay_dir)