From ba69c29e37cfd2e752ae185abffb64c8a68e3796 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 6 May 2009 15:55:17 -0300 Subject: [PATCH] mv scrapy/trunk to root as part of svn2hg migration --HG-- rename : scrapy/trunk/AUTHORS => AUTHORS rename : scrapy/trunk/INSTALL => INSTALL rename : scrapy/trunk/LICENSE => LICENSE rename : scrapy/trunk/README => README rename : scrapy/trunk/bin/runtests.sh => bin/runtests.sh rename : scrapy/trunk/docs/Makefile => docs/Makefile rename : scrapy/trunk/docs/README => docs/README rename : scrapy/trunk/docs/_ext/scrapydocs.py => docs/_ext/scrapydocs.py rename : scrapy/trunk/docs/_static/items_adaptors-sample1.html => docs/_static/items_adaptors-sample1.html rename : scrapy/trunk/docs/_static/scrapydoc.css => docs/_static/scrapydoc.css rename : scrapy/trunk/docs/_static/selectors-sample1.html => docs/_static/selectors-sample1.html rename : scrapy/trunk/docs/conf.py => docs/conf.py rename : scrapy/trunk/docs/faq.rst => docs/faq.rst rename : scrapy/trunk/docs/index.rst => docs/index.rst rename : scrapy/trunk/docs/intro/index.rst => docs/intro/index.rst rename : scrapy/trunk/docs/intro/install.rst => docs/intro/install.rst rename : scrapy/trunk/docs/intro/overview.rst => docs/intro/overview.rst rename : scrapy/trunk/docs/intro/tutorial.rst => docs/intro/tutorial.rst rename : scrapy/trunk/docs/media/scrapy-architecture.dia => docs/media/scrapy-architecture.dia rename : scrapy/trunk/docs/misc/api-stability.rst => docs/misc/api-stability.rst rename : scrapy/trunk/docs/misc/index.rst => docs/misc/index.rst rename : scrapy/trunk/docs/proposed/_images/scrapy_architecture.odg => docs/proposed/_images/scrapy_architecture.odg rename : scrapy/trunk/docs/proposed/_images/scrapy_architecture.png => docs/proposed/_images/scrapy_architecture.png rename : scrapy/trunk/docs/proposed/index.rst => docs/proposed/index.rst rename : scrapy/trunk/docs/proposed/introduction.rst => docs/proposed/introduction.rst rename : scrapy/trunk/docs/proposed/newitem.rst => docs/proposed/newitem.rst rename : scrapy/trunk/docs/proposed/spiders.rst => docs/proposed/spiders.rst rename : scrapy/trunk/docs/ref/downloader-middleware.rst => docs/ref/downloader-middleware.rst rename : scrapy/trunk/docs/ref/email.rst => docs/ref/email.rst rename : scrapy/trunk/docs/ref/exceptions.rst => docs/ref/exceptions.rst rename : scrapy/trunk/docs/ref/extension-manager.rst => docs/ref/extension-manager.rst rename : scrapy/trunk/docs/ref/extensions.rst => docs/ref/extensions.rst rename : scrapy/trunk/docs/ref/index.rst => docs/ref/index.rst rename : scrapy/trunk/docs/ref/link-extractors.rst => docs/ref/link-extractors.rst rename : scrapy/trunk/docs/ref/logging.rst => docs/ref/logging.rst rename : scrapy/trunk/docs/ref/request-response.rst => docs/ref/request-response.rst rename : scrapy/trunk/docs/ref/selectors.rst => docs/ref/selectors.rst rename : scrapy/trunk/docs/ref/settings.rst => docs/ref/settings.rst rename : scrapy/trunk/docs/ref/signals.rst => docs/ref/signals.rst rename : scrapy/trunk/docs/ref/spiders.rst => docs/ref/spiders.rst rename : scrapy/trunk/docs/topics/_images/adaptors_diagram.png => docs/topics/_images/adaptors_diagram.png rename : scrapy/trunk/docs/topics/_images/adaptors_diagram.svg => docs/topics/_images/adaptors_diagram.svg rename : scrapy/trunk/docs/topics/_images/firebug1.png => docs/topics/_images/firebug1.png rename : scrapy/trunk/docs/topics/_images/firebug2.png => docs/topics/_images/firebug2.png rename : scrapy/trunk/docs/topics/_images/firebug3.png => docs/topics/_images/firebug3.png rename : scrapy/trunk/docs/topics/_images/scrapy_architecture.odg => docs/topics/_images/scrapy_architecture.odg rename : scrapy/trunk/docs/topics/_images/scrapy_architecture.png => docs/topics/_images/scrapy_architecture.png rename : scrapy/trunk/docs/topics/adaptors.rst => docs/topics/adaptors.rst rename : scrapy/trunk/docs/topics/architecture.rst => docs/topics/architecture.rst rename : scrapy/trunk/docs/topics/downloader-middleware.rst => docs/topics/downloader-middleware.rst rename : scrapy/trunk/docs/topics/extensions.rst => docs/topics/extensions.rst rename : scrapy/trunk/docs/topics/firebug.rst => docs/topics/firebug.rst rename : scrapy/trunk/docs/topics/firefox.rst => docs/topics/firefox.rst rename : scrapy/trunk/docs/topics/index.rst => docs/topics/index.rst rename : scrapy/trunk/docs/topics/item-pipeline.rst => docs/topics/item-pipeline.rst rename : scrapy/trunk/docs/topics/items.rst => docs/topics/items.rst rename : scrapy/trunk/docs/topics/link-extractors.rst => docs/topics/link-extractors.rst rename : scrapy/trunk/docs/topics/robotstxt.rst => docs/topics/robotstxt.rst rename : scrapy/trunk/docs/topics/selectors.rst => docs/topics/selectors.rst rename : scrapy/trunk/docs/topics/settings.rst => docs/topics/settings.rst rename : scrapy/trunk/docs/topics/shell.rst => docs/topics/shell.rst rename : scrapy/trunk/docs/topics/spider-middleware.rst => docs/topics/spider-middleware.rst rename : scrapy/trunk/docs/topics/spiders.rst => docs/topics/spiders.rst rename : scrapy/trunk/docs/topics/stats.rst => docs/topics/stats.rst rename : scrapy/trunk/docs/topics/webconsole.rst => docs/topics/webconsole.rst rename : scrapy/trunk/examples/experimental/googledir/googledir/__init__.py => examples/experimental/googledir/googledir/__init__.py rename : scrapy/trunk/examples/experimental/googledir/googledir/items.py => examples/experimental/googledir/googledir/items.py rename : scrapy/trunk/examples/experimental/googledir/googledir/pipelines.py => examples/experimental/googledir/googledir/pipelines.py rename : scrapy/trunk/examples/experimental/googledir/googledir/settings.py => examples/experimental/googledir/googledir/settings.py rename : scrapy/trunk/examples/experimental/googledir/googledir/spiders/__init__.py => examples/experimental/googledir/googledir/spiders/__init__.py rename : scrapy/trunk/examples/experimental/googledir/googledir/spiders/google_directory.py => examples/experimental/googledir/googledir/spiders/google_directory.py rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_basic.tmpl => examples/experimental/googledir/googledir/templates/spider_basic.tmpl rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl => examples/experimental/googledir/googledir/templates/spider_crawl.tmpl rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl => examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl => examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl rename : scrapy/trunk/examples/experimental/googledir/scrapy-ctl.py => examples/experimental/googledir/scrapy-ctl.py rename : scrapy/trunk/examples/googledir/googledir/__init__.py => examples/googledir/googledir/__init__.py rename : scrapy/trunk/examples/googledir/googledir/items.py => examples/googledir/googledir/items.py rename : scrapy/trunk/examples/googledir/googledir/pipelines.py => examples/googledir/googledir/pipelines.py rename : scrapy/trunk/examples/googledir/googledir/settings.py => examples/googledir/googledir/settings.py rename : scrapy/trunk/examples/googledir/googledir/spiders/__init__.py => examples/googledir/googledir/spiders/__init__.py rename : scrapy/trunk/examples/googledir/googledir/spiders/google_directory.py => examples/googledir/googledir/spiders/google_directory.py rename : scrapy/trunk/examples/googledir/scrapy-ctl.py => examples/googledir/scrapy-ctl.py rename : scrapy/trunk/extras/sql/scraping.sql => extras/sql/scraping.sql rename : scrapy/trunk/profiling/priorityqueue/pq_classes.py => profiling/priorityqueue/pq_classes.py rename : scrapy/trunk/profiling/priorityqueue/run.py => profiling/priorityqueue/run.py rename : scrapy/trunk/profiling/priorityqueue/test_cases.py => profiling/priorityqueue/test_cases.py rename : scrapy/trunk/scrapy/__init__.py => scrapy/__init__.py rename : scrapy/trunk/scrapy/bin/scrapy-admin.py => scrapy/bin/scrapy-admin.py rename : scrapy/trunk/scrapy/command/__init__.py => scrapy/command/__init__.py rename : scrapy/trunk/scrapy/command/cmdline.py => scrapy/command/cmdline.py rename : scrapy/trunk/scrapy/command/commands/__init__.py => scrapy/command/commands/__init__.py rename : scrapy/trunk/scrapy/command/commands/crawl.py => scrapy/command/commands/crawl.py rename : scrapy/trunk/scrapy/command/commands/download.py => scrapy/command/commands/download.py rename : scrapy/trunk/scrapy/command/commands/genspider.py => scrapy/command/commands/genspider.py rename : scrapy/trunk/scrapy/command/commands/help.py => scrapy/command/commands/help.py rename : scrapy/trunk/scrapy/command/commands/list.py => scrapy/command/commands/list.py rename : scrapy/trunk/scrapy/command/commands/log.py => scrapy/command/commands/log.py rename : scrapy/trunk/scrapy/command/commands/parse.py => scrapy/command/commands/parse.py rename : scrapy/trunk/scrapy/command/commands/shell.py => scrapy/command/commands/shell.py rename : scrapy/trunk/scrapy/command/commands/start.py => scrapy/command/commands/start.py rename : scrapy/trunk/scrapy/command/commands/stats.py => scrapy/command/commands/stats.py rename : scrapy/trunk/scrapy/command/models.py => scrapy/command/models.py rename : scrapy/trunk/scrapy/conf/__init__.py => scrapy/conf/__init__.py rename : scrapy/trunk/scrapy/conf/commands/__init__.py => scrapy/conf/commands/__init__.py rename : scrapy/trunk/scrapy/conf/commands/crawl.py => scrapy/conf/commands/crawl.py rename : scrapy/trunk/scrapy/conf/commands/help.py => scrapy/conf/commands/help.py rename : scrapy/trunk/scrapy/conf/commands/list.py => scrapy/conf/commands/list.py rename : scrapy/trunk/scrapy/conf/commands/log.py => scrapy/conf/commands/log.py rename : scrapy/trunk/scrapy/conf/commands/scrape.py => scrapy/conf/commands/scrape.py rename : scrapy/trunk/scrapy/conf/commands/shell.py => scrapy/conf/commands/shell.py rename : scrapy/trunk/scrapy/conf/commands/stats.py => scrapy/conf/commands/stats.py rename : scrapy/trunk/scrapy/conf/commands/test.py => scrapy/conf/commands/test.py rename : scrapy/trunk/scrapy/conf/default_settings.py => scrapy/conf/default_settings.py rename : scrapy/trunk/scrapy/contrib/__init__.py => scrapy/contrib/__init__.py rename : scrapy/trunk/scrapy/contrib/aws.py => scrapy/contrib/aws.py rename : scrapy/trunk/scrapy/contrib/closedomain.py => scrapy/contrib/closedomain.py rename : scrapy/trunk/scrapy/contrib/cluster/__init__.py => scrapy/contrib/cluster/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/crawler/__init__.py => scrapy/contrib/cluster/crawler/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/crawler/manager.py => scrapy/contrib/cluster/crawler/manager.py rename : scrapy/trunk/scrapy/contrib/cluster/hooks/__init__.py => scrapy/contrib/cluster/hooks/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/hooks/svn.py => scrapy/contrib/cluster/hooks/svn.py rename : scrapy/trunk/scrapy/contrib/cluster/master/__init__.py => scrapy/contrib/cluster/master/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/master/manager.py => scrapy/contrib/cluster/master/manager.py rename : scrapy/trunk/scrapy/contrib/cluster/master/web.py => scrapy/contrib/cluster/master/web.py rename : scrapy/trunk/scrapy/contrib/cluster/master/ws_api.txt => scrapy/contrib/cluster/master/ws_api.txt rename : scrapy/trunk/scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py => scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py rename : scrapy/trunk/scrapy/contrib/cluster/tools/test-worker.py => scrapy/contrib/cluster/tools/test-worker.py rename : scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py => scrapy/contrib/cluster/worker/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/worker/manager.py => scrapy/contrib/cluster/worker/manager.py rename : scrapy/trunk/scrapy/contrib/codecs/__init__.py => scrapy/contrib/codecs/__init__.py rename : scrapy/trunk/scrapy/contrib/codecs/x_mac_roman.py => scrapy/contrib/codecs/x_mac_roman.py rename : scrapy/trunk/scrapy/contrib/debug.py => scrapy/contrib/debug.py rename : scrapy/trunk/scrapy/contrib/delayedclosedomain.py => scrapy/contrib/delayedclosedomain.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py => scrapy/contrib/downloadermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py => scrapy/contrib/downloadermiddleware/cache.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py => scrapy/contrib/downloadermiddleware/common.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py => scrapy/contrib/downloadermiddleware/cookies.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py => scrapy/contrib/downloadermiddleware/debug.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py => scrapy/contrib/downloadermiddleware/errorpages.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py => scrapy/contrib/downloadermiddleware/httpauth.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py => scrapy/contrib/downloadermiddleware/httpcompression.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py => scrapy/contrib/downloadermiddleware/redirect.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py => scrapy/contrib/downloadermiddleware/retry.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py => scrapy/contrib/downloadermiddleware/robotstxt.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py => scrapy/contrib/downloadermiddleware/stats.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py => scrapy/contrib/downloadermiddleware/useragent.py rename : scrapy/trunk/scrapy/contrib/groupsettings.py => scrapy/contrib/groupsettings.py rename : scrapy/trunk/scrapy/contrib/item/__init__.py => scrapy/contrib/item/__init__.py rename : scrapy/trunk/scrapy/contrib/item/models.py => scrapy/contrib/item/models.py rename : scrapy/trunk/scrapy/contrib/itemsampler.py => scrapy/contrib/itemsampler.py rename : scrapy/trunk/scrapy/contrib/link_extractors.py => scrapy/contrib/link_extractors.py rename : scrapy/trunk/scrapy/contrib/memdebug.py => scrapy/contrib/memdebug.py rename : scrapy/trunk/scrapy/contrib/memusage.py => scrapy/contrib/memusage.py rename : scrapy/trunk/scrapy/contrib/pipeline/__init__.py => scrapy/contrib/pipeline/__init__.py rename : scrapy/trunk/scrapy/contrib/pipeline/images.py => scrapy/contrib/pipeline/images.py rename : scrapy/trunk/scrapy/contrib/pipeline/media.py => scrapy/contrib/pipeline/media.py rename : scrapy/trunk/scrapy/contrib/pipeline/s3images.py => scrapy/contrib/pipeline/s3images.py rename : scrapy/trunk/scrapy/contrib/pipeline/show.py => scrapy/contrib/pipeline/show.py rename : scrapy/trunk/scrapy/contrib/prioritizers.py => scrapy/contrib/prioritizers.py rename : scrapy/trunk/scrapy/contrib/response/__init__.py => scrapy/contrib/response/__init__.py rename : scrapy/trunk/scrapy/contrib/response/soup.py => scrapy/contrib/response/soup.py rename : scrapy/trunk/scrapy/contrib/schedulermiddleware/__init__.py => scrapy/contrib/schedulermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py => scrapy/contrib/schedulermiddleware/duplicatesfilter.py rename : scrapy/trunk/scrapy/contrib/spider/__init__.py => scrapy/contrib/spider/__init__.py rename : scrapy/trunk/scrapy/contrib/spider/profiler.py => scrapy/contrib/spider/profiler.py rename : scrapy/trunk/scrapy/contrib/spider/reloader.py => scrapy/contrib/spider/reloader.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py => scrapy/contrib/spidermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py => scrapy/contrib/spidermiddleware/depth.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py => scrapy/contrib/spidermiddleware/duplicatesfilter.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py => scrapy/contrib/spidermiddleware/limit.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py => scrapy/contrib/spidermiddleware/offsite.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py => scrapy/contrib/spidermiddleware/referer.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py => scrapy/contrib/spidermiddleware/restrict.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py => scrapy/contrib/spidermiddleware/urlfilter.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py => scrapy/contrib/spidermiddleware/urllength.py rename : scrapy/trunk/scrapy/contrib/spiders/__init__.py => scrapy/contrib/spiders/__init__.py rename : scrapy/trunk/scrapy/contrib/spiders/crawl.py => scrapy/contrib/spiders/crawl.py rename : scrapy/trunk/scrapy/contrib/spiders/feed.py => scrapy/contrib/spiders/feed.py rename : scrapy/trunk/scrapy/contrib/spiders/generic.py => scrapy/contrib/spiders/generic.py rename : scrapy/trunk/scrapy/contrib/web/__init__.py => scrapy/contrib/web/__init__.py rename : scrapy/trunk/scrapy/contrib/web/http.py => scrapy/contrib/web/http.py rename : scrapy/trunk/scrapy/contrib/web/json.py => scrapy/contrib/web/json.py rename : scrapy/trunk/scrapy/contrib/web/service.py => scrapy/contrib/web/service.py rename : scrapy/trunk/scrapy/contrib/web/site.py => scrapy/contrib/web/site.py rename : scrapy/trunk/scrapy/contrib/web/stats.py => scrapy/contrib/web/stats.py rename : scrapy/trunk/scrapy/contrib/webconsole/__init__.py => scrapy/contrib/webconsole/__init__.py rename : scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py => scrapy/contrib/webconsole/enginestatus.py rename : scrapy/trunk/scrapy/contrib/webconsole/livestats.py => scrapy/contrib/webconsole/livestats.py rename : scrapy/trunk/scrapy/contrib/webconsole/scheduler.py => scrapy/contrib/webconsole/scheduler.py rename : scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py => scrapy/contrib/webconsole/spiderctl.py rename : scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py => scrapy/contrib/webconsole/spiderstats.py rename : scrapy/trunk/scrapy/contrib/webconsole/stats.py => scrapy/contrib/webconsole/stats.py rename : scrapy/trunk/scrapy/contrib_exp/__init__.py => scrapy/contrib_exp/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/__init__.py => scrapy/contrib_exp/adaptors/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/date.py => scrapy/contrib_exp/adaptors/date.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/extraction.py => scrapy/contrib_exp/adaptors/extraction.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/markup.py => scrapy/contrib_exp/adaptors/markup.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/misc.py => scrapy/contrib_exp/adaptors/misc.py rename : scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/__init__.py => scrapy/contrib_exp/downloadermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/decompression.py => scrapy/contrib_exp/downloadermiddleware/decompression.py rename : scrapy/trunk/scrapy/contrib_exp/history/__init__.py => scrapy/contrib_exp/history/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/history/history.py => scrapy/contrib_exp/history/history.py rename : scrapy/trunk/scrapy/contrib_exp/history/middleware.py => scrapy/contrib_exp/history/middleware.py rename : scrapy/trunk/scrapy/contrib_exp/history/scheduler.py => scrapy/contrib_exp/history/scheduler.py rename : scrapy/trunk/scrapy/contrib_exp/history/store.py => scrapy/contrib_exp/history/store.py rename : scrapy/trunk/scrapy/contrib_exp/link/__init__.py => scrapy/contrib_exp/link/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/__init__.py => scrapy/contrib_exp/newitem/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py => scrapy/contrib_exp/newitem/adaptors.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/fields.py => scrapy/contrib_exp/newitem/fields.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/models.py => scrapy/contrib_exp/newitem/models.py rename : scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py => scrapy/contrib_exp/pipeline/shoveitem.py rename : scrapy/trunk/scrapy/core/__init__.py => scrapy/core/__init__.py rename : scrapy/trunk/scrapy/core/downloader/__init__.py => scrapy/core/downloader/__init__.py rename : scrapy/trunk/scrapy/core/downloader/dnscache.py => scrapy/core/downloader/dnscache.py rename : scrapy/trunk/scrapy/core/downloader/handlers.py => scrapy/core/downloader/handlers.py rename : scrapy/trunk/scrapy/core/downloader/manager.py => scrapy/core/downloader/manager.py rename : scrapy/trunk/scrapy/core/downloader/middleware.py => scrapy/core/downloader/middleware.py rename : scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py => scrapy/core/downloader/responsetypes/__init__.py rename : scrapy/trunk/scrapy/core/downloader/responsetypes/mime.types => scrapy/core/downloader/responsetypes/mime.types rename : scrapy/trunk/scrapy/core/downloader/webclient.py => scrapy/core/downloader/webclient.py rename : scrapy/trunk/scrapy/core/engine.py => scrapy/core/engine.py rename : scrapy/trunk/scrapy/core/exceptions.py => scrapy/core/exceptions.py rename : scrapy/trunk/scrapy/core/manager.py => scrapy/core/manager.py rename : scrapy/trunk/scrapy/core/prioritizers.py => scrapy/core/prioritizers.py rename : scrapy/trunk/scrapy/core/scheduler/__init__.py => scrapy/core/scheduler/__init__.py rename : scrapy/trunk/scrapy/core/scheduler/middleware.py => scrapy/core/scheduler/middleware.py rename : scrapy/trunk/scrapy/core/scheduler/schedulers.py => scrapy/core/scheduler/schedulers.py rename : scrapy/trunk/scrapy/core/scheduler/store.py => scrapy/core/scheduler/store.py rename : scrapy/trunk/scrapy/core/signals.py => scrapy/core/signals.py rename : scrapy/trunk/scrapy/dupefilter/__init__.py => scrapy/dupefilter/__init__.py rename : scrapy/trunk/scrapy/extension/__init__.py => scrapy/extension/__init__.py rename : scrapy/trunk/scrapy/fetcher/__init__.py => scrapy/fetcher/__init__.py rename : scrapy/trunk/scrapy/http/__init__.py => scrapy/http/__init__.py rename : scrapy/trunk/scrapy/http/cookies.py => scrapy/http/cookies.py rename : scrapy/trunk/scrapy/http/headers.py => scrapy/http/headers.py rename : scrapy/trunk/scrapy/http/request/__init__.py => scrapy/http/request/__init__.py rename : scrapy/trunk/scrapy/http/request/form.py => scrapy/http/request/form.py rename : scrapy/trunk/scrapy/http/request/rpc.py => scrapy/http/request/rpc.py rename : scrapy/trunk/scrapy/http/response/__init__.py => scrapy/http/response/__init__.py rename : scrapy/trunk/scrapy/http/response/html.py => scrapy/http/response/html.py rename : scrapy/trunk/scrapy/http/response/text.py => scrapy/http/response/text.py rename : scrapy/trunk/scrapy/http/response/xml.py => scrapy/http/response/xml.py rename : scrapy/trunk/scrapy/http/url.py => scrapy/http/url.py rename : scrapy/trunk/scrapy/item/__init__.py => scrapy/item/__init__.py rename : scrapy/trunk/scrapy/item/adaptors.py => scrapy/item/adaptors.py rename : scrapy/trunk/scrapy/item/models.py => scrapy/item/models.py rename : scrapy/trunk/scrapy/item/pipeline.py => scrapy/item/pipeline.py rename : scrapy/trunk/scrapy/link/__init__.py => scrapy/link/__init__.py rename : scrapy/trunk/scrapy/link/extractors.py => scrapy/link/extractors.py rename : scrapy/trunk/scrapy/log/__init__.py => scrapy/log/__init__.py rename : scrapy/trunk/scrapy/mail/__init__.py => scrapy/mail/__init__.py rename : scrapy/trunk/scrapy/management/__init__.py => scrapy/management/__init__.py rename : scrapy/trunk/scrapy/management/telnet.py => scrapy/management/telnet.py rename : scrapy/trunk/scrapy/management/web.py => scrapy/management/web.py rename : scrapy/trunk/scrapy/patches/__init__.py => scrapy/patches/__init__.py rename : scrapy/trunk/scrapy/patches/monkeypatches.py => scrapy/patches/monkeypatches.py rename : scrapy/trunk/scrapy/spider/__init__.py => scrapy/spider/__init__.py rename : scrapy/trunk/scrapy/spider/manager.py => scrapy/spider/manager.py rename : scrapy/trunk/scrapy/spider/middleware.py => scrapy/spider/middleware.py rename : scrapy/trunk/scrapy/spider/models.py => scrapy/spider/models.py rename : scrapy/trunk/scrapy/stats/__init__.py => scrapy/stats/__init__.py rename : scrapy/trunk/scrapy/stats/corestats.py => scrapy/stats/corestats.py rename : scrapy/trunk/scrapy/stats/statscollector.py => scrapy/stats/statscollector.py rename : scrapy/trunk/scrapy/store/__init__.py => scrapy/store/__init__.py rename : scrapy/trunk/scrapy/store/db.py => scrapy/store/db.py rename : scrapy/trunk/scrapy/templates/project/module/__init__.py => scrapy/templates/project/module/__init__.py rename : scrapy/trunk/scrapy/templates/project/module/items.py.tmpl => scrapy/templates/project/module/items.py.tmpl rename : scrapy/trunk/scrapy/templates/project/module/pipelines.py.tmpl => scrapy/templates/project/module/pipelines.py.tmpl rename : scrapy/trunk/scrapy/templates/project/module/settings.py.tmpl => scrapy/templates/project/module/settings.py.tmpl rename : scrapy/trunk/scrapy/templates/project/module/spiders/__init__.py => scrapy/templates/project/module/spiders/__init__.py rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_basic.tmpl => scrapy/templates/project/module/templates/spider_basic.tmpl rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_crawl.tmpl => scrapy/templates/project/module/templates/spider_crawl.tmpl rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_csvfeed.tmpl => scrapy/templates/project/module/templates/spider_csvfeed.tmpl rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl => scrapy/templates/project/module/templates/spider_xmlfeed.tmpl rename : scrapy/trunk/scrapy/templates/project/root/scrapy-ctl.py => scrapy/templates/project/root/scrapy-ctl.py rename : scrapy/trunk/scrapy/tests/__init__.py => scrapy/tests/__init__.py rename : scrapy/trunk/scrapy/tests/run.py => scrapy/tests/run.py rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html => scrapy/tests/sample_data/adaptors/enc-ascii.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html => scrapy/tests/sample_data/adaptors/enc-cp1252.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html => scrapy/tests/sample_data/adaptors/enc-latin1.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html => scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html => scrapy/tests/sample_data/adaptors/enc-utf8.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml => scrapy/tests/sample_data/adaptors/extr_unquoted.xml rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.tar => scrapy/tests/sample_data/compressed/feed-sample1.tar rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml => scrapy/tests/sample_data/compressed/feed-sample1.xml rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 => scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz => scrapy/tests/sample_data/compressed/feed-sample1.xml.gz rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.zip => scrapy/tests/sample_data/compressed/feed-sample1.zip rename : scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin => scrapy/tests/sample_data/compressed/html-gzip.bin rename : scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin => scrapy/tests/sample_data/compressed/html-rawdeflate.bin rename : scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin => scrapy/tests/sample_data/compressed/html-zlibdeflate.bin rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml => scrapy/tests/sample_data/feeds/feed-sample1.xml rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml => scrapy/tests/sample_data/feeds/feed-sample2.xml rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.csv => scrapy/tests/sample_data/feeds/feed-sample3.csv rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv => scrapy/tests/sample_data/feeds/feed-sample4.csv rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv => scrapy/tests/sample_data/feeds/feed-sample5.csv rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html => scrapy/tests/sample_data/link_extractor/image_linkextractor.html rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html => scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html => scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html => scrapy/tests/sample_data/link_extractor/regex_linkextractor.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/index.html => scrapy/tests/sample_data/test_site/index.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html => scrapy/tests/sample_data/test_site/item1.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html => scrapy/tests/sample_data/test_site/item2.html rename : scrapy/trunk/scrapy/tests/test_adaptors.py => scrapy/tests/test_adaptors.py rename : scrapy/trunk/scrapy/tests/test_aws.py => scrapy/tests/test_aws.py rename : scrapy/trunk/scrapy/tests/test_c14nurls.py => scrapy/tests/test_c14nurls.py rename : scrapy/trunk/scrapy/tests/test_contrib_response_soup.py => scrapy/tests/test_contrib_response_soup.py rename : scrapy/trunk/scrapy/tests/test_dependencies.py => scrapy/tests/test_dependencies.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py => scrapy/tests/test_downloadermiddleware_cookies.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_decompression.py => scrapy/tests/test_downloadermiddleware_decompression.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py => scrapy/tests/test_downloadermiddleware_httpcompression.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py => scrapy/tests/test_downloadermiddleware_redirect.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_retry.py => scrapy/tests/test_downloadermiddleware_retry.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py => scrapy/tests/test_downloadermiddleware_useragent.py rename : scrapy/trunk/scrapy/tests/test_dupefilter.py => scrapy/tests/test_dupefilter.py rename : scrapy/trunk/scrapy/tests/test_engine.py => scrapy/tests/test_engine.py rename : scrapy/trunk/scrapy/tests/test_http_cookies.py => scrapy/tests/test_http_cookies.py rename : scrapy/trunk/scrapy/tests/test_http_headers.py => scrapy/tests/test_http_headers.py rename : scrapy/trunk/scrapy/tests/test_http_request.py => scrapy/tests/test_http_request.py rename : scrapy/trunk/scrapy/tests/test_http_response.py => scrapy/tests/test_http_response.py rename : scrapy/trunk/scrapy/tests/test_http_url.py => scrapy/tests/test_http_url.py rename : scrapy/trunk/scrapy/tests/test_item.py => scrapy/tests/test_item.py rename : scrapy/trunk/scrapy/tests/test_itemadaptor.py => scrapy/tests/test_itemadaptor.py rename : scrapy/trunk/scrapy/tests/test_libxml2.py => scrapy/tests/test_libxml2.py rename : scrapy/trunk/scrapy/tests/test_link.py => scrapy/tests/test_link.py rename : scrapy/trunk/scrapy/tests/test_newitem.py => scrapy/tests/test_newitem.py rename : scrapy/trunk/scrapy/tests/test_pipeline_images.py => scrapy/tests/test_pipeline_images.py rename : scrapy/trunk/scrapy/tests/test_responsetypes.py => scrapy/tests/test_responsetypes.py rename : scrapy/trunk/scrapy/tests/test_robustscrapeditem.py => scrapy/tests/test_robustscrapeditem.py rename : scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py => scrapy/tests/test_schedulermiddleware_duplicatesfilter.py rename : scrapy/trunk/scrapy/tests/test_serialization.py => scrapy/tests/test_serialization.py rename : scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py => scrapy/tests/test_spidermiddleware_duplicatesfilter.py rename : scrapy/trunk/scrapy/tests/test_spidermonkey.py => scrapy/tests/test_spidermonkey.py rename : scrapy/trunk/scrapy/tests/test_spiders/__init__.py => scrapy/tests/test_spiders/__init__.py rename : scrapy/trunk/scrapy/tests/test_spiders/testspider.py => scrapy/tests/test_spiders/testspider.py rename : scrapy/trunk/scrapy/tests/test_stats.py => scrapy/tests/test_stats.py rename : scrapy/trunk/scrapy/tests/test_storedb.py => scrapy/tests/test_storedb.py rename : scrapy/trunk/scrapy/tests/test_utils_datatypes.py => scrapy/tests/test_utils_datatypes.py rename : scrapy/trunk/scrapy/tests/test_utils_defer.py => scrapy/tests/test_utils_defer.py rename : scrapy/trunk/scrapy/tests/test_utils_iterators.py => scrapy/tests/test_utils_iterators.py rename : scrapy/trunk/scrapy/tests/test_utils_markup.py => scrapy/tests/test_utils_markup.py rename : scrapy/trunk/scrapy/tests/test_utils_misc.py => scrapy/tests/test_utils_misc.py rename : scrapy/trunk/scrapy/tests/test_utils_python.py => scrapy/tests/test_utils_python.py rename : scrapy/trunk/scrapy/tests/test_utils_request.py => scrapy/tests/test_utils_request.py rename : scrapy/trunk/scrapy/tests/test_utils_response.py => scrapy/tests/test_utils_response.py rename : scrapy/trunk/scrapy/tests/test_utils_url.py => scrapy/tests/test_utils_url.py rename : scrapy/trunk/scrapy/tests/test_webclient.py => scrapy/tests/test_webclient.py rename : scrapy/trunk/scrapy/tests/test_xpath.py => scrapy/tests/test_xpath.py rename : scrapy/trunk/scrapy/tests/test_xpath_extension.py => scrapy/tests/test_xpath_extension.py rename : scrapy/trunk/scrapy/utils/__init__.py => scrapy/utils/__init__.py rename : scrapy/trunk/scrapy/utils/c14n.py => scrapy/utils/c14n.py rename : scrapy/trunk/scrapy/utils/datatypes.py => scrapy/utils/datatypes.py rename : scrapy/trunk/scrapy/utils/db.py => scrapy/utils/db.py rename : scrapy/trunk/scrapy/utils/defer.py => scrapy/utils/defer.py rename : scrapy/trunk/scrapy/utils/display.py => scrapy/utils/display.py rename : scrapy/trunk/scrapy/utils/http.py => scrapy/utils/http.py rename : scrapy/trunk/scrapy/utils/iterators.py => scrapy/utils/iterators.py rename : scrapy/trunk/scrapy/utils/markup.py => scrapy/utils/markup.py rename : scrapy/trunk/scrapy/utils/misc.py => scrapy/utils/misc.py rename : scrapy/trunk/scrapy/utils/python.py => scrapy/utils/python.py rename : scrapy/trunk/scrapy/utils/request.py => scrapy/utils/request.py rename : scrapy/trunk/scrapy/utils/response.py => scrapy/utils/response.py rename : scrapy/trunk/scrapy/utils/serialization.py => scrapy/utils/serialization.py rename : scrapy/trunk/scrapy/utils/test.py => scrapy/utils/test.py rename : scrapy/trunk/scrapy/utils/url.py => scrapy/utils/url.py rename : scrapy/trunk/scrapy/xlib/BeautifulSoup.py => scrapy/xlib/BeautifulSoup.py rename : scrapy/trunk/scrapy/xlib/ClientForm.py => scrapy/xlib/ClientForm.py rename : scrapy/trunk/scrapy/xlib/__init__.py => scrapy/xlib/__init__.py rename : scrapy/trunk/scrapy/xlib/lrucache.py => scrapy/xlib/lrucache.py rename : scrapy/trunk/scrapy/xlib/lsprofcalltree.py => scrapy/xlib/lsprofcalltree.py rename : scrapy/trunk/scrapy/xlib/pydispatch/__init__.py => scrapy/xlib/pydispatch/__init__.py rename : scrapy/trunk/scrapy/xlib/pydispatch/dispatcher.py => scrapy/xlib/pydispatch/dispatcher.py rename : scrapy/trunk/scrapy/xlib/pydispatch/errors.py => scrapy/xlib/pydispatch/errors.py rename : scrapy/trunk/scrapy/xlib/pydispatch/license.txt => scrapy/xlib/pydispatch/license.txt rename : scrapy/trunk/scrapy/xlib/pydispatch/robust.py => scrapy/xlib/pydispatch/robust.py rename : scrapy/trunk/scrapy/xlib/pydispatch/robustapply.py => scrapy/xlib/pydispatch/robustapply.py rename : scrapy/trunk/scrapy/xlib/pydispatch/saferef.py => scrapy/xlib/pydispatch/saferef.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/INSTALL.scrapy => scrapy/xlib/spidermonkey/INSTALL.scrapy rename : scrapy/trunk/scrapy/xlib/spidermonkey/__init__.py => scrapy/xlib/spidermonkey/__init__.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/sm_settings.py => scrapy/xlib/spidermonkey/sm_settings.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/spidermonkey.py => scrapy/xlib/spidermonkey/spidermonkey.py rename : scrapy/trunk/scrapy/xpath/__init__.py => scrapy/xpath/__init__.py rename : scrapy/trunk/scrapy/xpath/constructors.py => scrapy/xpath/constructors.py rename : scrapy/trunk/scrapy/xpath/document.py => scrapy/xpath/document.py rename : scrapy/trunk/scrapy/xpath/extension.py => scrapy/xpath/extension.py rename : scrapy/trunk/scrapy/xpath/selector.py => scrapy/xpath/selector.py rename : scrapy/trunk/scrapy/xpath/types.py => scrapy/xpath/types.py rename : scrapy/trunk/scripts/rpm-install.sh => scripts/rpm-install.sh rename : scrapy/trunk/setup.cfg => setup.cfg rename : scrapy/trunk/setup.py => setup.py --- scrapy/trunk/AUTHORS => AUTHORS | 0 scrapy/trunk/INSTALL => INSTALL | 0 scrapy/trunk/LICENSE => LICENSE | 0 .../cluster-refactor/README => README | 0 {scrapy/trunk/bin => bin}/runtests.sh | 0 {scrapy/trunk/docs => docs}/Makefile | 0 {scrapy/trunk/docs => docs}/README | 0 .../trunk/docs => docs}/_ext/scrapydocs.py | 0 .../_static/items_adaptors-sample1.html | 0 .../trunk/docs => docs}/_static/scrapydoc.css | 0 .../_static/selectors-sample1.html | 0 {scrapy/trunk/docs => docs}/conf.py | 0 {scrapy/trunk/docs => docs}/faq.rst | 0 {scrapy/trunk/docs => docs}/index.rst | 0 {scrapy/trunk/docs => docs}/intro/index.rst | 0 {scrapy/trunk/docs => docs}/intro/install.rst | 0 .../trunk/docs => docs}/intro/overview.rst | 0 .../trunk/docs => docs}/intro/tutorial.rst | 0 .../media}/scrapy-architecture.dia | Bin .../docs => docs}/misc/api-stability.rst | 0 {scrapy/trunk/docs => docs}/misc/index.rst | 0 .../proposed/_images/scrapy_architecture.odg | Bin .../proposed/_images/scrapy_architecture.png | Bin .../trunk/docs => docs}/proposed/index.rst | 0 .../docs => docs}/proposed/introduction.rst | 0 .../trunk/docs => docs}/proposed/newitem.rst | 0 .../trunk/docs => docs}/proposed/spiders.rst | 0 .../ref/downloader-middleware.rst | 0 {scrapy/trunk/docs => docs}/ref/email.rst | 0 .../trunk/docs => docs}/ref/exceptions.rst | 0 .../docs => docs}/ref/extension-manager.rst | 0 .../trunk/docs => docs}/ref/extensions.rst | 0 {scrapy/trunk/docs => docs}/ref/index.rst | 0 .../docs => docs}/ref/link-extractors.rst | 0 {scrapy/trunk/docs => docs}/ref/logging.rst | 0 .../docs => docs}/ref/request-response.rst | 0 {scrapy/trunk/docs => docs}/ref/selectors.rst | 0 {scrapy/trunk/docs => docs}/ref/settings.rst | 0 {scrapy/trunk/docs => docs}/ref/signals.rst | 0 {scrapy/trunk/docs => docs}/ref/spiders.rst | 0 .../topics/_images/adaptors_diagram.png | Bin .../topics/_images/adaptors_diagram.svg | 0 .../docs => docs}/topics/_images/firebug1.png | Bin .../docs => docs}/topics/_images/firebug2.png | Bin .../docs => docs}/topics/_images/firebug3.png | Bin .../topics/_images/scrapy_architecture.odg | Bin .../topics/_images/scrapy_architecture.png | Bin .../trunk/docs => docs}/topics/adaptors.rst | 0 .../docs => docs}/topics/architecture.rst | 0 .../topics/downloader-middleware.rst | 0 .../trunk/docs => docs}/topics/extensions.rst | 0 .../trunk/docs => docs}/topics/firebug.rst | 0 .../trunk/docs => docs}/topics/firefox.rst | 0 {scrapy/trunk/docs => docs}/topics/index.rst | 0 .../docs => docs}/topics/item-pipeline.rst | 0 {scrapy/trunk/docs => docs}/topics/items.rst | 0 .../docs => docs}/topics/link-extractors.rst | 0 .../trunk/docs => docs}/topics/robotstxt.rst | 0 .../trunk/docs => docs}/topics/selectors.rst | 0 .../trunk/docs => docs}/topics/settings.rst | 0 {scrapy/trunk/docs => docs}/topics/shell.rst | 0 .../topics/spider-middleware.rst | 0 .../trunk/docs => docs}/topics/spiders.rst | 0 {scrapy/trunk/docs => docs}/topics/stats.rst | 0 .../trunk/docs => docs}/topics/webconsole.rst | 0 .../googledir/googledir}/__init__.py | 0 .../experimental/googledir/googledir/items.py | 0 .../googledir/googledir/pipelines.py | 0 .../googledir/googledir/settings.py | 0 .../googledir/googledir/spiders}/__init__.py | 0 .../googledir/spiders/google_directory.py | 0 .../googledir/templates/spider_basic.tmpl | 0 .../googledir/templates/spider_crawl.tmpl | 0 .../googledir/templates/spider_csvfeed.tmpl | 0 .../googledir/templates/spider_xmlfeed.tmpl | 0 .../experimental/googledir/scrapy-ctl.py | 0 .../googledir/googledir}/__init__.py | 0 .../googledir/googledir/items.py | 0 .../googledir/googledir/pipelines.py | 0 .../googledir/googledir/settings.py | 0 .../googledir/googledir}/spiders/__init__.py | 0 .../googledir/spiders/google_directory.py | 0 .../googledir/scrapy-ctl.py | 0 .../extras => extras}/sql/scraping.sql | 0 .../priorityqueue/pq_classes.py | 0 .../priorityqueue/run.py | 0 .../priorityqueue/test_cases.py | 0 scrapy/{trunk/scrapy => }/__init__.py | 0 scrapy/{trunk/scrapy => }/bin/scrapy-admin.py | 0 scrapy/branches/cluster-refactor/INSTALL | 32 - .../cluster-refactor/scrapy/__init__.py | 18 - .../scrapy/bin/scrapy-admin.py | 39 - .../scrapy/command/cmdline.py | 157 -- .../scrapy/command/commands/crawl.py | 40 - .../scrapy/command/commands/genspider.py | 64 - .../scrapy/command/commands/getattr.py | 25 - .../scrapy/command/commands/parse.py | 46 - .../scrapy/command/commands/replay.py | 184 -- .../scrapy/command/commands/shell.py | 158 -- .../scrapy/command/commands/stats.py | 30 - .../cluster-refactor/scrapy/command/models.py | 78 - .../cluster-refactor/scrapy/conf/__init__.py | 67 - .../scrapy/conf/core_settings.py | 30 - .../scrapy/conf/project_template/items.py | 6 - .../conf/project_template/scrapy-ctl.py | 4 - .../conf/project_template/scrapy_settings.py | 95 - .../scrapy/contrib/adaptorpipeline.py | 85 - .../scrapy/contrib/closedomain.py | 46 - .../cluster-refactor/scrapy/contrib/debug.py | 17 - .../contrib/downloadermiddleware/cache.py | 194 -- .../contrib/downloadermiddleware/common.py | 10 - .../downloadermiddleware/compression.py | 42 - .../contrib/downloadermiddleware/cookies.py | 27 - .../contrib/downloadermiddleware/debug.py | 20 - .../contrib/downloadermiddleware/httpauth.py | 8 - .../contrib/downloadermiddleware/redirect.py | 66 - .../contrib/downloadermiddleware/retry.py | 63 - .../contrib/downloadermiddleware/robots.py | 60 - .../contrib/downloadermiddleware/useragent.py | 7 - .../scrapy/contrib/history/middleware.py | 88 - .../scrapy/contrib/history/scheduler.py | 94 - .../scrapy/contrib/item/models.py | 208 -- .../scrapy/contrib/memdebug.py | 69 - .../scrapy/contrib/memusage.py | 122 - .../scrapy/contrib/pipeline/images.py | 181 -- .../scrapy/contrib/pipeline/media.py | 179 -- .../scrapy/contrib/pipeline/s3images.py | 154 -- .../scrapy/contrib/pipeline/shoveitem.py | 37 - .../scrapy/contrib/pipeline/show.py | 15 - .../scrapy/contrib/response/soup.py | 26 - .../scrapy/contrib/spidermiddleware/depth.py | 36 - .../scrapy/contrib/spidermiddleware/limit.py | 37 - .../contrib/spidermiddleware/offsite.py | 18 - .../contrib/spidermiddleware/referer.py | 15 - .../contrib/spidermiddleware/restrict.py | 21 - .../contrib/spidermiddleware/urlfilter.py | 17 - .../contrib/spidermiddleware/urllength.py | 24 - .../scrapy/contrib/spiders.py | 60 - .../scrapy/contrib/webconsole/livestats.py | 82 - .../scrapy/contrib/webconsole/schedstats.py | 41 - .../scrapy/contrib/webconsole/spiderctl.py | 208 -- .../scrapy/core/downloader/handlers.py | 78 - .../scrapy/core/downloader/manager.py | 196 -- .../scrapy/core/downloader/middleware.py | 173 -- .../cluster-refactor/scrapy/core/engine.py | 549 ---- .../scrapy/core/exceptions.py | 47 - .../cluster-refactor/scrapy/core/manager.py | 149 -- .../scrapy/core/scheduler/__init__.py | 3 - .../scrapy/core/scheduler/filter.py | 19 - .../scrapy/core/scheduler/schedulers.py | 173 -- .../cluster-refactor/scrapy/core/signals.py | 73 - .../scrapy/extension/__init__.py | 39 - .../cluster-refactor/scrapy/http/__init__.py | 11 - .../cluster-refactor/scrapy/http/headers.py | 97 - .../cluster-refactor/scrapy/http/request.py | 188 -- .../cluster-refactor/scrapy/http/response.py | 194 -- .../cluster-refactor/scrapy/item/__init__.py | 1 - .../cluster-refactor/scrapy/item/adaptors.py | 241 -- .../cluster-refactor/scrapy/item/models.py | 24 - .../cluster-refactor/scrapy/item/pipeline.py | 101 - .../cluster-refactor/scrapy/link/__init__.py | 68 - .../scrapy/link/extractors.py | 76 - .../cluster-refactor/scrapy/log/__init__.py | 57 - .../cluster-refactor/scrapy/mail/__init__.py | 54 - .../cluster-refactor/scrapy/management/web.py | 69 - .../scrapy/patches/monkeypatches.py | 45 - .../scrapy/replay/__init__.py | 187 -- .../cluster-refactor/scrapy/spider/manager.py | 140 - .../scrapy/spider/middleware.py | 122 - .../cluster-refactor/scrapy/spider/models.py | 89 - .../scrapy/stats/corestats.py | 69 - .../cluster-refactor/scrapy/tests/__init__.py | 1 - .../scrapy/tests/test_adaptors.py | 84 - .../scrapy/tests/test_decompress.py | 45 - .../scrapy/tests/test_defaultencoding.py | 9 - .../scrapy/tests/test_engine.py | 197 -- .../scrapy/tests/test_http_request.py | 230 -- .../scrapy/tests/test_http_response.py | 20 - .../scrapy/tests/test_http_url.py | 28 - .../scrapy/tests/test_libxml2.py | 43 - .../scrapy/tests/test_link.py | 35 - .../scrapy/tests/test_pipeline_images.py | 31 - .../scrapy/tests/test_serialization.py | 22 - .../scrapy/tests/test_spiders/testplugin.py | 41 - .../scrapy/tests/test_storedb.py | 76 - .../scrapy/tests/test_utils_datatypes.py | 26 - .../scrapy/tests/test_utils_markup.py | 113 - .../scrapy/tests/test_utils_url.py | 151 -- .../scrapy/tests/test_utils_xml.py | 37 - .../scrapy/tests/test_xpath.py | 225 -- .../scrapy/utils/datatypes.py | 458 ---- .../cluster-refactor/scrapy/utils/db.py | 39 - .../scrapy/utils/decompressor.py | 72 - .../cluster-refactor/scrapy/utils/defer.py | 81 - .../scrapy/utils/iterators.py | 23 - .../cluster-refactor/scrapy/utils/markup.py | 100 - .../cluster-refactor/scrapy/utils/misc.py | 97 - .../cluster-refactor/scrapy/utils/python.py | 55 - .../cluster-refactor/scrapy/utils/response.py | 15 - .../cluster-refactor/scrapy/utils/url.py | 147 -- .../cluster-refactor/scrapy/utils/xml.py | 28 - .../scrapy/xpath/constructors.py | 31 - .../scrapy/xpath/extension.py | 20 - .../cluster-refactor/scrapy/xpath/selector.py | 133 - scrapy/branches/cluster-refactor/setup.cfg | 9 - scrapy/branches/cluster-refactor/setup.py | 26 - .../scrapy => }/command/__init__.py | 0 scrapy/{trunk/scrapy => }/command/cmdline.py | 0 .../contrib => command/commands}/__init__.py | 0 .../scrapy => }/command/commands/crawl.py | 0 .../scrapy => }/command/commands/download.py | 0 .../scrapy => }/command/commands/genspider.py | 0 .../scrapy => }/command/commands/help.py | 0 .../scrapy => }/command/commands/list.py | 0 .../scrapy => }/command/commands/log.py | 0 .../scrapy => }/command/commands/parse.py | 0 .../scrapy => }/command/commands/shell.py | 0 .../scrapy => }/command/commands/start.py | 0 .../scrapy => }/command/commands/stats.py | 0 scrapy/{trunk/scrapy => }/command/models.py | 0 scrapy/{trunk/scrapy => }/conf/__init__.py | 0 .../commands}/__init__.py | 0 .../scrapy => }/conf/commands/crawl.py | 0 .../scrapy => }/conf/commands/help.py | 0 .../scrapy => }/conf/commands/list.py | 0 .../scrapy => }/conf/commands/log.py | 0 .../scrapy => }/conf/commands/scrape.py | 0 .../{trunk/scrapy => }/conf/commands/shell.py | 0 .../scrapy => }/conf/commands/stats.py | 0 .../scrapy => }/conf/commands/test.py | 0 .../scrapy => }/conf/default_settings.py | 0 .../contrib/pipeline => contrib}/__init__.py | 0 scrapy/{trunk/scrapy => }/contrib/aws.py | 0 .../{trunk/scrapy => }/contrib/closedomain.py | 0 .../scrapy => }/contrib/cluster/__init__.py | 0 .../cluster/crawler}/__init__.py | 0 .../contrib/cluster/crawler/manager.py | 0 .../contrib/cluster/hooks/__init__.py | 0 .../scrapy => }/contrib/cluster/hooks/svn.py | 0 .../cluster/master}/__init__.py | 0 .../contrib/cluster/master/manager.py | 0 .../scrapy => }/contrib/cluster/master/web.py | 0 .../contrib/cluster/master/ws_api.txt | 0 .../cluster/tools/scrapy-cluster-ctl.py | 0 .../contrib/cluster/tools/test-worker.py | 0 .../cluster/worker}/__init__.py | 0 .../contrib/cluster/worker/manager.py | 0 .../scrapy => }/contrib/codecs/__init__.py | 0 .../scrapy => }/contrib/codecs/x_mac_roman.py | 0 scrapy/{trunk/scrapy => }/contrib/debug.py | 0 .../scrapy => }/contrib/delayedclosedomain.py | 0 .../downloadermiddleware}/__init__.py | 0 .../contrib/downloadermiddleware/cache.py | 0 .../contrib/downloadermiddleware/common.py | 0 .../contrib/downloadermiddleware/cookies.py | 0 .../contrib/downloadermiddleware/debug.py | 0 .../downloadermiddleware/errorpages.py | 0 .../contrib/downloadermiddleware/httpauth.py | 0 .../downloadermiddleware/httpcompression.py | 0 .../contrib/downloadermiddleware/redirect.py | 0 .../contrib/downloadermiddleware/retry.py | 0 .../contrib/downloadermiddleware/robotstxt.py | 0 .../contrib/downloadermiddleware/stats.py | 0 .../contrib/downloadermiddleware/useragent.py | 0 .../scrapy => }/contrib/groupsettings.py | 0 .../scrapy => }/contrib/item/__init__.py | 0 .../{trunk/scrapy => }/contrib/item/models.py | 0 .../{trunk/scrapy => }/contrib/itemsampler.py | 0 .../scrapy => }/contrib/link_extractors.py | 0 scrapy/{trunk/scrapy => }/contrib/memdebug.py | 0 scrapy/{trunk/scrapy => }/contrib/memusage.py | 0 .../pipeline}/__init__.py | 0 .../scrapy => }/contrib/pipeline/images.py | 0 .../scrapy => }/contrib/pipeline/media.py | 0 .../scrapy => }/contrib/pipeline/s3images.py | 0 .../scrapy => }/contrib/pipeline/show.py | 0 .../scrapy => }/contrib/prioritizers.py | 0 .../response}/__init__.py | 0 .../scrapy => }/contrib/response/soup.py | 0 .../schedulermiddleware}/__init__.py | 0 .../schedulermiddleware/duplicatesfilter.py | 0 .../store => contrib/spider}/__init__.py | 0 .../scrapy => }/contrib/spider/profiler.py | 0 .../scrapy => }/contrib/spider/reloader.py | 0 .../spidermiddleware}/__init__.py | 0 .../contrib/spidermiddleware/depth.py | 0 .../spidermiddleware/duplicatesfilter.py | 0 .../contrib/spidermiddleware/limit.py | 0 .../contrib/spidermiddleware/offsite.py | 0 .../contrib/spidermiddleware/referer.py | 0 .../contrib/spidermiddleware/restrict.py | 0 .../contrib/spidermiddleware/urlfilter.py | 0 .../contrib/spidermiddleware/urllength.py | 0 .../scrapy => }/contrib/spiders/__init__.py | 0 .../scrapy => }/contrib/spiders/crawl.py | 0 .../scrapy => }/contrib/spiders/feed.py | 0 .../scrapy => }/contrib/spiders/generic.py | 0 .../scrapy/utils => contrib/web}/__init__.py | 0 .../scrapy => }/contrib/web/http.py | 0 .../scrapy => }/contrib/web/json.py | 0 .../scrapy => }/contrib/web/service.py | 0 .../scrapy => }/contrib/web/site.py | 0 .../scrapy => }/contrib/web/stats.py | 0 .../webconsole}/__init__.py | 0 .../contrib/webconsole/enginestatus.py | 0 .../contrib/webconsole/livestats.py | 0 .../contrib/webconsole/scheduler.py | 0 .../contrib/webconsole/spiderctl.py | 0 .../contrib/webconsole/spiderstats.py | 0 .../scrapy => }/contrib/webconsole/stats.py | 0 .../scrapy => }/contrib_exp/__init__.py | 0 .../contrib_exp/adaptors/__init__.py | 0 .../scrapy => }/contrib_exp/adaptors/date.py | 0 .../contrib_exp/adaptors/extraction.py | 0 .../contrib_exp/adaptors/markup.py | 0 .../scrapy => }/contrib_exp/adaptors/misc.py | 0 .../downloadermiddleware}/__init__.py | 0 .../downloadermiddleware/decompression.py | 0 .../history/__init__.py | 0 .../history/history.py | 0 .../contrib_exp/history/middleware.py | 0 .../contrib_exp/history/scheduler.py | 0 .../contrib => contrib_exp}/history/store.py | 0 .../scrapy => }/contrib_exp/link/__init__.py | 0 .../contrib_exp/newitem/__init__.py | 0 .../contrib_exp/newitem/adaptors.py | 0 .../scrapy => }/contrib_exp/newitem/fields.py | 0 .../scrapy => }/contrib_exp/newitem/models.py | 0 .../contrib_exp/pipeline/shoveitem.py | 0 .../scrapy => }/core/__init__.py | 0 .../scrapy => }/core/downloader/__init__.py | 0 .../scrapy => }/core/downloader/dnscache.py | 0 .../scrapy => }/core/downloader/handlers.py | 0 .../scrapy => }/core/downloader/manager.py | 0 .../scrapy => }/core/downloader/middleware.py | 0 .../core/downloader/responsetypes/__init__.py | 0 .../core/downloader/responsetypes/mime.types | 0 .../scrapy => }/core/downloader/webclient.py | 0 scrapy/{trunk/scrapy => }/core/engine.py | 0 scrapy/{trunk/scrapy => }/core/exceptions.py | 0 scrapy/{trunk/scrapy => }/core/manager.py | 0 .../scrapy => }/core/prioritizers.py | 0 .../scrapy => }/core/scheduler/__init__.py | 0 .../scrapy => }/core/scheduler/middleware.py | 0 .../scrapy => }/core/scheduler/schedulers.py | 0 .../scrapy => }/core/scheduler/store.py | 0 scrapy/{trunk/scrapy => }/core/signals.py | 0 .../{trunk/scrapy => }/dupefilter/__init__.py | 0 .../{trunk/scrapy => }/extension/__init__.py | 0 .../scrapy => }/fetcher/__init__.py | 0 scrapy/{trunk/scrapy => }/http/__init__.py | 0 scrapy/{trunk/scrapy => }/http/cookies.py | 0 scrapy/{trunk/scrapy => }/http/headers.py | 0 .../scrapy => }/http/request/__init__.py | 0 .../{trunk/scrapy => }/http/request/form.py | 0 scrapy/{trunk/scrapy => }/http/request/rpc.py | 0 .../scrapy => }/http/response/__init__.py | 0 .../{trunk/scrapy => }/http/response/html.py | 0 .../{trunk/scrapy => }/http/response/text.py | 0 .../{trunk/scrapy => }/http/response/xml.py | 0 .../cluster-refactor/scrapy => }/http/url.py | 0 scrapy/{trunk/scrapy => }/item/__init__.py | 0 scrapy/{trunk/scrapy => }/item/adaptors.py | 0 scrapy/{trunk/scrapy => }/item/models.py | 0 scrapy/{trunk/scrapy => }/item/pipeline.py | 0 scrapy/{trunk/scrapy => }/link/__init__.py | 0 scrapy/{trunk/scrapy => }/link/extractors.py | 0 scrapy/{trunk/scrapy => }/log/__init__.py | 0 scrapy/{trunk/scrapy => }/mail/__init__.py | 0 .../googledir => management}/__init__.py | 0 .../scrapy => }/management/telnet.py | 0 scrapy/{trunk/scrapy => }/management/web.py | 0 .../command/commands => patches}/__init__.py | 0 .../scrapy => }/patches/monkeypatches.py | 0 .../scrapy => }/spider/__init__.py | 0 scrapy/{trunk/scrapy => }/spider/manager.py | 0 .../{trunk/scrapy => }/spider/middleware.py | 0 scrapy/{trunk/scrapy => }/spider/models.py | 0 .../scrapy => }/stats/__init__.py | 0 scrapy/{trunk/scrapy => }/stats/corestats.py | 0 .../scrapy => }/stats/statscollector.py | 0 .../conf/commands => store}/__init__.py | 0 .../cluster-refactor/scrapy => }/store/db.py | 0 .../project/module}/__init__.py | 0 .../templates/project/module/items.py.tmpl | 0 .../project/module/pipelines.py.tmpl | 0 .../templates/project/module/settings.py.tmpl | 0 .../project/module}/spiders/__init__.py | 0 .../module/templates/spider_basic.tmpl | 0 .../module/templates/spider_crawl.tmpl | 0 .../module/templates/spider_csvfeed.tmpl | 0 .../module/templates/spider_xmlfeed.tmpl | 0 .../templates/project/root/scrapy-ctl.py | 0 scrapy/{trunk/scrapy => }/tests/__init__.py | 0 scrapy/{trunk/scrapy => }/tests/run.py | 0 .../tests/sample_data/adaptors/enc-ascii.html | 0 .../sample_data/adaptors/enc-cp1252.html | 0 .../sample_data/adaptors/enc-latin1.html | 0 .../adaptors/enc-utf8-meta-latin1.html | 0 .../tests/sample_data/adaptors/enc-utf8.html | 0 .../sample_data/adaptors/extr_unquoted.xml | 0 .../sample_data/compressed/feed-sample1.tar | Bin .../sample_data/compressed/feed-sample1.xml | 0 .../compressed/feed-sample1.xml.bz2 | Bin .../compressed/feed-sample1.xml.gz | Bin .../sample_data/compressed/feed-sample1.zip | Bin .../sample_data/compressed/html-gzip.bin | Bin .../compressed/html-rawdeflate.bin | Bin .../compressed/html-zlibdeflate.bin | Bin .../tests/sample_data/feeds/feed-sample1.xml | 0 .../tests/sample_data/feeds/feed-sample2.xml | 0 .../tests/sample_data/feeds/feed-sample3.csv | 0 .../tests/sample_data/feeds/feed-sample4.csv | 0 .../tests/sample_data/feeds/feed-sample5.csv | 0 .../link_extractor/image_linkextractor.html | 0 .../link_extractor/linkextractor_latin1.html | 0 .../link_extractor/linkextractor_noenc.html | 0 .../link_extractor/regex_linkextractor.html | 0 .../tests/sample_data/test_site/index.html | 0 .../tests/sample_data/test_site/item1.html | 0 .../tests/sample_data/test_site/item2.html | 0 .../{trunk/scrapy => }/tests/test_adaptors.py | 0 scrapy/{trunk/scrapy => }/tests/test_aws.py | 0 .../scrapy => }/tests/test_c14nurls.py | 0 .../tests/test_contrib_response_soup.py | 0 .../scrapy => }/tests/test_dependencies.py | 0 .../test_downloadermiddleware_cookies.py | 0 ...test_downloadermiddleware_decompression.py | 0 ...st_downloadermiddleware_httpcompression.py | 0 .../test_downloadermiddleware_redirect.py | 0 .../tests/test_downloadermiddleware_retry.py | 0 .../test_downloadermiddleware_useragent.py | 0 .../scrapy => }/tests/test_dupefilter.py | 0 .../{trunk/scrapy => }/tests/test_engine.py | 0 .../scrapy => }/tests/test_http_cookies.py | 0 .../scrapy => }/tests/test_http_headers.py | 0 .../scrapy => }/tests/test_http_request.py | 0 .../scrapy => }/tests/test_http_response.py | 0 .../{trunk/scrapy => }/tests/test_http_url.py | 0 scrapy/{trunk/scrapy => }/tests/test_item.py | 0 .../scrapy => }/tests/test_itemadaptor.py | 0 .../{trunk/scrapy => }/tests/test_libxml2.py | 0 scrapy/{trunk/scrapy => }/tests/test_link.py | 0 .../{trunk/scrapy => }/tests/test_newitem.py | 0 .../scrapy => }/tests/test_pipeline_images.py | 0 .../scrapy => }/tests/test_responsetypes.py | 0 .../tests/test_robustscrapeditem.py | 0 ...st_schedulermiddleware_duplicatesfilter.py | 0 .../scrapy => }/tests/test_serialization.py | 0 .../test_spidermiddleware_duplicatesfilter.py | 0 .../scrapy => }/tests/test_spidermonkey.py | 0 .../test_spiders}/__init__.py | 0 .../tests/test_spiders/testspider.py | 0 .../scrapy => }/tests/test_stats.py | 0 .../{trunk/scrapy => }/tests/test_storedb.py | 0 .../scrapy => }/tests/test_utils_datatypes.py | 0 .../scrapy => }/tests/test_utils_defer.py | 0 .../scrapy => }/tests/test_utils_iterators.py | 0 .../scrapy => }/tests/test_utils_markup.py | 0 .../scrapy => }/tests/test_utils_misc.py | 0 .../scrapy => }/tests/test_utils_python.py | 0 .../scrapy => }/tests/test_utils_request.py | 0 .../scrapy => }/tests/test_utils_response.py | 0 .../scrapy => }/tests/test_utils_url.py | 0 .../scrapy => }/tests/test_webclient.py | 0 scrapy/{trunk/scrapy => }/tests/test_xpath.py | 0 .../scrapy => }/tests/test_xpath_extension.py | 0 scrapy/trunk/README | 3 - .../trunk/docs/media/scrapy-architecture.dia | Bin 2751 -> 0 bytes scrapy/trunk/extras/sql/scraping.sql | 72 - scrapy/trunk/scrapy/command/__init__.py | 1 - .../trunk/scrapy/command/commands/download.py | 28 - scrapy/trunk/scrapy/command/commands/help.py | 26 - scrapy/trunk/scrapy/command/commands/list.py | 16 - scrapy/trunk/scrapy/command/commands/log.py | 46 - scrapy/trunk/scrapy/command/commands/start.py | 16 - scrapy/trunk/scrapy/conf/commands/crawl.py | 1 - scrapy/trunk/scrapy/conf/commands/help.py | 1 - scrapy/trunk/scrapy/conf/commands/list.py | 1 - scrapy/trunk/scrapy/conf/commands/log.py | 1 - scrapy/trunk/scrapy/conf/commands/scrape.py | 2 - scrapy/trunk/scrapy/conf/commands/stats.py | 1 - scrapy/trunk/scrapy/conf/commands/test.py | 1 - .../scrapy/contrib/cluster/worker/__init__.py | 0 .../contrib/downloadermiddleware/__init__.py | 0 .../downloadermiddleware/errorpages.py | 13 - scrapy/trunk/scrapy/contrib/groupsettings.py | 26 - scrapy/trunk/scrapy/contrib/item/__init__.py | 1 - .../trunk/scrapy/contrib/pipeline/__init__.py | 0 scrapy/trunk/scrapy/contrib/prioritizers.py | 34 - .../trunk/scrapy/contrib/response/__init__.py | 0 .../contrib/schedulermiddleware/__init__.py | 0 .../trunk/scrapy/contrib/spider/__init__.py | 0 .../trunk/scrapy/contrib/spider/profiler.py | 60 - .../trunk/scrapy/contrib/spider/reloader.py | 19 - .../contrib/spidermiddleware/__init__.py | 0 scrapy/trunk/scrapy/contrib/web/__init__.py | 0 scrapy/trunk/scrapy/contrib/web/http.py | 115 - scrapy/trunk/scrapy/contrib/web/json.py | 99 - scrapy/trunk/scrapy/contrib/web/service.py | 89 - scrapy/trunk/scrapy/contrib/web/site.py | 133 - scrapy/trunk/scrapy/contrib/web/stats.py | 55 - .../scrapy/contrib/webconsole/__init__.py | 0 .../scrapy/contrib/webconsole/enginestatus.py | 27 - .../scrapy/contrib/webconsole/spiderstats.py | 80 - .../trunk/scrapy/contrib/webconsole/stats.py | 26 - .../downloadermiddleware/__init__.py | 0 .../scrapy/contrib_exp/history/__init__.py | 3 - .../scrapy/contrib_exp/history/history.py | 162 -- .../trunk/scrapy/contrib_exp/history/store.py | 47 - scrapy/trunk/scrapy/core/__init__.py | 3 - .../trunk/scrapy/core/downloader/__init__.py | 1 - scrapy/trunk/scrapy/core/prioritizers.py | 45 - scrapy/trunk/scrapy/core/scheduler/store.py | 35 - scrapy/trunk/scrapy/fetcher/__init__.py | 51 - scrapy/trunk/scrapy/http/url.py | 57 - scrapy/trunk/scrapy/management/__init__.py | 0 scrapy/trunk/scrapy/management/telnet.py | 42 - scrapy/trunk/scrapy/patches/__init__.py | 0 scrapy/trunk/scrapy/spider/__init__.py | 4 - scrapy/trunk/scrapy/stats/__init__.py | 3 - scrapy/trunk/scrapy/stats/statscollector.py | 77 - scrapy/trunk/scrapy/store/__init__.py | 0 scrapy/trunk/scrapy/store/db.py | 130 - .../templates/project/module/__init__.py | 0 .../project/module/spiders/__init__.py | 1 - .../sample_data/compressed/feed-sample1.tar | Bin 20480 -> 0 bytes .../sample_data/compressed/feed-sample1.xml | 100 - .../compressed/feed-sample1.xml.bz2 | Bin 1430 -> 0 bytes .../compressed/feed-sample1.xml.gz | Bin 1131 -> 0 bytes .../sample_data/compressed/feed-sample1.zip | Bin 1260 -> 0 bytes .../tests/sample_data/feeds/feed-sample1.xml | 100 - .../tests/sample_data/feeds/feed-sample2.xml | 100 - .../tests/sample_data/test_site/index.html | 18 - .../tests/sample_data/test_site/item1.html | 17 - .../tests/sample_data/test_site/item2.html | 17 - scrapy/trunk/scrapy/tests/test_c14nurls.py | 28 - .../trunk/scrapy/tests/test_dependencies.py | 16 - .../trunk/scrapy/tests/test_spidermonkey.py | 24 - .../scrapy/tests/test_spiders/__init__.py | 0 scrapy/trunk/scrapy/tests/test_stats.py | 52 - scrapy/trunk/scrapy/utils/__init__.py | 0 scrapy/trunk/scrapy/utils/c14n.py | 110 - scrapy/trunk/scrapy/utils/display.py | 49 - scrapy/trunk/scrapy/utils/serialization.py | 74 - scrapy/trunk/scrapy/xlib/BeautifulSoup.py | 1931 -------------- scrapy/trunk/scrapy/xlib/__init__.py | 1 - scrapy/trunk/scrapy/xlib/lrucache.py | 213 -- scrapy/trunk/scrapy/xlib/lsprofcalltree.py | 119 - .../trunk/scrapy/xlib/pydispatch/__init__.py | 6 - .../scrapy/xlib/pydispatch/dispatcher.py | 497 ---- scrapy/trunk/scrapy/xlib/pydispatch/errors.py | 10 - .../trunk/scrapy/xlib/pydispatch/license.txt | 34 - scrapy/trunk/scrapy/xlib/pydispatch/robust.py | 57 - .../scrapy/xlib/pydispatch/robustapply.py | 49 - .../trunk/scrapy/xlib/pydispatch/saferef.py | 165 -- .../scrapy/xlib/spidermonkey/INSTALL.scrapy | 27 - .../scrapy/xlib/spidermonkey/__init__.py | 1 - .../scrapy/xlib/spidermonkey/sm_settings.py | 2 - .../scrapy/xlib/spidermonkey/spidermonkey.py | 2254 ----------------- scrapy/trunk/scrapy/xpath/__init__.py | 13 - scrapy/trunk/scrapy/xpath/document.py | 22 - scrapy/trunk/scrapy/xpath/types.py | 11 - scrapy/trunk/scripts/rpm-install.sh | 10 - .../cluster/master => utils}/__init__.py | 0 .../scrapy => }/utils/c14n.py | 0 scrapy/{trunk/scrapy => }/utils/datatypes.py | 0 scrapy/{trunk/scrapy => }/utils/db.py | 0 scrapy/{trunk/scrapy => }/utils/defer.py | 0 .../scrapy => }/utils/display.py | 0 scrapy/{trunk/scrapy => }/utils/http.py | 0 scrapy/{trunk/scrapy => }/utils/iterators.py | 0 scrapy/{trunk/scrapy => }/utils/markup.py | 0 scrapy/{trunk/scrapy => }/utils/misc.py | 0 scrapy/{trunk/scrapy => }/utils/python.py | 0 scrapy/{trunk/scrapy => }/utils/request.py | 0 scrapy/{trunk/scrapy => }/utils/response.py | 0 .../scrapy => }/utils/serialization.py | 0 scrapy/{trunk/scrapy => }/utils/test.py | 0 scrapy/{trunk/scrapy => }/utils/url.py | 0 .../scrapy => }/xlib/BeautifulSoup.py | 0 scrapy/{trunk/scrapy => }/xlib/ClientForm.py | 0 .../scrapy => }/xlib/__init__.py | 0 .../scrapy => }/xlib/lrucache.py | 0 .../scrapy => }/xlib/lsprofcalltree.py | 0 .../scrapy => }/xlib/pydispatch/__init__.py | 0 .../scrapy => }/xlib/pydispatch/dispatcher.py | 0 .../scrapy => }/xlib/pydispatch/errors.py | 0 .../scrapy => }/xlib/pydispatch/license.txt | 0 .../scrapy => }/xlib/pydispatch/robust.py | 0 .../xlib/pydispatch/robustapply.py | 0 .../scrapy => }/xlib/pydispatch/saferef.py | 0 .../xlib/spidermonkey/INSTALL.scrapy | 0 .../scrapy => }/xlib/spidermonkey/__init__.py | 0 .../xlib/spidermonkey/sm_settings.py | 0 .../xlib/spidermonkey/spidermonkey.py | 0 .../scrapy => }/xpath/__init__.py | 0 .../{trunk/scrapy => }/xpath/constructors.py | 0 .../scrapy => }/xpath/document.py | 0 scrapy/{trunk/scrapy => }/xpath/extension.py | 0 scrapy/{trunk/scrapy => }/xpath/selector.py | 0 .../scrapy => }/xpath/types.py | 0 .../scripts => scripts}/rpm-install.sh | 0 scrapy/trunk/setup.cfg => setup.cfg | 0 scrapy/trunk/setup.py => setup.py | 0 sites/scrapy.org/Makefile | 13 - sites/scrapy.org/README | 7 - sites/scrapy.org/docs/old/basics.rst | 47 - sites/scrapy.org/docs/old/tutorial/index.rst | 9 - .../docs/old/tutorial/tutorial1.rst | 74 - .../docs/old/tutorial/tutorial2.rst | 173 -- .../docs/old/tutorial/tutorial3.rst | 173 -- sites/scrapy.org/scrapyorg/__init__.py | 0 .../scrapy.org/scrapyorg/article/__init__.py | 0 sites/scrapy.org/scrapyorg/article/urls.py | 9 - sites/scrapy.org/scrapyorg/article/views.py | 21 - sites/scrapy.org/scrapyorg/blog/__init__.py | 0 sites/scrapy.org/scrapyorg/blog/admin.py | 17 - sites/scrapy.org/scrapyorg/blog/feeds.py | 42 - sites/scrapy.org/scrapyorg/blog/managers.py | 9 - sites/scrapy.org/scrapyorg/blog/models.py | 80 - sites/scrapy.org/scrapyorg/blog/sitemap.py | 13 - .../scrapyorg/blog/templates/base.html | 21 - .../blog/templates/blog/base_blog.html | 5 - .../blog/templates/blog/category_detail.html | 25 - .../blog/templates/blog/category_list.html | 20 - .../blog/templates/blog/post_archive_day.html | 23 - .../templates/blog/post_archive_month.html | 23 - .../templates/blog/post_archive_year.html | 21 - .../blog/templates/blog/post_detail.html | 67 - .../blog/templates/blog/post_list.html | 35 - .../blog/templates/blog/post_search.html | 37 - .../templates/feeds/posts_description.html | 1 - .../blog/templates/feeds/posts_title.html | 1 - .../admin/blog/post/change_form.html | 56 - .../templates_backup/inlines/default.html | 7 - .../scrapyorg/blog/templatetags/__init__.py | 0 .../scrapyorg/blog/templatetags/blog.py | 103 - sites/scrapy.org/scrapyorg/blog/tests.py | 66 - sites/scrapy.org/scrapyorg/blog/urls.py | 41 - sites/scrapy.org/scrapyorg/blog/views.py | 160 -- sites/scrapy.org/scrapyorg/docs/__init__.py | 0 sites/scrapy.org/scrapyorg/docs/models.py | 3 - sites/scrapy.org/scrapyorg/docs/urls.py | 20 - sites/scrapy.org/scrapyorg/docs/views.py | 33 - .../scrapy.org/scrapyorg/download/__init__.py | 0 sites/scrapy.org/scrapyorg/download/admin.py | 11 - sites/scrapy.org/scrapyorg/download/models.py | 51 - .../download/templatetags/__init__.py | 0 .../download/templatetags/download_tags.py | 39 - sites/scrapy.org/scrapyorg/download/urls.py | 8 - sites/scrapy.org/scrapyorg/download/views.py | 12 - sites/scrapy.org/scrapyorg/lib/__init__.py | 0 .../scrapy.org/scrapyorg/lib/templatetags.py | 8 - .../scrapyorg/local_settings.py.template | 12 - sites/scrapy.org/scrapyorg/manage.py | 11 - sites/scrapy.org/scrapyorg/settings.py | 98 - sites/scrapy.org/scrapyorg/urls.py | 37 - .../static/images/box-borders-bottom.gif | Bin 84 -> 0 bytes .../static/images/box-borders-bottom.png | Bin 2852 -> 0 bytes .../static/images/box-borders-top.png | Bin 181 -> 0 bytes sites/scrapy.org/static/images/footer-bg.jpg | Bin 338 -> 0 bytes sites/scrapy.org/static/images/icon-arrow.gif | Bin 57 -> 0 bytes .../scrapy.org/static/images/icon-author.gif | Bin 57 -> 0 bytes sites/scrapy.org/static/images/logo.jpg | Bin 27114 -> 0 bytes sites/scrapy.org/static/images/main-bg.jpg | Bin 371 -> 0 bytes sites/scrapy.org/static/style/pygments.css | 61 - sites/scrapy.org/static/style/style.css | 154 -- .../scrapy.org/templates/admin/base_site.html | 37 - .../templates/articles/community.html | 28 - sites/scrapy.org/templates/articles/doc.html | 29 - .../templates/articles/download.html | 47 - sites/scrapy.org/templates/articles/home.html | 56 - sites/scrapy.org/templates/base.html | 35 - sites/scrapy.org/templates/base_doc.html | 29 - sites/scrapy.org/templates/base_home.html | 36 - sites/scrapy.org/templates/base_weblog.html | 6 - .../scrapy.org/templates/blog/base_blog.html | 5 - .../templates/blog/category_detail.html | 25 - .../templates/blog/category_list.html | 20 - .../templates/blog/post_archive_day.html | 23 - .../templates/blog/post_archive_month.html | 23 - .../templates/blog/post_archive_year.html | 21 - .../templates/blog/post_detail.html | 76 - .../scrapy.org/templates/blog/post_list.html | 35 - .../templates/blog/post_search.html | 37 - .../templates/blog_back/entry_archive.html | 15 - .../blog_back/entry_archive_day.html | 16 - .../blog_back/entry_archive_month.html | 16 - .../blog_back/entry_archive_year.html | 15 - .../templates/blog_back/entry_detail.html | 34 - sites/scrapy.org/templates/docs/doc.html | 34 - sites/scrapy.org/templates/footer.html | 10 - sites/scrapy.org/templates/header.html | 14 - sites/static.scrapy.org/asc.png | Bin 222 -> 0 bytes sites/static.scrapy.org/attachment.png | Bin 280 -> 0 bytes sites/static.scrapy.org/changeset.png | Bin 294 -> 0 bytes sites/static.scrapy.org/closedticket.png | Bin 297 -> 0 bytes sites/static.scrapy.org/collapsed.png | Bin 218 -> 0 bytes sites/static.scrapy.org/css/about.css | 18 - sites/static.scrapy.org/css/admin.css | 66 - sites/static.scrapy.org/css/browser.css | 183 -- sites/static.scrapy.org/css/changeset.css | 42 - sites/static.scrapy.org/css/code.css | 164 -- sites/static.scrapy.org/css/diff.css | 186 -- sites/static.scrapy.org/css/prefs.css | 25 - sites/static.scrapy.org/css/report.css | 127 - sites/static.scrapy.org/css/roadmap.css | 84 - sites/static.scrapy.org/css/search.css | 14 - sites/static.scrapy.org/css/ticket.css | 115 - sites/static.scrapy.org/css/timeline.css | 77 - .../css/trac-0.11rc2.css.diff | 13 - sites/static.scrapy.org/css/trac.css | 587 ----- sites/static.scrapy.org/css/wiki.css | 44 - sites/static.scrapy.org/desc.png | Bin 222 -> 0 bytes sites/static.scrapy.org/dots.gif | Bin 50 -> 0 bytes sites/static.scrapy.org/draft.png | Bin 4953 -> 0 bytes sites/static.scrapy.org/edit_toolbar.png | Bin 1449 -> 0 bytes sites/static.scrapy.org/editedticket.png | Bin 241 -> 0 bytes sites/static.scrapy.org/envelope.png | Bin 180 -> 0 bytes sites/static.scrapy.org/expanded.png | Bin 227 -> 0 bytes sites/static.scrapy.org/expander_normal.png | Bin 241 -> 0 bytes .../expander_normal_hover.png | Bin 212 -> 0 bytes sites/static.scrapy.org/expander_open.png | Bin 223 -> 0 bytes .../static.scrapy.org/expander_open_hover.png | Bin 209 -> 0 bytes sites/static.scrapy.org/extlink.gif | Bin 90 -> 0 bytes sites/static.scrapy.org/feed.png | Bin 638 -> 0 bytes sites/static.scrapy.org/file.png | Bin 285 -> 0 bytes sites/static.scrapy.org/folder.png | Bin 357 -> 0 bytes .../guide/basic-workflow.png | Bin 9028 -> 0 bytes .../guide/original-workflow.png | Bin 5688 -> 0 bytes sites/static.scrapy.org/ics.png | Bin 347 -> 0 bytes sites/static.scrapy.org/imggrid.png | Bin 205 -> 0 bytes sites/static.scrapy.org/js/blame.js | 102 - sites/static.scrapy.org/js/diff.js | 113 - sites/static.scrapy.org/js/expand_dir.js | 116 - sites/static.scrapy.org/js/folding.js | 26 - sites/static.scrapy.org/js/ie_pre7_hacks.js | 7 - sites/static.scrapy.org/js/jquery.js | 11 - sites/static.scrapy.org/js/keyboard_nav.js | 61 - sites/static.scrapy.org/js/noconflict.js | 3 - sites/static.scrapy.org/js/query.js | 278 -- sites/static.scrapy.org/js/search.js | 59 - sites/static.scrapy.org/js/suggest.js | 144 -- sites/static.scrapy.org/js/trac.js | 75 - sites/static.scrapy.org/js/wikitoolbar.js | 93 - sites/static.scrapy.org/loading.gif | Bin 2209 -> 0 bytes sites/static.scrapy.org/lock-locked.png | Bin 358 -> 0 bytes sites/static.scrapy.org/logo.jpg | Bin 27114 -> 0 bytes sites/static.scrapy.org/main-bg.jpg | Bin 371 -> 0 bytes sites/static.scrapy.org/milestone.png | Bin 245 -> 0 bytes sites/static.scrapy.org/newticket.png | Bin 227 -> 0 bytes sites/static.scrapy.org/parent.png | Bin 228 -> 0 bytes sites/static.scrapy.org/python.png | Bin 6111 -> 0 bytes sites/static.scrapy.org/topbar_gradient.png | Bin 350 -> 0 bytes sites/static.scrapy.org/topbar_gradient2.png | Bin 309 -> 0 bytes sites/static.scrapy.org/trac.ico | Bin 3638 -> 0 bytes sites/static.scrapy.org/trac_banner.png | Bin 2161 -> 0 bytes sites/static.scrapy.org/trac_logo_mini.png | Bin 689 -> 0 bytes sites/static.scrapy.org/vgradient.png | Bin 159 -> 0 bytes sites/static.scrapy.org/wiki.png | Bin 233 -> 0 bytes 761 files changed, 22823 deletions(-) rename scrapy/trunk/AUTHORS => AUTHORS (100%) rename scrapy/trunk/INSTALL => INSTALL (100%) rename scrapy/trunk/LICENSE => LICENSE (100%) rename scrapy/branches/cluster-refactor/README => README (100%) rename {scrapy/trunk/bin => bin}/runtests.sh (100%) rename {scrapy/trunk/docs => docs}/Makefile (100%) rename {scrapy/trunk/docs => docs}/README (100%) rename {scrapy/trunk/docs => docs}/_ext/scrapydocs.py (100%) rename {scrapy/trunk/docs => docs}/_static/items_adaptors-sample1.html (100%) rename {scrapy/trunk/docs => docs}/_static/scrapydoc.css (100%) rename {scrapy/trunk/docs => docs}/_static/selectors-sample1.html (100%) rename {scrapy/trunk/docs => docs}/conf.py (100%) rename {scrapy/trunk/docs => docs}/faq.rst (100%) rename {scrapy/trunk/docs => docs}/index.rst (100%) rename {scrapy/trunk/docs => docs}/intro/index.rst (100%) rename {scrapy/trunk/docs => docs}/intro/install.rst (100%) rename {scrapy/trunk/docs => docs}/intro/overview.rst (100%) rename {scrapy/trunk/docs => docs}/intro/tutorial.rst (100%) rename {scrapy/branches/cluster-refactor/docs => docs/media}/scrapy-architecture.dia (100%) rename {scrapy/trunk/docs => docs}/misc/api-stability.rst (100%) rename {scrapy/trunk/docs => docs}/misc/index.rst (100%) rename {scrapy/trunk/docs => docs}/proposed/_images/scrapy_architecture.odg (100%) rename {scrapy/trunk/docs => docs}/proposed/_images/scrapy_architecture.png (100%) rename {scrapy/trunk/docs => docs}/proposed/index.rst (100%) rename {scrapy/trunk/docs => docs}/proposed/introduction.rst (100%) rename {scrapy/trunk/docs => docs}/proposed/newitem.rst (100%) rename {scrapy/trunk/docs => docs}/proposed/spiders.rst (100%) rename {scrapy/trunk/docs => docs}/ref/downloader-middleware.rst (100%) rename {scrapy/trunk/docs => docs}/ref/email.rst (100%) rename {scrapy/trunk/docs => docs}/ref/exceptions.rst (100%) rename {scrapy/trunk/docs => docs}/ref/extension-manager.rst (100%) rename {scrapy/trunk/docs => docs}/ref/extensions.rst (100%) rename {scrapy/trunk/docs => docs}/ref/index.rst (100%) rename {scrapy/trunk/docs => docs}/ref/link-extractors.rst (100%) rename {scrapy/trunk/docs => docs}/ref/logging.rst (100%) rename {scrapy/trunk/docs => docs}/ref/request-response.rst (100%) rename {scrapy/trunk/docs => docs}/ref/selectors.rst (100%) rename {scrapy/trunk/docs => docs}/ref/settings.rst (100%) rename {scrapy/trunk/docs => docs}/ref/signals.rst (100%) rename {scrapy/trunk/docs => docs}/ref/spiders.rst (100%) rename {scrapy/trunk/docs => docs}/topics/_images/adaptors_diagram.png (100%) rename {scrapy/trunk/docs => docs}/topics/_images/adaptors_diagram.svg (100%) rename {scrapy/trunk/docs => docs}/topics/_images/firebug1.png (100%) rename {scrapy/trunk/docs => docs}/topics/_images/firebug2.png (100%) rename {scrapy/trunk/docs => docs}/topics/_images/firebug3.png (100%) rename {scrapy/trunk/docs => docs}/topics/_images/scrapy_architecture.odg (100%) rename {scrapy/trunk/docs => docs}/topics/_images/scrapy_architecture.png (100%) rename {scrapy/trunk/docs => docs}/topics/adaptors.rst (100%) rename {scrapy/trunk/docs => docs}/topics/architecture.rst (100%) rename {scrapy/trunk/docs => docs}/topics/downloader-middleware.rst (100%) rename {scrapy/trunk/docs => docs}/topics/extensions.rst (100%) rename {scrapy/trunk/docs => docs}/topics/firebug.rst (100%) rename {scrapy/trunk/docs => docs}/topics/firefox.rst (100%) rename {scrapy/trunk/docs => docs}/topics/index.rst (100%) rename {scrapy/trunk/docs => docs}/topics/item-pipeline.rst (100%) rename {scrapy/trunk/docs => docs}/topics/items.rst (100%) rename {scrapy/trunk/docs => docs}/topics/link-extractors.rst (100%) rename {scrapy/trunk/docs => docs}/topics/robotstxt.rst (100%) rename {scrapy/trunk/docs => docs}/topics/selectors.rst (100%) rename {scrapy/trunk/docs => docs}/topics/settings.rst (100%) rename {scrapy/trunk/docs => docs}/topics/shell.rst (100%) rename {scrapy/trunk/docs => docs}/topics/spider-middleware.rst (100%) rename {scrapy/trunk/docs => docs}/topics/spiders.rst (100%) rename {scrapy/trunk/docs => docs}/topics/stats.rst (100%) rename {scrapy/trunk/docs => docs}/topics/webconsole.rst (100%) rename {scrapy/branches/cluster-refactor/scrapy/command/commands => examples/experimental/googledir/googledir}/__init__.py (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/items.py (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/pipelines.py (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/settings.py (100%) rename {scrapy/branches/cluster-refactor/scrapy/conf/commands => examples/experimental/googledir/googledir/spiders}/__init__.py (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/spiders/google_directory.py (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/templates/spider_basic.tmpl (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/templates/spider_crawl.tmpl (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/templates/spider_csvfeed.tmpl (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl (100%) rename {scrapy/trunk/examples => examples}/experimental/googledir/scrapy-ctl.py (100%) rename {scrapy/branches/cluster-refactor/scrapy/conf/project_template => examples/googledir/googledir}/__init__.py (100%) rename {scrapy/trunk/examples => examples}/googledir/googledir/items.py (100%) rename {scrapy/trunk/examples => examples}/googledir/googledir/pipelines.py (100%) rename {scrapy/trunk/examples => examples}/googledir/googledir/settings.py (100%) rename {scrapy/branches/cluster-refactor/scrapy/conf/project_template => examples/googledir/googledir}/spiders/__init__.py (100%) rename {scrapy/trunk/examples => examples}/googledir/googledir/spiders/google_directory.py (100%) rename {scrapy/trunk/examples => examples}/googledir/scrapy-ctl.py (100%) rename {scrapy/branches/cluster-refactor/extras => extras}/sql/scraping.sql (100%) rename {scrapy/trunk/profiling => profiling}/priorityqueue/pq_classes.py (100%) rename {scrapy/trunk/profiling => profiling}/priorityqueue/run.py (100%) rename {scrapy/trunk/profiling => profiling}/priorityqueue/test_cases.py (100%) rename scrapy/{trunk/scrapy => }/__init__.py (100%) rename scrapy/{trunk/scrapy => }/bin/scrapy-admin.py (100%) delete mode 100644 scrapy/branches/cluster-refactor/INSTALL delete mode 100644 scrapy/branches/cluster-refactor/scrapy/__init__.py delete mode 100755 scrapy/branches/cluster-refactor/scrapy/bin/scrapy-admin.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/cmdline.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/crawl.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/genspider.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/getattr.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/parse.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/replay.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/shell.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/commands/stats.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/command/models.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/conf/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/conf/core_settings.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/conf/project_template/items.py delete mode 100755 scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy-ctl.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy_settings.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/adaptorpipeline.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/closedomain.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/debug.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cache.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/common.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/compression.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cookies.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/debug.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/httpauth.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/redirect.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/retry.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/robots.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/useragent.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/history/middleware.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/history/scheduler.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/item/models.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/memdebug.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/memusage.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/images.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/media.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/s3images.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/shoveitem.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/show.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/response/soup.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/depth.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/limit.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/offsite.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/referer.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/restrict.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urlfilter.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urllength.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/spiders.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/livestats.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/schedstats.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderctl.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/downloader/handlers.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/downloader/manager.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/downloader/middleware.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/engine.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/exceptions.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/manager.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/scheduler/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/scheduler/filter.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/scheduler/schedulers.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/core/signals.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/extension/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/http/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/http/headers.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/http/request.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/http/response.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/item/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/item/adaptors.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/item/models.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/item/pipeline.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/link/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/link/extractors.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/log/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/mail/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/management/web.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/patches/monkeypatches.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/replay/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/spider/manager.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/spider/middleware.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/spider/models.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/stats/corestats.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/__init__.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_adaptors.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_decompress.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_defaultencoding.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_engine.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_http_request.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_http_response.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_http_url.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_libxml2.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_link.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_pipeline_images.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_serialization.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/testplugin.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_storedb.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_utils_datatypes.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_utils_markup.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_utils_url.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_utils_xml.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/tests/test_xpath.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/datatypes.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/db.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/decompressor.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/defer.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/iterators.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/markup.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/misc.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/python.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/response.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/url.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/utils/xml.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/xpath/constructors.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/xpath/extension.py delete mode 100644 scrapy/branches/cluster-refactor/scrapy/xpath/selector.py delete mode 100644 scrapy/branches/cluster-refactor/setup.cfg delete mode 100644 scrapy/branches/cluster-refactor/setup.py rename scrapy/{branches/cluster-refactor/scrapy => }/command/__init__.py (100%) rename scrapy/{trunk/scrapy => }/command/cmdline.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib => command/commands}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/command/commands/crawl.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/command/commands/download.py (100%) rename scrapy/{trunk/scrapy => }/command/commands/genspider.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/command/commands/help.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/command/commands/list.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/command/commands/log.py (100%) rename scrapy/{trunk/scrapy => }/command/commands/parse.py (100%) rename scrapy/{trunk/scrapy => }/command/commands/shell.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/command/commands/start.py (100%) rename scrapy/{trunk/scrapy => }/command/commands/stats.py (100%) rename scrapy/{trunk/scrapy => }/command/models.py (100%) rename scrapy/{trunk/scrapy => }/conf/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/downloadermiddleware => conf/commands}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/crawl.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/help.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/list.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/log.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/scrape.py (100%) rename scrapy/{trunk/scrapy => }/conf/commands/shell.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/stats.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/conf/commands/test.py (100%) rename scrapy/{trunk/scrapy => }/conf/default_settings.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/pipeline => contrib}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/aws.py (100%) rename scrapy/{trunk/scrapy => }/contrib/closedomain.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/response => contrib/cluster/crawler}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/crawler/manager.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/hooks/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/hooks/svn.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/spider => contrib/cluster/master}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/master/manager.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/master/web.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/master/ws_api.txt (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/tools/scrapy-cluster-ctl.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/tools/test-worker.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/spidermiddleware => contrib/cluster/worker}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/cluster/worker/manager.py (100%) rename scrapy/{trunk/scrapy => }/contrib/codecs/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/codecs/x_mac_roman.py (100%) rename scrapy/{trunk/scrapy => }/contrib/debug.py (100%) rename scrapy/{trunk/scrapy => }/contrib/delayedclosedomain.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/web => contrib/downloadermiddleware}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/cache.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/common.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/cookies.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/debug.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/downloadermiddleware/errorpages.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/httpauth.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/httpcompression.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/redirect.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/retry.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/robotstxt.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/stats.py (100%) rename scrapy/{trunk/scrapy => }/contrib/downloadermiddleware/useragent.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/groupsettings.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/item/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/item/models.py (100%) rename scrapy/{trunk/scrapy => }/contrib/itemsampler.py (100%) rename scrapy/{trunk/scrapy => }/contrib/link_extractors.py (100%) rename scrapy/{trunk/scrapy => }/contrib/memdebug.py (100%) rename scrapy/{trunk/scrapy => }/contrib/memusage.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib/webconsole => contrib/pipeline}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/pipeline/images.py (100%) rename scrapy/{trunk/scrapy => }/contrib/pipeline/media.py (100%) rename scrapy/{trunk/scrapy => }/contrib/pipeline/s3images.py (100%) rename scrapy/{trunk/scrapy => }/contrib/pipeline/show.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/prioritizers.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/management => contrib/response}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/response/soup.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/patches => contrib/schedulermiddleware}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/schedulermiddleware/duplicatesfilter.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/store => contrib/spider}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/spider/profiler.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/spider/reloader.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/tests/test_spiders => contrib/spidermiddleware}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/depth.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/duplicatesfilter.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/limit.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/offsite.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/referer.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/restrict.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/urlfilter.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spidermiddleware/urllength.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spiders/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spiders/crawl.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spiders/feed.py (100%) rename scrapy/{trunk/scrapy => }/contrib/spiders/generic.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/utils => contrib/web}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/web/http.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/web/json.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/web/service.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/web/site.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/web/stats.py (100%) rename scrapy/{trunk/examples/experimental/googledir/googledir => contrib/webconsole}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/webconsole/enginestatus.py (100%) rename scrapy/{trunk/scrapy => }/contrib/webconsole/livestats.py (100%) rename scrapy/{trunk/scrapy => }/contrib/webconsole/scheduler.py (100%) rename scrapy/{trunk/scrapy => }/contrib/webconsole/spiderctl.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/webconsole/spiderstats.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/contrib/webconsole/stats.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/adaptors/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/adaptors/date.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/adaptors/extraction.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/adaptors/markup.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/adaptors/misc.py (100%) rename scrapy/{trunk/examples/experimental/googledir/googledir/spiders => contrib_exp/downloadermiddleware}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/downloadermiddleware/decompression.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib => contrib_exp}/history/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib => contrib_exp}/history/history.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/history/middleware.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/history/scheduler.py (100%) rename scrapy/{branches/cluster-refactor/scrapy/contrib => contrib_exp}/history/store.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/link/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/newitem/__init__.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/newitem/adaptors.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/newitem/fields.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/newitem/models.py (100%) rename scrapy/{trunk/scrapy => }/contrib_exp/pipeline/shoveitem.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/core/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/core/downloader/__init__.py (100%) rename scrapy/{trunk/scrapy => }/core/downloader/dnscache.py (100%) rename scrapy/{trunk/scrapy => }/core/downloader/handlers.py (100%) rename scrapy/{trunk/scrapy => }/core/downloader/manager.py (100%) rename scrapy/{trunk/scrapy => }/core/downloader/middleware.py (100%) rename scrapy/{trunk/scrapy => }/core/downloader/responsetypes/__init__.py (100%) rename scrapy/{trunk/scrapy => }/core/downloader/responsetypes/mime.types (100%) rename scrapy/{trunk/scrapy => }/core/downloader/webclient.py (100%) rename scrapy/{trunk/scrapy => }/core/engine.py (100%) rename scrapy/{trunk/scrapy => }/core/exceptions.py (100%) rename scrapy/{trunk/scrapy => }/core/manager.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/core/prioritizers.py (100%) rename scrapy/{trunk/scrapy => }/core/scheduler/__init__.py (100%) rename scrapy/{trunk/scrapy => }/core/scheduler/middleware.py (100%) rename scrapy/{trunk/scrapy => }/core/scheduler/schedulers.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/core/scheduler/store.py (100%) rename scrapy/{trunk/scrapy => }/core/signals.py (100%) rename scrapy/{trunk/scrapy => }/dupefilter/__init__.py (100%) rename scrapy/{trunk/scrapy => }/extension/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/fetcher/__init__.py (100%) rename scrapy/{trunk/scrapy => }/http/__init__.py (100%) rename scrapy/{trunk/scrapy => }/http/cookies.py (100%) rename scrapy/{trunk/scrapy => }/http/headers.py (100%) rename scrapy/{trunk/scrapy => }/http/request/__init__.py (100%) rename scrapy/{trunk/scrapy => }/http/request/form.py (100%) rename scrapy/{trunk/scrapy => }/http/request/rpc.py (100%) rename scrapy/{trunk/scrapy => }/http/response/__init__.py (100%) rename scrapy/{trunk/scrapy => }/http/response/html.py (100%) rename scrapy/{trunk/scrapy => }/http/response/text.py (100%) rename scrapy/{trunk/scrapy => }/http/response/xml.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/http/url.py (100%) rename scrapy/{trunk/scrapy => }/item/__init__.py (100%) rename scrapy/{trunk/scrapy => }/item/adaptors.py (100%) rename scrapy/{trunk/scrapy => }/item/models.py (100%) rename scrapy/{trunk/scrapy => }/item/pipeline.py (100%) rename scrapy/{trunk/scrapy => }/link/__init__.py (100%) rename scrapy/{trunk/scrapy => }/link/extractors.py (100%) rename scrapy/{trunk/scrapy => }/log/__init__.py (100%) rename scrapy/{trunk/scrapy => }/mail/__init__.py (100%) rename scrapy/{trunk/examples/googledir/googledir => management}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/management/telnet.py (100%) rename scrapy/{trunk/scrapy => }/management/web.py (100%) rename scrapy/{trunk/scrapy/command/commands => patches}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/patches/monkeypatches.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/spider/__init__.py (100%) rename scrapy/{trunk/scrapy => }/spider/manager.py (100%) rename scrapy/{trunk/scrapy => }/spider/middleware.py (100%) rename scrapy/{trunk/scrapy => }/spider/models.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/stats/__init__.py (100%) rename scrapy/{trunk/scrapy => }/stats/corestats.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/stats/statscollector.py (100%) rename scrapy/{trunk/scrapy/conf/commands => store}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/store/db.py (100%) rename scrapy/{trunk/scrapy/contrib => templates/project/module}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/items.py.tmpl (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/pipelines.py.tmpl (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/settings.py.tmpl (100%) rename scrapy/{trunk/examples/googledir/googledir => templates/project/module}/spiders/__init__.py (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/templates/spider_basic.tmpl (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/templates/spider_crawl.tmpl (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/templates/spider_csvfeed.tmpl (100%) rename scrapy/{trunk/scrapy => }/templates/project/module/templates/spider_xmlfeed.tmpl (100%) rename scrapy/{trunk/scrapy => }/templates/project/root/scrapy-ctl.py (100%) rename scrapy/{trunk/scrapy => }/tests/__init__.py (100%) rename scrapy/{trunk/scrapy => }/tests/run.py (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/adaptors/enc-ascii.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/adaptors/enc-cp1252.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/adaptors/enc-latin1.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/adaptors/enc-utf8-meta-latin1.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/adaptors/enc-utf8.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/adaptors/extr_unquoted.xml (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/compressed/feed-sample1.tar (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/compressed/feed-sample1.xml (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/compressed/feed-sample1.xml.bz2 (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/compressed/feed-sample1.xml.gz (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/compressed/feed-sample1.zip (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/compressed/html-gzip.bin (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/compressed/html-rawdeflate.bin (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/compressed/html-zlibdeflate.bin (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/feeds/feed-sample1.xml (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/feeds/feed-sample2.xml (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/feeds/feed-sample3.csv (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/feeds/feed-sample4.csv (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/feeds/feed-sample5.csv (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/link_extractor/image_linkextractor.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/link_extractor/linkextractor_latin1.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/link_extractor/linkextractor_noenc.html (100%) rename scrapy/{trunk/scrapy => }/tests/sample_data/link_extractor/regex_linkextractor.html (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/test_site/index.html (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/test_site/item1.html (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/sample_data/test_site/item2.html (100%) rename scrapy/{trunk/scrapy => }/tests/test_adaptors.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_aws.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/test_c14nurls.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_contrib_response_soup.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/test_dependencies.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_downloadermiddleware_cookies.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_downloadermiddleware_decompression.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_downloadermiddleware_httpcompression.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_downloadermiddleware_redirect.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_downloadermiddleware_retry.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_downloadermiddleware_useragent.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_dupefilter.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_engine.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_http_cookies.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_http_headers.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_http_request.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_http_response.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_http_url.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_item.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_itemadaptor.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_libxml2.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_link.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_newitem.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_pipeline_images.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_responsetypes.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_robustscrapeditem.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_schedulermiddleware_duplicatesfilter.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_serialization.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_spidermiddleware_duplicatesfilter.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/test_spidermonkey.py (100%) rename scrapy/{trunk/scrapy/contrib/cluster/crawler => tests/test_spiders}/__init__.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_spiders/testspider.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/tests/test_stats.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_storedb.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_datatypes.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_defer.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_iterators.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_markup.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_misc.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_python.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_request.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_response.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_utils_url.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_webclient.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_xpath.py (100%) rename scrapy/{trunk/scrapy => }/tests/test_xpath_extension.py (100%) delete mode 100644 scrapy/trunk/README delete mode 100644 scrapy/trunk/docs/media/scrapy-architecture.dia delete mode 100644 scrapy/trunk/extras/sql/scraping.sql delete mode 100644 scrapy/trunk/scrapy/command/__init__.py delete mode 100644 scrapy/trunk/scrapy/command/commands/download.py delete mode 100644 scrapy/trunk/scrapy/command/commands/help.py delete mode 100644 scrapy/trunk/scrapy/command/commands/list.py delete mode 100644 scrapy/trunk/scrapy/command/commands/log.py delete mode 100644 scrapy/trunk/scrapy/command/commands/start.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/crawl.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/help.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/list.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/log.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/scrape.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/stats.py delete mode 100644 scrapy/trunk/scrapy/conf/commands/test.py delete mode 100644 scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py delete mode 100644 scrapy/trunk/scrapy/contrib/groupsettings.py delete mode 100644 scrapy/trunk/scrapy/contrib/item/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/pipeline/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/prioritizers.py delete mode 100644 scrapy/trunk/scrapy/contrib/response/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/schedulermiddleware/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/spider/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/spider/profiler.py delete mode 100644 scrapy/trunk/scrapy/contrib/spider/reloader.py delete mode 100644 scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/web/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/web/http.py delete mode 100644 scrapy/trunk/scrapy/contrib/web/json.py delete mode 100644 scrapy/trunk/scrapy/contrib/web/service.py delete mode 100644 scrapy/trunk/scrapy/contrib/web/site.py delete mode 100644 scrapy/trunk/scrapy/contrib/web/stats.py delete mode 100644 scrapy/trunk/scrapy/contrib/webconsole/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py delete mode 100644 scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py delete mode 100644 scrapy/trunk/scrapy/contrib/webconsole/stats.py delete mode 100644 scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib_exp/history/__init__.py delete mode 100644 scrapy/trunk/scrapy/contrib_exp/history/history.py delete mode 100644 scrapy/trunk/scrapy/contrib_exp/history/store.py delete mode 100644 scrapy/trunk/scrapy/core/__init__.py delete mode 100644 scrapy/trunk/scrapy/core/downloader/__init__.py delete mode 100644 scrapy/trunk/scrapy/core/prioritizers.py delete mode 100644 scrapy/trunk/scrapy/core/scheduler/store.py delete mode 100644 scrapy/trunk/scrapy/fetcher/__init__.py delete mode 100644 scrapy/trunk/scrapy/http/url.py delete mode 100644 scrapy/trunk/scrapy/management/__init__.py delete mode 100644 scrapy/trunk/scrapy/management/telnet.py delete mode 100644 scrapy/trunk/scrapy/patches/__init__.py delete mode 100644 scrapy/trunk/scrapy/spider/__init__.py delete mode 100644 scrapy/trunk/scrapy/stats/__init__.py delete mode 100644 scrapy/trunk/scrapy/stats/statscollector.py delete mode 100644 scrapy/trunk/scrapy/store/__init__.py delete mode 100644 scrapy/trunk/scrapy/store/db.py delete mode 100644 scrapy/trunk/scrapy/templates/project/module/__init__.py delete mode 100644 scrapy/trunk/scrapy/templates/project/module/spiders/__init__.py delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.tar delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.zip delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/test_site/index.html delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html delete mode 100644 scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html delete mode 100644 scrapy/trunk/scrapy/tests/test_c14nurls.py delete mode 100644 scrapy/trunk/scrapy/tests/test_dependencies.py delete mode 100644 scrapy/trunk/scrapy/tests/test_spidermonkey.py delete mode 100644 scrapy/trunk/scrapy/tests/test_spiders/__init__.py delete mode 100644 scrapy/trunk/scrapy/tests/test_stats.py delete mode 100644 scrapy/trunk/scrapy/utils/__init__.py delete mode 100644 scrapy/trunk/scrapy/utils/c14n.py delete mode 100644 scrapy/trunk/scrapy/utils/display.py delete mode 100644 scrapy/trunk/scrapy/utils/serialization.py delete mode 100644 scrapy/trunk/scrapy/xlib/BeautifulSoup.py delete mode 100644 scrapy/trunk/scrapy/xlib/__init__.py delete mode 100644 scrapy/trunk/scrapy/xlib/lrucache.py delete mode 100644 scrapy/trunk/scrapy/xlib/lsprofcalltree.py delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/__init__.py delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/dispatcher.py delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/errors.py delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/license.txt delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/robust.py delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/robustapply.py delete mode 100644 scrapy/trunk/scrapy/xlib/pydispatch/saferef.py delete mode 100644 scrapy/trunk/scrapy/xlib/spidermonkey/INSTALL.scrapy delete mode 100644 scrapy/trunk/scrapy/xlib/spidermonkey/__init__.py delete mode 100644 scrapy/trunk/scrapy/xlib/spidermonkey/sm_settings.py delete mode 100644 scrapy/trunk/scrapy/xlib/spidermonkey/spidermonkey.py delete mode 100644 scrapy/trunk/scrapy/xpath/__init__.py delete mode 100644 scrapy/trunk/scrapy/xpath/document.py delete mode 100644 scrapy/trunk/scrapy/xpath/types.py delete mode 100644 scrapy/trunk/scripts/rpm-install.sh rename scrapy/{trunk/scrapy/contrib/cluster/master => utils}/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/utils/c14n.py (100%) rename scrapy/{trunk/scrapy => }/utils/datatypes.py (100%) rename scrapy/{trunk/scrapy => }/utils/db.py (100%) rename scrapy/{trunk/scrapy => }/utils/defer.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/utils/display.py (100%) rename scrapy/{trunk/scrapy => }/utils/http.py (100%) rename scrapy/{trunk/scrapy => }/utils/iterators.py (100%) rename scrapy/{trunk/scrapy => }/utils/markup.py (100%) rename scrapy/{trunk/scrapy => }/utils/misc.py (100%) rename scrapy/{trunk/scrapy => }/utils/python.py (100%) rename scrapy/{trunk/scrapy => }/utils/request.py (100%) rename scrapy/{trunk/scrapy => }/utils/response.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/utils/serialization.py (100%) rename scrapy/{trunk/scrapy => }/utils/test.py (100%) rename scrapy/{trunk/scrapy => }/utils/url.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/BeautifulSoup.py (100%) rename scrapy/{trunk/scrapy => }/xlib/ClientForm.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/lrucache.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/lsprofcalltree.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/dispatcher.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/errors.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/license.txt (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/robust.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/robustapply.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/pydispatch/saferef.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/spidermonkey/INSTALL.scrapy (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/spidermonkey/__init__.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/spidermonkey/sm_settings.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xlib/spidermonkey/spidermonkey.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xpath/__init__.py (100%) rename scrapy/{trunk/scrapy => }/xpath/constructors.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xpath/document.py (100%) rename scrapy/{trunk/scrapy => }/xpath/extension.py (100%) rename scrapy/{trunk/scrapy => }/xpath/selector.py (100%) rename scrapy/{branches/cluster-refactor/scrapy => }/xpath/types.py (100%) rename {scrapy/branches/cluster-refactor/scripts => scripts}/rpm-install.sh (100%) rename scrapy/trunk/setup.cfg => setup.cfg (100%) rename scrapy/trunk/setup.py => setup.py (100%) delete mode 100644 sites/scrapy.org/Makefile delete mode 100644 sites/scrapy.org/README delete mode 100644 sites/scrapy.org/docs/old/basics.rst delete mode 100644 sites/scrapy.org/docs/old/tutorial/index.rst delete mode 100644 sites/scrapy.org/docs/old/tutorial/tutorial1.rst delete mode 100644 sites/scrapy.org/docs/old/tutorial/tutorial2.rst delete mode 100644 sites/scrapy.org/docs/old/tutorial/tutorial3.rst delete mode 100644 sites/scrapy.org/scrapyorg/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/article/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/article/urls.py delete mode 100644 sites/scrapy.org/scrapyorg/article/views.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/admin.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/feeds.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/managers.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/models.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/sitemap.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/base.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/base_blog.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/category_detail.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/category_list.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/post_archive_day.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/post_archive_month.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/post_archive_year.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/post_detail.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/post_list.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/blog/post_search.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/feeds/posts_description.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates/feeds/posts_title.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates_backup/admin/blog/post/change_form.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templates_backup/inlines/default.html delete mode 100644 sites/scrapy.org/scrapyorg/blog/templatetags/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/templatetags/blog.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/tests.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/urls.py delete mode 100644 sites/scrapy.org/scrapyorg/blog/views.py delete mode 100644 sites/scrapy.org/scrapyorg/docs/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/docs/models.py delete mode 100644 sites/scrapy.org/scrapyorg/docs/urls.py delete mode 100644 sites/scrapy.org/scrapyorg/docs/views.py delete mode 100644 sites/scrapy.org/scrapyorg/download/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/download/admin.py delete mode 100644 sites/scrapy.org/scrapyorg/download/models.py delete mode 100644 sites/scrapy.org/scrapyorg/download/templatetags/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/download/templatetags/download_tags.py delete mode 100644 sites/scrapy.org/scrapyorg/download/urls.py delete mode 100644 sites/scrapy.org/scrapyorg/download/views.py delete mode 100644 sites/scrapy.org/scrapyorg/lib/__init__.py delete mode 100644 sites/scrapy.org/scrapyorg/lib/templatetags.py delete mode 100644 sites/scrapy.org/scrapyorg/local_settings.py.template delete mode 100755 sites/scrapy.org/scrapyorg/manage.py delete mode 100644 sites/scrapy.org/scrapyorg/settings.py delete mode 100644 sites/scrapy.org/scrapyorg/urls.py delete mode 100644 sites/scrapy.org/static/images/box-borders-bottom.gif delete mode 100644 sites/scrapy.org/static/images/box-borders-bottom.png delete mode 100644 sites/scrapy.org/static/images/box-borders-top.png delete mode 100644 sites/scrapy.org/static/images/footer-bg.jpg delete mode 100644 sites/scrapy.org/static/images/icon-arrow.gif delete mode 100644 sites/scrapy.org/static/images/icon-author.gif delete mode 100644 sites/scrapy.org/static/images/logo.jpg delete mode 100644 sites/scrapy.org/static/images/main-bg.jpg delete mode 100644 sites/scrapy.org/static/style/pygments.css delete mode 100644 sites/scrapy.org/static/style/style.css delete mode 100644 sites/scrapy.org/templates/admin/base_site.html delete mode 100644 sites/scrapy.org/templates/articles/community.html delete mode 100644 sites/scrapy.org/templates/articles/doc.html delete mode 100644 sites/scrapy.org/templates/articles/download.html delete mode 100644 sites/scrapy.org/templates/articles/home.html delete mode 100644 sites/scrapy.org/templates/base.html delete mode 100644 sites/scrapy.org/templates/base_doc.html delete mode 100644 sites/scrapy.org/templates/base_home.html delete mode 100644 sites/scrapy.org/templates/base_weblog.html delete mode 100644 sites/scrapy.org/templates/blog/base_blog.html delete mode 100644 sites/scrapy.org/templates/blog/category_detail.html delete mode 100644 sites/scrapy.org/templates/blog/category_list.html delete mode 100644 sites/scrapy.org/templates/blog/post_archive_day.html delete mode 100644 sites/scrapy.org/templates/blog/post_archive_month.html delete mode 100644 sites/scrapy.org/templates/blog/post_archive_year.html delete mode 100644 sites/scrapy.org/templates/blog/post_detail.html delete mode 100644 sites/scrapy.org/templates/blog/post_list.html delete mode 100644 sites/scrapy.org/templates/blog/post_search.html delete mode 100644 sites/scrapy.org/templates/blog_back/entry_archive.html delete mode 100644 sites/scrapy.org/templates/blog_back/entry_archive_day.html delete mode 100644 sites/scrapy.org/templates/blog_back/entry_archive_month.html delete mode 100644 sites/scrapy.org/templates/blog_back/entry_archive_year.html delete mode 100644 sites/scrapy.org/templates/blog_back/entry_detail.html delete mode 100644 sites/scrapy.org/templates/docs/doc.html delete mode 100644 sites/scrapy.org/templates/footer.html delete mode 100644 sites/scrapy.org/templates/header.html delete mode 100644 sites/static.scrapy.org/asc.png delete mode 100644 sites/static.scrapy.org/attachment.png delete mode 100644 sites/static.scrapy.org/changeset.png delete mode 100644 sites/static.scrapy.org/closedticket.png delete mode 100644 sites/static.scrapy.org/collapsed.png delete mode 100644 sites/static.scrapy.org/css/about.css delete mode 100644 sites/static.scrapy.org/css/admin.css delete mode 100644 sites/static.scrapy.org/css/browser.css delete mode 100644 sites/static.scrapy.org/css/changeset.css delete mode 100644 sites/static.scrapy.org/css/code.css delete mode 100644 sites/static.scrapy.org/css/diff.css delete mode 100644 sites/static.scrapy.org/css/prefs.css delete mode 100644 sites/static.scrapy.org/css/report.css delete mode 100644 sites/static.scrapy.org/css/roadmap.css delete mode 100644 sites/static.scrapy.org/css/search.css delete mode 100644 sites/static.scrapy.org/css/ticket.css delete mode 100644 sites/static.scrapy.org/css/timeline.css delete mode 100644 sites/static.scrapy.org/css/trac-0.11rc2.css.diff delete mode 100644 sites/static.scrapy.org/css/trac.css delete mode 100644 sites/static.scrapy.org/css/wiki.css delete mode 100644 sites/static.scrapy.org/desc.png delete mode 100644 sites/static.scrapy.org/dots.gif delete mode 100644 sites/static.scrapy.org/draft.png delete mode 100644 sites/static.scrapy.org/edit_toolbar.png delete mode 100644 sites/static.scrapy.org/editedticket.png delete mode 100644 sites/static.scrapy.org/envelope.png delete mode 100644 sites/static.scrapy.org/expanded.png delete mode 100644 sites/static.scrapy.org/expander_normal.png delete mode 100644 sites/static.scrapy.org/expander_normal_hover.png delete mode 100644 sites/static.scrapy.org/expander_open.png delete mode 100644 sites/static.scrapy.org/expander_open_hover.png delete mode 100644 sites/static.scrapy.org/extlink.gif delete mode 100644 sites/static.scrapy.org/feed.png delete mode 100644 sites/static.scrapy.org/file.png delete mode 100644 sites/static.scrapy.org/folder.png delete mode 100644 sites/static.scrapy.org/guide/basic-workflow.png delete mode 100644 sites/static.scrapy.org/guide/original-workflow.png delete mode 100644 sites/static.scrapy.org/ics.png delete mode 100644 sites/static.scrapy.org/imggrid.png delete mode 100644 sites/static.scrapy.org/js/blame.js delete mode 100644 sites/static.scrapy.org/js/diff.js delete mode 100644 sites/static.scrapy.org/js/expand_dir.js delete mode 100644 sites/static.scrapy.org/js/folding.js delete mode 100644 sites/static.scrapy.org/js/ie_pre7_hacks.js delete mode 100644 sites/static.scrapy.org/js/jquery.js delete mode 100644 sites/static.scrapy.org/js/keyboard_nav.js delete mode 100644 sites/static.scrapy.org/js/noconflict.js delete mode 100644 sites/static.scrapy.org/js/query.js delete mode 100644 sites/static.scrapy.org/js/search.js delete mode 100644 sites/static.scrapy.org/js/suggest.js delete mode 100644 sites/static.scrapy.org/js/trac.js delete mode 100644 sites/static.scrapy.org/js/wikitoolbar.js delete mode 100644 sites/static.scrapy.org/loading.gif delete mode 100644 sites/static.scrapy.org/lock-locked.png delete mode 100644 sites/static.scrapy.org/logo.jpg delete mode 100644 sites/static.scrapy.org/main-bg.jpg delete mode 100644 sites/static.scrapy.org/milestone.png delete mode 100644 sites/static.scrapy.org/newticket.png delete mode 100644 sites/static.scrapy.org/parent.png delete mode 100644 sites/static.scrapy.org/python.png delete mode 100644 sites/static.scrapy.org/topbar_gradient.png delete mode 100644 sites/static.scrapy.org/topbar_gradient2.png delete mode 100644 sites/static.scrapy.org/trac.ico delete mode 100644 sites/static.scrapy.org/trac_banner.png delete mode 100644 sites/static.scrapy.org/trac_logo_mini.png delete mode 100644 sites/static.scrapy.org/vgradient.png delete mode 100644 sites/static.scrapy.org/wiki.png diff --git a/scrapy/trunk/AUTHORS b/AUTHORS similarity index 100% rename from scrapy/trunk/AUTHORS rename to AUTHORS diff --git a/scrapy/trunk/INSTALL b/INSTALL similarity index 100% rename from scrapy/trunk/INSTALL rename to INSTALL diff --git a/scrapy/trunk/LICENSE b/LICENSE similarity index 100% rename from scrapy/trunk/LICENSE rename to LICENSE diff --git a/scrapy/branches/cluster-refactor/README b/README similarity index 100% rename from scrapy/branches/cluster-refactor/README rename to README diff --git a/scrapy/trunk/bin/runtests.sh b/bin/runtests.sh similarity index 100% rename from scrapy/trunk/bin/runtests.sh rename to bin/runtests.sh diff --git a/scrapy/trunk/docs/Makefile b/docs/Makefile similarity index 100% rename from scrapy/trunk/docs/Makefile rename to docs/Makefile diff --git a/scrapy/trunk/docs/README b/docs/README similarity index 100% rename from scrapy/trunk/docs/README rename to docs/README diff --git a/scrapy/trunk/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py similarity index 100% rename from scrapy/trunk/docs/_ext/scrapydocs.py rename to docs/_ext/scrapydocs.py diff --git a/scrapy/trunk/docs/_static/items_adaptors-sample1.html b/docs/_static/items_adaptors-sample1.html similarity index 100% rename from scrapy/trunk/docs/_static/items_adaptors-sample1.html rename to docs/_static/items_adaptors-sample1.html diff --git a/scrapy/trunk/docs/_static/scrapydoc.css b/docs/_static/scrapydoc.css similarity index 100% rename from scrapy/trunk/docs/_static/scrapydoc.css rename to docs/_static/scrapydoc.css diff --git a/scrapy/trunk/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html similarity index 100% rename from scrapy/trunk/docs/_static/selectors-sample1.html rename to docs/_static/selectors-sample1.html diff --git a/scrapy/trunk/docs/conf.py b/docs/conf.py similarity index 100% rename from scrapy/trunk/docs/conf.py rename to docs/conf.py diff --git a/scrapy/trunk/docs/faq.rst b/docs/faq.rst similarity index 100% rename from scrapy/trunk/docs/faq.rst rename to docs/faq.rst diff --git a/scrapy/trunk/docs/index.rst b/docs/index.rst similarity index 100% rename from scrapy/trunk/docs/index.rst rename to docs/index.rst diff --git a/scrapy/trunk/docs/intro/index.rst b/docs/intro/index.rst similarity index 100% rename from scrapy/trunk/docs/intro/index.rst rename to docs/intro/index.rst diff --git a/scrapy/trunk/docs/intro/install.rst b/docs/intro/install.rst similarity index 100% rename from scrapy/trunk/docs/intro/install.rst rename to docs/intro/install.rst diff --git a/scrapy/trunk/docs/intro/overview.rst b/docs/intro/overview.rst similarity index 100% rename from scrapy/trunk/docs/intro/overview.rst rename to docs/intro/overview.rst diff --git a/scrapy/trunk/docs/intro/tutorial.rst b/docs/intro/tutorial.rst similarity index 100% rename from scrapy/trunk/docs/intro/tutorial.rst rename to docs/intro/tutorial.rst diff --git a/scrapy/branches/cluster-refactor/docs/scrapy-architecture.dia b/docs/media/scrapy-architecture.dia similarity index 100% rename from scrapy/branches/cluster-refactor/docs/scrapy-architecture.dia rename to docs/media/scrapy-architecture.dia diff --git a/scrapy/trunk/docs/misc/api-stability.rst b/docs/misc/api-stability.rst similarity index 100% rename from scrapy/trunk/docs/misc/api-stability.rst rename to docs/misc/api-stability.rst diff --git a/scrapy/trunk/docs/misc/index.rst b/docs/misc/index.rst similarity index 100% rename from scrapy/trunk/docs/misc/index.rst rename to docs/misc/index.rst diff --git a/scrapy/trunk/docs/proposed/_images/scrapy_architecture.odg b/docs/proposed/_images/scrapy_architecture.odg similarity index 100% rename from scrapy/trunk/docs/proposed/_images/scrapy_architecture.odg rename to docs/proposed/_images/scrapy_architecture.odg diff --git a/scrapy/trunk/docs/proposed/_images/scrapy_architecture.png b/docs/proposed/_images/scrapy_architecture.png similarity index 100% rename from scrapy/trunk/docs/proposed/_images/scrapy_architecture.png rename to docs/proposed/_images/scrapy_architecture.png diff --git a/scrapy/trunk/docs/proposed/index.rst b/docs/proposed/index.rst similarity index 100% rename from scrapy/trunk/docs/proposed/index.rst rename to docs/proposed/index.rst diff --git a/scrapy/trunk/docs/proposed/introduction.rst b/docs/proposed/introduction.rst similarity index 100% rename from scrapy/trunk/docs/proposed/introduction.rst rename to docs/proposed/introduction.rst diff --git a/scrapy/trunk/docs/proposed/newitem.rst b/docs/proposed/newitem.rst similarity index 100% rename from scrapy/trunk/docs/proposed/newitem.rst rename to docs/proposed/newitem.rst diff --git a/scrapy/trunk/docs/proposed/spiders.rst b/docs/proposed/spiders.rst similarity index 100% rename from scrapy/trunk/docs/proposed/spiders.rst rename to docs/proposed/spiders.rst diff --git a/scrapy/trunk/docs/ref/downloader-middleware.rst b/docs/ref/downloader-middleware.rst similarity index 100% rename from scrapy/trunk/docs/ref/downloader-middleware.rst rename to docs/ref/downloader-middleware.rst diff --git a/scrapy/trunk/docs/ref/email.rst b/docs/ref/email.rst similarity index 100% rename from scrapy/trunk/docs/ref/email.rst rename to docs/ref/email.rst diff --git a/scrapy/trunk/docs/ref/exceptions.rst b/docs/ref/exceptions.rst similarity index 100% rename from scrapy/trunk/docs/ref/exceptions.rst rename to docs/ref/exceptions.rst diff --git a/scrapy/trunk/docs/ref/extension-manager.rst b/docs/ref/extension-manager.rst similarity index 100% rename from scrapy/trunk/docs/ref/extension-manager.rst rename to docs/ref/extension-manager.rst diff --git a/scrapy/trunk/docs/ref/extensions.rst b/docs/ref/extensions.rst similarity index 100% rename from scrapy/trunk/docs/ref/extensions.rst rename to docs/ref/extensions.rst diff --git a/scrapy/trunk/docs/ref/index.rst b/docs/ref/index.rst similarity index 100% rename from scrapy/trunk/docs/ref/index.rst rename to docs/ref/index.rst diff --git a/scrapy/trunk/docs/ref/link-extractors.rst b/docs/ref/link-extractors.rst similarity index 100% rename from scrapy/trunk/docs/ref/link-extractors.rst rename to docs/ref/link-extractors.rst diff --git a/scrapy/trunk/docs/ref/logging.rst b/docs/ref/logging.rst similarity index 100% rename from scrapy/trunk/docs/ref/logging.rst rename to docs/ref/logging.rst diff --git a/scrapy/trunk/docs/ref/request-response.rst b/docs/ref/request-response.rst similarity index 100% rename from scrapy/trunk/docs/ref/request-response.rst rename to docs/ref/request-response.rst diff --git a/scrapy/trunk/docs/ref/selectors.rst b/docs/ref/selectors.rst similarity index 100% rename from scrapy/trunk/docs/ref/selectors.rst rename to docs/ref/selectors.rst diff --git a/scrapy/trunk/docs/ref/settings.rst b/docs/ref/settings.rst similarity index 100% rename from scrapy/trunk/docs/ref/settings.rst rename to docs/ref/settings.rst diff --git a/scrapy/trunk/docs/ref/signals.rst b/docs/ref/signals.rst similarity index 100% rename from scrapy/trunk/docs/ref/signals.rst rename to docs/ref/signals.rst diff --git a/scrapy/trunk/docs/ref/spiders.rst b/docs/ref/spiders.rst similarity index 100% rename from scrapy/trunk/docs/ref/spiders.rst rename to docs/ref/spiders.rst diff --git a/scrapy/trunk/docs/topics/_images/adaptors_diagram.png b/docs/topics/_images/adaptors_diagram.png similarity index 100% rename from scrapy/trunk/docs/topics/_images/adaptors_diagram.png rename to docs/topics/_images/adaptors_diagram.png diff --git a/scrapy/trunk/docs/topics/_images/adaptors_diagram.svg b/docs/topics/_images/adaptors_diagram.svg similarity index 100% rename from scrapy/trunk/docs/topics/_images/adaptors_diagram.svg rename to docs/topics/_images/adaptors_diagram.svg diff --git a/scrapy/trunk/docs/topics/_images/firebug1.png b/docs/topics/_images/firebug1.png similarity index 100% rename from scrapy/trunk/docs/topics/_images/firebug1.png rename to docs/topics/_images/firebug1.png diff --git a/scrapy/trunk/docs/topics/_images/firebug2.png b/docs/topics/_images/firebug2.png similarity index 100% rename from scrapy/trunk/docs/topics/_images/firebug2.png rename to docs/topics/_images/firebug2.png diff --git a/scrapy/trunk/docs/topics/_images/firebug3.png b/docs/topics/_images/firebug3.png similarity index 100% rename from scrapy/trunk/docs/topics/_images/firebug3.png rename to docs/topics/_images/firebug3.png diff --git a/scrapy/trunk/docs/topics/_images/scrapy_architecture.odg b/docs/topics/_images/scrapy_architecture.odg similarity index 100% rename from scrapy/trunk/docs/topics/_images/scrapy_architecture.odg rename to docs/topics/_images/scrapy_architecture.odg diff --git a/scrapy/trunk/docs/topics/_images/scrapy_architecture.png b/docs/topics/_images/scrapy_architecture.png similarity index 100% rename from scrapy/trunk/docs/topics/_images/scrapy_architecture.png rename to docs/topics/_images/scrapy_architecture.png diff --git a/scrapy/trunk/docs/topics/adaptors.rst b/docs/topics/adaptors.rst similarity index 100% rename from scrapy/trunk/docs/topics/adaptors.rst rename to docs/topics/adaptors.rst diff --git a/scrapy/trunk/docs/topics/architecture.rst b/docs/topics/architecture.rst similarity index 100% rename from scrapy/trunk/docs/topics/architecture.rst rename to docs/topics/architecture.rst diff --git a/scrapy/trunk/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst similarity index 100% rename from scrapy/trunk/docs/topics/downloader-middleware.rst rename to docs/topics/downloader-middleware.rst diff --git a/scrapy/trunk/docs/topics/extensions.rst b/docs/topics/extensions.rst similarity index 100% rename from scrapy/trunk/docs/topics/extensions.rst rename to docs/topics/extensions.rst diff --git a/scrapy/trunk/docs/topics/firebug.rst b/docs/topics/firebug.rst similarity index 100% rename from scrapy/trunk/docs/topics/firebug.rst rename to docs/topics/firebug.rst diff --git a/scrapy/trunk/docs/topics/firefox.rst b/docs/topics/firefox.rst similarity index 100% rename from scrapy/trunk/docs/topics/firefox.rst rename to docs/topics/firefox.rst diff --git a/scrapy/trunk/docs/topics/index.rst b/docs/topics/index.rst similarity index 100% rename from scrapy/trunk/docs/topics/index.rst rename to docs/topics/index.rst diff --git a/scrapy/trunk/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst similarity index 100% rename from scrapy/trunk/docs/topics/item-pipeline.rst rename to docs/topics/item-pipeline.rst diff --git a/scrapy/trunk/docs/topics/items.rst b/docs/topics/items.rst similarity index 100% rename from scrapy/trunk/docs/topics/items.rst rename to docs/topics/items.rst diff --git a/scrapy/trunk/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst similarity index 100% rename from scrapy/trunk/docs/topics/link-extractors.rst rename to docs/topics/link-extractors.rst diff --git a/scrapy/trunk/docs/topics/robotstxt.rst b/docs/topics/robotstxt.rst similarity index 100% rename from scrapy/trunk/docs/topics/robotstxt.rst rename to docs/topics/robotstxt.rst diff --git a/scrapy/trunk/docs/topics/selectors.rst b/docs/topics/selectors.rst similarity index 100% rename from scrapy/trunk/docs/topics/selectors.rst rename to docs/topics/selectors.rst diff --git a/scrapy/trunk/docs/topics/settings.rst b/docs/topics/settings.rst similarity index 100% rename from scrapy/trunk/docs/topics/settings.rst rename to docs/topics/settings.rst diff --git a/scrapy/trunk/docs/topics/shell.rst b/docs/topics/shell.rst similarity index 100% rename from scrapy/trunk/docs/topics/shell.rst rename to docs/topics/shell.rst diff --git a/scrapy/trunk/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst similarity index 100% rename from scrapy/trunk/docs/topics/spider-middleware.rst rename to docs/topics/spider-middleware.rst diff --git a/scrapy/trunk/docs/topics/spiders.rst b/docs/topics/spiders.rst similarity index 100% rename from scrapy/trunk/docs/topics/spiders.rst rename to docs/topics/spiders.rst diff --git a/scrapy/trunk/docs/topics/stats.rst b/docs/topics/stats.rst similarity index 100% rename from scrapy/trunk/docs/topics/stats.rst rename to docs/topics/stats.rst diff --git a/scrapy/trunk/docs/topics/webconsole.rst b/docs/topics/webconsole.rst similarity index 100% rename from scrapy/trunk/docs/topics/webconsole.rst rename to docs/topics/webconsole.rst diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/__init__.py b/examples/experimental/googledir/googledir/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/commands/__init__.py rename to examples/experimental/googledir/googledir/__init__.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/items.py b/examples/experimental/googledir/googledir/items.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/items.py rename to examples/experimental/googledir/googledir/items.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/pipelines.py b/examples/experimental/googledir/googledir/pipelines.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/pipelines.py rename to examples/experimental/googledir/googledir/pipelines.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/settings.py rename to examples/experimental/googledir/googledir/settings.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/__init__.py b/examples/experimental/googledir/googledir/spiders/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/__init__.py rename to examples/experimental/googledir/googledir/spiders/__init__.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/spiders/google_directory.py rename to examples/experimental/googledir/googledir/spiders/google_directory.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_basic.tmpl b/examples/experimental/googledir/googledir/templates/spider_basic.tmpl similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_basic.tmpl rename to examples/experimental/googledir/googledir/templates/spider_basic.tmpl diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl b/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl rename to examples/experimental/googledir/googledir/templates/spider_crawl.tmpl diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl b/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl rename to examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl b/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl rename to examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl diff --git a/scrapy/trunk/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/scrapy-ctl.py rename to examples/experimental/googledir/scrapy-ctl.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/__init__.py b/examples/googledir/googledir/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/project_template/__init__.py rename to examples/googledir/googledir/__init__.py diff --git a/scrapy/trunk/examples/googledir/googledir/items.py b/examples/googledir/googledir/items.py similarity index 100% rename from scrapy/trunk/examples/googledir/googledir/items.py rename to examples/googledir/googledir/items.py diff --git a/scrapy/trunk/examples/googledir/googledir/pipelines.py b/examples/googledir/googledir/pipelines.py similarity index 100% rename from scrapy/trunk/examples/googledir/googledir/pipelines.py rename to examples/googledir/googledir/pipelines.py diff --git a/scrapy/trunk/examples/googledir/googledir/settings.py b/examples/googledir/googledir/settings.py similarity index 100% rename from scrapy/trunk/examples/googledir/googledir/settings.py rename to examples/googledir/googledir/settings.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/spiders/__init__.py b/examples/googledir/googledir/spiders/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/project_template/spiders/__init__.py rename to examples/googledir/googledir/spiders/__init__.py diff --git a/scrapy/trunk/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py similarity index 100% rename from scrapy/trunk/examples/googledir/googledir/spiders/google_directory.py rename to examples/googledir/googledir/spiders/google_directory.py diff --git a/scrapy/trunk/examples/googledir/scrapy-ctl.py b/examples/googledir/scrapy-ctl.py similarity index 100% rename from scrapy/trunk/examples/googledir/scrapy-ctl.py rename to examples/googledir/scrapy-ctl.py diff --git a/scrapy/branches/cluster-refactor/extras/sql/scraping.sql b/extras/sql/scraping.sql similarity index 100% rename from scrapy/branches/cluster-refactor/extras/sql/scraping.sql rename to extras/sql/scraping.sql diff --git a/scrapy/trunk/profiling/priorityqueue/pq_classes.py b/profiling/priorityqueue/pq_classes.py similarity index 100% rename from scrapy/trunk/profiling/priorityqueue/pq_classes.py rename to profiling/priorityqueue/pq_classes.py diff --git a/scrapy/trunk/profiling/priorityqueue/run.py b/profiling/priorityqueue/run.py similarity index 100% rename from scrapy/trunk/profiling/priorityqueue/run.py rename to profiling/priorityqueue/run.py diff --git a/scrapy/trunk/profiling/priorityqueue/test_cases.py b/profiling/priorityqueue/test_cases.py similarity index 100% rename from scrapy/trunk/profiling/priorityqueue/test_cases.py rename to profiling/priorityqueue/test_cases.py diff --git a/scrapy/trunk/scrapy/__init__.py b/scrapy/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/__init__.py rename to scrapy/__init__.py diff --git a/scrapy/trunk/scrapy/bin/scrapy-admin.py b/scrapy/bin/scrapy-admin.py similarity index 100% rename from scrapy/trunk/scrapy/bin/scrapy-admin.py rename to scrapy/bin/scrapy-admin.py diff --git a/scrapy/branches/cluster-refactor/INSTALL b/scrapy/branches/cluster-refactor/INSTALL deleted file mode 100644 index b101fd26f..000000000 --- a/scrapy/branches/cluster-refactor/INSTALL +++ /dev/null @@ -1,32 +0,0 @@ -You need: - * Python 2.5 - * Twisted - * libxml2 python bindings. version 2.6.28 or above is highly recommended. - * pyopenssl for HTTPS support - * for win32: http://webcleaner.sourceforge.net/pyOpenSSL-0.6.win32-py2.5.exe - * spidermonkey (optional) - required for JSParser - * simplejson (optional) - required for url2guid web service - -The python interpreter should have sys.getdefaultencoding() == 'utf-8' -If this is not the case, you can add the following lines to a file -[PYTHON_INSTALL]/site-packages/sitecustomize.py: -import sys -sys.setdefaultencoding('utf-8') - - -INSTALLING LIBRARIES --------------------- - -The procedure for installing the required third party libraries (twisted, -libxml2 and pyopenssl) depends on the platform and OS you use. - -Visit the project homepages for more information about it: - - * Twisted: http://twistedmatrix.com/ - * libxml2: http://xmlsoft.org/ - * releases available here: ftp://xmlsoft.org/libxml2/ - * pyopenssl: http://pyopenssl.sourceforge.net/ - - -In Debian/Ubuntu Linux it's as easy as: -apt-get install python-twisted python-libxml2 python-pyopenssl diff --git a/scrapy/branches/cluster-refactor/scrapy/__init__.py b/scrapy/branches/cluster-refactor/scrapy/__init__.py deleted file mode 100644 index 8b3215d1e..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Scrapy - a screen scraping framework written in Python -""" - -__version__ = "0.1.0" - -import sys, os - -if sys.version_info < (2,5): - print "Scrapy %s requires Python 2.5 or above" % __version__ - sys.exit(1) - -# add external python libraries bundled into scrapy -sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "xlib")) - -# monkey patches to fix external library issues -from scrapy.patches import monkeypatches -monkeypatches.apply_patches() diff --git a/scrapy/branches/cluster-refactor/scrapy/bin/scrapy-admin.py b/scrapy/branches/cluster-refactor/scrapy/bin/scrapy-admin.py deleted file mode 100755 index f2dc1d5c3..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/bin/scrapy-admin.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -"""Scrapy admin script is used to create new scrapy projects and similar -tasks""" - -import os -import shutil -from optparse import OptionParser - -import scrapy - -usage = """ -scrapy-admin.py [options] [command] - -Available commands: - - startproject - Starts a new project with name 'project_name' -""" - -def main(): - parser = OptionParser(usage=usage) - opts, args = parser.parse_args() - - if not args: - parser.print_help() - - cmd = args[0] - if cmd == "startproject": - if len(args) >= 2: - project_name = args[1] - project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template") - shutil.copytree(project_tplpath, project_name) - else: - print "scrapy-admin.py: missing project name" - else: - print "scrapy-admin.py: unknown command: %s" % cmd - -if __name__ == '__main__': - main() diff --git a/scrapy/branches/cluster-refactor/scrapy/command/cmdline.py b/scrapy/branches/cluster-refactor/scrapy/command/cmdline.py deleted file mode 100644 index 0ef238448..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/cmdline.py +++ /dev/null @@ -1,157 +0,0 @@ -from __future__ import with_statement - -import sys -import os -import optparse - -import scrapy -from scrapy import log -from scrapy.spider import spiders -from scrapy.conf import settings - -def find_commands(dir): - try: - return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and f.endswith('.py')] - except OSError: - return [] - -def builtin_commands_dict(): - d = {} - scrapy_dir = scrapy.__path__[0] - commands_dir = os.path.join(scrapy_dir, 'command', 'commands') - for cmdname in find_commands(commands_dir): - modname = 'scrapy.command.commands.%s' % cmdname - command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) - if callable(command): - d[cmdname] = command() - else: - print 'WARNING: Builtin command module %s exists but Command class not found' % modname - return d - -def custom_commands_dict(): - d = {} - cmdsmod = settings['COMMANDS_MODULE'] - if cmdsmod: - mod = __import__(cmdsmod, {}, {}, ['']) - for cmdname in find_commands(mod.__path__[0]): - modname = '%s.%s' % (cmdsmod, cmdname) - command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) - if callable(command): - d[cmdname] = command() - else: - print 'WARNING: Custom command module %s exists but Command class not found' % modname - return d - -def getcmdname(argv): - for arg in argv[1:]: - if not arg.startswith('-'): - return arg - -def usage(argv): - s = "usage: %s [options] [args]\n" % argv[0] - s += " %s -h\n\n" % argv[0] - s += "Built-in subcommands:\n" - - builtin_cmds = builtin_commands_dict() - custom_cmds = custom_commands_dict() - - filtered_builtin_cmds = [(name, cls) for name, cls in builtin_cmds.iteritems() if name not in custom_cmds] - - for cmdname, cmdclass in filtered_builtin_cmds: - s += " %s %s\n" % (cmdname, cmdclass.syntax()) - s += " %s\n" % cmdclass.short_desc() - - if custom_cmds: - s += "\n" - s += "Custom (or overloaded) subcommands:\n" - for cmdname, cmdclass in custom_cmds.iteritems(): - s += " %s %s\n" % (cmdname, cmdclass.syntax()) - s += " %s\n" % cmdclass.short_desc() - - return s - - -def update_defaults(defaults, module): - settingsdict = vars(module) - for k, v in settingsdict.iteritems(): - if not k.startswith("_"): - defaults[k] = v - -def command_settings(cmdname): - try: - module = __import__('%s.%s' % ('scrapy.conf.commands', cmdname), {}, {}, ['']) - update_defaults(settings.defaults, module) - except ImportError: - pass - - basepath = settings['COMMANDS_SETTINGS_MODULE'] - if basepath: - try: - module = __import__('%s.%s' % (basepath, cmdname), {}, {}, ['']) - update_defaults(settings.defaults, module) - except ImportError: - pass - -# This dict holds information about the executed command for later use -command_executed = {} - -def execute(): - execute_with_args(sys.argv) - -def execute_with_args(argv): - spiders.load() - cmds = builtin_commands_dict() - cmds.update(custom_commands_dict()) - - cmdname = getcmdname(argv) - command_settings(cmdname) - - if not cmdname: - print "Scrapy %s\n" % scrapy.__version__ - print usage(argv) - sys.exit() - - parser = optparse.OptionParser() - - if cmdname in cmds: - cmd = cmds[cmdname] - cmd.add_options(parser) - parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) - parser.description = cmd.long_desc() - else: - print "Scrapy %s\n" % scrapy.__version__ - print "Unknown command: %s\n" % cmdname - print 'Type "%s -h" for help' % argv[0] - sys.exit() - - (opts, args) = parser.parse_args(args=argv[1:]) - del args[0] # args[0] is cmdname - - # storing command executed info for later reference - command_executed['name'] = cmdname - command_executed['class'] = cmd - command_executed['args'] = args[:] - command_executed['opts'] = opts.__dict__.copy() - - cmd.process_options(args, opts) - log.start() # start logging - if opts.profile: - log.msg("Profiling enabled. Analyze later with: python -m pstats %s" % opts.profile) - import cProfile - loc = locals() - p = cProfile.Profile() - p.runctx('ret = cmd.run(args, opts)', globals(), loc) - p.dump_stats(opts.profile) - try: - import lsprofcalltree - fn = opts.profile + ".cachegrind" - k = lsprofcalltree.KCacheGrind(p) - with open(fn, 'w') as f: - k.output(f) - except ImportError: - pass - ret = loc['ret'] - else: - ret = cmd.run(args, opts) - if ret is False: - parser.print_help() diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/crawl.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/crawl.py deleted file mode 100644 index 866ce16c9..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/crawl.py +++ /dev/null @@ -1,40 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.core.manager import scrapymanager -from scrapy.replay import Replay -from scrapy.conf import settings - - -class Command(ScrapyCommand): - def syntax(self): - return "[options] [domain|url] ..." - - def short_desc(self): - return "Run the web scraping engine from the command line" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("--nocache", dest="nocache", action="store_true", help="disable HTTP cache") - parser.add_option("--nopipeline", dest="nopipeline", action="store_true", help="disable scraped item pipeline") - parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls") - parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE") - parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - if opts.nopipeline: - settings.overrides['ITEM_PIPELINES'] = [] - - if opts.nocache: - settings.overrides['CACHE2_DIR'] = None - - if opts.restrict: - settings.overrides['RESTRICT_TO_URLS'] = args - - if opts.record or opts.recorddir: - # self.replay is used for preventing Replay signals handler from - # disconnecting since pydispatcher uses weak references - self.replay = Replay(opts.record or opts.recorddir, mode='record', usedir=bool(opts.recorddir)) - self.replay.record(args=args, opts=opts.__dict__) - - def run(self, args, opts): - scrapymanager.runonce(*args, **opts.__dict__) diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/genspider.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/genspider.py deleted file mode 100644 index d442c500a..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/genspider.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import string - -from scrapy.spider import spiders -from scrapy.command import ScrapyCommand -from scrapy.conf import settings - -class Command(ScrapyCommand): - - """ Default template file name """ - template_name = 'spider.tmpl' - - """ Childs can define custom tvars """ - custom_tvars = {} - - def syntax(self): - return " " - - def short_desc(self): - return "Generate new spider based on predefined template" - - def run(self, args, opts): - if len(args) != 2: - return False - - name = args[0] - site = args[1] - spiders_dict = spiders.asdict() - if not name in spiders_dict.keys(): - self._genspider(name, site) - else: - print "Spider '%s' exist" % name - - def _genspider(self, name, site): - """ Generate spider """ - tvars = { - 'name': name, - 'site': site, - 'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('-')]) - } - tvars.update(self.custom_tvars) - - spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, ['']) - spidersdir = os.path.abspath(os.path.dirname(spiders_module.__file__)) - if name[0] not in string.letters: # must start with a letter, for valid python modules - name = "a" + name - name = name.replace('-', '_') # - are replaced by _, for valid python modules - self._genfiles(self.template_name, '%s/%s.py' % (spidersdir, name), tvars) - - def _genfiles(self, template_name, source_name, tvars): - """ Generate source from template, substitute variables """ - template_file = os.path.join(settings['TEMPLATES_DIR'], template_name) - tmpl = open(template_file) - clines = [] - for l in tmpl.readlines(): - for key, val in tvars.items(): - l = l.replace('@%s@' % key, val) - clines.append(l) - tmpl.close() - source = ''.join(clines) - if not os.path.exists(source_name): - sfile = open(source_name, "w") - sfile.write(source) - sfile.close() diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/getattr.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/getattr.py deleted file mode 100644 index ac639bf3d..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/getattr.py +++ /dev/null @@ -1,25 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.fetcher import fetch -from scrapy.spider import spiders -from scrapy.item import ScrapedItem - -def get_item_attr(pagedata, attr="guid"): - spider = spiders.fromurl(pagedata.url) - items = spider.parse(pagedata) - attrs = [getattr(i, attr) for i in items if isinstance(i, ScrapedItem)] - return attrs - -def get_attr(url, attr="guid"): - pagedatas = fetch([url]) - if pagedatas: - return get_item_attr(pagedatas[0], attr) - -class Command(ScrapyCommand): - def syntax(self): - return " " - - def short_desc(self): - return "Print an attribute from the item scraped in the given URL" - - def run(self, args, opts): - print get_attr(args[0], args[1]) diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/parse.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/parse.py deleted file mode 100644 index 79860b9af..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/parse.py +++ /dev/null @@ -1,46 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.fetcher import fetch -from scrapy.http import Request -from scrapy.item import ScrapedItem -from scrapy.spider import spiders -from scrapy.utils import display - -class Command(ScrapyCommand): - def syntax(self): - return "[options] " - - def short_desc(self): - return "Parse the URL and print their results" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links") - parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items") - parser.add_option("--identify", dest="identify", action="store_true", help="try to use identify instead of parse") - parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output") - - def pipeline_process(self, item, opts): - return item - - def run(self, args, opts): - if not args: - print "A URL is required" - return - - responses = fetch([args[0]]) - if responses: - response = responses[0] - spider = spiders.fromurl(response.url) - result = spider.parse(response) if not opts.identify else spider.identify(response) - - items = [self.pipeline_process(i, opts) for i in result if isinstance(i, ScrapedItem)] - links = [i for i in result if isinstance(i, Request)] - - display.nocolour = opts.nocolour - if not opts.noitems: - print "# Scraped Items", "-"*60 - display.pprint(items) - - if not opts.nolinks: - print "# Links", "-"*68 - display.pprint(links) diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/replay.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/replay.py deleted file mode 100644 index 590f976a9..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/replay.py +++ /dev/null @@ -1,184 +0,0 @@ -import os - -from scrapy.command import ScrapyCommand -from scrapy.replay import Replay -from scrapy.utils import display -from scrapy.conf import settings -from scrapy.command import cmdline - -class Command(ScrapyCommand): - def syntax(self): - return "[options] [action]" - - def short_desc(self): - return "Replay a session previously recorded with crawl --record" - - def help(self): - s = "Replay a session previously recorded with crawl --record\n" - s += "\n" - s += "Available actions:\n" - s += " crawl: just replay the crawl (default if action omitted)\n" - s += " diff: replay the crawl and show differences in items scraped/passed\n" - s += " update: replay the crawl and update both scraped and passed items\n" - s += " showitems: show items stored\n" - s += " showpages: show all responses downloaded (not only HTML pages)\n" - return s - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show verbose output (full items/responses)") - parser.add_option("-t", "--item-type", dest="itype", help="item type (scraped, passed). default: scraped", metavar="TYPE") - parser.add_option("--output", dest="outfile", help="write output to FILE. if omitted uses stdout", metavar="FILE") - parser.add_option("--nocolour", dest="nocolour", action="store_true", help="disable colorized output (for console only)") - parser.add_option("-i", "--ignore", dest="ignores", action="append", help="item attribute to ignore. can be passed multiple times", metavar="ATTR") - parser.add_option("--target", dest="targets", action="append", help="crawl TARGET instead of recorded urls/domains. can be passed multiple times") - # adding option to update - parser.add_option("--pages", dest="pages", action="store_true", help="update all the pages in the replay file, recording it again.") - parser.add_option("-q", "--quiet", dest="quiet", action="store_true", help="no verbose mode in option diff.") - - def process_options(self, args, opts): - if args: - ScrapyCommand.process_options(self, args, opts) - self.opts = opts - self.action = args[1] if len(args) > 1 else 'crawl' - mode = 'update' if self.action == 'update' else 'play' - usedir = args and os.path.isdir(args[0]) - self.replay = Replay(args[0], mode=mode, usedir=usedir) - if self.action not in ['crawl', 'diff', 'update']: - settings.overrides['LOG_ENABLED'] = False - - def run(self, args, opts): - if not args: - print "A is required" - return - - display.nocolour = opts.nocolour - - if opts.itype == 'passed': - self.before_db = self.replay.passed_old - self.now_db = self.replay.passed_new - else: # default is 'scraped' - opts.itype = 'scraped' - self.before_db = self.replay.scraped_old - self.now_db = self.replay.scraped_new - - actionfunc = getattr(self, 'action_%s' % self.action, None) - if actionfunc: - rep = actionfunc(opts) - if rep: - if opts.outfile: - f = open(opts.outfile, "w") - f.write(rep) - f.close() - else: - print rep, - self.replay.cleanup() - else: - print "Unknown replay action: %s" % self.action - - def action_crawl(self, opts): - self.replay.play(args=opts.targets) - - def action_update(self, opts): - self.replay.update(args=opts.targets, opts=opts.__dict__) - if (opts.pages): - args = ['scrapy-crawl', 'crawl'] - args.extend(self.replay.options['args']) - for k in self.replay.options['opts']: - if self.replay.options['opts'][k]: - args.append("--%s" % k) - if self.replay.options['opts'][k] != True: - args.append(self.replay.options['opts'][k]) - cmdline.execute_with_args(args) - - def action_showitems(self, opts): - s = "" - s += self._format_items(self.before_db.values()) - s += ">>> Total: %d items %s\n" % (len(self.before_db), opts.itype) - return s - - def action_showpages(self, opts): - s = "" - if self.opts.verbose: - for r in self.replay.responses_old.values(): - s += ">>> %s\n" % str(r) - s += display.pformat(r) - else: - s += "\n".join([str(r) for r in self.replay.responses_old.values()]) + "\n" - s += ">>> Total: %d responses received\n" % len(self.replay.responses_old) - return s - - def action_diff(self, opts): - self.action_crawl(opts) - - guids_before = set(self.before_db.keys()) - guids_now = set(map(str, self.now_db.keys())) - - guids_new = guids_now - guids_before - guids_missing = guids_before - guids_now - guids_both = guids_now & guids_before - - changed_items, chreport = self._report_differences(self.before_db, self.now_db, guids_both) - - ok_items = len(guids_both) - changed_items - new_items = len(guids_new) - missing_items = len(guids_missing) - - if (new_items - missing_items - changed_items) == 0 and opts.quiet: - s = "" - else: - s = "CRAWLING DIFFERENCES REPORT\n\n" - - s += "Total items : %d\n" % (len(guids_both) + new_items + missing_items) - s += " Items OK : %d\n" % ok_items - s += " New items : %d\n" % new_items - s += " Missing items : %d\n" % missing_items - s += " Changed items : %d\n" % changed_items - - s += "\n" - s += "- NEW ITEMS (%d) -----------------------------------------\n" % new_items - s += self._format_items([self.now_db[g] for g in guids_new]) - - s += "\n" - s += "- MISSING ITEMS (%d) -------------------------------------\n" % missing_items - s += self._format_items([self.before_db[g] for g in guids_missing]) - - s += "\n" - s += "- CHANGED ITEMS (%d) -------------------------------------\n" % changed_items - - s += chreport - - return s - - def _report_differences(self, old_items, new_items, guids): - items_old = [old_items[g] for g in guids] - items_new = [new_items[g] for g in guids] - - c = 0 - s = "" - for old, new in zip(items_old, items_new): - d = self._item_diff(old, new) - if d: - c += 1 - s += d - return c, s - - def _item_diff(self, old, new): - delta = new - old - - s = "" - if delta.diff: - s += ">>> Item guid=%s name=%s\n" % (old.guid, old.name) - s += display.pformat(delta.diff) + "\n" - return s - - def _format_items(self, items): - if self.opts.verbose: - s = display.pformat(items) - else: - s = "" - for i in items: - s += "%s\n" % str(i) - s += " <%s>\n" % i.url - return s - diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/shell.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/shell.py deleted file mode 100644 index 474b53a08..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/shell.py +++ /dev/null @@ -1,158 +0,0 @@ -from twisted.internet import reactor - -import scrapy -from scrapy.command import ScrapyCommand -from scrapy.spider import spiders -from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector -from scrapy.utils.misc import load_class -from scrapy.extension import extensions -from scrapy.conf import settings -from scrapy.core.manager import scrapymanager -from scrapy.http import Request, Response -from scrapy.core.downloader.handlers import download_any -from scrapy.fetcher import get_or_create_spider -from scrapy.utils.decompressor import Decompressor - -#This code comes from twisted 8. We define here while -#using old twisted version. -def blockingCallFromThread(reactor, f, *a, **kw): - """ - Run a function in the reactor from a thread, and wait for the result - synchronously, i.e. until the callback chain returned by the function - get a result. - - @param reactor: The L{IReactorThreads} provider which will be used to - schedule the function call. - @param f: the callable to run in the reactor thread - @type f: any callable. - @param a: the arguments to pass to C{f}. - @param kw: the keyword arguments to pass to C{f}. - - @return: the result of the callback chain. - @raise: any error raised during the callback chain. - """ - import Queue - from twisted.python import failure - from twisted.internet import defer - queue = Queue.Queue() - def _callFromThread(): - result = defer.maybeDeferred(f, *a, **kw) - result.addBoth(queue.put) - reactor.callFromThread(_callFromThread) - result = queue.get() - if isinstance(result, failure.Failure): - result.raiseException() - return result - -class Command(ScrapyCommand): - def syntax(self): - return "[url]" - - def short_desc(self): - return "Interactive scraping console" - - def long_desc(self): - return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html" - - def update_vars(self): - """ You can use this function to update the local variables that will be available in the scrape console """ - pass - - def get_url(self, url, decompress=False): - - print "Downloading URL... ", - r = Request(url) - spider = get_or_create_spider(url) - try: - result = blockingCallFromThread(reactor, download_any, r, spider) - if isinstance(result, Response): - print "Done." - if decompress: - print "Decompressing response...", - d = Decompressor() - result = d.extract(result) - print "Done." - result.request = r - self.generate_vars(url, result) - return True - except Exception, e: - print "Error: %s" % e - - def generate_vars(self, url, response): - itemcls = load_class(settings['DEFAULT_ITEM_CLASS']) - item = itemcls() - self.vars['item'] = item - if url: - self.vars['xxs'] = XmlXPathSelector(response) - self.vars['hxs'] = HtmlXPathSelector(response) - self.vars['url'] = url - self.vars['response'] = response - self.vars['spider'] = spiders.fromurl(url) - self.update_vars() - self.user_ns.update(self.vars) - self.print_vars() - - def print_vars(self): - print '-' * 78 - print "Available local variables:" - for key, val in self.vars.iteritems(): - if isinstance(val, basestring): - print " %s: %s" % (key, val) - else: - print " %s: %s" % (key, val.__class__) - print "Available commands:" - print " get : Fetches an url and updates all variables." - print " getd : Similar to get, but filter with decompress." - print " scrapehelp: Prints this help." - print '-' * 78 - - def run(self, args, opts): - self.vars = {} - self.user_ns = {} - url = None - if args: - url = args[0] - - print "Scrapy %s - Interactive scraping console\n" % scrapy.__version__ - - print "Enabling Scrapy extensions...", - extensions.load() - print "done" - - def _console_thread(): - - def _get_magic(shell, arg): - self.get_url(arg.strip()) - def _help_magic(shell, _): - self.print_vars() - def _getd_magic(shell, arg): - self.get_url(arg.strip(), decompress=True) - - if url: - result = self.get_url(url) - if not result: - self.generate_vars(None, None) - else: - self.generate_vars(None, None) - try: # use IPython if available - import IPython - shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns) - ip = shell.IP.getapi() - ip.expose_magic("get", _get_magic) - ip.expose_magic("getd", _getd_magic) - ip.expose_magic("scrapehelp", _help_magic) - shell.mainloop() - reactor.callFromThread(scrapymanager.stop) - except ImportError: - import code - try: # readline module is only available on unix systems - import readline - except ImportError: - pass - else: - import rlcompleter - readline.parse_and_bind("tab:complete") - code.interact(local=self.vars) - - reactor.callInThread(_console_thread) - scrapymanager.start() \ No newline at end of file diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/stats.py b/scrapy/branches/cluster-refactor/scrapy/command/commands/stats.py deleted file mode 100644 index e062ce6a8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/commands/stats.py +++ /dev/null @@ -1,30 +0,0 @@ -import pprint -from scrapy.command import ScrapyCommand -from scrapy.conf import settings - -class Command(ScrapyCommand): - def syntax(self): - return " [domain ...]" - - def short_desc(self): - return "Show all stats history stored for the given domain(s)" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-p", "--path", dest="path", help="restrict stats to PATH", metavar="PATH") - - def run(self, args, opts): - if not args: - print "A domain is required" - return - if not settings['SCRAPING_DB']: - print "SCRAPING_DB setting is required for this command" - return - - from scrapy.store.db import DomainDataHistory - ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - - for domain in args: - print "# %s" % domain - pprint.pprint(list(ddh.getall(domain, opts.path))) - diff --git a/scrapy/branches/cluster-refactor/scrapy/command/models.py b/scrapy/branches/cluster-refactor/scrapy/command/models.py deleted file mode 100644 index b4e247d4b..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/command/models.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Base class for Scrapy commands -""" -import os -from scrapy.conf import settings - -class ScrapyCommand(object): - def syntax(self): - """ - Command syntax (preferably one-line). Do not include command name. - """ - return "" - - def short_desc(self): - """ - A short description of the command - """ - return "" - - def long_desc(self): - """ - A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted - by optparser which removes newlines and wraps text. - """ - return self.short_desc() - - def help(self): - """ - An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since not post-formatting will - be applied to its contents. - """ - return self.long_desc() - - def add_options(self, parser): - """ - Populate option parse with options available for this command - """ - parser.add_option("-f", "--logfile", dest="logfile", help="logfile to use. if omitted stderr will be used", metavar="FILE") - parser.add_option("-o", "--loglevel", dest="loglevel", default=None, help="log level") - parser.add_option("--default-spider", dest="default_spider", default=None, help="default spider (domain) to use if no spider is found") - parser.add_option("--spider", dest="spider", default=None, help="Force using the given spider when the arguments are urls") - parser.add_option("--nolog", dest="nolog", action="store_true", help="disable all log messages") - parser.add_option("--profile", dest="profile", default=None, help="write profiling stats in FILE, to analyze later with: python -m pstats FILE", metavar="FILE") - parser.add_option("--pidfile", dest="pidfile", help="Write process pid to file FILE", metavar="FILE") - - def process_options(self, args, opts): - if opts.logfile: - settings.overrides['LOG_ENABLED'] = True - settings.overrides['LOGFILE'] = opts.logfile - - if opts.loglevel: - settings.overrides['LOG_ENABLED'] = True - settings.overrides['LOGLEVEL'] = opts.loglevel - - if opts.nolog: - settings.overrides['LOG_ENABLED'] = False - - if opts.default_spider: - from scrapy.spider import spiders - spiders.default_domain = opts.default_spider - - if opts.spider: - from scrapy.spider import spiders - spiders.force_domain = opts.spider - - if opts.pidfile: - pid = os.getpid() - open(opts.pidfile, "w").write(str(pid)) - - def run(self, args, opts): - """ - Entry point for running commands - """ - raise NotImplementedError - - diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/__init__.py b/scrapy/branches/cluster-refactor/scrapy/conf/__init__.py deleted file mode 100644 index 6bb2f4c6d..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/conf/__init__.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import cPickle as pickle - -SETTINGS_MODULE = os.environ.get('SCRAPYSETTINGS_MODULE', 'scrapy_settings') - -class Settings(object): - """Class to obtain configuration values from settings module - which can be overriden by environment variables prepended by SCRAPY_""" - - # settings in precedence order - overrides = None - settings = None - defaults = None - core = None - - def __init__(self): - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - self.overrides = pickle.loads(pickled_settings) if pickled_settings else {} - self.settings = self._import(SETTINGS_MODULE) - self.defaults = {} - self.core = self._import('scrapy.conf.core_settings') - - def _import(self, modulepath): - return __import__(modulepath, {}, {}, ['']) - - def __getitem__(self, opt_name): - if opt_name in self.overrides: - return self.overrides[opt_name] - - if 'SCRAPY_' + opt_name in os.environ: - return os.environ['SCRAPY_' + opt_name] - - if hasattr(self.settings, opt_name): - return getattr(self.settings, opt_name) - - if opt_name in self.defaults: - return self.defaults[opt_name] - - if hasattr(self.core, opt_name): - return getattr(self.core, opt_name) - - def get(self, name, default=None): - return self[name] if self[name] is not None else default - - def getbool(self, name, default=False): - """ - True is: 1, '1', True - False is: 0, '0', False, None - """ - return bool(int(self.get(name, default))) - - def getint(self, name, default=0): - return int(self.get(name, default)) - - def getfloat(self, name, default=0.0): - return float(self.get(name, default)) - - def getlist(self, name, default=None): - value = self.get(name) - if value is None: - return [] - elif hasattr(value, '__iter__'): - return value - else: - return str(value).split(',') - -settings = Settings() diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/core_settings.py b/scrapy/branches/cluster-refactor/scrapy/conf/core_settings.py deleted file mode 100644 index ad140af03..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/conf/core_settings.py +++ /dev/null @@ -1,30 +0,0 @@ -import scrapy -# Scrapy core settings - -BOT_NAME = 'scrapy' -BOT_VERSION = scrapy.__version__ - -ENGINE_DEBUG = False - -# Download configuration options -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) -DOWNLOAD_TIMEOUT = 180 # 3mins -CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel -REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain -CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid (a negative value means "never expires") - -LOG_ENABLED = True # -LOGLEVEL = 'DEBUG' # default loglevel -LOGFILE = None # None means sys.stderr by default -LOG_STDOUT = False # - -DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' -SCHEDULER = 'scrapy.core.scheduler.Scheduler' -MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore' -PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' - -EXTENSIONS = [] - -# contrib.middleware.retry.RetryMiddleware default settings -RETRY_TIMES = 3 -RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/items.py b/scrapy/branches/cluster-refactor/scrapy/conf/project_template/items.py deleted file mode 100644 index d78c4d149..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/items.py +++ /dev/null @@ -1,6 +0,0 @@ -# Define here the models for your scraped items - -from scrapy.item import ScrapedItem - -class MyItem(ScrapedItem): - pass diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy-ctl.py b/scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy-ctl.py deleted file mode 100755 index e8e56eaf2..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy-ctl.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from scrapy.command.cmdline import execute -execute() diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy_settings.py b/scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy_settings.py deleted file mode 100644 index e3712975c..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy_settings.py +++ /dev/null @@ -1,95 +0,0 @@ -import $project_name - -# --------------------------------------------------------------------------- -# - Scrapy settings for $project_name - -# --------------------------------------------------------------------------- - -BOT_NAME = 'scrapybot' -BOT_VERSION = '1.0' - -SPIDER_MODULES = ['$project_name.spiders'] -NEWSPIDER_MODULE = '$project_name.spiders' -TEMPLATES_DIR = '%s/templates' % $project_name.__path__[0] -ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % $project_name.__path__[0] -DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) -DOWNLOAD_TIMEOUT = 600 - -# uncomment if you want to add your own custom scrapy commands -#COMMANDS_MODULE = '$project_name.commands' -#COMMANDS_SETTINGS_MODULE = '$project_name.conf.commands' - -#Global timeout between sucessive downloads (can be overrided by spider -#attribute download_timeout -#DOWNLOAD_TIMEOUT = 0 - -MYSQL_CONNECTION_SETTINGS = {"charset": "utf8" } -MYSQL_CONNECTION_PING_PERIOD = 600 - -SCHEDULER = 'scrapy.core.scheduler.Scheduler' -SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO - -#CACHE2_DIR = '/tmp/cache2' # if set, enables HTTP cache -#CACHE2_IGNORE_MISSING = 0 # ignore requests not in cache -#CACHE2_SECTORIZE = 1 # sectorize domains to distribute storage among servers - -#STATS_ENABLED = 1 # enable stats -#STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory) -#STATS_DEBUG = 0 # log stats on domain closed - -EXTENSIONS = ( - 'scrapy.management.web.WebConsole', - 'scrapy.management.telnet.TelnetConsole', -) - -DOWNLOADER_MIDDLEWARES = ( - # Engine side - 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', - 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', - 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', - 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', - 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', - 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', - 'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware', - 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', - 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', - # Downloader side -) - -SPIDER_MIDDLEWARES = ( - # Engine side - 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', - 'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware', - # Spider side -) - -# Item pipelines are usually configured by commands (see conf/commands) -#ITEM_PIPELINES = ( -#) - -#DEPTH_LIMIT = 10 # limit the maximum link depth to follow -#DEPTH_STATS = 1 # enable depth stats - -# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html -URLLENGTH_LIMIT = 2083 - -#WEBCONSOLE_ENABLED = 1 -#WEBCONSOLE_PORT = 8060 # if not set uses a dynamic port - -#TELNETCONSOLE_ENABLED = 1 -#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port - -# global mail sending settings -#MAIL_HOST = 'localhost' -#MAIL_FROM = 'scrapybot@localhost' - -# scrapy webservice -WS_ENABLED = 0 - -SPIDERPROFILER_ENABLED = 0 diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/adaptorpipeline.py b/scrapy/branches/cluster-refactor/scrapy/contrib/adaptorpipeline.py deleted file mode 100644 index 9bac76b31..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/adaptorpipeline.py +++ /dev/null @@ -1,85 +0,0 @@ -from scrapy.item.adaptors import AdaptorPipe as BaseAdaptorPipe -from scrapy.utils.serialization import serialize - -class DuplicatedAdaptorName(Exception): pass - -class _Adaptor(object): - """ - Adaptors instances should be instantiated and used only - inside the AdaptorPipe. - """ - def __init__(self, function, match_function): - self.basefunction = function - self.match_function = match_function - def __repr__(self): - return self.basefunction.func_name - def __call__(self, *args): - return self.basefunction(*args) - -class AdaptorPipe(BaseAdaptorPipe): - - def __init__(self, attribute_names, adaptors=None): - """ - If "adaptors" is given, constructs pipeline from this. - "adaptors" is an ordered tuple of 2-elements tuples, each of which - has the same parameters you give to the insertadaptor method, except - 'after' and 'before', because you define the adaptors order in the tuple. - Example: - ( - (my_function, lambda x: x in my_list) - ... - ) - """ - self.__attribute_names = [ n for n in attribute_names ] - self.__adaptorspipe = [] - self.pipes = {} - if adaptors: - for entry in adaptors: - self.insertadaptor(compile_pipe=False, *entry) - self._compile_pipe() - - @property - def adaptors_names(self): - _adaptors = [] - for a in self.__adaptorspipe: - _adaptors.append(a.basefunction.func_name) - return _adaptors - - def insertadaptor(self, function, match_function=lambda x: True, compile_pipe=True, after=None, before=None): - """ - Inserts a "function" as an adaptor that will apply when match_function returns True (by - default always apply) - If "after" is given, inserts the adaptor after the already inserted adaptor - of the name given in this parameter, If "before" is given, inserts it before - the adaptor of the given name. "name" is the name of the adaptor. - """ - if function.func_name in self.adaptors_names: - raise DuplicatedAdaptorName(function.func_name) - else: - adaptor = _Adaptor(function, match_function) - #by default append adaptor at end of pipe - pos = len(self.adaptors_names) - if after: - pos = self.adaptors_names.index(after) + 1 - elif before: - pos = self.adaptors_names.index(before) - self.__adaptorspipe.insert(pos, adaptor) - if compile_pipe: - self._compile_pipe() - return pos - - def removeadaptor(self, adaptorname): - pos = self.adaptors_names.index(adaptorname) - self.__adaptorspipe.pop(pos) - self._compile_pipe() - - def _compile_pipe(self): - for attrname in self.__attribute_names: - adaptors_pipe = [] - for adaptor in self.__adaptorspipe: - if adaptor.match_function(attrname): - adaptors_pipe.append(adaptor) - self.pipes[attrname] = adaptors_pipe - - def __repr__(self): - return serialize(self.pipes, "pprint") \ No newline at end of file diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/closedomain.py b/scrapy/branches/cluster-refactor/scrapy/contrib/closedomain.py deleted file mode 100644 index bfba4a721..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/closedomain.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -CloseDomain is an extension that forces spiders to be closed after a given -time has expired. - -""" -import datetime -import pprint - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.core.engine import scrapyengine -from scrapy.core.exceptions import NotConfigured -from scrapy.mail import MailSender -from scrapy.stats import stats -from scrapy.conf import settings - -class CloseDomain(object): - def __init__(self): - self.timeout = settings.getint('CLOSEDOMAIN_TIMEOUT') - if not self.timeout: - raise NotConfigured - - self.tasks = {} - self.mail = MailSender() - self.notify = settings.getlist('CLOSEDOMAIN_NOTIFY') - - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) - - def domain_opened(self, domain): - self.tasks[domain] = scrapyengine.addtask(self.close_domain, self.timeout, args=[domain]) - - def close_domain(self, domain): - log.msg("Domain was opened for more than %d seconds, closing it..." % self.timeout, domain=domain) - scrapyengine.close_domain(domain) - if self.notify: - body = "Closed domain %s because it remained opened for more than %s\n\n" % (domain, datetime.timedelta(seconds=self.timeout)) - body += "DOMAIN STATS ------------------------------------------------------\n\n" - body += pprint.pformat(stats.get(domain, None)) - subj = "Closed domain by timeout: %s" % domain - self.mail.send(self.notify, subj, body) - - def domain_closed(self, domain): - scrapyengine.removetask(self.tasks[domain]) diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/debug.py b/scrapy/branches/cluster-refactor/scrapy/contrib/debug.py deleted file mode 100644 index 6638960a8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/debug.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Extensions for debugging Scrapy -""" -import signal -import traceback - -class StackTraceDebug(object): - def __init__(self): - try: - signal.signal(signal.SIGUSR1, self.dump_stacktrace) - except AttributeError: - # win32 platforms don't support SIGUSR signals - pass - - def dump_stacktrace(self, signum, frame): - print "Got signal. Dumping stack trace..." - traceback.print_stack(frame) diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cache.py deleted file mode 100644 index 946983361..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cache.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import with_statement - -import os -import sys -import hashlib -import datetime -import urlparse -import cPickle as pickle -from pydispatch import dispatcher -from twisted.internet import defer - -from scrapy.core import signals -from scrapy import log -from scrapy.core.engine import scrapyengine -from scrapy.http import Response, Headers -from scrapy.http.headers import headers_dict_to_raw -from scrapy.core.exceptions import UsageError, NotConfigured, HttpException, IgnoreRequest -from scrapy.conf import settings - - -class CacheMiddleware(object): - def __init__(self): - if not settings['CACHE2_DIR']: - raise NotConfigured - self.cache = Cache(settings['CACHE2_DIR'], sectorize=settings.getbool('CACHE2_SECTORIZE')) - self.ignore_missing = settings.getbool('CACHE2_IGNORE_MISSING') - dispatcher.connect(self.open_domain, signal=signals.domain_open) - - def open_domain(self, domain): - self.cache.open_domain(domain) - - def process_request(self, request, spider): - if not is_cacheable(request): - return - - key = request.fingerprint() - domain = spider.domain_name - try: - response = self.cache.retrieve_response(domain, key) - except: - log.msg("Corrupt cache for %s" % request.url, log.WARNING) - response = False - if response: - response.cached = True - if not 200 <= int(response.status) < 300: - raise HttpException(response.status, None, response) - return response - elif self.ignore_missing: - raise IgnoreRequest("Ignored request not in cache: %s" % request) - - def process_response(self, request, response, spider): - if not is_cacheable(request): - return response - - if isinstance(response, Response) and not response.cached: - key = request.fingerprint() - domain = spider.domain_name - self.cache.store(domain, key, request, response) - - return response - - def process_exception(self, request, exception, spider): - if not is_cacheable(request): - return - - if isinstance(exception, HttpException) and isinstance(exception.response, Response): - key = request.fingerprint() - domain = spider.domain_name - self.cache.store(domain,key, request, exception.response) - -def is_cacheable(request): - scheme, _, _, _, _ = urlparse.urlsplit(request.url) - return scheme in ['http', 'https'] - - -class Cache(object): - DOMAIN_SECTORDIR = 'data' - DOMAIN_LINKDIR = 'domains' - - def __init__(self, cachedir, sectorize=False): - self.cachedir = cachedir - self.sectorize = sectorize - - self.baselinkpath = os.path.join(self.cachedir, self.DOMAIN_LINKDIR) - if not os.path.exists(self.baselinkpath): - os.makedirs(self.baselinkpath) - - self.basesectorpath = os.path.join(self.cachedir, self.DOMAIN_SECTORDIR) - if not os.path.exists(self.basesectorpath): - os.makedirs(self.basesectorpath) - - def domainsectorpath(self, domain): - sector = hashlib.sha1(domain).hexdigest()[0] - return os.path.join(self.basesectorpath, sector, domain) - - def domainlinkpath(self, domain): - return os.path.join(self.baselinkpath, domain) - - def requestpath(self, domain, key): - linkpath = self.domainlinkpath(domain) - return os.path.join(linkpath, key[0:2], key) - - def open_domain(self, domain): - if domain: - linkpath = self.domainlinkpath(domain) - if self.sectorize: - sectorpath = self.domainsectorpath(domain) - if not os.path.exists(sectorpath): - os.makedirs(sectorpath) - if not os.path.exists(linkpath): - try: - os.symlink(sectorpath, linkpath) - except: - os.makedirs(linkpath) # windows filesystem - else: - if not os.path.exists(linkpath): - os.makedirs(linkpath) - - def is_cached(self, domain, key): - requestpath = self.requestpath(domain, key) - if os.path.exists(requestpath): - with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: - metadata = pickle.load(f) - expiration_secs = settings.getint('CACHE2_EXPIRATION_SECS') - if expiration_secs >= 0: - if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs): - return True - else: - log.msg('dropping old cached response from %s' % metadata['timestamp']) - return False - else: - # disabled cache expiration - return True - else: - return False - - def retrieve_response(self, domain, key): - """ - Return response dictionary if request has correspondent cache record; - return None if not. - """ - if not self.is_cached(domain, key): - return None # not cached - - requestpath = self.requestpath(domain, key) - metadata = responsebody = responseheaders = None - with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: - metadata = pickle.load(f) - with open(os.path.join(requestpath, 'response_body')) as f: - responsebody = f.read() - with open(os.path.join(requestpath, 'response_headers')) as f: - responseheaders = f.read() - - url = metadata['url'] - original_url = metadata.get('original_url', url) - headers = Headers(responseheaders) - status = metadata['status'] - - response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody) - response.cached = True - return response - - def store(self, domain, key, request, response): - requestpath = self.requestpath(domain, key) - if not os.path.exists(requestpath): - os.makedirs(requestpath) - - metadata = { - 'url':request.url, - 'method': request.method, - 'status': response.status, - 'domain': response.domain, - 'original_url': response.original_url, - 'timestamp': datetime.datetime.utcnow(), - } - - # metadata - with open(os.path.join(requestpath, 'meta_data'), 'w') as f: - f.write(repr(metadata)) - # pickled metadata (to recover without using eval) - with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f: - pickle.dump(metadata, f) - # response - with open(os.path.join(requestpath, 'response_headers'), 'w') as f: - f.write(headers_dict_to_raw(response.headers)) - with open(os.path.join(requestpath, 'response_body'), 'w') as f: - f.write(response.body.get_content()) - # request - with open(os.path.join(requestpath, 'request_headers'), 'w') as f: - f.write(headers_dict_to_raw(request.headers)) - if request.body: - with open(os.path.join(requestpath, 'request_body'), 'w') as f: - f.write(request.body) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/common.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/common.py deleted file mode 100644 index 063ea72b7..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/common.py +++ /dev/null @@ -1,10 +0,0 @@ -class CommonMiddleware(object): - """This middleware provides common/basic functionality, and should always - be enabled""" - - def process_request(self, request, spider): - request.headers.setdefault('Accept-Language', 'en') - request.headers.setdefault('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') - if request.method == 'POST': - request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded') - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/compression.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/compression.py deleted file mode 100644 index 9b23f33f1..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/compression.py +++ /dev/null @@ -1,42 +0,0 @@ -import zlib -from gzip import GzipFile -from cStringIO import StringIO - -from scrapy.http import Response, ResponseBody - - -class CompressionMiddleware(object): - """This middleware allows compressed (gzip, deflate) traffic to be - sent/received from web sites""" - - def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') - - def process_response(self, request, response, spider): - if isinstance(response, Response): - content_encoding = response.headers.get('Content-Encoding') - if content_encoding: - encoding = content_encoding[0].lower() - raw_body = response.body.get_content() - declared_encoding = response.body.declared_encoding - decoded_body = self._decode(raw_body, encoding) - response.body = ResponseBody(decoded_body, declared_encoding) - response.headers['Content-Encoding'] = content_encoding[1:] - return response - - def _decode(self, body, encoding): - if encoding == 'gzip': - body = GzipFile(fileobj=StringIO(body)).read() - - if encoding == 'deflate': - try: - body = zlib.decompress(body) - except zlib.error: - # ugly hack to work with raw deflate content that may - # be sent by microsof servers. For more information, see: - # http://carsten.codimi.de/gzip.yaws/ - # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx - # http://www.gzip.org/zlib/zlib_faq.html#faq38 - body = zlib.decompress(body, -15) - return body - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cookies.py deleted file mode 100644 index 9816dc521..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cookies.py +++ /dev/null @@ -1,27 +0,0 @@ -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy.core.engine import scrapyengine -from scrapy.utils.misc import dict_updatedefault - -class CookiesMiddleware(object): - """This middleware enables working with sites that need cookies""" - - def __init__(self): - self.cookies = {} - dispatcher.connect(self.domain_open, signals.domain_open) - dispatcher.connect(self.domain_closed, signals.domain_closed) - - def process_request(self, request, spider): - dict_updatedefault(request.cookies, self.cookies[spider.domain_name]) - - def process_response(self, request, response, spider): - cookies = self.cookies[spider.domain_name] - cookies.update(request.cookies) - return response - - def domain_open(self, domain): - self.cookies[domain] = {} - - def domain_closed(self, domain): - del self.cookies[domain] diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/debug.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/debug.py deleted file mode 100644 index 6e13dc1b8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/debug.py +++ /dev/null @@ -1,20 +0,0 @@ -from scrapy import log -from scrapy.conf import settings - -class CrawlDebug(object): - def __init__(self): - self.enabled = settings.getbool('CRAWL_DEBUG') - - def process_request(self, request, spider): - if self.enabled: - log.msg("Crawling %s" % repr(request), domain=spider.domain_name, level=log.DEBUG) - - def process_exception(self, request, exception, spider): - if self.enabled: - log.msg("Crawl exception %s in %s" % (exception, repr(request)), domain=spider.domain_name, level=log.DEBUG) - - def process_response(self, request, response, spider): - if self.enabled: - log.msg("Fetched %s from %s" % (response.info(), repr(request)), domain=spider.domain_name, level=log.DEBUG) - return response - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/httpauth.py deleted file mode 100644 index fcc9ebb8c..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/httpauth.py +++ /dev/null @@ -1,8 +0,0 @@ -class HttpAuthMiddleware(object): - """This middleware allows spiders to use HTTP auth in a cleaner way - (http_user and http_pass spider class attributes)""" - - def process_request(self, request, spider): - if getattr(spider, 'http_user', None) or getattr(spider, 'http_pass', None): - request.httpauth(spider.http_user, spider.http_pass) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/redirect.py deleted file mode 100644 index bc334f08c..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/redirect.py +++ /dev/null @@ -1,66 +0,0 @@ -import re - -from scrapy import log -from scrapy.http import Request, Response -from scrapy.core.exceptions import HttpException -from scrapy.utils.url import urljoin_rfc as urljoin - -class RedirectLoop(Exception): - pass - -META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);url=([^"\']+)', re.IGNORECASE) -# some sites use meta-refresh for redirecting to a session expired page, so we -# restrict automatic redirection to a maximum delay (in number of seconds) -META_REFRESH_MAXSEC = 100 -MAX_REDIRECT_LOOP = 10 - -class RedirectMiddleware(object): - def process_exception(self, request, exception, spider): - if isinstance(exception, HttpException): - status = exception.status - response = exception.response - - if status in ['302', '303']: - redirected_url = urljoin(request.url, response.headers['location'][0]) - if not getattr(spider, "no_redirect", False): - redirected = request.copy() - redirected.url = redirected_url - redirected.method = 'GET' - redirected.body = None - # This is needed to avoid redirection loops with requests that contain dont_filter = True - # Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks - if isinstance(redirected.dont_filter, int): - if not hasattr(redirected, "original_dont_filter"): - redirected.original_dont_filter = redirected.dont_filter - if redirected.dont_filter <= -MAX_REDIRECT_LOOP: - raise RedirectLoop("Exited redirect loop with %s consecutive visits to the same url." % (redirected.original_dont_filter + MAX_REDIRECT_LOOP) ) - redirected.dont_filter -= 1 - else: - redirected.dont_filter = False - log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) - return redirected - log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name) - return response - - if status in ['301', '307']: - redirected_url = urljoin(request.url, response.headers['location'][0]) - if not getattr(spider, "no_redirect", False): - redirected = request.copy() - redirected.url = redirected_url - # This is needed to avoid redirection loops with requests that contain dont_filter = True - # Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks - redirected.dont_filter = False - log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) - return redirected - log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name) - return response - - def process_response(self, request, response, spider): - if isinstance(response, Response): - m = META_REFRESH_RE.search(response.body.to_string()[0:4096]) - if m and int(m.group(1)) < META_REFRESH_MAXSEC: - redirected = request.copy() - redirected.url = urljoin(request.url, m.group(2)) - log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name) - return redirected - return response diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/retry.py deleted file mode 100644 index d21deb160..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/retry.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -An extension to retry failed requests that are potentially caused by temporary -problems such as a connection timeout or HTTP 500 error. - -You can change the behaviour of this moddileware by modifing the scraping settings: -RETRY_TIMES - how many times to retry a failed page -RETRY_HTTP_CODES - which HTTP response codes to retry - -Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. Once -there is no more failed pages to retry this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. - -Default values are located in scrapy.conf.core_settings - -About HTTP errors to consider: - -- You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP - protocol. It's included by default because it's a common code used to - indicate server overload, which would be something we want to retry -- 200 is included by default (and shoudln't be removed) to check for partial - downloads errors, which means the TCP connection has broken in the middle of - a HTTP download -""" - -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError -from twisted.internet.defer import TimeoutError as UserTimeoutError - -from scrapy import log -from scrapy.core.exceptions import HttpException -from scrapy.conf import settings - -class RetryMiddleware(object): - - EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, - ConnectionRefusedError, ConnectionDone, ConnectError) - - def __init__(self): - self.failed_count = {} - self.max_retries = settings.getint('RETRY_TIMES') - - def process_exception(self, request, exception, spider): - retry = False - - if isinstance(exception, self.EXCEPTIONS_TO_RETRY): - retry = True - elif isinstance(exception, HttpException): - if exception.status in settings.getlist('RETRY_HTTP_CODES'): - retry = True - - if retry: - fp = request.fingerprint() - count = self.failed_count[fp] = self.failed_count.get(fp, 0) + 1 - - if self.failed_count[fp] < self.max_retries: - log.msg("Retrying %s (failed %d times): %s" % (request, count, exception), level=log.DEBUG, domain=spider.domain_name) - retryreq = request.copy() - retryreq.dont_filter = True - return retryreq - else: - log.msg("Discarding %s (failed %d times): %s" % (request, count, exception), domain=spider.domain_name, level=log.DEBUG) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/robots.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/robots.py deleted file mode 100644 index d4e245eb8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/robots.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -jhis is an expertimental middleware to respect robots.txt policies. The biggest -problem it has is that it uses urllib directly (in RobotFileParser.read() -method) and that conflicts with twisted networking, so it should be ported to -use twisted networking API, but that is not as trivial as it may seem. - -This code is left here for future reference, when we resume the work on this -subject. - -""" - -import re -import urlparse -import robotparser - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.core.exceptions import IgnoreRequest -from scrapy.conf import settings - -BASEURL_RE = re.compile("http://.*?/") - -class RobotsMiddleware(object): - - def __init__(self): - self._parsers = {} - self._spiderdomains = {} - self._pending = {} - dispatcher.connect(self.domain_open, signals.domain_open) - dispatcher.connect(self.domain_closed, signals.domain_closed) - - def process_request(self, request, spider): - agent = getattr(spider, 'user_agent', None) or settings['USER_AGENT'] - rp = self.robot_parser(request.url, spider.domain_name) - if rp and not rp.can_fetch(agent, request.url): - raise IgnoreRequest("URL forbidden by robots.txt: %s" % request.url) - - def robot_parser(self, url, spiderdomain): - urldomain = urlparse.urlparse(url).hostname - if urldomain in self._parsers: - rp = self._parsers[urldomain] - else: - rp = robotparser.RobotFileParser() - m = BASEURL_RE.search(url) - if m: - rp.set_url("%srobots.txt" % m.group()) - rp.read() - self._parsers[urldomain] = rp - self._spiderdomains[spiderdomain].add(urldomain) - return rp - - def domain_open(self, domain): - self._spiderdomains[domain] = set() - - def domain_closed(self, domain): - for urldomain in self._spiderdomains[domain]: - del self._parsers[urldomain] - del self._spiderdomains[domain] diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/useragent.py deleted file mode 100644 index 6bda3e85f..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/useragent.py +++ /dev/null @@ -1,7 +0,0 @@ -class UserAgentMiddleware(object): - """This middleware allows spiders to override the user_agent""" - - def process_request(self, request, spider): - if getattr(spider, 'user_agent', None): - request.headers.setdefault('User-Agent', spider.user_agent) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/history/middleware.py b/scrapy/branches/cluster-refactor/scrapy/contrib/history/middleware.py deleted file mode 100644 index 9fae23bdd..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/history/middleware.py +++ /dev/null @@ -1,88 +0,0 @@ -import hashlib -from datetime import datetime - -from pydispatch import dispatcher - -from scrapy.utils.misc import load_class -from scrapy.core import signals -from scrapy import log -from scrapy.core.exceptions import NotConfigured, IgnoreRequest -from scrapy.conf import settings - -class HistoryMiddleware(object): - # How often we should re-check links we know about - MIN_CHECK_DAYS = 4 - # How often we should process pages that have not changed (need to include depth) - MIN_PROCESS_UNCHANGED_DAYS = 12 - - def __init__(self): - historycls = load_class(settings['MEMORYSTORE']) - if not historycls: - raise NotConfigured - self.historydata = historycls() - dispatcher.connect(self.open_domain, signal=signals.domain_open) - dispatcher.connect(self.close_domain, signal=signals.domain_closed) - - def process_request(self, request, spider): - key = urlkey(request.url) - status = self.historydata.status(domain, key) - if status: - _url, version, last_checked = status - d = datetime.now() - last_checked - if d.days < self.MIN_CHECK_DAYS: - raise IgnoreRequest("Not scraping %s (scraped %s ago)" % (request.url, d)) - request.context['history_response_version'] = version - - def process_response(self, request, response, spider): - version = request.context.get('history_response_version') - if version == response.version(): - del request.content['history_response_version'] - hist = self.historydata.version_info(domain, version) - if hist: - versionkey, created = hist - # if versionkey != urlkey(url) this means - # the same content is available on a different url - delta = datetime.now() - created - if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS: - message = "skipping %s: unchanged for %s" % (response.url, delta) - raise IgnoreRequest(message) - self.record_visit(domain, request, response) - return response - - def process_exception(self, request, exception, spider): - self.record_visit(spider.domain_name, request, None) - - def open_domain(self, domain): - self.historydata.open(domain) - - def close_domain(self, domain): - self.historydata.close_site(domain) - - def record_visit(self, domain, request, response): - """record the fact that the url has been visited""" - url = request.url - post_version = hash(request.body) - key = urlkey(url) - if response: - redirect_url = response.url - parentkey = urlkey(response.request.headers.get('referer')) if response.request else None - version = response.version() - else: - redirect_url, parentkey, version = url, None, None - self.historydata.store(domain, key, url, parentkey, version, post_version) - - -def urlkey(url): - """Generate a 'key' for a given url - - >>> urlkey("http://www.example.com/") - '89e6a0649e06d83370cdf2cbfb05f363934a8d0c' - >>> urlkey("http://www.example.com/") == urlkey("http://www.example.com/?") - True - """ - from scrapy.utils.c14n import canonicalize - return hash(canonicalize(url)) - - -def hash(value): - return hashlib.sha1(value).hexdigest() if value else None diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/history/scheduler.py b/scrapy/branches/cluster-refactor/scrapy/contrib/history/scheduler.py deleted file mode 100644 index e8f20f032..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/history/scheduler.py +++ /dev/null @@ -1,94 +0,0 @@ -from datetime import datetime - -from twisted.internet import defer - -from scrapy import log -from scrapy.core.scheduler import Scheduler -from scrapy.core.exceptions import IgnoreRequest - -class RulesScheduler(Scheduler): - """Scheduler that uses rules to determine if we should follow links - - TODO: - * take into account where in chain of links we are (less depth should - be crawled more often) - * Be more strict about scraping product pages that rarely lead to new - versions of products. The same applies to pages with links. Particularly - useful for filtering out when there are many urls for the same product. - (but be careful to also filter out pages that almost always lead to new - output). - """ - - # if these parameters change, then update bin/unavailable.py - - # How often we should re-check links we know about - MIN_CHECK_DAYS = 4 - - # How often we should process pages that have not changed (need to include depth) - MIN_PROCESS_UNCHANGED_DAYS = 12 - - def enqueue_request(self, domain, request, priority=1): - """Add a page to be scraped for a domain that is currently being scraped. - - The url will only be added if we have not checked it already within - a specified time period. - """ - requestid = request.fingerprint() - added = self.groupfilter.add(domain, requestid) - - if request.dont_filter or added: - key = urlkey(request.url) # we can not use fingerprint unless lost crawled history - status = self.historydata.status(domain, key) - now = datetime.now() - version = None - if status: - _url, version, last_checked = status - d = now - last_checked - if d.days < self.MIN_CHECK_DAYS: - log.msg("Not scraping %s (scraped %s ago)" % (request.url, d), level=log.DEBUG) - return - # put the version in the pending pages to avoid querying DB again - record = (request, version, now) - self.pending_requests[domain].put(record, priority) - - def next_request(self, domain): - """Get the next page from the superclass. This will add a callback - to prevent processing the page unless its content has been - changed. - - In the event that it a page is not processed, the record_visit method - is called to update the last_checked time. - """ - pending_list = self.pending_requests.get(domain) - if not pending_list : - return None - request, version, timestamp = pending_list.get_nowait()[1] - post_version = hash(request.body) - - def callback(pagedata): - """process other callback if we pass the checks""" - if version == pagedata.version(): - hist = self.historydata.version_info(domain, version) - if hist: - versionkey, created = hist - # if versionkey != urlkey(url) this means - # the same content is available on a different url - delta = timestamp - created - if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS: - message = "skipping %s: unchanged for %s" % (pagedata.url, delta) - raise IgnoreRequest(message) - self.record_visit(domain, request.url, pagedata.url, - pagedata.parent, pagedata.version(), - post_version) - return pagedata - - def errback(error) : - self.record_visit(domain, request.url, request.url, None, None, - post_version) - return error - - d = defer.Deferred() - d.addCallbacks(callback, errback) - request.prepend_callback(d) - - return request diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/item/models.py b/scrapy/branches/cluster-refactor/scrapy/contrib/item/models.py deleted file mode 100644 index 451d90b2a..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/item/models.py +++ /dev/null @@ -1,208 +0,0 @@ -""" -This module contains some extra base models for scraped items which could be -useful in some Scrapy implementations -""" - -import hashlib - -from pprint import PrettyPrinter -from scrapy.item import ScrapedItem -from scrapy.core.exceptions import UsageError, DropItem - -class ValidationError(DropItem): - """Indicates a data validation error""" - def __init__(self,problem,value=None): - self.problem = problem - self.value = value - - def __str__(self): - if self.value is not None: - return '%s "%s"' % (self.problem, self.value) - else: - return '%s' % (self.problem) - -class ValidationPipeline(object): - def process_item(self, domain, response, item): - item.validate() - return item - -class RobustScrapedItem(ScrapedItem): - """ - A more robust scraped item class with a built-in validation mechanism and - minimal versioning support - """ - - ATTRIBUTES = { - 'guid': basestring, # a global unique identifier - 'url': basestring, # the main URL where this item was scraped from - } - - def __init__(self, data=None): - """ - A scraped item can be initialised with a dictionary that will be - squirted directly into the object. - """ - if isinstance(data, dict): - for attr, value in data.iteritems(): - setattr(self, attr, value) - elif data is not None: - raise UsageError("Initialize with dict, not %s" % data.__class__.__name__) - - self.__dict__['_version'] = None - - def __getattr__(self, attr): - # Return None for valid attributes not set, raise AttributeError for invalid attributes - # Note that this method is called only when the attribute is not found in - # self.__dict__ or the class/instance methods. - if attr in self.ATTRIBUTES: - return None - else: - raise AttributeError(attr) - - def __setattr__(self, attr, value): - """ - Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES - """ - if not attr.startswith('_') and attr not in self.ATTRIBUTES: - raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__)) - - if value is None: - self.__dict__.pop(attr, None) - return - - type1 = self.ATTRIBUTES[attr] - if hasattr(type1, '__iter__'): - if not hasattr(value, '__iter__'): - raise TypeError('Attribute "%s" must be a sequence' % attr) - type2 = type1[0] - for i in value: - if not isinstance(i, type2): - raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__)) - else: - if not isinstance(value, type1): - raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__)) - - self.__dict__[attr] = value - self.__dict__['_version'] = None - - def __delattr__(self, attr): - """ - Delete an attribute from the ScrapedItem instance if it exists. - If not, raise an AttributeError. - """ - if attr in self.__dict__: - del self.__dict__[attr] - self.__dict__['_version'] = None - else: - raise AttributeError("Attribute '%s' doesn't exist" % attr) - - def __eq__(self, other): - if isinstance(other, type(self)): - return self.version == other.version - - def __ne__(self, other): - return self.version != other.version - - def __sub__(self, other): - return RobustItemDelta(other, self) - - def __repr__(self): - # Generate this format so that it can be deserialized easily: - # ClassName({...}) - reprdict = {} - for k, v in self.__dict__.iteritems(): - if not k.startswith('_'): - reprdict[k] = v - return "%s(%s)" % (self.__class__.__name__, repr(reprdict)) - - def __str__(self) : - return "%s: GUID=%s, url=%s" % ( self.__class__.__name__ , self.guid, self.url ) - - def validate(self): - """Method used to validate item attributes data""" - if not self.guid: - raise ValidationError('A guid is required') - - def copy(self): - """Create a new ScrapedItem object based on the current one""" - import copy - return copy.deepcopy(self) - - @property - def version(self): - """ - Return a (cached) 40 char hash of all the item attributes. - - WARNING: This cached version won't work if mutable products are - modified directly like: - - item.features.append('feature') - """ - if self._version: - return self._version - hash_ = hashlib.sha1() - hash_.update("".join(["".join([n, str(v)]) for n,v in sorted(self.__dict__.iteritems())])) - return hash_.hexdigest() - - -class RobustItemDelta(object): - """ - This class represents the difference between - a pair of RobustScrapedItems. - """ - - def __init__(self, old_item, new_item): - if not isinstance(old_item, RobustScrapedItem) or \ - not isinstance(new_item, RobustScrapedItem): - raise TypeError("Both arguments must be RobustScrapedItem instances") - - if old_item.guid != new_item.guid: - raise AttributeError("Item GUIDs must be equal in order to create a RobustItemDelta object") - - self.old_item = old_item - self.new_item = new_item - self.diff = self.do_diff() - - def do_diff(self): - """ - This method should retreive a dictionary - containing the changes between both items - as in this example: - - >>> delta.do_diff() - >>> {'attrib': {'new': 'New value', 'old': 'Old value'}, # Common attributes - 'attrib2': {'new': 'New value 2', 'old': 'Old value 2'}, - 'attrib3': [{'new': 'New list value', 'old': 'Old list value'}, # List attributes - {'new': 'New list value 2', 'old': 'Old list value 2'}]} - """ - - if self.old_item == self.new_item: - return {} - - diff = {} - for key, value in self.old_item.__dict__.items(): - if key in self.old_item.ATTRIBUTES.keys(): - new_value = getattr(self.new_item, key) - if value != new_value: - diff[key] = {'new': new_value, 'old': value} - for key, value in self.new_item.__dict__.items(): - if value and key in self.new_item.ATTRIBUTES.keys(): - if not getattr(self.old_item, key): - diff[key] = {'new': value, 'old': None} - return diff - - def __eq__(self, other): - if isinstance(other, RobustItemDelta): - if other.old_item == self.old_item and \ - other.new_item == self.new_item and \ - other.diff == self.diff: - return True - return False - - def __repr__(self): - if self.diff: - pp = PrettyPrinter(indent=3) - return pp.pformat(self.diff) - else: - return 'No differences found between the provided items.' - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/memdebug.py b/scrapy/branches/cluster-refactor/scrapy/contrib/memdebug.py deleted file mode 100644 index a73801a07..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/memdebug.py +++ /dev/null @@ -1,69 +0,0 @@ -import pprint -import gc -import socket - -import libxml2 -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy.core.exceptions import NotConfigured -from scrapy.mail import MailSender -from scrapy.stats import stats -from scrapy.extension import extensions -from scrapy.conf import settings - -class MemoryDebugger(object): - - def __init__(self): - if not settings.getbool('MEMDEBUG_ENABLED'): - raise NotConfigured - - self.mail = MailSender() - self.rcpts = settings.getlist('MEMDEBUG_NOTIFY') - - self.domains_scraped = [] - dispatcher.connect(self.domain_opened, signals.domain_opened) - dispatcher.connect(self.engine_started, signals.engine_started) - dispatcher.connect(self.engine_stopped, signals.engine_stopped) - - def engine_started(self): - libxml2.debugMemory(1) - - def engine_stopped(self): - figures = self.collect_figures() - report = self.create_report(figures) - self.print_or_send_report(report) - - def collect_figures(self): - libxml2.cleanupParser() - gc.collect() - - figures = [] - if 'MemoryUsage' in extensions.enabled: - memusage = extensions.enabled['MemoryUsage'] - memusage.update() - figures.append(("Memory usage at startup", int(memusage.data['startup']/1024/1024), "Mb")) - figures.append(("Maximum memory usage", int(memusage.data['max']/1024/1024), "Mb")) - figures.append(("Memory usage at shutdown", int(memusage.virtual/1024/1024), "Mb")) - figures.append(("Objects in gc.garbage", len(gc.garbage), "")) - figures.append(("libxml2 memory leak", libxml2.debugMemory(1), "bytes")) - return figures - - def create_report(self, figures): - s = "" - s += "SCRAPY MEMORY DEBUGGER RESULTS\n\n" - for f in figures: - s += "%-30s : %s %s\n" % f - s += "\n" - if stats: - s += "SCRAPING STATS --------------------------------------------------\n\n" - s += pprint.pformat(stats) - return s - - def print_or_send_report(self, report): - if self.rcpts: - self.mail.send(self.rcpts, "Scrapy Memory Debugger results at %s" % socket.gethostname(), report) - print report - - def domain_opened(self, domain): - self.domains_scraped.append(domain) diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/memusage.py b/scrapy/branches/cluster-refactor/scrapy/contrib/memusage.py deleted file mode 100644 index 760e92314..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/memusage.py +++ /dev/null @@ -1,122 +0,0 @@ -import sys -import os -import pprint -import socket - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.core.exceptions import NotConfigured -from scrapy.mail import MailSender -from scrapy.stats import stats -from scrapy.conf import settings - -class MemoryUsage(object): - - _proc_status = '/proc/%d/status' % os.getpid() - _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0, - 'KB': 1024.0, 'MB': 1024.0*1024.0} - - def __init__(self): - if not settings.getbool('MEMUSAGE_ENABLED'): - raise NotConfigured - if sys.platform != 'linux2': - raise NotConfigured("MemoryUsage extension is only available on Linux") - - self.warned = False - - self.data = {} - self.data['startup'] = 0 - self.data['max'] = 0 - - scrapyengine.addtask(self.update, 60.0, now=True) - - self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY') - self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 - self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 - self.report = settings.getbool('MEMUSAGE_REPORT') - - if self.limit: - scrapyengine.addtask(self._check_limit, 60.0, now=True) - if self.warning: - scrapyengine.addtask(self._check_warning, 60.0, now=True) - - self.mail = MailSender() - - dispatcher.connect(self.engine_started, signal=signals.engine_started) - - - @property - def virtual(self): - return self._vmvalue('VmSize:') - - @property - def resident(self): - return self._vmvalue('VmRSS:') - - @property - def stacksize(self): - return self._vmvalue('VmStk:') - - def engine_started(self): - self.data['startup'] = self.virtual - - def update(self): - if self.virtual > self.data['max']: - self.data['max'] = self.virtual - - def _vmvalue(self, VmKey): - # get pseudo file /proc//status - try: - t = open(self._proc_status) - v = t.read() - t.close() - except: - return 0.0 # non-Linux? - # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' - i = v.index(VmKey) - v = v[i:].split(None, 3) # whitespace - if len(v) < 3: - return 0.0 # invalid format? - # convert Vm value to bytes - return float(v[1]) * self._scale[v[2]] - - def _check_limit(self): - if self.virtual > self.limit: - mem = self.limit/1024/1024 - log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR) - if self.notify_mails: - subj = "%s terminated: memory usage exceeded %dM at %s" % (settings['BOT_NAME'], mem, socket.gethostname()) - self._send_report(self.notify_mails, subj) - scrapymanager.stop() - - def _check_warning(self): - if self.warned: # warn only once - return - if self.virtual > self.warning: - mem = self.warning/1024/1024 - log.msg("Memory usage reached %dM" % mem, level=log.WARNING) - if self.notify_mails: - subj = "%s warning: memory usage reached %dM at %s" % (settings['BOT_NAME'], mem, socket.gethostname()) - self._send_report(self.notify_mails, subj) - self.warned = True - - def _send_report(self, rcpts, subject): - """send notification mail with some additional useful info""" - s = "Memory usage at engine startup : %dM\r\n" % (self.data['startup']/1024/1024) - s += "Maximum memory usage : %dM\r\n" % (self.data['max']/1024/1024) - s += "Current memory usage : %dM\r\n" % (self.virtual/1024/1024) - - s += "ENGINE STATUS ------------------------------------------------------- \r\n" - s += "\r\n" - s += scrapyengine.getstatus() - s += "\r\n" - - if stats: - s += "SCRAPING STATS ------------------------------------------------------ \r\n" - s += "\r\n" - s += pprint.pformat(stats) - self.mail.send(rcpts, subject, s) diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/images.py b/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/images.py deleted file mode 100644 index e9f06cc03..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/images.py +++ /dev/null @@ -1,181 +0,0 @@ -import re -import os -import time -import hashlib -import urllib -import urlparse -from cStringIO import StringIO - -import Image - -from scrapy import log -from scrapy.stats import stats -from scrapy.core.exceptions import DropItem, NotConfigured -from scrapy.core.exceptions import HttpException -from scrapy.conf import settings - -from scrapy.contrib.pipeline.media import MediaPipeline - -# the age at which we download images again -IMAGE_EXPIRES = settings.getint('IMAGES_EXPIRES', 90) - -class NoimagesDrop(DropItem): - pass - -class ImageException(Exception): - """General image error exception""" - - -class ImagesPipeline(MediaPipeline): - MEDIA_TYPE = 'image' - THUMBS = None -# THUMBS = ( -# ("50", (50, 50)), -# ("110", (110, 110)), -# ("270", (270, 270)) -# ) - MIN_WIDTH = 0 - MIN_HEIGHT = 0 - - def __init__(self): - if not settings['IMAGES_DIR']: - raise NotConfigured - - self.BASEDIRNAME = settings['IMAGES_DIR'] - self.mkdir(self.BASEDIRNAME) - - self.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0) - self.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0) - MediaPipeline.__init__(self) - - def media_to_download(self, request, info): - relative, absolute = self._get_paths(request) - if not should_download(absolute): - self.inc_stats(info.domain, 'uptodate') - referer = request.headers.get('Referer') - log.msg('Image (uptodate): Downloaded %s from %s referred in <%s>' % \ - (self.MEDIA_TYPE, request, referer), level=log.DEBUG, domain=info.domain) - return relative - - def media_downloaded(self, response, request, info): - mtype = self.MEDIA_TYPE - referer = request.headers.get('Referer') - - if not response or not response.body.to_string(): - msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: Empty image (no-content)' % (mtype, request, referer) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - result = self.save_image(response, request, info) # save and thumbs response - - status = 'cached' if getattr(response, 'cached', False) else 'downloaded' - msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer) - log.msg(msg, level=log.DEBUG, domain=info.domain) - self.inc_stats(info.domain, status) - return result - - def media_failed(self, failure, request, info): - referer = request.headers.get('Referer') - errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure) - msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' % (self.MEDIA_TYPE, request, referer, errmsg) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - def save_image(self, response, request, info): - mtype = self.MEDIA_TYPE - relpath, abspath = self._get_paths(request) - dirname = os.path.dirname(abspath) - self.mkdir(dirname, info) - - try: - save_image_with_thumbnails(response, abspath, self.THUMBS, self.MIN_WIDTH, self.MIN_HEIGHT) - except ImageException, ex: - log.msg(str(ex), level=log.WARNING, domain=info.domain) - raise ex - except Exception, ex: - referer = request.headers.get('Referer') - msg = 'Image (processing-error): Error thumbnailing %s from %s referred in <%s>: %s' % (mtype, request, referer, ex) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - return relpath # success value sent as input result for item_media_downloaded - - def _get_paths(self, request): - relative = image_path(request.url) - absolute = os.path.join(self.BASEDIRNAME, relative) - return relative, absolute - - def mkdir(self, dirname, info=None): - already_created = info.extra.setdefault('created_directories', set()) if info else set() - if dirname not in already_created: - if not os.path.exists(dirname): - os.makedirs(dirname) - already_created.add(dirname) - - def inc_stats(self, domain, status): - stats.incpath('%s/image_count' % domain) - stats.incpath('%s/image_status_count/%s' % (domain, status)) - - - -def should_download(path): - """Should the image downloader download the image to the location specified - """ - try: - mtime = os.path.getmtime(path) - age_seconds = time.time() - mtime - age_days = age_seconds / 60 / 60 / 24 - return age_days > IMAGE_EXPIRES - except: - return True - -_MULTIPLE_SLASHES_REGEXP = re.compile(r"\/{2,}") -_FINAL_SLASH_REGEXP = re.compile(r"\/$") -def image_path(url): - """Return the relative path on the target filesystem for an image to be - downloaded to. - """ - _, netloc, urlpath, query, _ = urlparse.urlsplit(url) - urlpath = _MULTIPLE_SLASHES_REGEXP.sub('/', urlpath) - urlpath = _FINAL_SLASH_REGEXP.sub('.jpg', urlpath) - if os.sep != '/': - urlpath.replace('/', os.sep) - if query: - img_path = os.path.join(netloc, hashlib.sha1(url).hexdigest()) - else: - img_path = os.path.join(netloc, urlpath[1:]) - return urllib.unquote(img_path) - - -def thumbnail_name(image, sizestr): - """Get the name of a thumbnail image given the name of the original file. - - There will can be many types of thumbnails, so we will have a "name" for - each type. - """ - return os.path.splitext(image)[0] + '_' + sizestr + '.jpg' - -def save_scaled_image(image, img_path, name, size): - thumb = image.copy() if image.mode == 'RGB' else image.convert('RGB') - thumb.thumbnail(size, Image.ANTIALIAS) - filename = thumbnail_name(img_path, name) - thumb.save(filename, 'JPEG') - -def save_image_with_thumbnails(response, path, thumbsizes, min_width=0, min_height=0): - memoryfile = StringIO(response.body.to_string()) - im = Image.open(memoryfile) - if im.mode != 'RGB': - log.msg("Found non-RGB image during scraping %s" % path, level=log.WARNING) - for name, size in thumbsizes or []: - save_scaled_image(im, path, name, size) - try: - im.save(path) - except Exception, ex: - log.msg("Image (processing-error): cannot process %s, so writing direct file: Error: %s" % (path, ex)) - f = open(path, 'wb') - f.write(response.body.to_string()) - f.close() - width, height = im.size - if width < min_width or height < min_height: - raise ImageException("Image too small (%dx%d < %dx%d): %s" % (width, height, min_width, min_height, response.url)) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/media.py b/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/media.py deleted file mode 100644 index 0c5c59d2c..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/media.py +++ /dev/null @@ -1,179 +0,0 @@ -from twisted.internet import defer - -from scrapy.utils.defer import mustbe_deferred, defer_result -from scrapy import log -from scrapy.core.engine import scrapyengine -from scrapy.spider import spiders - - -class DomainInfo(object): - def __init__(self, domain): - self.domain = domain - self.spider = spiders.fromdomain(domain) - self.downloading = {} - self.downloaded = {} - self.waiting = {} - self.extra = {} - - -class MediaPipeline(object): - def __init__(self): - self.domaininfo = {} - - def open_domain(self, domain): - self.domaininfo[domain] = DomainInfo(domain) - - def close_domain(self, domain): - del self.domaininfo[domain] - - def process_item(self, domain, response, item): - info = self.domaininfo[domain] - requests = self.get_media_requests(item, info) - assert requests is None or hasattr(requests, '__iter__'), \ - 'get_media_requests should return None or iterable' - - def _bugtrap(_failure, request): - log.msg('Unhandled ERROR in MediaPipeline.item_media_{downloaded,failed} for %s: %s' % (request, _failure), log.ERROR, domain=domain) - - lst = [] - for request in requests or (): - dfd = self._enqueue(request, info) - dfd.addCallbacks( - callback=self.item_media_downloaded, - callbackArgs=(item, request, info), - errback=self.item_media_failed, - errbackArgs=(item, request, info), - ) - dfd.addErrback(_bugtrap, request) - lst.append(dfd) - - dlst = defer.DeferredList(lst, consumeErrors=False) - dlst.addBoth(lambda _: self.item_completed(item, info)) - return dlst - - def _enqueue(self, request, info): - wad = request.deferred or defer.Deferred() - - fp = request.fingerprint() - if fp in info.downloaded: - cached = info.downloaded[fp] - defer_result(cached).chainDeferred(wad) - else: - info.waiting.setdefault(fp, []).append(wad) - if fp not in info.downloading: - self._download(request, info, fp) - - return wad - - def _download(self, request, info, fp): - def _bugtrap(_failure): - log.msg('Unhandled ERROR in MediaPipeline._downloaded: %s' % (_failure), log.ERROR, domain=info.domain) - - result = self.media_to_download(request, info) - if result is not None: - dwld = defer_result(result) - else: - dwld = mustbe_deferred(self.download, request, info) - dwld.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - errback=self.media_failed, - errbackArgs=(request, info), - ) - - dwld.addBoth(self._downloaded, info, fp) - dwld.addErrback(_bugtrap) - info.downloading[fp] = (request, dwld) - - def _downloaded(self, result, info, fp): - info.downloaded[fp] = result # cache result - waiting = info.waiting[fp] # client list - del info.waiting[fp] - del info.downloading[fp] - for wad in waiting: - defer_result(result).chainDeferred(wad) - - - ### Overradiable Interface - def download(self, request, info): - """ Defines how to request the download of media - - Default gives high priority to media requests and use scheduler, - shouldn't be necessary to override. - - This methods is called only if result for request isn't cached, - request fingerprint is used as cache key. - - """ - return scrapyengine.schedule(request, info.spider, priority=0) - - def media_to_download(self, request, info): - """ Ongoing request hook pre-cache - - This method is called every time a media is requested for download, and - only once for the same request because return value is cached as media - result. - - returning a non-None value implies: - - the return value is cached and piped into `item_media_downloaded` or `item_media_failed` - - prevents downloading, this means calling `download` method. - - `media_downloaded` or `media_failed` isn't called. - - """ - - def get_media_requests(self, item, info): - """ Return a list of Request objects to download for this item - - Should return None or an iterable - - Defaults return None (no media to download) - - """ - - def media_downloaded(self, response, request, info): - """ Method called on success download of media request - - Return value is cached and used as input for `item_media_downloaded` method. - Default implementation returns None. - - WARNING: returning the response object can eat your memory. - - """ - - def media_failed(self, failure, request, info): - """ Method called when media request failed due to any kind of download error. - - Return value is cached and used as input for `item_media_failed` method. - Default implementation returns same Failure object. - """ - return failure - - def item_media_downloaded(self, result, item, request, info): - """ Method to handle result of requested media for item. - - result is the return value of `media_downloaded` hook, or the non-Failure instance - returned by `media_failed` hook. - - return value of this method isn't important and is recommended to return None. - """ - - def item_media_failed(self, failure, item, request, info): - """ Method to handle failed result of requested media for item. - - result is the returned Failure instance of `media_failed` hook, or Failure instance - of an exception raised by `media_downloaded` hook. - - return value of this method isn't important and is recommended to return None. - """ - - def item_completed(self, item, info): - """ Method called when all media requests for a single item has returned a result or failure. - - The return value of this method is used as output of pipeline stage. - - `item_completed` can return item itself or raise DropItem exception. - - Default returns item - """ - return item - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/s3images.py b/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/s3images.py deleted file mode 100644 index 573daf294..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/s3images.py +++ /dev/null @@ -1,154 +0,0 @@ -import time -import hashlib -import rfc822 -from cStringIO import StringIO - -import Image -import boto - -from scrapy import log -from scrapy.stats import stats -from scrapy.core.exceptions import DropItem, NotConfigured -from scrapy.core.exceptions import HttpException -from scrapy.contrib.pipeline.media import MediaPipeline -from scrapy.conf import settings - -class NoimagesDrop(DropItem): - """Product with no images exception""" - -class ImageException(Exception): - """General image error exception""" - -class S3ImagesPipeline(MediaPipeline): - MEDIA_TYPE = 'image' - THUMBS = ( - ("50", (50, 50)), - ("110", (110, 110)), - ("270", (270, 270)) - ) - - def __init__(self): - if not settings['S3_IMAGES']: - raise NotConfigured - - # days to wait before redownloading images - self.image_refresh_days = settings.getint('IMAGES_REFRESH_DAYS', 90) - - self.bucket_name = settings['S3_BUCKET'] - self.prefix = settings['S3_PREFIX'] - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] - conn = boto.connect_s3(access_key, secret_key) - self.bucket = conn.get_bucket(self.bucket_name) - - MediaPipeline.__init__(self) - - def media_to_download(self, request, info): - key = self.s3_image_key(request.url) - if not self.s3_should_download(request.url): - self.inc_stats(info.domain, 'uptodate') - referer = request.headers.get('Referer') - log.msg('Image (uptodate) type=%s at <%s> referred from <%s>' % \ - (self.MEDIA_TYPE, request.url, referer), level=log.DEBUG, domain=info.domain) - return key - - def media_downloaded(self, response, request, info): - mtype = self.MEDIA_TYPE - referer = request.headers.get('Referer') - - if not response or not response.body.to_string(): - msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: Empty image (no-content)' % (mtype, request, referer) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - result = self.save_image(response, request, info) # save and thumbs response - - status = 'cached' if getattr(response, 'cached', False) else 'downloaded' - msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer) - log.msg(msg, level=log.DEBUG, domain=info.domain) - self.inc_stats(info.domain, status) - return result - - def media_failed(self, failure, request, info): - referer = request.headers.get('Referer') - errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure) - msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' % (self.MEDIA_TYPE, request, referer, errmsg) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - def save_image(self, response, request, info): - try: - key = self.s3_image_key(request.url) - self.s3_store_image(response, request.url) - except ImageException, ex: - log.msg(str(ex), level=log.WARNING, domain=info.domain) - raise ex - except Exception, ex: - log.msg(str(ex), level=log.WARNING, domain=info.domain) - raise ex - - return key # success value sent as input result for item_media_downloaded - - def inc_stats(self, domain, status): - stats.incpath('%s/image_count' % domain) - stats.incpath('%s/image_status_count/%s' % (domain, status)) - - def s3_image_key(self, url): - """Return the relative path on the target filesystem for an image to be - downloaded to. - """ - image_guid = hashlib.sha1(url).hexdigest() - return '%s/full/%s.jpg' % (self.prefix, image_guid) - - def s3_thumb_key(self, url, thumb_id): - """Return the relative path on the target filesystem for an image to be - downloaded to. - """ - image_guid = hashlib.sha1(url).hexdigest() - return '%s/thumbs/%s/%s.jpg' % (self.prefix, thumb_id, image_guid) - - def s3_should_download(self, url): - """Return if the image should be downloaded by checking if it's already in - the S3 storage and not too old""" - key = self.s3_image_key(url) - k = self.bucket.get_key(key) - if k is None: - return True - modified_tuple = rfc822.parsedate_tz(k.last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - age_seconds = time.time() - modified_stamp - age_days = age_seconds / 60 / 60 / 24 - return age_days > self.image_refresh_days - - def s3_store_image(self, response, url): - """Upload image to S3 storage""" - buf = StringIO(response.body.to_string()) - image = Image.open(buf) - key = self.s3_image_key(url) - self._s3_put_image(image, key) - self.s3_store_thumbnails(image, url) - - def s3_store_thumbnails(self, image, url): - """Upload image thumbnails to S3 storage""" - for thumb_id, size in self.THUMBS or []: - thumb = image.copy() if image.mode == 'RGB' else image.convert('RGB') - thumb.thumbnail(size, Image.ANTIALIAS) - key = self.s3_thumb_key(url, thumb_id) - self._s3_put_image(thumb, key) - - def s3_public_url(self, key): - return "http://%s.s3.amazonaws.com/%s" % (self.bucket_name, key) - - def _s3_put_image(self, image, key): - buf = StringIO() - try: - image.save(buf, 'JPEG') - except Exception, ex: - raise ImageException("Cannot process image. Error: %s" % ex) - - buf.seek(0) - k = self.bucket.new_key(key) - k.content_type = 'image/jpeg' - k.set_contents_from_file(buf, policy='public-read') - log.msg("Uploaded to S3: %s" % self.s3_public_url(key), level=log.DEBUG) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/shoveitem.py b/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/shoveitem.py deleted file mode 100644 index e4050bf88..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/shoveitem.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -A pipeline to persist objects using shove. - -New is a "new generation" shelve. For more information see: -http://pypi.python.org/pypi/shove -""" - -from string import Template - -from shove import Shove -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy.conf import settings - -class ShoveItemPipeline(object): - - def __init__(self): - self.uritpl = settings['SHOVEITEM_STORE_URI'] - if not self.uritpl: - raise NotConfigured - self.opts = settings['SHOVEITEM_STORE_OPT'] or {} - self.stores = {} - - dispatcher.connect(self.domain_open, signal=signals.domain_open) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) - - def process_item(self, domain, response, item): - self.stores[domain][str(item.guid)] = item - return item - - def domain_open(self, domain): - uri = Template(self.uritpl).substitute(domain=domain) - self.stores[domain] = Shove(uri, **self.opts) - - def domain_closed(self, domain): - self.stores[domain].sync() diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/show.py b/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/show.py deleted file mode 100644 index f08234be6..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/show.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Pipeline to print Items -""" -from scrapy import log -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings - -class ShowItemPipeline(object): - def __init__(self): - if not settings['DEBUG_SHOWITEM']: - raise NotConfigured - - def process_item(self, domain, response, item): - log.msg("Scraped: \n%s" % repr(item), log.DEBUG, domain=domain) - return item diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/response/soup.py b/scrapy/branches/cluster-refactor/scrapy/contrib/response/soup.py deleted file mode 100644 index 9708b21fe..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/response/soup.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -The ResponseSoup extension causes the Response objects to grow a new method -("getsoup") which returns a (cached) BeautifulSoup object of its body, and a -"soup" attribute with the same effect. The soup argument is provided for -convenience, but you cannot pass any BeautifulSoup constructor arguments (which -you can do with the getsoup() method). - -For more information about BeautifulSoup see: -http://www.crummy.com/software/BeautifulSoup/documentation.html -""" - -from BeautifulSoup import BeautifulSoup - -from scrapy.http import Response - - -class ResponseSoup(object): - def __init__(self): - setattr(Response, 'getsoup', getsoup) - setattr(Response, 'soup', property(getsoup)) - -def getsoup(response, **kwargs): - if not hasattr(response, '_soup'): - body = response.body.to_string() if response.body is not None else "" - setattr(response, '_soup', BeautifulSoup(body, **kwargs)) - return response._soup diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/depth.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/depth.py deleted file mode 100644 index 1727052c2..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/depth.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -DepthMiddleware is a scrape middleware used for tracking the depth of each -Request inside the site being scraped. It can be used to limit the maximum -depth to scrape or things like that -""" - -from scrapy import log -from scrapy.http import Request -from scrapy.stats import stats -from scrapy.conf import settings - -class DepthMiddleware(object): - - def __init__(self): - self.maxdepth = settings.getint('DEPTH_LIMIT') - self.stats = settings.getbool('DEPTH_STATS') - if self.stats and self.maxdepth: - stats.setpath('_envinfo/request_depth_limit', self.maxdepth) - - def process_result(self, response, result, spider): - def _filter(request): - if isinstance(request, Request): - request.depth = response.request.depth + 1 - if self.maxdepth and request.depth > self.maxdepth: - log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name) - return False - elif self.stats: - stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, request.depth)) - if request.depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0): - stats.setpath('%s/request_depth_max' % spider.domain_name, request.depth) - return True - - if self.stats and response.request.depth == 0: # otherwise we loose stats for depth=0 - stats.incpath('%s/request_depth_count/0' % spider.domain_name) - - return (r for r in result or () if _filter(r)) diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/limit.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/limit.py deleted file mode 100644 index fedc7d2d1..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/limit.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -RequestLimitMiddleware: Limits the scheduler request queue from the point of -view of the spider. That is, if the scheduler queue contains an equal or -greater ammount of requests than the specified limit, the new requests -(generated by the spider) will be ignored. - -The limit is setted from the spider attribute "requests_queue_size". If not -found, from the scrapy setting "REQUESTS_QUEUE_SIZE". If not found, no limit -will be applied. If given a value of 0, no limit will be applied. -""" - -from scrapy.core.engine import scrapyengine -from scrapy.conf import settings -from scrapy.http import Request -from scrapy import log - -class RequestLimitMiddleware(object): - #_last_queue_size = 0 - def process_result(self, response, result, spider): - requests = [] - other = [] - [requests.append(r) if isinstance(r, Request) else other.append(r) for r in result] - - max_pending = spider.requests_queue_size if hasattr(spider,"requests_queue_size") else settings.getint("REQUESTS_QUEUE_SIZE") - if not max_pending: - accepted = requests - else: - free_slots = max_pending - len(scrapyengine.scheduler.pending_requests[spider.domain_name]) - accepted = requests[:free_slots] - dropped = set(requests) - set(accepted) - if dropped: - for r in dropped: - log.msg("Ignoring link (max schedule queue size reached): %s " % r.url, level=log.WARNING, domain=spider.domain_name) - #actual_size = len(scrapyengine.scheduler.pending_requests[spider.domain_name]) - #log.msg("queue size: %d (%+d)" % (actual_size, actual_size - self._last_queue_size) ) - #self._last_queue_size = actual_size - return accepted + other diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/offsite.py deleted file mode 100644 index 7a0644d04..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/offsite.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by -the spider. -""" - -from scrapy import log -from scrapy.http import Request -from scrapy.utils.url import url_is_from_spider - -class OffsiteMiddleware(object): - def process_result(self, response, result, spider): - def _filter(r): - if isinstance(r, Request) and not url_is_from_spider(r.url, spider): - log.msg("Ignoring link (offsite): %s " % r.url, level=log.DEBUG, domain=spider.domain_name) - return False - return True - return (r for r in result or () if _filter(r)) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/referer.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/referer.py deleted file mode 100644 index ee05f6918..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/referer.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -RefererMiddleware: populates Request referer field, based on the Response which -originated it. -""" - -from scrapy.http import Request - -class RefererMiddleware(object): - def process_result(self, response, result, spider): - def _set_referer(r): - if isinstance(r, Request): - r.headers.setdefault('Referer', response.url) - return r - return (_set_referer(r) for r in result or ()) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/restrict.py deleted file mode 100644 index b4eceef31..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/restrict.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -RestrictMiddleware: restricts crawling to fixed set of particular URLs -""" - -from scrapy.http import Request -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings - -class RestrictMiddleware(object): - def __init__(self): - self.allowed_urls = set(settings.getlist('RESTRICT_TO_URLS')) - if not self.allowed_urls: - raise NotConfigured - - def process_result(self, response, result, spider): - def _filter(r): - if isinstance(r, Request) and r.url not in self.allowed_urls: - return False - return True - return (r for r in result or () if _filter(r)) - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urlfilter.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urlfilter.py deleted file mode 100644 index 1034f747d..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urlfilter.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -UrlFilterMiddleware: canonicalizes URLs to filter out duplicated ones -""" - -from scrapy.http import Request -from scrapy.utils.url import canonicalize_url - -class UrlFilterMiddleware(object): - def process_result(self, response, result, spider): - disabled = getattr(spider, 'urlfilter_disabled', False) - for r in result: - if isinstance(r, Request) and not disabled: - curl = canonicalize_url(r.url) - # only assign if different to avoid re-calculating fingerprint - if curl != r.url: - r.url = curl - yield r diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urllength.py deleted file mode 100644 index cfca458ef..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urllength.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -UrlLengthMiddleware: Filters out requests with URLs longer than URLLENGTH_LIMIT -""" - -from scrapy import log -from scrapy.http import Request -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings - -class UrlLengthMiddleware(object): - def __init__(self): - self.maxlength = settings.getint('URLLENGTH_LIMIT') - if not self.maxlength: - raise NotConfigured - - def process_result(self, response, result, spider): - def _filter(request): - if isinstance(request, Request) and len(request.url) > self.maxlength: - log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name) - return False - else: - return True - - return (r for r in result or () if _filter(r)) diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spiders.py b/scrapy/branches/cluster-refactor/scrapy/contrib/spiders.py deleted file mode 100644 index e03a86cde..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/spiders.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -This module contains BasicSpider, a spider class which provides support for -basic crawling. -""" - -from scrapy.http import Request -from scrapy.spider import BaseSpider -from scrapy.core.exceptions import UsageError - -class BasicSpider(BaseSpider): - """BasicSpider extends BaseSpider by providing support for simple crawling - by following links contained in web pages. - - With BasicSpider you can write a basic spider very easily and quickly. For - more information refer to the Scrapy tutorial""" - - def __init__(self): - super(BaseSpider, self).__init__() - - self._links_callback = [] - - for attr in dir(self): - if attr.startswith('links_'): - suffix = attr.split('_', 1)[1] - value = getattr(self, attr) - try: - callback = getattr(self, 'parse_%s' % suffix) - except AttributeError: - raise UsageError("%s defines links_%s but doesn't provide a parse_%s method" % \ - (type(self).__name__, suffix, suffix)) - self._links_callback.append((value, callback)) - - def parse(self, response): - """This function is called by the core for all the start_urls. Do not - override this function, override parse_start_url instead.""" - return self._parse_wrapper(response, self.parse_start_url) - - def parse_start_url(self, response): - """Callback function for processing start_urls. It must return a list - of ScrapedItems and/or Requests.""" - return [] - - def _links_to_follow(self, response): - res = [] - links_to_follow = {} - for lx, callback in self._links_callback: - for url, link_text in lx.extract_urls(response).iteritems(): - links_to_follow[url] = (callback, link_text) - - for url, cb_link in links_to_follow.iteritems(): - request = Request(url=url, link_text=link_text) - request.append_callback(self._parse_wrapper, callback) - res.append(request) - return res - - def _parse_wrapper(self, response, callback): - res = self._links_to_follow(response) - res += callback(response) or () - return res - diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/livestats.py b/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/livestats.py deleted file mode 100644 index cf38e24a0..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/livestats.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Live statistics extension -""" -from datetime import datetime -from pydispatch import dispatcher -from scrapy.core import signals -from scrapy.core.engine import scrapyengine -from scrapy.management.web import banner - -class SpiderStats(object): - def __init__(self): - self.scraped = 0 - self.crawled = 0 - self.started = None - self.finished = None - -class LiveStats(object): - webconsole_id = 'livestats' - webconsole_name = 'Spider live statistics of current run' - - def __init__(self): - self.domains = {} - dispatcher.connect(self.domain_open, signal=signals.domain_open) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) - - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def domain_open(self, domain, spider): - pstats = SpiderStats() - self.domains[spider.domain_name] = pstats - pstats.started = datetime.now() - pstats.finished = None - - def domain_closed(self, domain, spider): - self.domains[spider.domain_name].finished = datetime.now() - - def item_scraped(self, item, spider): - self.domains[spider.domain_name].scraped += 1 - - def response_downloaded(self, response, spider): - #sometimes we download responses without opening/closing domains, - #for example from scrapy shell - if self.domains.get(spider.domain_name): - self.domains[spider.domain_name].crawled += 1 - - def webconsole_render(self, wc_request): - sch = scrapyengine.scheduler - dwl = scrapyengine.downloader - - totdomains = totscraped = totcrawled = totscheduled = totactive = totpending = 0 - s = banner(self) - s += "\n" - s += "\n" - for d in sorted(self.domains.keys()): - scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0 - active, pending = len(dwl.active_requests(d)), len(dwl.request_queue(d)) - stats = self.domains[d] - runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started - - s += '\n' % \ - (d, stats.scraped, stats.crawled, scheduled, pending, active, str(stats.started), str(stats.finished), str(runtime)) - - totdomains += 1 - totscraped += stats.scraped - totcrawled += stats.crawled - totscheduled += scheduled - totactive += active - totpending += pending - s += '\n' % \ - (totdomains, totscraped, totcrawled, totscheduled, totactive, totpending) - s += "
DomainItems
Scraped		Pages
Crawled		Scheduler
Pending		Downloader
Pending		Downloader
Active		Start timeFinish timeRun time
%s%d%d%d%d%d%s%s%s
%d domains%d%d%d%d%d
\n" - - s += "\n" - s += "\n" - - return s - - def webconsole_discover_module(self): - return self diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/schedstats.py b/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/schedstats.py deleted file mode 100644 index 36a207bf3..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/schedstats.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -Scheduler information module for Scrapy webconsole -""" -from pydispatch import dispatcher -from scrapy.core import signals -from scrapy.core.engine import scrapyengine -from scrapy.spider import spiders -from scrapy.management.web import banner - -class SchedulerStats(object): - webconsole_id = 'scheduler' - webconsole_name = 'Scheduler queue' - - def __init__(self): - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - s = banner(self) - s += "
    \n" - for domain, requests in scrapyengine.scheduler.pending_requests.iteritems(): - s += "
  • \n" - s += "%s (%s pages)\n" % (domain, len(requests)) - s += "
      \n" - # requests is a tuple of request and deffered now, as I understand - for r, d in requests: - if hasattr(r, 'url'): - s += "
    • %s
      • \n" % (r.url, r.url) - #else: - # s += "
    • %s
      • \n" % (repr(r)) - s += "
    \n" - s += "
    • \n" - s += "
\n" - - s += "\n" - s += "\n" - - return s - - def webconsole_discover_module(self): - return self diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderctl.py b/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderctl.py deleted file mode 100644 index ac93defac..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderctl.py +++ /dev/null @@ -1,208 +0,0 @@ -""" -Extensions for allowing spider control from web console -""" - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.spider import spiders -from scrapy.management.web import banner -from scrapy.conf import settings - -class Spiderctl(object): - webconsole_id = 'spiderctl' - webconsole_name = 'Spider control panel' - - def __init__(self): - self.running = set() - self.finished = set() - dispatcher.connect(self.domain_open, signal=signals.domain_open) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) - - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def domain_open(self, domain, spider): - self.running.add(domain) - - def domain_closed(self, domain, spider): - if domain in self.running: - self.running.remove(domain) - self.finished.add(domain) - - def webconsole_render(self, wc_request): - if wc_request.args: - changes = self.webconsole_control(wc_request) - - enabled_domains = spiders.asdict(include_disabled=False).keys() - self.scheduled = set(scrapyengine.scheduler.pending_domains_count) - self.not_scheduled = [d for d in enabled_domains if d not in self.scheduled - and d not in self.running - and d not in self.finished] - - s = banner(self) - s += '

Reload spiders

\n' - s += '\n' - s += "\n" % \ - (len(self.running), - settings['CONCURRENT_DOMAINS'], - len(self.scheduled), - len(self.finished), - len(self.not_scheduled)) - s += "\n" - - # running - s += "\n" - - # scheduled - s += "\n" - - # finished - s += "\n" - - # not scheduled - s += "\n" - - s += "\n" - s += "\n" - - s += "\n" - - s += "\n' - - s += "\n" - - s += "\n" - - s += "\n" - - s += "
Running (%d/%d)Scheduled (%d)Finished (%d)Not scheduled (%d)
\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - - s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "
 \n" - s += '
\n' - s += "\n" - s += '
\n' - s += '\n' - s += '
\n' - s += '		 \n" - s += '
\n' - s += "\n" - s += '
\n' - s += '\n' - s += '
\n' - s += "
\n" - - if wc_request.args: - s += changes - - s += "\n" - s += "\n" - - return s - - def webconsole_control(self, wc_request): - args = wc_request.args - s = "
\n" - if "reloadspiders" in args: - scrapymanager.reload_spiders() - s += "

Spiders reloaded

\n" - return s - - if "stop_running_domains" in args: - s += "

" - s += "Stopped spiders:

  • %s
" % "
  • ".join(args["stop_running_domains"]) - for domain in args["stop_running_domains"]: - scrapyengine.close_domain(domain) - s += "

    " - if "remove_pending_domains" in args: - removed = [] - for domain in args["remove_pending_domains"]: - if scrapyengine.scheduler.remove_pending_domain(domain): - removed.append(domain) - if removed: - s += "

    " - s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_domains"]) - s += "

    " - if "bulk_remove_domains" in args: - scheduled = [] - to_remove = set([d.strip() for d in args["bulk_remove_domains"][0].split("\n")]) - removed = set([d for d in scrapyengine.scheduler.pending_domains if d in to_remove]) - scrapyengine.scheduler.pending_domains = [d for d in execengine.scheduler.pending_domains if d not in removed] - if removed: - s += "

    " - s += "Removed:

    • %s
    " % "
  • ".join(removed) - s += "

    " - s += "

    " - s += "Not removed:

    • %s
    " % "
  • ".join(to_remove - removed) - s += "

    " - if "add_pending_domains" in args: - for domain in args["add_pending_domains"]: - if domain not in scrapyengine.scheduler.pending_domains_count: - scrapymanager.crawl(domain) - s += "

    " - s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_domains"]) - s += "

    " - if "bulk_schedule_domains" in args: - to_schedule = set([d.strip() for d in args["bulk_schedule_domains"][0].split("\n")]) - scheduled = set() - for domain in to_schedule: - if domain in enabled_domains and domain not in scrapyengine.scheduler.pending_domains_count: - scrapymanager.crawl(domain) - scheduled.add(domain) - if scheduled: - s += "

    " - s += "Scheduled:

    • %s
    " % "
  • ".join(scheduled) - s += "

    " - s += "

    " - s += "Not scheduled:

    • %s
    " % "
  • ".join(to_schedule - scheduled) - s += "

    " - if "rerun_finished_domains" in args: - for domain in args["rerun_finished_domains"]: - if domain not in scrapyengine.scheduler.pending_domains_count: - scrapymanager.crawl(domain) - self.finished.remove(domain) - s += "

    " - s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) - s += "

    " - - return s - - def webconsole_discover_module(self): - return self diff --git a/scrapy/branches/cluster-refactor/scrapy/core/downloader/handlers.py b/scrapy/branches/cluster-refactor/scrapy/core/downloader/handlers.py deleted file mode 100644 index e63e501c4..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/downloader/handlers.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Download handlers for different schemes -""" -from __future__ import with_statement - -import os -import urlparse - -from twisted.web.client import HTTPClientFactory -from twisted.internet import defer, reactor -from twisted.web import error as web_error - -from scrapy.core import signals -from scrapy.http import Request, Response, Headers -from scrapy.core.exceptions import UsageError, HttpException -from scrapy.utils.defer import defer_succeed -from scrapy.conf import settings - -def download_any(request, spider): - u = urlparse.urlparse(request.url) - if u.scheme == 'file': - return download_file(request, spider) - elif u.scheme in ('http', 'https'): - return download_http(request, spider) - else: - raise UsageError("Unsupported scheme '%s' in URL: <%s>" % (u.scheme, request.url)) - -def download_http(request, spider): - """This functions handles http/https downloads""" - url = urlparse.urldefrag(request.url)[0] - - agent = request.headers.get('user-agent', settings.get('USER_AGENT')) - request.headers.pop('user-agent', None) # remove user-agent if already present - factory = HTTPClientFactory(url=str(url), # never pass unicode urls to twisted - method=request.method, - postdata=request.body, - headers=request.headers, - agent=agent, - cookies=request.cookies, - timeout=getattr(spider, "download_timeout", None) or settings.getint('DOWNLOAD_TIMEOUT'), - followRedirect=False) - - def _response(body): - body = body or '' - status = factory.status - headers = Headers(factory.response_headers) - r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body) - signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider) - signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider) - return r - - def _on_success(body): - return _response(body) - - def _on_error(_failure): - ex = _failure.value - if isinstance(ex, web_error.Error): # HttpException - raise HttpException(ex.status, ex.message, _response(ex.response)) - return _failure - - factory.noisy = False - factory.deferred.addCallbacks(_on_success, _on_error) - - u = urlparse.urlparse(request.url) - if u.scheme == 'https' : - from twisted.internet import ssl - contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(u.hostname, u.port or 443, factory, contextFactory) - else: - reactor.connectTCP(u.hostname, u.port or 80, factory) - return factory.deferred - -def download_file(request, spider) : - """Return a deferred for a file download.""" - filepath = request.url.split("file://")[1] - with open(filepath) as f: - response = Response(domain=spider.domain_name, url=request.url, body=f.read()) - return defer_succeed(response) diff --git a/scrapy/branches/cluster-refactor/scrapy/core/downloader/manager.py b/scrapy/branches/cluster-refactor/scrapy/core/downloader/manager.py deleted file mode 100644 index ce8130687..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/downloader/manager.py +++ /dev/null @@ -1,196 +0,0 @@ -""" -Download web pages using asynchronous IO -""" - -import datetime - -from twisted.internet import reactor, defer - -from scrapy.core.exceptions import IgnoreRequest -from scrapy.spider import spiders -from scrapy.core.downloader.handlers import download_any -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy import log -from scrapy.conf import settings -from scrapy.utils.defer import chain_deferred, mustbe_deferred - - -class SiteDetails(object): - """This is a simple data record that encapsulates the details we hold on - each domain which we are scraping. - """ - def __init__(self, download_delay=None, max_concurrent_requests=2): - self.download_delay = download_delay - self.max_concurrent_requests = max_concurrent_requests if not download_delay else 1 - - self.queue = [] - self.active = set() - self.downloading = set() - self.closed = False - self.lastseen = None - - def is_idle(self): - return not (self.active or self.downloading) - - def capacity(self): - return self.max_concurrent_requests - len(self.downloading) - - -class Downloader(object): - """Maintain many concurrent downloads and provide an HTTP abstraction - We will have a limited number of connections per domain and scrape many domains in - parallel. - - request(..) should be called to request resources using http, https or file - protocols. - """ - - def __init__(self) : - self.sites = {} - self.middleware = DownloaderMiddlewareManager() - self.middleware.download_function = self.enqueue - self.download_function = download_any - - def fetch(self, request, spider): - """ Main method to use to request a download - - This method includes middleware mangling. Middleware can returns a - Response object, then request never reach downloader queue, and it will - not be downloaded from site. - """ - domain = spider.domain_name - site = self.sites[domain] - if not site or site.closed: - raise IgnoreRequest('Unable to fetch (domain already closed): %s' % request) - - site.active.add(request) - def _deactivate(_): - site.active.remove(request) - return _ - dwld = self.middleware.download(request, spider) - dwld.addBoth(_deactivate) - return dwld - - def enqueue(self, request, spider): - """ Enqueue a Request for a effective download from site - """ - domain = spider.domain_name - site = self.sites.get(domain) - if not site or site.closed: - raise IgnoreRequest('Trying to enqueue %s from closed site %s' % (request, domain)) - - deferred = defer.Deferred() - site.queue.append((request, deferred)) - self.process_queue(spider) - return deferred - - def process_queue(self, spider): - """ Effective download requests from site queue - """ - domain = spider.domain_name - site = self.sites.get(domain) - if not site: - return - - # download delay handling - now = datetime.datetime.now() - if site.download_delay and site.lastseen: - delta = now - site.lastseen - penalty = site.download_delay - delta.seconds - if penalty > 0: - reactor.callLater(penalty, self.process_queue, spider=spider) - return - site.lastseen = now - - while site.queue and site.capacity()>0: - request, deferred = site.queue.pop(0) - self._download(request, spider, deferred) - - if site.closed and site.is_idle(): - # XXX: Remove scrapyengine reference - del self.sites[domain] - from scrapy.core.engine import scrapyengine - scrapyengine.closed_domain(domain=domain) - - def _download(self, request, spider, deferred): - log.msg('Activating %s' % request.traceinfo(), log.TRACE) - domain = spider.domain_name - site = self.sites.get(domain) - site.downloading.add(request) - - def _remove(result): - log.msg('Deactivating %s' % request.traceinfo(), log.TRACE) - site.downloading.remove(request) - return result - - def _finish(result): - self.process_queue(spider) - - dwld = mustbe_deferred(self.download_function, request, spider) - dwld.addBoth(_remove) - chain_deferred(dwld, deferred) - dwld.addBoth(_finish) - - def open_domain(self, domain): - """Allocate resources to begin processing a domain""" - spider = spiders.fromdomain(domain) - if domain in self.sites: # reopen - self.sites[domain].closed = False - return - - # Instanciate site specific handling based on info provided by spider - delay = getattr(spider, 'download_delay', None) or settings.getint("DOWNLOAD_DELAY", 0) - maxcr = getattr(spider, 'max_concurrent_requests', settings.getint('REQUESTS_PER_DOMAIN')) - site = SiteDetails(download_delay=delay, max_concurrent_requests=maxcr) - self.sites[domain] = site - - def close_domain(self, domain): - """Free any resources associated with the given domain""" - log.msg("Downloader closing domain %s" % domain, log.TRACE, domain=domain) - site = self.sites.get(domain) - if site: - site.closed = True - spider = spiders.fromdomain(domain) - self.process_queue(spider) - else: - log.msg('Domain %s already closed' % domain, log.TRACE, domain=domain) - - # Most of the following functions must be reviewed to decide if are really needed - def domain_is_open(self, domain): - return domain in self.sites - - def lastseen(self, domain): - if domain in self.sites: - return self.sites[domain].lastseen - - def outstanding(self, domain): - """The number of outstanding requests for a domain - This includes both active requests and pending requests. - """ - site = self.sites.get(domain) - if site: - return len(site.active) + len(site.queue) - - def domain_is_idle(self, domain): - return not self.outstanding(domain) - - def request_queue(self, domain): - site = self.sites.get(domain) - return site.queue if site else [] - - def active_requests(self, domain): - site = self.sites.get(domain) - return site.active if site else [] - - def has_capacity(self): - """Does the downloader have capacity to handle more domains""" - return len(self.sites) < settings.getint('CONCURRENT_DOMAINS') - - def is_idle(self): - return not self.sites - - # deprecated - def clear_requests(self, domain): - log.msg("Downloader clearing request for domain %s" % domain, log.TRACE, domain=domain) - self.sites[domain].queue = [] - diff --git a/scrapy/branches/cluster-refactor/scrapy/core/downloader/middleware.py b/scrapy/branches/cluster-refactor/scrapy/core/downloader/middleware.py deleted file mode 100644 index 32081e66f..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/downloader/middleware.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -request-response middleware extension -""" -from scrapy.core import signals -from scrapy import log -from scrapy.http import Request, Response -from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class -from scrapy.utils.defer import mustbe_deferred -from scrapy.core.downloader.handlers import download_any -from scrapy.conf import settings - -class DownloaderMiddlewareManager(object): - """Request-Response Middleware Manager - - Middleware is a framework of hooks into Scrapy's request/response - processing. It's a light, low-level "spider" system for globally altering - Scrapy's input and/or output. - - Middleware is heavily based on Django middleware system, at the point that - it tries to mimic Django middleware behaviour. For Scrapy, the Django's - view function has the same meaning of the final download handler function - to use for the request's url. - - To activate a middleware component, add it to the DOWNLOADER_MIDDLEWARES list - in your Scrapy settings. In DOWNLOADER_MIDDLEWARES, each middleware component - is represented by a string: the full Python path to the middleware's class - name. For example: - - DOWNLOADER_MIDDLEWARES = ( - 'scrapy.contrib.middleware.common.SpiderMiddleware', - 'scrapy.contrib.middleware.common.CommonMiddleware', - 'scrapy.contrib.middleware.redirect.RedirectMiddleware', - 'scrapy.contrib.middleware.cache.CacheMiddleware', - ) - - Writing your own middleware is easy. Each middleware component is a single - Python class that defines one or more of the following methods: - - - process_request(self, request, spider) - - `request` is a Request object. - `spider` is a BaseSpider object - - This method is called in each request until scrapy decides which - download function to use. - - process_request() should return either None, Response or Request. - - If returns None, Scrapy will continue processing this request, - executing any other middleware and, then, the appropiate download - function. - - If returns a Response object, Scrapy won't bother calling ANY other - request or exception middleware, or the appropiate download function; - it'll return that Response. Response middleware is always called on - every response. - - If returns a Request object, returned request is used to instruct a - redirection. Redirection is handled inside middleware scope, and - original request don't finish until redirected request is completed. - - - process_response(self, request, response, spider): - - `request` is a Request object - `response` is a Response object - `spider` is a BaseSpider object - - process_response MUST return a Response object. It could alter the given - response, or it could create a brand-new Response. - - - process_exception(self, request, exception, spider) - - `request` is a Request object. - `exception` is an Exception object - `spider` is a BaseSpider object - - Scrapy calls process_exception() when a download handler or - process_request middleware raises an exception. - - process_exception() should return either None, Response or Request object. - - if it returns None, Scrapy will continue processing this exception, - executing any other exception middleware, until no middleware left and - default exception handling kicks in. - - If it returns a Response object, the response middleware kicks in, and - won't bother calling ANY other exception middleware. - - If it returns a Request object, returned request is used to instruct a - immediate redirection. Redirection is handled inside middleware scope, - and original request don't finish until redirected request is - completed. This stop process_exception middleware as returning - Response does. - - """ - def __init__(self): - self.loaded = False - self.request_middleware = [] - self.response_middleware = [] - self.exception_middleware = [] - self.load() - self.download_function = download_any - - def _add_middleware(self, mw): - if hasattr(mw, 'process_request'): - self.request_middleware.append(mw.process_request) - if hasattr(mw, 'process_response'): - self.response_middleware.insert(0, mw.process_response) - if hasattr(mw, 'process_exception'): - self.exception_middleware.insert(0, mw.process_exception) - - def load(self): - """Load middleware defined in settings module - """ - mws = [] - for mwpath in settings.getlist('DOWNLOADER_MIDDLEWARES') or (): - cls = load_class(mwpath) - if cls: - try: - mw = cls() - self._add_middleware(mw) - mws.append(mw) - except NotConfigured: - pass - log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) - self.loaded = True - - def download(self, request, spider): - def process_request(request): - for method in self.request_middleware: - response = method(request=request, spider=spider) - assert response is None or isinstance(response, (Response, Request)), \ - 'Middleware %s.process_request must return None, Response or Request, got %s' % \ - (method.im_self.__class__.__name__, response.__class__.__name__) - if response: - return response - return self.download_function(request=request, spider=spider) - - def process_response(response): - assert response is not None, 'Received None in process_response' - if isinstance(response, Request): - return response - - for method in self.response_middleware: - response = method(request=request, response=response, spider=spider) - assert isinstance(response, (Response, Request)), \ - 'Middleware %s.process_response must return Response or Request, got %s' % \ - (method.im_self.__class__.__name__, type(response)) - if isinstance(response, Request): - signals.send_catch_log(signal=signals.response_received, sender=self.__class__, response=response, spider=spider) - return response - signals.send_catch_log(signal=signals.response_received, sender=self.__class__, response=response, spider=spider) - return response - - def process_exception(_failure): - exception = _failure.value - for method in self.exception_middleware: - response = method(request=request, exception=exception, spider=spider) - assert response is None or isinstance(response, (Response, Request)), \ - 'Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (method.im_self.__class__.__name__, type(response)) - if response: - return response - return _failure - - deferred = mustbe_deferred(process_request, request) - deferred.addErrback(process_exception) - deferred.addCallback(process_response) - return deferred diff --git a/scrapy/branches/cluster-refactor/scrapy/core/engine.py b/scrapy/branches/cluster-refactor/scrapy/core/engine.py deleted file mode 100644 index 325a97dbb..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/engine.py +++ /dev/null @@ -1,549 +0,0 @@ -""" -Core execution engine for the web crawling framework. This controls the -scheduler, downloader and spiders. -""" -from datetime import datetime - -from twisted.internet import defer, reactor, task -from twisted.python.failure import Failure -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.core.scheduler import Scheduler -from scrapy.core.downloader import Downloader -from scrapy.http import Response, Request -from scrapy.core.exceptions import IgnoreRequest, HttpException, DontCloseDomain -from scrapy.item import ScrapedItem -from scrapy.item.pipeline import ItemPipeline -from scrapy.spider import spiders -from scrapy.spider.middleware import SpiderMiddlewareManager -from scrapy.utils.defer import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate -from scrapy.conf import settings - - -class ExecutionEngine(object): - """ - The Execution Engine controls execution of the scraping process. - - The process begins with the _mainloop() method, which is called - periodically to add more domains to scrape. It adds the first page for a - domain to the scheduler by calling _schedule_page and calls - process_scheduled_requests, which starts the scraping process for a domain. - - The process_scheduled_requests method asks the scheduler for the next - available page for a given domain and requests it from the downloader. The - downloader will execute a callback function that was added in the - _schedule_page method. This callback with process the output from the - spider and then call process_scheduled_requests to continue the scraping - process for that domain. - """ - - # back off downloading more pages if we exceed this backlog size - DOWNLOADER_BACKLOG = 10 - - def __init__(self): - self.configured = False - self.keep_alive = False - self.domain_close_delay = settings.getint('DOMAIN_CLOSE_DELAY') # seconds after an idle domain will be closed - self.initializing = set() # domais in intialization state - self.cancelled = set() # domains in cancelation state - self.debug_mode = settings.getbool('ENGINE_DEBUG') - self.tasks = [] - self.ports = [] - self.running = False - self.paused = False - - def configure(self, scheduler=None, downloader=None): - """ - Configure execution engine with the given scheduling policy and downloader. - """ - self.scheduler = scheduler or Scheduler() - self.downloader = downloader or Downloader() - self.spidermiddleware = SpiderMiddlewareManager() - self._scraping = {} - self.pipeline = ItemPipeline() - # key dictionary of per domain lists of initial requests to scrape - self.starters = {} - - self.configured = True - - def addtask(self, function, interval, args=None, kwargs=None, now=False): - """ - Adds a looping task. Use this instead of twisted task.LooopingCall to - make sure the reactor is left in a clean state after the engine is - stopped. - """ - if not args: - args = [] - if not kwargs: - kwargs = {} - tsk = task.LoopingCall(function, *args, **kwargs) - self.tasks.append((tsk, interval, now)) - if self.running: - tsk.start(interval, now) - return tsk - - def removetask(self, tsk): - """Remove a looping task previously added with addtask() method""" - self.tasks = [(t, i, n) for (t, i, n) in self.tasks if t is not tsk] - if tsk.running: - tsk.stop() - - def listenTCP(self, *args, **kwargs): - if self.running: - self.ports.append(reactor.listenTCP(*args, **kwargs)) - else: - self.ports.append((args, kwargs)) - - def clean_reactor(self): - """Leaves the reactor in a clean state by removing all pending tasks - and listening ports. It can only be called when the engine is not - running. - """ - if not self.running: - for tsk, _, _ in self.tasks: - if tsk.running: - tsk.stop() - self.tasks = [] - for p in [p for p in self.ports if not isinstance(p, tuple)]: - p.stopListening() - self.ports = [] - - def start(self): - """Start the execution engine""" - if not self.running: - reactor.callWhenRunning(self._mainloop) - self.start_time = datetime.now() - signals.send_catch_log(signal=signals.engine_started, sender=self.__class__) - self.addtask(self._mainloop, 5.0) - for tsk, interval, now in self.tasks: - tsk.start(interval, now) - for args, kwargs in [t for t in self.ports if isinstance(t, tuple)]: - reactor.listenTCP(*args, **kwargs) - self.running = True - reactor.run() # blocking call - - def stop(self): - """Stop the execution engine""" - if self.running: - self.running = False - for domain in self.open_domains: - spider = spiders.fromdomain(domain) - signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, status='cancelled') - for tsk, _, _ in self.tasks: # stop looping calls - if tsk.running: - tsk.stop() - self.tasks = [] - for p in [p for p in self.ports if not isinstance(p, tuple)]: - p.stopListening() - if reactor.running: - reactor.stop() - signals.send_catch_log(signal=signals.engine_stopped, sender=self.__class__) - - def pause(self): - """Pause the execution engine""" - self.paused = True - - def resume(self): - """Resume the execution engine""" - self.paused = False - - def is_idle(self): - return self.scheduler.is_idle() and self.pipeline.is_idle() and self.downloader.is_idle() and not self._scraping - - def next_domain(self): - domain = self.scheduler.next_domain() - if domain: - spider = spiders.fromdomain(domain) - self.open_domain(domain, spider) - return domain - - def next_request(self, spider, breakloop=True): - """Scrape the next request for the domain passed. - - The next request to be scraped is retrieved from the scheduler and - requested from the downloader. - - The domain is closed if there are no more pages to scrape. - """ - if self.paused: - return reactor.callLater(5, self.next_request, spider) - - if breakloop: - # delaying make reentrant call to next_request safe - return reactor.callLater(0, self.next_request, spider, breakloop=False) - - domain = spider.domain_name - - # check that the engine is still running and domain is open - if not self.running: - return - - # outstanding requests in downloader queue - outstanding = self.downloader.outstanding(domain) - if outstanding > self.DOWNLOADER_BACKLOG: - return - - # Next pending request from scheduler - request, deferred = self.scheduler.next_request(domain) - if request: - try: - dwld = self.download(request, spider) - except IgnoreRequest, ex: - log.msg(ex.message, log.WARNING, domain=domain) - except Exception, ex: - log.exc("Bug in download code: %s" % request, domain=domain) - self._domain_idle(domain) - else: - chain_deferred(dwld, deferred) - else: - if self.domain_is_idle(domain): - self._domain_idle(domain) - - def domain_is_idle(self, domain): - scraping = self._scraping.get(domain) - pending = self.scheduler.domain_has_pending(domain) - downloading = not self.downloader.domain_is_idle(domain) - haspipe = not self.pipeline.domain_is_idle(domain) - oninit = domain in self.initializing - if pending or downloading or haspipe or oninit or scraping: - return False - elif self.domain_close_delay: - lastseen = self.downloader.lastseen(domain) - if not lastseen: # domain not yet started - return False - now = datetime.now() - delta = now - lastseen - return (delta.seconds >= self.domain_close_delay) - else: - return True - - def domain_is_open(self, domain): - return domain in self.downloader.sites - - @property - def open_domains(self): - return self.downloader.sites.keys() - - def crawl(self, request, spider, priority=1, domain_priority=1): - domain = spider.domain_name - - def _process(response): - assert isinstance(response, (Response, Exception)) - - def _onpipelinefinish(pipe_result, item): - # _ can only be an item or a DropItem failure, since other - # failures are caught in ItemPipeine (item/pipeline.py) - if isinstance(pipe_result, Failure): - signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, item=item, spider=spider, response=response, exception=pipe_result.value) - else: - signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, item=item, spider=spider, response=response, pipe_output=pipe_result) - self.next_request(spider) - - def _onsuccess_per_item(item): - if isinstance(item, ScrapedItem): - log.msg("Scraped %s in <%s>" % (item, request.url), log.DEBUG, domain=domain) - signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, item=item, spider=spider, response=response) - piped = self.pipeline.pipe(item, spider, response) # TODO: remove response - piped.addBoth(_onpipelinefinish, item) - elif isinstance(item, Request): - signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response) - self.crawl(request=item, spider=spider, priority=priority) - else: - log.msg('Garbage found in spider output while processing %s, got type %s' % (request, type(item)), log.TRACE, domain=domain) - - class _ResultContainer(object): - def append(self, item): - _onsuccess_per_item(item) - - def _onsuccess(result): - return deferred_degenerate(result, _ResultContainer()) - - def _onerror(_failure): - if not isinstance(_failure.value, IgnoreRequest): - referer = None if not isinstance(response, Response) else response.request.headers.get('Referer', None) - log.msg("Error while spider was processing <%s> from <%s>: %s" % (request.url, referer, _failure), log.ERROR, domain=domain) - - def _bugtrap(_failure): - log.msg('FRAMEWORK BUG processing %s: %s' % (request, _failure), log.ERROR, domain=domain) - - - scd = self.scrape(request, response, spider) - scd.addCallbacks(_onsuccess, _onerror) - scd.addErrback(_bugtrap) - - self._scraping[domain].add(response) - scd.addBoth(lambda _: self._scraping[domain].remove(response)) - return scd - - def _cleanfailure(_failure): - ex = _failure.value - if not isinstance(ex, IgnoreRequest): - log.msg("Unknown error propagated in %s: %s" % (request, _failure), log.ERROR, domain=domain) - request.deferred.addErrback(lambda _:None) - request.deferred.errback(_failure) #TODO: merge into spider middleware. - - schd = self.schedule(request, spider, priority, domain_priority) - schd.addCallbacks(_process, _cleanfailure) - return schd - - def scrape(self, request, response, spider): - return self.spidermiddleware.scrape(request, response, spider) - - def schedule(self, request, spider, priority=1, domain_priority=1): - domain = spider.domain_name - if not self.scheduler.domain_is_open(domain): - if self.debug_mode: - log.msg('Scheduling %s (delayed)' % request.traceinfo(), log.TRACE) - return self._add_starter(request, spider, domain_priority) - if self.debug_mode: - log.msg('Scheduling %s (now)' % request.traceinfo(), log.TRACE) - schd = self.scheduler.enqueue_request(domain, request, priority) - self.next_request(spider) - return schd - - def _mainloop(self): - """Add more domains to be scraped if the downloader has the capacity. - - If there is nothing else scheduled then stop the execution engine. - """ - if not self.running or self.paused: - return - - # main domain starter loop - while self.running and self.downloader.has_capacity(): - if not self.next_domain(): - return self._stop_if_idle() - - # purge idle domains - (domain_close_delay support) - if self.domain_close_delay: - for domain in self.downloader.sites: - if self.domain_is_idle(domain): - self._domain_idle(domain) - - def _add_starter(self, request, spider, domain_priority): - domain = spider.domain_name - if not self.scheduler.is_pending(domain): - self.scheduler.add_domain(domain, priority=domain_priority) - self.starters[domain] = [] - deferred = defer.Deferred() - self.starters[domain] += [(request, deferred)] - return deferred - - def _run_starters(self, spider): - domain = spider.domain_name - starters = self.starters.get(domain, []) - while starters: - request, deferred = starters.pop(0) - schd = self.schedule(request, spider) - chain_deferred(schd, deferred) - del self.starters[domain] - - def download(self, request, spider): - log.msg('Downloading %s' % request.traceinfo(), log.TRACE) - domain = spider.domain_name - - def _on_success(response): - """handle the result of a page download""" - assert isinstance(response, (Response, Request)) - log.msg("Requested %s" % request.traceinfo(), level=log.TRACE, domain=domain) - if isinstance(response, Response): - response.request = request # tie request to obtained response - cached = 'cached' if response.cached else 'live' - log.msg("Crawled %s <%s> from <%s>" % (cached, response.url, request.headers.get('referer')), level=log.DEBUG, domain=domain) - return response - elif isinstance(response, Request): - redirected = response # proper alias - schd = self.schedule(redirected, spider, priority=4) - chain_deferred(schd, redirected.deferred) - return schd - - def _on_error(_failure): - """handle an error processing a page""" - ex = _failure.value - if isinstance(ex, IgnoreRequest): - log.msg(_failure.getErrorMessage(), level=log.DEBUG, domain=domain) - return _failure - referer = request.headers.get('Referer', None) - errmsg = str(ex) if isinstance(ex, HttpException) else str(_failure) - log.msg("Downloading <%s> from <%s>: %s" % (request.url, referer, errmsg), log.ERROR, domain=domain) - return Failure(IgnoreRequest(str(ex))) - - def _on_complete(_): - self.next_request(spider) - - dwld = self.downloader.fetch(request, spider) - dwld.addCallbacks(_on_success, _on_error) - deferred = defer.Deferred() - chain_deferred(dwld, deferred) - dwld.addBoth(_on_complete) - return deferred - - def initialize(self, spider): - domain = spider.domain_name - if not hasattr(spider, 'init_domain'): - return defer_succeed(True) - - def _initialize(req): - if isinstance(req, Request): - _response = None - def _referer(response): - req.deferred.addCallback(_setreferer, response) - return response - - def _setreferer(result, response): - if isinstance(result, Request): - result.headers.setdefault('Referer', response.url) - return result - - def _onerror(_failure): - ex = _failure.value - if isinstance(ex, IgnoreRequest): - log.msg(ex.message, log.DEBUG, domain=domain) - else: - return _failure - - schd = self.schedule(req, spider) - schd.addCallback(_referer) - chain_deferred(schd, req.deferred) - schd.addErrback(_onerror) - schd.addBoth(_initialize) - return schd - return req - - def _bugtrap(_failure): - log.msg("Bug in %s init_domain code: %s" % (domain, _failure), log.ERROR, domain=domain) - - def _state(state): - self.initializing.remove(domain) - if state is True: - log.msg('Succeded initialization for %s' % domain, log.INFO, domain=domain) - else: - log.msg('Failed initialization for %s' % domain, log.INFO, domain=domain) - return state - - log.msg('Started initialization for %s' % domain, log.INFO, domain=domain) - self.initializing.add(domain) - req = spider.init_domain() - deferred = mustbe_deferred(_initialize, req) - deferred.addErrback(_bugtrap) - deferred.addCallback(_state) - return deferred - - def open_domain(self, domain, spider=None): - log.msg("Domain opened", domain=domain) - spider = spider or spiders.fromdomain(domain) - - self.scheduler.open_domain(domain) - self.downloader.open_domain(domain) - self.pipeline.open_domain(domain) - self._scraping[domain] = set() - signals.send_catch_log(signals.domain_open, sender=self.__class__, domain=domain, spider=spider) - - # init_domain - dfd = self.initialize(spider) - def _state(state): - if state is True: - signals.send_catch_log(signals.domain_opened, sender=self.__class__, domain=domain, spider=spider) - self._run_starters(spider) - else: - self._domain_idle(domain) - dfd.addCallback(_state) - - def close_domain(self, domain): - """Close (cancel) domain and clear all its outstanding requests""" - if domain not in self.cancelled: - self.cancelled.add(domain) - self._close_domain(domain) - - def _close_domain(self, domain): - self.downloader.close_domain(domain) - - def _domain_idle(self, domain): - """ - Called when a domain gets idle. This function is called when there are no - remaining pages to download or scheduled. It can be called multiple - times. If the some extensions raises a DontCloseDomain exception the - domain won't be closed and this function is garanteed to be called - again (at least once) for this domain. - """ - # we get a callback from the downloader which completes closing - #log.msg("Finishing scraping %s" % domain, domain=domain) - #from traceback import print_stack; print_stack() - - spider = spiders.fromdomain(domain) - try: - dispatcher.send(signal=signals.domain_idle, sender=self.__class__, domain=domain, spider=spider) - except DontCloseDomain: - return - except: - log.exc("Exception catched on domain_idle signal dispatch") - - if self.domain_is_idle(domain): - self._close_domain(domain) - - def _stop_if_idle(self): - """Call the stop method if the system has no outstanding tasks. """ - if self.is_idle() and not self.keep_alive: - self.stop() - - def closed_domain(self, domain): - """ - This function is called after the domain has been closed, and throws - the domain_closed signal which is meant to be used for cleaning up - purposes. In contrast to domain_idle, this function is called only - ONCE for each domain run. - """ - spider = spiders.fromdomain(domain) - self.scheduler.close_domain(domain) - self.pipeline.close_domain(domain) - del self._scraping[domain] - status = 'cancelled' if domain in self.cancelled else 'finished' - signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, status=status) - log.msg("Domain closed (%s)" % status, domain=domain) - self.cancelled.discard(domain) - self._mainloop() - - def getstatus(self): - """ - Return a report of the current engine status - """ - s = "Execution engine status\n\n" - - global_tests = [ - "datetime.now()-self.start_time", - "self.is_idle()", - "self.scheduler.is_idle()", - "len(self.scheduler.pending_domains_count)", - "self.downloader.is_idle()", - "len(self.downloader.sites)", - "self.downloader.has_capacity()", - "self.pipeline.is_idle()", - "len(self.pipeline.domaininfo)", - ] - domain_tests = [ - "self.domain_is_idle(domain)", - "self.scheduler.domain_has_pending(domain)", - "len(self.scheduler.pending_requests[domain])", - "self.downloader.outstanding(domain)", - "len(self.downloader.request_queue(domain))", - "len(self.downloader.active_requests(domain))", - "self.pipeline.domain_is_idle(domain)", - "len(self.pipeline.domaininfo[domain])", - ] - - for test in global_tests: - s += "%-47s : %s\n" % (test, eval(test)) - s += "\n" - for domain in self.downloader.sites: - s += "%s\n" % domain - for test in domain_tests: - s += " %-45s : %s\n" % (test, eval(test)) - - return s - - def st(self): # shortcut for printing engine status (useful in telnet console) - print self.getstatus() - -scrapyengine = ExecutionEngine() diff --git a/scrapy/branches/cluster-refactor/scrapy/core/exceptions.py b/scrapy/branches/cluster-refactor/scrapy/core/exceptions.py deleted file mode 100644 index bf4fd7fd5..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/exceptions.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Scrapy core exceptions -""" - -# Internal - -class UsageError(Exception): - """Incorrect usage of the core API""" - pass - -class NotConfigured(Exception): - """Indicates a missing configuration situation""" - pass - -# HTTP and crawling - -class IgnoreRequest(Exception): - """Indicates a decision was made not to process a request""" - pass - -class DontCloseDomain(Exception): - """Request the domain not to be closed yet""" - pass - -class HttpException(Exception): - def __init__(self, status, message, response): - if not message: - from twisted.web import http - message = http.responses.get(int(status)) - - self.status = status - self.message = message - self.response = response - Exception.__init__(self, status, message, response) - - def __str__(self): - return '%s %s' % (self.status, self.message) - -# Items - -class DropItem(Exception): - """Drop item from the item pipeline""" - pass - -class NotSupported(Exception): - """Indicates a feature or method is not supported""" - pass \ No newline at end of file diff --git a/scrapy/branches/cluster-refactor/scrapy/core/manager.py b/scrapy/branches/cluster-refactor/scrapy/core/manager.py deleted file mode 100644 index 7872b784e..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/manager.py +++ /dev/null @@ -1,149 +0,0 @@ -import signal - -from scrapy.extension import extensions -from scrapy import log -from scrapy.http import Request -from scrapy.core.engine import scrapyengine -from scrapy.spider import spiders -from scrapy.utils.misc import load_class -from scrapy.utils.url import is_url -from scrapy.conf import settings - -class ExecutionManager(object): - """Process a list of sites or urls. - - This class should be used in a main for process a list of sites/urls. - - It extracts products and could be used to store results in a database or - just for testing spiders. - """ - def __init__(self): - self.interrupted = False - self.configured = False - - def configure(self, *args, **opts): - self._install_signals() - - extensions.load() - log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys())) - - scheduler = load_class(settings['SCHEDULER'])() - - scrapyengine.configure(scheduler=scheduler) - - self.prioritizer_class = load_class(settings['PRIORITIZER']) - - requests = self._parse_args(args) - self.priorities = self.prioritizer_class(requests.keys()) - - def crawl(self, *args): - """Schedule the given args for crawling. args is a list of urls or domains""" - - requests = self._parse_args(args) - # schedule initial requets to be scraped at engine start - for domain in requests or (): - spider = spiders.fromdomain(domain) - priority = self.priorities.get_priority(domain) - for request in requests[domain]: - scrapyengine.crawl(request, spider, domain_priority=priority) - - def runonce(self, *args, **opts): - """Run the engine until it finishes scraping all domains and then exit""" - self.configure(*args, **opts) - self.crawl(*args) - scrapyengine.start() - - def start(self, **opts): - """Start the scrapy server, without scheduling any domains""" - self.configure(**opts) - scrapyengine.keep_alive = True - scrapyengine.start()# blocking call - self.stop() - - def stop(self): - """Stop the scrapy server, shutting down the execution engine""" - self.interrupted = True - scrapyengine.stop() - log.log_level = -999 # disable logging - signal.signal(signal.SIGTERM, signal.SIG_IGN) - signal.signal(signal.SIGINT, signal.SIG_IGN) - if hasattr(signal, "SIGBREAK"): - signal.signal(signal.SIGBREAK, signal.SIG_IGN) - - def reload_spiders(self): - """ - Reload all enabled spiders except for the ones that are currently - running. - - """ - spiders.reload(skip_domains=scrapyengine.open_domains) - # reload priorities for the new domains - self.priorities = self.prioritizer_class(spiders.asdict(include_disabled=False).keys()) - - def _install_signals(self): - def sig_handler_terminate(signalinfo, param): - log.msg('Received shutdown request, waiting for deferreds to finish...', log.INFO) - self.stop() - - signal.signal(signal.SIGTERM, sig_handler_terminate) - # only handle SIGINT if there isn't already a handler (e.g. for Pdb) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler: - signal.signal(signal.SIGINT, sig_handler_terminate) - # Catch Ctrl-Break in windows - if hasattr(signal, "SIGBREAK"): - signal.signal(signal.SIGBREAK, sig_handler_terminate) - - def _parse_args(self, args): - """ Parse crawl arguments and return a dict domains -> [requests] """ - if not args: - args = [p.domain_name for p in spiders.enabled] - - requests, urls, sites = set(), set(), set() - for a in args: - if isinstance(a, Request): - requests.add(a) - elif is_url(a): - urls.add(a) - else: - sites.add(a) - - perdomain = {} - def _add(domain, request): - if domain not in perdomain: - perdomain[domain] = [] - perdomain[domain] += [request] - - # sites - for domain in sites: - spider = spiders.fromdomain(domain) - if not spider: - log.msg('Could not found spider for %s' % domain, log.ERROR) - continue - for url in self._start_urls(spider): - request = Request(url, callback=spider.parse, dont_filter=True) - _add(domain, request) - # urls - for url in urls: - spider = spiders.fromurl(url) - if spider: - request = Request(url=url, callback=spider.parse, dont_filter=True) - _add(spider.domain_name, request) - else: - log.msg('Could not found spider for <%s>' % url, log.ERROR) - - # requests - for request in requests: - if request.domain: - spider = spiders.fromdomain(request.domain) - else: - spider = spiders.fromurl(request.url) - if not spider: - log.msg('Could not found spider for %s' % request, log.ERROR) - continue - _add(spider.domain_name, request) - return perdomain - - def _start_urls(self, spider): - return spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls] - -scrapymanager = ExecutionManager() diff --git a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/__init__.py b/scrapy/branches/cluster-refactor/scrapy/core/scheduler/__init__.py deleted file mode 100644 index 1027c0382..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.core.scheduler.schedulers import Scheduler -from scrapy.core.scheduler.filter import GroupFilter -from scrapy.core.scheduler.store import MemoryStore diff --git a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/filter.py b/scrapy/branches/cluster-refactor/scrapy/core/scheduler/filter.py deleted file mode 100644 index 9008a4d03..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/filter.py +++ /dev/null @@ -1,19 +0,0 @@ -class GroupFilter(dict): - """Filter groups of keys""" - def open(self, group): - self[group] = set() - - def close(self, group): - del self[group] - - def add(self, group, key): - """Add a key to the group if an equivalent key has not already been added. - This method will return true if the key was added and false otherwise. - """ - if key not in self[group]: - self[group].add(key) - return True - return False - - def has(self, group, key): - return key in self[group] diff --git a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/schedulers.py b/scrapy/branches/cluster-refactor/scrapy/core/scheduler/schedulers.py deleted file mode 100644 index 81d2c9f82..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/schedulers.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -The Scrapy Scheduler -""" - -from twisted.internet import defer - -from scrapy.core.scheduler.filter import GroupFilter -from scrapy import log -from scrapy.core.exceptions import IgnoreRequest -from scrapy.utils.datatypes import PriorityQueue, PriorityStack -from scrapy.utils.defer import defer_fail -from scrapy.conf import settings - - -class Scheduler(object) : - """ - The scheduler decides what to scrape, how fast, and in what order. - The scheduler schedules websites and pages to be scraped. Individual - web pages that are to be scraped are batched up into a "run" for a website. - As the domain is being scraped, pages that are discovered are added to the - scheduler. The scheduler must not allow the same page to be requested - multiple times within the same batch. - - Typical usage: - - * next_availble_domain() called to find out when there is something to do - * begin_domain() called to commence scraping a website - * enqueue_request() called multiple times when new links found - * next_request() called multiple times when there is capacity to process urls - * close_domain() called when there are no more pages or upon error - - Note a couple things: - 1) The order in which you get back the list of pages to scrape is not - necesarily the order you put them in. - 2) A canonical URL is calculated for each url for each domain to check that - it is unique, however the actual url passed in is returned when - get_next_page is called. - This is for two main reasons: - * To be nice to the screen scraped site just incase the specific - format of the url is significant. - * To take advantage of any caching that uses the URL/URI as a key - - all_domains contains the names of all domains that are to be scheduled. - - Two crawling orders are available by default, which can be set with the - SCHEDULER_ORDER settings: - - * BFO - breath-first order (default). Consumes more memory than DFO but reaches - most relevant pages faster. - * DFO - depth-first order. Consumes less memory than BFO but usually takes - longer to reach the most relevant pages. - """ - - def __init__(self): - self.pending_domains_count = {} - self.domains_queue = PriorityQueue() - self.pending_requests = {} - self.groupfilter = GroupFilter() - self.dfo = settings.get('SCHEDULER_ORDER', '').upper() == 'DFO' - - def domain_is_open(self, domain): - return domain in self.pending_requests - - def is_pending(self, domain): - return domain in self.pending_domains_count - - def domain_has_pending(self, domain): - if domain in self.pending_requests: - return not self.pending_requests[domain].empty() - - def next_domain(self) : - """ - Return next domain available to scrape and remove it from available domains queue - """ - if self.pending_domains_count: - priority, domain = self.domains_queue.get_nowait() - if self.pending_domains_count[domain] == 1: - del self.pending_domains_count[domain] - else: - self.pending_domains_count[domain] -= 1 - return domain - return None - - def add_domain(self, domain, priority=1): - """ - This functions schedules a new domain to be scraped, with the given - priority. It doesn't check if the domain is already scheduled. A - domain can be scheduled twice, either with the same or with different - priority. - """ - self.domains_queue.put(domain, priority=priority) - if domain not in self.pending_domains_count: - self.pending_domains_count[domain] = 1 - else: - self.pending_domains_count[domain] += 1 - - def open_domain(self, domain): - """ - Allocates resources for maintaining a schedule for domain. - """ - if self.dfo: - self.pending_requests[domain] = PriorityStack() - else: - self.pending_requests[domain] = PriorityQueue() - - self.groupfilter.open(domain) - - def enqueue_request(self, domain, request, priority=1): - """ - Add a page to be scraped for a domain that is currently being scraped. - """ - requestid = request.fingerprint() - added = self.groupfilter.add(domain, requestid) - - if request.dont_filter or added: - deferred = defer.Deferred() - self.pending_requests[domain].put((request, deferred), priority) - return deferred - else: - return defer_fail(IgnoreRequest('Skipped (already visited): %s' % request)) - - def request_seen(self, domain, request): - """ - Returns True if the given Request was scheduled before for the given - domain - """ - return self.groupfilter.has(domain, request.fingerprint()) - - def next_request(self, domain): - """ - Get the next request to be scraped. - - None should be returned if there are no more request pending for the domain passed. - """ - pending_list = self.pending_requests.get(domain) - if pending_list and not pending_list.empty(): - return pending_list.get_nowait()[1] - else: - return (None, None) - - def close_domain(self, domain) : - """ - Called once we are finished scraping a domain. The scheduler will - free any resources associated with the domain. - """ - try : - del self.pending_requests[domain] - except Exception, inst: - msg = "Could not clear pending pages for domain %s, %s" % (domain, inst) - log.msg(msg, level=log.WARNING) - - try : - self.groupfilter.close(domain) - except Exception, inst: - msg = "Could not clear url filter for domain %s, %s" % (domain, inst) - log.msg(msg, level=log.WARNING) - - def remove_pending_domain(self, domain): - """ - Remove a pending domain not yet started. If the domain was enqueued - several times, all those instances are removed. - - Returns the number of times the domain was enqueued. 0 if domains was - not pending. - - If domain is open (not pending) it is not removed and returns None. You - need to call close_domain for open domains. - """ - if not self.domain_is_open(domain): - return self.pending_domains_count.pop(domain, 0) - - def is_idle(self): - return not self.pending_requests diff --git a/scrapy/branches/cluster-refactor/scrapy/core/signals.py b/scrapy/branches/cluster-refactor/scrapy/core/signals.py deleted file mode 100644 index d84d702b3..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/core/signals.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Scrapy core signals -""" -from pydispatch import dispatcher - -from scrapy import log - -# After the execution engine has started -# args: None -engine_started = object() - -# After the execution engine has stopped -# args: None -engine_stopped = object() - -# Before opening a new domain for crawling -# args: domain, spider -domain_open = object() - -# After a domain has been opened for crawling -# args: domain, spider -domain_opened = object() - -# When a domain has no remaining requests to process -# args: domain, spider -domain_idle = object() - -# After a domain has been closed -# args: domain, spider, status -# status is a string and its possible values are: "finished" or "cancelled" -domain_closed = object() - -# domain has been initialized (successful or not) -# args: domain, spider, status -domain_initialized = object() - -# New request received from spiders -# args: request, spider, response -# response is the response (fed to the spider) which generated the request -request_received = object() - -# When request is sent in the downloader -# args: request, spider -request_uploaded = object() - -# When new response is received (by the engine) from the downloader (middleware) -# args: response, spider -response_received = object() - -# When response arrives from the downloader -# args: response, spider -response_downloaded = object() - -# After item is returned from spiders -# args: item, spider, response -item_scraped = object() - -# After item is processed by pipeline and pass all its stages -# args: item, spider, response, pipe_output -item_passed = object() - -# After item is dropped by pipeline -# args: item, spider, response, exception -item_dropped = object() - -def send_catch_log(signal, sender=None, **kwargs): - """ - Send a signal and log any exceptions raised by its listeners - """ - try: - dispatcher.send(signal=signal, sender=sender, **kwargs) - except: - log.exc("Exception catched on signal dispatch") diff --git a/scrapy/branches/cluster-refactor/scrapy/extension/__init__.py b/scrapy/branches/cluster-refactor/scrapy/extension/__init__.py deleted file mode 100644 index dc96e1194..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/extension/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -This module contains the ExtensionManager which takes care of loading and -keeping track of all enabled extensions. It also contains an instantiated -ExtensionManager (extensions) to be used as singleton. -""" -from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class -from scrapy import log -from scrapy.conf import settings - -class ExtensionManager(object): - - def __init__(self): - self.loaded = False - self.enabled = {} - self.disabled = {} - - def load(self): - """ - Load enabled extensions in settings module - """ - - self.loaded = False - self.enabled.clear() - self.disabled.clear() - for extension_path in settings.getlist('EXTENSIONS'): - try: - cls = load_class(extension_path) - self.enabled[cls.__name__] = cls() - except NotConfigured, e: - self.disabled[cls.__name__] = extension_path - if e.args: - log.msg(e) - self.loaded = True - - def reload(self): - self.load() - -extensions = ExtensionManager() diff --git a/scrapy/branches/cluster-refactor/scrapy/http/__init__.py b/scrapy/branches/cluster-refactor/scrapy/http/__init__.py deleted file mode 100644 index d9fef2626..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/http/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Module containing all HTTP related classes - -Use this module (instead of the more specific ones) when importing Headers, -Request, Response, ResponseBody and Url outside this module. -""" - -from scrapy.http.url import Url -from scrapy.http.headers import Headers -from scrapy.http.request import Request -from scrapy.http.response import Response, ResponseBody diff --git a/scrapy/branches/cluster-refactor/scrapy/http/headers.py b/scrapy/branches/cluster-refactor/scrapy/http/headers.py deleted file mode 100644 index 54d74a9ef..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/http/headers.py +++ /dev/null @@ -1,97 +0,0 @@ -from scrapy.utils.datatypes import CaselessDict - -def headers_raw_to_dict(headers_raw): - """ - Convert raw headers (single multi-line string) - to the dictionary. - - For example: - >>> headers_raw_to_dict("Content-type: text/html\\n\\rAccept: gzip\\n\\n") - {'Content-type': ['text/html'], 'Accept': ['gzip']} - - Incorrect input: - >>> headers_raw_to_dict("Content-typt gzip\\n\\n") - {} - - Argument is None: - >>> headers_raw_to_dict(None) - """ - if headers_raw is None: - return None - return dict([ - (header_item[0].strip(), [header_item[1].strip()]) - for header_item - in [ - header.split(':', 1) - for header - in headers_raw.splitlines()] - if len(header_item) == 2]) - -def headers_dict_to_raw(headers_dict): - """ - Returns a raw HTTP headers representation of headers - - For example: - >>> headers_dict_to_raw({'Content-type': 'text/html', 'Accept': 'gzip'}) - 'Content-type: text/html\\r\\nAccept: gzip' - >>> from twisted.python.util import InsensitiveDict - >>> td = InsensitiveDict({'Content-type': ['text/html'], 'Accept': ['gzip']}) - >>> headers_dict_to_raw(td) - 'Content-type: text/html\\r\\nAccept: gzip' - - Argument is None: - >>> headers_dict_to_raw(None) - - """ - if headers_dict is None: - return None - raw_lines = [] - for key, value in headers_dict.items(): - if isinstance(value, (str, unicode)): - raw_lines.append("%s: %s" % (key, value)) - elif isinstance(value, (list, tuple)): - for v in value: - raw_lines.append("%s: %s" % (key, v)) - return '\r\n'.join(raw_lines) - -class Headers(CaselessDict): - def __init__(self, dictorstr=None, fromdict=None, fromstr=None, encoding='utf-8'): - self.encoding = encoding - - if dictorstr is not None: - if isinstance(dictorstr, dict): - d = dictorstr - elif isinstance(dictorstr, basestring): - d = headers_raw_to_dict(dictorstr) - elif fromdict is not None: - d = fromdict - elif fromstr is not None: - d = headers_raw_to_dict(fromstr) - else: - d = {} - - # can't use CaselessDict.__init__(self, d) because it doesn't call __setitem__ - for k,v in d.iteritems(): - self.__setitem__(k.lower(), v) - - def normkey(self, key): - return key.title() - - def __setitem__(self, key, value): - """Headers must not be unicode""" - if isinstance(key, unicode): - key = key.encode(self.encoding) - if isinstance(value, unicode): - value = value.encode(self.encoding) - super(Headers, self).__setitem__(key, value) - - def tostring(self): - return headers_dict_to_raw(self) - - def rawsize(self): - """Estimated size of raw HTTP headers, in bytes""" - # For each header line you have 4 extra chars: ": " and CRLF - return sum([len(k)+len(v)+4 for k, v in self.iteritems()]) - - def to_string(self): - return headers_dict_to_raw(self) diff --git a/scrapy/branches/cluster-refactor/scrapy/http/request.py b/scrapy/branches/cluster-refactor/scrapy/http/request.py deleted file mode 100644 index 5b9e5841a..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/http/request.py +++ /dev/null @@ -1,188 +0,0 @@ -import urllib -import warnings -import hashlib -from copy import copy -from base64 import urlsafe_b64encode - -from twisted.internet import defer - -from scrapy.http.url import Url -from scrapy.http.headers import Headers -from scrapy.utils.url import safe_url_string -from scrapy.utils.c14n import canonicalize -from scrapy.utils.defer import chain_deferred - -class Request(object): - def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None, - referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None, - fingerprint_params=None, domain=None): - - self.encoding = url_encoding # this one has to be set first - self.set_url(url) - - # method - if method is None and body is not None: - method = 'POST' # backwards compatibility - self.method = method.upper() if method else 'GET' - assert isinstance(self.method, basestring), 'Request method argument must be str or unicode, got %s: %s' % (type(method), method) - - # body - if isinstance(body, dict): - body = urllib.urlencode(body) - self.body = body - - # callback / deferred - if callable(callback): - callback = defer.Deferred().addCallback(callback) - self.deferred = callback or defer.Deferred() - - # request cookies - self.cookies = cookies or {} - # request headers - self.headers = Headers(headers or {}, encoding=url_encoding) - # persistent context across requests - self.context = context or {} - # dont_filter be filtered by scheduler - self.dont_filter = dont_filter - # fingerprint parameters - self.fingerprint_params = fingerprint_params or {} - self._fingerprint = None - # shortcut for setting referer - if referer is not None: - self.headers['referer'] = referer - # http auth - if http_user or http_pass: - self.httpauth(http_user, http_pass) - self.depth = 0 - self.link_text = link_text - #allows to directly specify the spider for the request - self.domain = domain - - def append_callback(self, callback, *args, **kwargs): - if isinstance(callback, defer.Deferred): - return chain_deferred(self.deferred, callback) - return self.deferred.addCallback(callback, *args, **kwargs) - - def prepend_callback(self, func, *args, **kwargs): - if callable(func): - func = defer.Deferred().addCallback(func, *args, **kwargs) - assert isinstance(func, defer.Deferred), 'prepend_callback expects a callable or defer.Deferred instance, got %s' % type(func) - self.deferred = chain_deferred(func, self.deferred) - return self.deferred - - def set_url(self, url): - assert isinstance(url, basestring), 'Request url argument must be str or unicode, got %s:' % type(url).__name__ - decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding) - self._url = Url(safe_url_string(decoded_url, self.encoding)) - self._fingerprint = None # invalidate cached fingerprint - url = property(lambda x: x._url, set_url) - - def httpauth(self, http_user, http_pass): - if not http_user: - http_user = '' - if not http_pass: - http_pass = '' - self.headers['Authorization'] = 'Basic ' + urlsafe_b64encode("%s:%s" % (http_user, http_pass)) - - def __getitem__(self, key): - warnings.warn("Request tuples have been deprecated, Request objects are now used. Index mappings: 0 -> r.url, 1 -> r.body, 2 -> r.deferred, 3 -> r.options ", DeprecationWarning) - return (self.url, self.body, self.deferred, {})[key] - - def __str__(self): - if self.method != 'GET': - return "<(%s) %s>" % (self.method, self.url) - return "<%s>" % self.url - - def __len__(self): - """Return raw HTTP request size""" - return len(self.to_string()) - - def traceinfo(self): - fp = self.fingerprint() - version = '%s..%s' % (fp[:4], fp[-4:]) - return "" % (self.method, self.url, version) - - def __repr__(self): - d = { - 'method': self.method, - 'url': self.url, - 'headers': self.headers, - 'cookies': self.cookies, - 'body': self.body, - 'context': self.context - } - return "%s(%s)" % (self.__class__.__name__, repr(d)) - - def copy(self): - """Clone request except `context` attribute""" - new = copy(self) - for att in self.__dict__: - if att not in ['context', 'url', 'deferred', '_fingerprint']: - value = getattr(self, att) - setattr(new, att, copy(value)) - new.deferred = defer.Deferred() - new.context = self.context # requests shares same context dictionary - new._fingerprint = None # reset fingerprint - return new - - def fingerprint(self): - """Returns unique resource fingerprint with caching support""" - if not self._fingerprint or self.fingerprint_params: - self.update_fingerprint() - return self._fingerprint - - def update_fingerprint(self): - """Update request fingerprint, based on its current data. A request - fingerprint is a hash which uniquely identifies the HTTP resource""" - - headers = {} - if self.fingerprint_params: - if 'tamperfunc' in self.fingerprint_params: - tamperfunc = self.fingerprint_params['tamperfunc'] - assert callable(tamperfunc) - req = tamperfunc(self.copy()) - assert isinstance(req, Request) - try: - del req.fingerprint_params['tamperfunc'] - except KeyError: - pass - return req.fingerprint() - - if self.headers: - if 'include_headers' in self.fingerprint_params: - keys = [k.lower() for k in self.fingerprint_params['include_headers']] - headers = dict([(k, v) for k, v in self.headers.items() if k.lower() in keys]) - elif 'exclude_headers' in self.fingerprint_params: - keys = [k.lower() for k in self.fingerprint_params['exclude_headers']] - headers = dict([(k, v) for k, v in self.headers.items() if k.lower() not in keys]) - - # fingerprint generation - fp = hashlib.sha1() - fp.update(canonicalize(self.url)) - fp.update(self.method) - - if self.body and self.method in ['POST', 'PUT']: - fp.update(self.body) - - if headers: - for k, v in sorted([(k.lower(), v) for k, v in headers.items()]): - fp.update(k) - fp.update(v) - - self._fingerprint = fp.hexdigest() - - def to_string(self): - """ Return raw HTTP request representation (as string). This is - provided only for reference since it's not the actual stream of bytes - that will be send when performing the request (that's controlled by - Twisted). - """ - - s = "%s %s HTTP/1.1\r\n" % (self.method, self.url) - s += "Host: %s\r\n" % self.url.hostname - s += self.headers.to_string() + "\r\n" - s += "\r\n" - if self.body: - s += self.body - s += "\r\n" - return s diff --git a/scrapy/branches/cluster-refactor/scrapy/http/response.py b/scrapy/branches/cluster-refactor/scrapy/http/response.py deleted file mode 100644 index 0ec3a2f4b..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/http/response.py +++ /dev/null @@ -1,194 +0,0 @@ -import re -import hashlib -import copy - -from BeautifulSoup import UnicodeDammit - -from scrapy.http.url import Url -from scrapy.http.headers import Headers - -from twisted.web import http -reason_phrases = http.RESPONSES - -class Response(object) : - """HTTP responses - - Arguments: - * Domain - the spider domain for the page - * url - the final url for the resource - * original_url - the url requested - * headers - HTTP headers - * status - HTTP status code - * body - Body object containing the content of the response - """ - _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - - def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None): - self.domain = domain - self.url = Url(url) - self.original_url = Url(original_url) if original_url else url # different if redirected or escaped - self.headers = Headers(headers or {}) - self.status = status - if isinstance(body, str): - self.body = ResponseBody(body, self.headers_encoding()) - else: - self.body = body - self.cached = False - self.request = None # request which originated this response - - def version(self): - """A hash of the contents of this response""" - if not hasattr(self, '_version'): - self._version = hashlib.sha1(self.body.to_string()).hexdigest() - return self._version - - def headers_encoding(self): - content_type = self.headers.get('Content-Type') - if content_type: - encoding = self._ENCODING_RE.search(content_type[0]) - if encoding: - return encoding.group(1) - - def __repr__(self): - return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s)" % \ - (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body)) - - def __str__(self): - version = '%s..%s' % (self.version()[:4], self.version()[-4:]) - return "" % (self.status, self.url, version) - - def __len__(self): - """Return raw HTTP response size""" - return len(self.to_string()) - - def info(self): - return "[^;]+);\s*charset=(?P[\w-]+)') - _encoding_re = _template % ('encoding', r'(?P[\w-]+)') - - XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) - - METATAG_RE = re.compile(r'" % function.func_name, repr(value) - value = function(value) - if debug: - print " %07s | output>" % function.func_name, repr(value) - - except Exception, e: - print "Error in '%s' adaptor. Traceback text:" % function.func_name - print format_exc() - return - - return value - - -############ -# Adaptors # -############ -def extract(location): - """ - This adaptor extracts a list of strings - from 'location', which can be either a list (or tuple), - or an XPathSelector. - - This function *always* returns a list. - """ - if not location: - return [] - elif isinstance(location, XPathSelectorList): - return flatten([extract(o) for o in location]) - elif isinstance(location, XPathSelector): - return location.extract() - elif isinstance(location, (list, tuple)): - return flatten(location) - elif isinstance(location, basestring): - return [location] - -def _absolutize_links(rel_links, current_url, base_url): - abs_links = [] - for link in rel_links: - if link.startswith('/'): - abs_links.append('%s%s' % (base_url, link)) - elif link.startswith('http://'): - abs_links.append(link) - else: - abs_links.append('%s/%s' % (current_url, link)) - return abs_links - -def extract_links(locations): - """ - This adaptor receives either an XPathSelector containing - the desired locations for finding urls, or a tuple like (xpath, regexp) - containing the xpath locations to look in, and a regular expression - to parse those locations. - - In any case, this adaptor returns a list of absolute urls extracted. - """ - ret = [] - if locations: - regexp = None - if isinstance(locations, XPathSelector): - locations = XPathSelectorList([locations]) - elif isinstance(locations, tuple): - locations, regexp = locations - - if isinstance(locations, XPathSelectorList): - if regexp: - ret = locations.re(regexp) - else: - for selector in locations: - if selector.xmlNode.type == 'element': - if selector.xmlNode.name == 'a': - children = selector.x('child::*') - if len(children) > 1: - ret.extend(selector.x('.//@href')) - ret.extend(selector.x('.//@src')) - elif len(children) == 1 and children[0].xmlNode.name == 'img': - ret.extend(children.x('@src')) - else: - ret.extend(selector.x('@href')) - elif selector.xmlNode.name == 'img': - ret.extend(selector.x('@src')) - else: - ret.extend(selector.x('.//@href')) - ret.extend(selector.x('.//@src')) - elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']: - ret.append(selector) - ret = [selector.extract() for selector in ret] - current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups() - ret = _absolutize_links(ret, current_url, base_url) - return ret - -def to_unicode(value): - """ - Receives a list of strings, converts - it to unicode, and returns a new list. - """ - if isinstance(value, (list, tuple)): - return [ unicode(v) for v in value ] - else: - raise TypeError('to_unicode adaptor must receive a list or a tuple.') - -def regex(expr): - """ - This factory function returns a ready-to-use - adaptor for the specified regular expression. - This adaptor will accept either an XPathSelectorList - or a list of strings, and will apply the provided regular - expression to each of its members. - - This adaptor always returns a list of strings. - """ - def _regex(value): - if isinstance(value, (XPathSelector, XPathSelectorList)): - return value.re(expr) - elif isinstance(value, list) and value: - return flatten([extract_regex(expr, string, 'utf-8') for string in value]) - return value - return _regex - -def unquote_all(value): - """ - Receives a list of strings, removes all of the - entities the strings may have, and returns - a new list - """ - return [ remove_entities(v) for v in value ] - -def unquote(value): - """ - Receives a list of strings, removes all of the entities - the strings may have (except for < and &), and - returns a new list - """ - return [ remove_entities(v, keep=['lt', 'amp']) for v in value ] - -def remove_tags(value): - return [ replace_tags(v) for v in value ] - -_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) -def _remove_root(value): - m = _remove_root_re.search(value) - if m: - value = m.group(1) - return value - -def remove_root(value): - return [ _remove_root(v) for v in value ] - -_clean_spaces_re = re.compile("\s+", re.U) -def remove_multispaces(value): - return [ _clean_spaces_re.sub(' ', v) for v in value ] - -def strip(value): - return [ v.strip() for v in value ] - -def drop_empty_elements(value): - return [ v for v in value if v ] - -def delist(value): - return ' '.join(value) - - - -############# -# Pipelines # -############# -""" -The following methods automatically generate adaptor pipelines -for some basic datatypes, according to the parameters you pass them. -""" -def single_pipeline(remove_root=True, remove_tags=True, do_unquote=True): - pipe = [ extract, - unique, - to_unicode, - drop_empty_elements, - remove_multispaces, - strip, - ] - if remove_root: - pipe.insert(4, remove_root) - if remove_tags: - pipe.insert(5, remove_tags) - if do_unquote: - pipe.append(unquote) - return pipe + [delist] - -def url_pipeline(): - return [ extract_links, - unique, - to_unicode, - drop_empty_elements, - ] - -def list_pipeline(do_extract=True): - pipe = [] - if do_extract: - pipe.append(extract) - return pipe + [ unique, - to_unicode, - drop_empty_elements, - unquote, - remove_tags, - remove_root, - strip, - ] - -def list_join_pipeline(delimiter='\t'): - return list_pipeline() + [delimiter.join] - diff --git a/scrapy/branches/cluster-refactor/scrapy/item/models.py b/scrapy/branches/cluster-refactor/scrapy/item/models.py deleted file mode 100644 index a68568104..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/item/models.py +++ /dev/null @@ -1,24 +0,0 @@ -from scrapy.item.adaptors import AdaptorPipe - -class ScrapedItem(object): - """ - This is the base class for all scraped items. - 'guid' attribute is required, and is an attribute - that identifies uniquely the given scraped item. - """ - - def setadaptors(self, adaptors_pipe): - """ - Set adaptors to use for this item. Receives a dict of adaptors and - returns the item itself. - """ - object.__setattr__(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe)) - return self - - def append_adaptor(self, attrname, adaptor): - self._adaptors_pipe.append_adaptor(attrname, adaptor) - - def attribute(self, attrname, value): - value = self._adaptors_pipe.execute(attrname, value) - if not hasattr(self, attrname) or not getattr(self, attrname): - setattr(self, attrname, value) diff --git a/scrapy/branches/cluster-refactor/scrapy/item/pipeline.py b/scrapy/branches/cluster-refactor/scrapy/item/pipeline.py deleted file mode 100644 index 5adc0555e..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/item/pipeline.py +++ /dev/null @@ -1,101 +0,0 @@ -from scrapy import log -from scrapy.core.exceptions import DropItem, NotConfigured -from scrapy.item import ScrapedItem -from scrapy.utils.misc import load_class -from scrapy.utils.defer import defer_succeed, mustbe_deferred -from scrapy.conf import settings - -class ItemPipeline(object): - - def __init__(self): - self.loaded = False - self.pipeline = [] - self.domaininfo = {} - self.load() - - def load(self): - """ - Load pipelines stages defined in settings module - """ - for stage in settings.getlist('ITEM_PIPELINES') or (): - cls = load_class(stage) - if cls: - try: - stageinstance = cls() - self.pipeline.append(stageinstance) - except NotConfigured: - pass - log.msg("Enabled item pipelines: %s" % ", ".join([type(p).__name__ for p in self.pipeline])) - self.loaded = True - - def open_domain(self, domain): - self.domaininfo[domain] = set() - for pipe in self.pipeline: - if hasattr(pipe, 'open_domain'): - pipe.open_domain(domain) - - def close_domain(self, domain): - for pipe in self.pipeline: - if hasattr(pipe, 'close_domain'): - pipe.close_domain(domain) - del self.domaininfo[domain] - - def is_idle(self): - return not self.domaininfo - - def domain_is_idle(self, domain): - return not self.domaininfo.get(domain) - - def pipe(self, item, spider, response): - """ - item pipelines are instanceable classes that defines a `pipeline` method - that takes ScrapedItem as input and returns ScrapedItem. - - The output from one stage is the input of the next. - - Raising DropItem stops pipeline. - - This pipeline is configurable with the ITEM_PIPELINES setting - """ - if not self.pipeline: - return defer_succeed(item) - - domain = spider.domain_name - pipeline = self.pipeline[:] - current_stage = pipeline[0] - info = self.domaininfo[domain] - info.add(item) - - def _next_stage(item): - assert isinstance(item, ScrapedItem), 'Pipeline stages must return a ScrapedItem or raise DropItem' - if not pipeline: - return item - - current_stage = pipeline.pop(0) - log.msg("_%s_ Pipeline stage: %s" % (item, type(current_stage).__name__), log.TRACE, domain=domain) - - d = mustbe_deferred(current_stage.process_item, domain, response, item) - d.addCallback(_next_stage) - return d - - def _ondrop(_failure): - ex = _failure.value - if isinstance(ex, DropItem): - # TODO: current_stage is not working, check why - #log.msg("%s: Dropped %s - %s" % (type(current_stage).__name__, item, str(ex)), log.DEBUG, domain=domain) - log.msg("Dropped %s - %s" % (item, str(ex)), log.DEBUG, domain=domain) - return _failure - else: - # TODO: current_stage is not working, check why - #log.msg('%s: Error processing %s - %s' % (type(current_stage).__name__, item, _failure), log.ERROR, domain=domain) - log.msg('Error processing %s - %s' % (item, _failure), log.ERROR, domain=domain) - - def _pipeline_finished(_): - log.msg("_%s_ Pipeline finished" % item, log.TRACE, domain=domain) - info.remove(item) - return _ - - deferred = mustbe_deferred(_next_stage, item) - deferred.addErrback(_ondrop) - deferred.addBoth(_pipeline_finished) - return deferred diff --git a/scrapy/branches/cluster-refactor/scrapy/link/__init__.py b/scrapy/branches/cluster-refactor/scrapy/link/__init__.py deleted file mode 100644 index 069d22d19..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/link/__init__.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -LinkExtractor provides en efficient way to extract links from pages -""" - -from scrapy.utils.python import FixedSGMLParser -from scrapy.utils.url import urljoin_rfc as urljoin - -class LinkExtractor(FixedSGMLParser): - """LinkExtractor are used to extract links from web pages. They are - instantiated and later "applied" to a Response using the extract_urls - method which must receive a Response object and return a dict whoose keys - are the (absolute) urls to follow, and its values any arbitrary data. In - this case the values are the text of the hyperlink. - - This is the base LinkExtractor class that provides enough basic - functionality for extracting links to follow, but you could override this - class or create a new one if you need some additional functionality. The - only requisite is that the new (or overrided) class must provide a - extract_urls method that receives a Response and returns a dict with the - links to follow as its keys. - - The constructor arguments are: - - * tag (string or function) - * a tag name which is used to search for links (defaults to "a") - * a function which receives a tag name and returns whether to scan it - * attr (string or function) - * an attribute name which is used to search for links (defaults to "href") - * a function which receives an attribute name and returns whether to scan it - """ - - def __init__(self, tag="a", attr="href"): - FixedSGMLParser.__init__(self) - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.inside_link = False - - def extract_urls(self, response): - self.reset() - self.feed(response.body.to_string()) - self.close() - - base_url = self.base_url if self.base_url else response.url - urls = {} - for link, text in self.links.iteritems(): - urls[urljoin(base_url, link)] = text - return urls - - def reset(self): - FixedSGMLParser.reset(self) - self.links = {} - self.base_url = None - - def unknown_starttag(self, tag, attrs): - if tag == 'base': - self.base_url = dict(attrs).get('href') - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - self.links[value] = "" - self.inside_link = value - - def unknown_endtag(self, tag): - self.inside_link = False - - def handle_data(self, data): - if self.inside_link and not self.links.get(self.inside_link, None): - self.links[self.inside_link] = data diff --git a/scrapy/branches/cluster-refactor/scrapy/link/extractors.py b/scrapy/branches/cluster-refactor/scrapy/link/extractors.py deleted file mode 100644 index 2153da7e1..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/link/extractors.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -This module provides some LinkExtractors, which extend that base LinkExtractor -(scrapy.link.LinkExtractor) with some useful features. - -""" - -import re - -from scrapy.link import LinkExtractor -from scrapy.utils.url import canonicalize_url, url_is_from_any_domain -from scrapy.utils.response import new_response_from_xpaths -from scrapy.utils.misc import dict_updatedefault - -_re_type = type(re.compile("", 0)) - -_matches = lambda url, regexs: any((r.search(url) for r in regexs)) -_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file']) - -class RegexLinkExtractor(LinkExtractor): - """RegexLinkExtractor implements extends the base LinkExtractor by - providing several mechanisms to extract the links. - - It's constructor parameters are: - - allow - list of regexes that the (absolute urls) must match to be extracted - deny - ignore urls that match any of these regexes - allow_domains - only extract urls from these domains - deny_domains - ignore urls from these dmoains - tags - look for urls in this tags - attrs - look for urls in this attrs - canonicalize - canonicalize all extracted urls using scrapy.utils.url.canonicalize_url - - Both 'allow' and 'deny' arguments can be a list of regexes strings or regex - python objects (already compiled) - - Url matching is always performed against the absolute urls, never the - relative urls found in pages. - - """ - - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href'), canonicalize=True): - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in allow] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in deny] - self.allow_domains = set(allow_domains) - self.deny_domains = set(deny_domains) - self.restrict_xpaths = restrict_xpaths - self.canonicalize = canonicalize - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs - LinkExtractor.__init__(self, tag=tag_func, attr=attr_func) - - def extract_urls(self, response): - if self.restrict_xpaths: - response = new_response_from_xpaths(response, self.restrict_xpaths) - - url_text = LinkExtractor.extract_urls(self, response) - urls = [u for u in url_text.iterkeys() if _is_valid_url(u)] - - if self.allow_res: - urls = [u for u in urls if _matches(u, self.allow_res)] - if self.deny_res: - urls = [u for u in urls if not _matches(u, self.deny_res)] - if self.allow_domains: - urls = [u for u in urls if url_is_from_any_domain(u, self.allow_domains)] - if self.deny_domains: - urls = [u for u in urls if not url_is_from_any_domain(u, self.deny_domains)] - - res = {} - if self.canonicalize: - for u in urls: - res[canonicalize_url(u)] = url_text[u] - else: - for u in urls: - res[u] = url_text[u] - return res diff --git a/scrapy/branches/cluster-refactor/scrapy/log/__init__.py b/scrapy/branches/cluster-refactor/scrapy/log/__init__.py deleted file mode 100644 index 4c03a9121..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/log/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Crawler logging functionality -""" -import sys - -from twisted.python import log - -from scrapy.conf import settings - -# Logging levels -levels = { - 0: "SILENT", - 1: "CRITICAL", - 2: "ERROR", - 3: "WARNING", - 4: "INFO", - 5: "DEBUG", - 6: "TRACE", -} - -BOT_NAME = settings['BOT_NAME'] - -# Set logging level module attributes -for v, k in levels.items(): - setattr(sys.modules[__name__], k, v) - -# default logging level -log_level = DEBUG - -started = False - -def start(logfile=None, loglevel=None, log_stdout=None): - """ Init logging """ - if started or not settings.getbool('LOG_ENABLED'): - return - - logfile = logfile or settings['LOGFILE'] - loglevel = loglevel or settings['LOGLEVEL'] - log_stdout = log_stdout or settings.getbool('LOG_STDOUT') - - file = open(logfile, 'a') if logfile else sys.stderr - level = int(getattr(sys.modules[__name__], loglevel)) if loglevel else DEBUG - log.startLogging(file, setStdout=log_stdout) - setattr(sys.modules[__name__], 'log_level', level) - setattr(sys.modules[__name__], 'started', True) - -def msg(message, level=INFO, component=BOT_NAME, domain=None): - """ Log message according to the level """ - component = "%s/%s" % (BOT_NAME, domain) if domain else component - if level <= log_level: - log.msg("%s: %s" % (levels[level], message), system=component) - -def exc(message, level=ERROR, component=BOT_NAME, domain=None): - from traceback import format_exc - message = message + '\n' + format_exc() - msg(message, level, component, domain) - diff --git a/scrapy/branches/cluster-refactor/scrapy/mail/__init__.py b/scrapy/branches/cluster-refactor/scrapy/mail/__init__.py deleted file mode 100644 index 1078366e9..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/mail/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -import smtplib - -from email.MIMEMultipart import MIMEMultipart -from email.MIMEBase import MIMEBase -from email.MIMEText import MIMEText -from email.Utils import COMMASPACE, formatdate -from email import Encoders - -from scrapy import log -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings - -class MailSender(object): - - def __init__(self, smtphost=None, mailfrom=None): - self.smtphost = smtphost if smtphost else settings['MAIL_HOST'] - self.mailfrom = mailfrom if mailfrom else settings['MAIL_FROM'] - - if not self.smtphost or not self.mailfrom: - raise NotConfigured("MAIL_HOST and MAIL_FROM settings are required") - - def send(self, to, subject, body, cc=None, attachs=None): - """ - Send mail to the given recipients - - - to: must be a list of email recipients - - attachs must be a list of tuples: (attach_name, mimetype, file_object) - - body and subjet must be a string - """ - - msg = MIMEMultipart() - msg['From'] = self.mailfrom - msg['To'] = COMMASPACE.join(to) - msg['Date'] = formatdate(localtime=True) - msg['Subject'] = subject - rcpts = to[:] - if cc: - rcpts.extend(cc) - msg['Cc'] = COMMASPACE.join(cc) - - msg.attach(MIMEText(body)) - - for attach_name, mimetype, f in (attachs or []): - part = MIMEBase(*mimetype.split('/')) - part.set_payload(f.read()) - Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="%s"' % attach_name) - msg.attach(part) - - smtp = smtplib.SMTP(self.smtphost) - smtp.sendmail(self.mailfrom, rcpts, msg.as_string()) - log.msg('Mail sent: To=%s Cc=%s Subject="%s"' % (to, cc, subject)) - smtp.close() - diff --git a/scrapy/branches/cluster-refactor/scrapy/management/web.py b/scrapy/branches/cluster-refactor/scrapy/management/web.py deleted file mode 100644 index f9d828546..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/management/web.py +++ /dev/null @@ -1,69 +0,0 @@ -import re -import socket -from datetime import datetime - -from twisted.internet import reactor -from twisted.web import server, resource -from pydispatch import dispatcher - -from scrapy.core.engine import scrapyengine -from scrapy.conf import settings - -# web management signals -webconsole_discover_module = object() - -urlpath_re = re.compile(r"^/(\w+)/") - -def banner(module=None): - s = "\n" - s += "Scrapy\n" - s += "\n" - s += "

    Scrapy web console

    \n" - now = datetime.now() - uptime = now - scrapyengine.start_time - s += "

    Bot: %s | Host: %s | Uptime: %s | Time: %s

    \n" % (settings['BOT_NAME'], socket.gethostname(), str(uptime), str(now)) - if module: - s += "

    %s

    \n" % (module.webconsole_id, module.webconsole_name) - return s - -class WebConsoleResource(resource.Resource): - isLeaf = True - - @property - def modules(self): - if not hasattr(self, '_modules'): - self._modules = {} - for _, obj in dispatcher.send(signal=webconsole_discover_module, sender=self.__class__): - self._modules[obj.webconsole_id] = obj - return self._modules - - def render_GET(self, request): - m = urlpath_re.search(request.path) - if m: - return self.modules[m.group(1)].webconsole_render(request) - else: - return self.module_list() - - render_POST = render_GET - - def module_list(self): - s = banner() - s += "

    Available modules:

    \n" - s += "
      \n" - for name, obj in self.modules.iteritems(): - s += "
    • %s
      • \n" % (name, obj.webconsole_name) - s += "
    \n" - s += "\n" - s += "\n" - return s - -class WebConsole(server.Site): - - def __init__(self): - if not settings.getbool('WEBCONSOLE_ENABLED'): - return - - server.Site.__init__(self, WebConsoleResource()) - self.noisy = False - port = settings.getint('WEBCONSOLE_PORT') - scrapyengine.listenTCP(port, self) diff --git a/scrapy/branches/cluster-refactor/scrapy/patches/monkeypatches.py b/scrapy/branches/cluster-refactor/scrapy/patches/monkeypatches.py deleted file mode 100644 index bad1edebd..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/patches/monkeypatches.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Monkey patches - -These are generally a bad idea. -""" -from twisted.web.client import HTTPClientFactory - - -# Extend limit for BeautifulSoup parsing loops -#import sys -#sys.setrecursionlimit(7400) - -def apply_patches(): - patch_HTTPClientFactory_gotHeaders() - - -# XXX: Monkeypatch for twisted.web.client-HTTPClientFactory -# HTTPClientFactory.gotHeaders dies when parsing malformed cookies, -# and the crawler is getting malformed cookies from this site. -_old_gotHeaders = HTTPClientFactory.gotHeaders -# Cookies format: http://www.ietf.org/rfc/rfc2109.txt -# I have choosen not to filter based on this, so we don't filter invalid -# values that could be managed correctly by twisted. -#_COOKIES = re.compile(r'^[^=;]+=[^=;]*(;[^=;]+=[^=;])*$') - -def _is_good_cookie(cookie): - """ Check if a given cookie would make gotHeaders raise an exception """ - cookparts = cookie.split(';') - cook = cookparts[0] - return len(cook.split('=', 1)) == 2 - -def _new_gotHeaders(self, headers): - """ Remove cookies that would make twisted raise an exception """ - if headers.has_key('set-cookie'): - cookies = [cookie for cookie in headers['set-cookie'] - if _is_good_cookie(cookie)] - if cookies: - headers['set-cookie'] = cookies - else: - del headers['set-cookie'] - - return _old_gotHeaders(self, headers) - -def patch_HTTPClientFactory_gotHeaders(): - setattr(HTTPClientFactory, 'gotHeaders', _new_gotHeaders) diff --git a/scrapy/branches/cluster-refactor/scrapy/replay/__init__.py b/scrapy/branches/cluster-refactor/scrapy/replay/__init__.py deleted file mode 100644 index 38512e824..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/replay/__init__.py +++ /dev/null @@ -1,187 +0,0 @@ -from __future__ import with_statement - -import os -import cPickle as pickle -import shutil -import tempfile -import tarfile - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.core.manager import scrapymanager -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings - -class Replay(object): - """ - This is the Replay extension, used to replay a crawling and see if the - items scraped are the same. - """ - - def __init__(self, repfile, mode='play', usedir=False): - """ - Available modes: record, play, update - - repfile can be either a dir (is usedir=True) or a tar.gz file (if - usedir=False) - """ - - # XXX: this is ugly, and should be removed. but how? - cachemw = 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware' - if not cachemw in settings['DOWNLOADER_MIDDLEWARES']: - raise NotConfigured("Cache middleware must be enabled to use Replay") - - self.recording = mode == 'record' - self.updating = mode == 'update' - self.playing = mode == 'play' - - self.options = {} - self.scraped_old = {} - self.scraped_new = {} - self.passed_old = {} - self.passed_new = {} - self.responses_old = {} - self.responses_new = {} - - #self._opendb(repfile, usedir) - self._load(repfile, usedir) - - settings.overrides['CACHE2_DIR'] = self.cache2dir - settings.overrides['CACHE2_EXPIRATION_SECS'] = -1 - settings.overrides['CACHE2_IGNORE_MISSING'] = self.playing or self.updating - settings.overrides['CACHE2_SECTORIZE'] = False - - dispatcher.connect(self.engine_started, signal=signals.engine_started) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.item_passed, signal=signals.item_passed) - dispatcher.connect(self.response_received, signal=signals.response_received) - - - def play(self, args=None, opts=None): - if self.recording: - raise ValueError("Replay.play() not available in record mode") - - args = args or self.options['args'] - opts = opts or self.options['opts'] - scrapymanager.runonce(*args, **opts) - - def record(self, args=None, opts=None): - self.recording = True - self.options.clear() - self.options['args'] = args or [] - self.options['opts'] = opts or {} - - if os.path.exists(self.cache2dir): - shutil.rmtree(self.cache2dir) - else: - os.mkdir(self.cache2dir) - - def update(self, args=None, opts=None): - self.updating = True - if (opts['pages']): - args = self.options['args'] - opts = self.options['opts'] - settings.overrides['CACHE2_EXPIRATION_SECS'] = 0 - settings.overrides['CACHE2_IGNORE_MISSING'] = False - else: - self.play(args, opts) - - def engine_started(self): - log.msg("Replay: recording session in %s" % self.repfile) - - def engine_stopped(self): - if self.recording or self.updating: - log.msg("Replay: recorded in %s: %d/%d scraped/passed items, %d received responses" % \ - (self.repfile, len(self.scraped_old), len(self.passed_old), len(self.responses_old))) - self._save() - self.cleanup() - - def item_scraped(self, item, spider): - if self.recording or self.updating: - self.scraped_old[str(item.guid)] = item.copy() - else: - self.scraped_new[str(item.guid)] = item.copy() - - def item_passed(self, item, spider): - if self.recording or self.updating: - self.passed_old[str(item.guid)] = item.copy() - else: - self.passed_new[str(item.guid)] = item.copy() - - def response_received(self, response, spider): - #key = response.request.fingerprint() - key = response.version() - if (self.recording or self.updating) and key: - self.responses_old[key] = response.copy() - elif key: - self.responses_new[key] = response.copy() - - def _load(self, repfile, usedir): - if usedir: - if os.path.isdir(repfile): - replay_dir = repfile - elif self.recording: - os.makedirs(repfile) - replay_dir = repfile - else: - raise IOError("No such dir: " % repfile) - elif os.path.exists(repfile) and (self.playing or self.updating): - if tarfile.is_tarfile(repfile): - replay_dir = tempfile.mkdtemp(prefix="replay-") - tar = tarfile.open(repfile) - tar.extractall(replay_dir) - tar.close() - else: - raise IOError("Wrong tarfile: %s" % repfile) - elif self.recording: - replay_dir = tempfile.mkdtemp(prefix="replay-") - else: - raise IOError("No such file: %s" % repfile) - - self.cache2dir = os.path.join(replay_dir, "httpcache") - self.replay_dir = replay_dir - self.repfile = repfile - self.usedir = usedir - - self.options_path = os.path.join(replay_dir, "options.db") - self.scraped_path = os.path.join(replay_dir, "items_scraped.db") - self.passed_path = os.path.join(replay_dir, "items_passed.db") - self.responses_path = os.path.join(replay_dir, "responses.db") - - if self.updating: - with open(self.options_path, 'r') as f: - self.options = pickle.load(f) - if self.playing: - with open(self.options_path, 'r') as f: - self.options = pickle.load(f) - with open(self.responses_path, 'r') as f: - self.responses_old = pickle.load(f) - with open(self.scraped_path, 'r') as f: - self.scraped_old = pickle.load(f) - with open(self.passed_path, 'r') as f: - self.passed_old = pickle.load(f) - - def _save(self): - if self.recording or self.updating: - with open(self.options_path, 'w') as f: - pickle.dump(self.options, f) - with open(self.responses_path, 'w') as f: - pickle.dump(self.responses_old, f) - with open(self.scraped_path, 'w') as f: - pickle.dump(self.scraped_old, f) - with open(self.passed_path, 'w') as f: - pickle.dump(self.passed_old, f) - - if not self.usedir: - if self.recording or self.updating: - tar = tarfile.open(self.repfile, "w:gz") - for name in os.listdir(self.replay_dir): - tar.add(os.path.join(self.replay_dir, name), name) - tar.close() - - def cleanup(self): - if not self.usedir and os.path.exists(self.replay_dir): - shutil.rmtree(self.replay_dir) diff --git a/scrapy/branches/cluster-refactor/scrapy/spider/manager.py b/scrapy/branches/cluster-refactor/scrapy/spider/manager.py deleted file mode 100644 index 96203eee0..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/spider/manager.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -SpiderManager is the class which locates and manages all website-specific -spiders -""" -import sys -import os -import urlparse - -from twisted.plugin import getCache - -from scrapy.spider.models import ISpider -from scrapy import log -from scrapy.conf import settings -from scrapy.utils.url import url_is_from_spider - -class SpiderManager(object): - """Spider locator and manager""" - - def __init__(self): - self.loaded = False - self.default_domain = None - self.spider_modules = settings.getlist('SPIDER_MODULES') - self.force_domain = None - - @property - def all(self): - self._load_on_demand() - return self._alldict.values() - - @property - def enabled(self): - self._load_on_demand() - return self._enableddict.values() - - def fromdomain(self, domain_name, include_disabled=True): - return self.asdict(include_disabled=include_disabled).get(domain_name) - - def fromurl(self, url, include_disabled=True): - self._load_on_demand() - if self.force_domain: - return self._alldict.get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self._alldict: # try first locating by domain - return self._alldict[domain] - else: # else search spider by spider - plist = self.all if include_disabled else self.enabled - for p in plist: - if url_is_from_spider(url, p): - return p - return self._alldict.get(self.default_domain) - - def asdict(self, include_disabled=True): - self._load_on_demand() - return self._alldict if include_disabled else self._enableddict - - def _load_on_demand(self): - if not self.loaded: - self.load() - - def _enabled_spiders(self): - if settings['ENABLED_SPIDERS']: - return set(settings['ENABLED_SPIDERS']) - elif settings['ENABLED_SPIDERS_FILE']: - if os.path.exists(settings['ENABLED_SPIDERS_FILE']): - lines = open(settings['ENABLED_SPIDERS_FILE']).readlines() - return set([l.strip() for l in lines if not l.strip().startswith('#')]) - else: - return set() - else: - return set() - - def load(self): - self._invaliddict = {} - self._alldict = {} - self._enableddict = {} - enabled_spiders = self._enabled_spiders() - - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for module in modules: - for spider in self._getspiders(ISpider, module): - try: - ISpider.validateInvariants(spider) - self._alldict[spider.domain_name] = spider - if spider.domain_name in enabled_spiders: - self._enableddict[spider.domain_name] = spider - except Exception, e: - self._invaliddict[spider.domain_name] = spider - # we can't use the log module here because it may not be available yet - print "WARNING: Could not load spider %s: %s" % (spider, e) - self.loaded = True - - def reload(self, skip_domains=None): - """ - Reload all enabled spiders. - - This discovers any spiders added under the spiders module/packages, - removes any spiders removed, updates all enabled spiders code and - updates list of enabled spiders from ENABLED_SPIDERS_FILE or - ENABLED_SPIDERS setting. - - Disabled spiders are intentionally excluded to avoid - syntax/initialization errors. Currently running spiders are also - excluded to avoid inconsistent behaviours. - - If skip_domains is passed those spiders won't be reloaded. - - """ - skip_domains = set(skip_domains or []) - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for m in modules: - reload(m) - self.load() # first call to update list of enabled spiders - reloaded = 0 - pdict = self.asdict(include_disabled=False) - for domain, spider in pdict.iteritems(): - if not domain in skip_domains: - reload(sys.modules[spider.__module__]) - reloaded += 1 - self.load() # second call to update spider instances - log.msg("Reloaded %d/%d scrapy spiders" % (reloaded, len(pdict)), level=log.DEBUG) - - def _getspiders(self, interface, package): - """ - This is an override of twisted.plugin.getPlugin, because we're - interested in catching exceptions thrown when loading spiders such as - KeyboardInterrupt - """ - - try: - allDropins = getCache(package) - for dropin in allDropins.itervalues(): - for plugin in dropin.plugins: - adapted = interface(plugin, None) - if adapted is not None: - yield adapted - except KeyboardInterrupt: - sys.stderr.write("Interrupted while loading Scrapy spiders\n") - sys.exit(2) diff --git a/scrapy/branches/cluster-refactor/scrapy/spider/middleware.py b/scrapy/branches/cluster-refactor/scrapy/spider/middleware.py deleted file mode 100644 index 782f099e5..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/spider/middleware.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Spider middleware manager -""" - - -from twisted.python.failure import Failure - -from scrapy import log -from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class -from scrapy.utils.defer import mustbe_deferred -from scrapy.conf import settings - -def _isiterable(possible_iterator): - try: - return iter(possible_iterator) - except TypeError: - return None - -class SpiderMiddlewareManager(object): - def __init__(self, callback=None, errback=None): - self.loaded = False - self.spider_middleware = [] - self.result_middleware = [] - self.exception_middleware = [] - self.filter_middleware = [] - self.domaininfo = {} - - self.callback = callback or self._callback - self.errback = errback or self._errback - self.load() - - def _add_middleware(self, mw): - if hasattr(mw, 'process_scrape'): - self.spider_middleware.append(mw.process_scrape) - if hasattr(mw, 'process_result'): - self.result_middleware.insert(0, mw.process_result) - if hasattr(mw, 'process_exception'): - self.exception_middleware.insert(0, mw.process_exception) - if hasattr(mw, 'filter'): - self.filter_middleware.insert(0, mw.filter) - - def load(self): - """Load middleware defined in settings module""" - mws = [] - for mwpath in settings.getlist('SPIDER_MIDDLEWARES') or (): - cls = load_class(mwpath) - if cls: - try: - mw = cls() - self._add_middleware(mw) - mws.append(mw) - except NotConfigured: - pass - log.msg("Enabled spider middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) - self.loaded = True - - def scrape(self, request, response, spider): - fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__) - - def process_scrape(response): - for method in self.spider_middleware: - result = method(response=response, spider=spider) - assert result is None or _isiterable(result), \ - 'Middleware %s must returns None or an iterable object, got %s ' % \ - (fname(method), type(result)) - if result is not None: - return result - return self.call(request=request, response=response, spider=spider) - - - def process_result(result): - for method in self.result_middleware: - result = method(response=response, result=result, spider=spider) - assert _isiterable(result), \ - 'Middleware %s must returns an iterable object, got %s ' % \ - (fname(method), type(result)) - return result - - def process_exception(_failure): - exception = _failure.value - for method in self.exception_middleware: - result = method(response=response, exception=exception, spider=spider) - assert result is None or _isiterable(result), \ - 'Middleware %s must returns None, or an iterable object, got %s ' % \ - (fname(method), type(result)) - if result is not None: - return result - return _failure - - dfd = mustbe_deferred(process_scrape, response) - dfd.addErrback(process_exception) - dfd.addCallback(process_result) - return dfd - - def call(self, request, response, spider): - if isinstance(response, Exception): - return self.errback(request, Failure(response), spider) - return self.callback(request, response, spider) - - def _callback(self, request, response, spider): - request.deferred.callback(response) - return request.deferred - - def _errback(self, request, failure, spider): - request.deferred.errback(failure) - return request.deferred - - -class DummyMiddleware(object): - def process_scrape(self, response, spider): - pass - - def process_result(self, response, result, spider): - return result - - def process_exception(self, response, exception, spider): - pass - - def filter(self, item): - return True - diff --git a/scrapy/branches/cluster-refactor/scrapy/spider/models.py b/scrapy/branches/cluster-refactor/scrapy/spider/models.py deleted file mode 100644 index d680b93e9..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/spider/models.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Base class for scrapy spiders -""" -from zope.interface import Interface, Attribute, invariant, implements -from twisted.plugin import IPlugin - -from scrapy.core.exceptions import UsageError - -def _valid_start_urls(obj): - """Check the start urls specified are valid""" - if not obj.start_urls: - raise UsageError("A start url is required") - -def _valid_domain_name(obj): - """Check the domain name specified is valid""" - if not obj.domain_name: - raise UsageError("A site domain name is required") - -def _valid_download_delay(obj): - """Check the download delay is valid, if specified""" - delay = getattr(obj, 'download_delay', 0) - if not type(delay) in (int, long, float): - raise UsageError("download_delay must be numeric") - if float(delay) < 0.0: - raise UsageError("download_delay must be positive") - -class ISpider(Interface, IPlugin) : - """Interface to be implemented by site-specific web spiders""" - - start_urls = Attribute( - """A sequence of URLs to retrieve to initiate the spider for this - site. A single URL may also be provided here.""") - - domain_name = Attribute( - """The domain name of the site to be scraped.""") - - download_delay = Attribute( - """Optional delay in seconds to wait between web page downloads. - Note that this delay does not apply to image downloads. - A delay of less than a second can be specified.""") - - user_agent = Attribute( - """Optional User-Agent to use for this domain""") - - invariant(_valid_start_urls) - invariant(_valid_domain_name) - invariant(_valid_download_delay) - - def parse(self, pagedata) : - """This is first called with the data corresponding to start_url. It - must return a (possibly empty) sequence where each element is either: - * A Request object for further processing. - * An object that extends ScrapedItem (defined in scrapeditem module) - * or None (this will be ignored) - - When a Request object is returned, the Request is scheduled, then - downloaded and finally its results is handled to the Request callback. - That callback must behave the same way as this function. - - When a ScrapedItem is returned, it is passed to the transformation pipeline - and finally the destination systems are updated. - - The simplest way to use this is to have a method in your class for each - page type. So each function knows the layout and how to extract data - for a single page (or set of similar pages). A typical class might work - like: - * parse() parses the landing page and returns Requests - for a category() function to parse category pages. - * category() parses the category pages and returns links and callbacks - for a item() function to parse item pages. - * item() parses the item details page and returns objects that - extend ScrapedItem - """ - pass - - def init_domain(self): - """This is first called to initialize domain specific quirks, like - session cookies or login stuff - """ - pass - - -class BaseSpider(object): - """Base class for scrapy spiders. All spiders must inherit from this - class.""" - - implements(ISpider) - domain_name = None - extra_domain_names = [] diff --git a/scrapy/branches/cluster-refactor/scrapy/stats/corestats.py b/scrapy/branches/cluster-refactor/scrapy/stats/corestats.py deleted file mode 100644 index 20435a04c..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/stats/corestats.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Scrapy extension for collecting scraping stats -""" -import os -import getpass -import socket -import datetime - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy.stats import stats -from scrapy.conf import settings - -class CoreStats(object): - """Scrapy core stats collector""" - - def __init__(self): - stats.setpath('_envinfo/user', getpass.getuser()) - stats.setpath('_envinfo/host', socket.gethostname()) - stats.setpath('_envinfo/logfile', settings['LOGFILE']) - stats.setpath('_envinfo/pid', os.getpid()) - - dispatcher.connect(self.stats_domain_open, signal=stats.domain_open) - dispatcher.connect(self.stats_domain_closing, signal=stats.domain_closing) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.item_passed, signal=signals.item_passed) - dispatcher.connect(self.item_dropped, signal=signals.item_dropped) - dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) - dispatcher.connect(self.request_uploaded, signal=signals.request_uploaded) - - def stats_domain_open(self, domain, spider): - stats.setpath('%s/start_time' % domain, datetime.datetime.now()) - stats.setpath('%s/envinfo' % domain, stats.getpath('_envinfo')) - stats.incpath('_global/domain_count/opened') - - def stats_domain_closing(self, domain, spider, status): - stats.setpath('%s/finish_time' % domain, datetime.datetime.now()) - stats.setpath('%s/finish_status' % domain, 'OK' if status == 'finished' else status) - stats.incpath('_global/domain_count/%s' % status) - - def item_scraped(self, item, spider): - stats.incpath('%s/item_scraped_count' % spider.domain_name) - stats.incpath('_global/item_scraped_count') - - def item_passed(self, item, spider, pipe_output): - stats.incpath('%s/item_passed_count' % spider.domain_name) - stats.incpath('_global/item_passed_count') - - def item_dropped(self, item, spider, exception): - reason = exception.__class__.__name__ - stats.incpath('%s/item_dropped_count' % spider.domain_name) - stats.incpath('%s/item_dropped_reasons_count/%s' % (spider.domain_name, reason)) - stats.incpath('_global/item_dropped_count') - - def response_downloaded(self, response, spider): - stats.incpath('%s/response_count' % spider.domain_name) - stats.incpath('%s/response_status_count/%s' % (spider.domain_name, response.status)) - stats.incpath('_global/response_downloaded_count') - - reslen = len(response) - stats.incpath('%s/transfer/downloaded_bytes' % spider.domain_name, reslen) - stats.incpath('_global/transfer/downloaded_bytes', reslen) - - def request_uploaded(self, request, spider): - reqlen = len(request) - stats.incpath('%s/transfer/uploaded_bytes' % spider.domain_name, reqlen) - stats.incpath('_global/transfer/uploaded_bytes', reqlen) - diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/__init__.py b/scrapy/branches/cluster-refactor/scrapy/tests/__init__.py deleted file mode 100644 index b72b1493f..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -""" """ \ No newline at end of file diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_adaptors.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_adaptors.py deleted file mode 100644 index df1627b5a..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_adaptors.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf8 -*- -import unittest -from scrapy.item import adaptors -from scrapy.xpath.selector import XmlXPathSelector -from scrapy.http import Response, ResponseBody - -class AdaptorsTestCase(unittest.TestCase): - def test_extract(self): - sample_xsel = XmlXPathSelector(text='foobartest') - self.assertEqual(adaptors.extract(sample_xsel.x('/')), - ['foobartest']) - self.assertEqual(adaptors.extract(sample_xsel.x('xml/*')), - ['foobar', 'test']) - self.assertEqual(adaptors.extract(sample_xsel.x('xml/@id')), ['2']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag1')), ['foobar']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag1//text()')), - ['foo', 'bar']) - self.assertEqual(adaptors.extract(sample_xsel.x('//text()')), - ['foo', 'bar', 'test']) - self.assertEqual(adaptors.extract(sample_xsel.x('//tag3/@value')), ['mytag']) - - def test_extract_links(self): - test_data = """ - - """ - sample_response = Response('foobar.com', 'http://foobar.com/dummy', body=ResponseBody(test_data)) - sample_xsel = XmlXPathSelector(sample_response) - self.assertEqual(adaptors.extract_links(sample_xsel.x('//@href')), - [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', - u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.html']) - self.assertEqual(adaptors.extract_links(sample_xsel.x('//a')), - [u'http://foobar.com/dummy/lala1.html', u'http://foobar.com/lala2.html', - u'http://foobar.com/lala3.html', u'http://foobar.com/dummy/lala4.jpg']) - self.assertEqual(adaptors.extract_links((sample_xsel.x('//a[@onclick]'), r'opensomething\(\'(.*?)\'\)')), - [u'http://foobar.com/my_html1.html', u'http://foobar.com/dummy/my_html2.html']) - - def test_to_unicode(self): - self.assertEqual(adaptors.to_unicode(['lala', 'lele', 'luluñ', 1, 'áé']), - [u'lala', u'lele', u'lulu\xf1', u'1', u'\xe1\xe9']) - - def test_regex(self): - func = adaptors.regex(r'href="(.*?)"') - self.assertEqual(func(['dsa', - 'href="lelelel.net"']), - ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) - - def test_unquote_all(self): - self.assertEqual(adaptors.unquote_all([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
    &']) - - def test_unquote(self): - self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&']) - - def test_remove_tags(self): - test_data = ['adsaas
    ', '
    dsadasf
    '] - self.assertEqual(adaptors.remove_tags(test_data), ['adsaas', 'dsadasf']) - - def test_remove_root(self): - self.assertEqual(adaptors.remove_root(['
    lallaadsfsdfdspepepep
    ']), - ['lallaadsfsdfdspepepep
    ']) - - def test_remove_multispaces(self): - self.assertEqual(adaptors.remove_multispaces([' hello, whats up?', 'testing testingtesting testing']), - [' hello, whats up?', 'testing testingtesting testing']) - - def test_strip(self): - self.assertEqual(adaptors.strip([' hi there, sweety ;D ', ' I CAN HAZ TEST?? ']), - ['hi there, sweety ;D', 'I CAN HAZ TEST??']) - - def test_drop_empty_elements(self): - self.assertEqual(adaptors.drop_empty_elements([1, 2, None, 5, None, 6, None, 'hi']), - [1, 2, 5, 6, 'hi']) - - def test_delist(self): - self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']), - 'hi there fellas. this is my test.') - - diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_decompress.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_decompress.py deleted file mode 100644 index 784495b76..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_decompress.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -from unittest import TestCase, main -from scrapy.http import Response, ResponseBody -from scrapy.utils.decompressor import Decompressor - -class ScrapyDecompressTest(TestCase): - uncompressed_body = '' - test_responses = {} - decompressor = Decompressor() - - def setUp(self): - self.datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'compressed') - formats = ['tar', 'xml.bz2', 'xml.gz', 'zip'] - uncompressed_fd = open(os.path.join(self.datadir, 'feed-sample1.xml'), 'r') - self.uncompressed_body = uncompressed_fd.read() - uncompressed_fd.close() - - for format in formats: - fd = open(os.path.join(self.datadir, 'feed-sample1.' + format), 'r') - body = ResponseBody(fd.read()) - fd.close() - self.test_responses[format] = Response('foo.com', 'http://foo.com/bar', body=body) - - def test_tar(self): - ret = self.decompressor.extract(self.test_responses['tar']) - if ret: - self.assertEqual(ret.body.to_string(), self.uncompressed_body) - - def test_zip(self): - ret = self.decompressor.extract(self.test_responses['zip']) - if ret: - self.assertEqual(ret.body.to_string(), self.uncompressed_body) - - def test_gz(self): - ret = self.decompressor.extract(self.test_responses['xml.gz']) - if ret: - self.assertEqual(ret.body.to_string(), self.uncompressed_body) - - def test_bz2(self): - ret = self.decompressor.extract(self.test_responses['xml.bz2']) - if ret: - self.assertEqual(ret.body.to_string(), self.uncompressed_body) - -if __name__ == '__main__': - main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_defaultencoding.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_defaultencoding.py deleted file mode 100644 index 36c5b782d..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_defaultencoding.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys -from unittest import TestCase, main - -class DefaultEncodingTest(TestCase): - def test_defaultencoding(self): - self.assertEqual(sys.getdefaultencoding(), 'utf-8') - -if __name__ == "__main__": - main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_engine.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_engine.py deleted file mode 100644 index f64a8062a..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_engine.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -Scrapy engine tests -""" - -import sys -import os -import urlparse -import unittest - -from twisted.internet import reactor -from twisted.web import server, resource, static, util - -#class TestResource(resource.Resource): -# isLeaf = True -# -# def render_GET(self, request): -# return "hello world!" - -def start_test_site(): - root_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "test_site") - r = static.File(root_dir) -# r.putChild("test", TestResource()) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) - - port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") - return port - - -class CrawlingSession(object): - - def __init__(self): - self.domain = 'scrapytest.org' - self.spider = None - self.respplug = [] - self.reqplug = [] - self.itemresp = [] - self.signals_catched = {} - self.wasrun = False - - def run(self): - self.port = start_test_site() - self.portno = self.port.getHost().port - - from scrapy.spider import spiders - spiders.spider_modules = ['scrapy.tests.test_spiders'] - spiders.reload() - - self.spider = spiders.fromdomain(self.domain) - if self.spider: - self.spider.start_urls = [ - self.geturl("/"), - self.geturl("/redirect"), - ] - - from scrapy.core import signals - from scrapy.core.manager import scrapymanager - from scrapy.core.engine import scrapyengine - from pydispatch import dispatcher - - dispatcher.connect(self.record_signal, signals.engine_started) - dispatcher.connect(self.record_signal, signals.engine_stopped) - dispatcher.connect(self.record_signal, signals.domain_opened) - dispatcher.connect(self.record_signal, signals.domain_idle) - dispatcher.connect(self.record_signal, signals.domain_closed) - dispatcher.connect(self.item_scraped, signals.item_scraped) - dispatcher.connect(self.request_received, signals.request_received) - dispatcher.connect(self.response_downloaded, signals.response_downloaded) - - scrapymanager.runonce(self.domain) - self.port.stopListening() - self.wasrun = True - - def geturl(self, path): - return "http://localhost:%s%s" % (self.portno, path) - - def getpath(self, url): - u = urlparse.urlparse(url) - return u.path - - def item_scraped(self, item, spider, response): - self.itemresp.append((item, response)) - - def request_received(self, request, spider): - self.reqplug.append((request, spider)) - - def response_downloaded(self, response, spider): - self.respplug.append((response, spider)) - - def record_signal(self, *args, **kwargs): - """Record a signal and its parameters""" - signalargs = kwargs.copy() - sig = signalargs.pop('signal') - signalargs.pop('sender', None) - self.signals_catched[sig] = signalargs - -session = CrawlingSession() - - -class EngineTest(unittest.TestCase): - - def setUp(self): - if not session.wasrun: - session.run() - - # disable extensions that cause problems with tests (probably - # because they leave the reactor in an unclean state) - from scrapy.conf import settings - settings.overrides['CLUSTER_MANAGER_ENABLED'] = 0 - - def test_spider_locator(self): - """ - Check the spider is loaded and located properly via the SpiderLocator - """ - assert session.spider is not None - self.assertEqual(session.spider.domain_name, session.domain) - - def test_visited_urls(self): - """ - Make sure certain URls were actually visited - """ - # expected urls that should be visited - must_be_visited = ["/", "/redirect", "/redirected", - "/item1.html", "/item2.html", "/item999.html"] - - urls_visited = set([rp[0].url for rp in session.respplug]) - urls_expected = set([session.geturl(p) for p in must_be_visited]) - assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) - - def test_requests_received(self): - """ - Check requests received - """ - # 3 requests should be received from the spider. start_urls and redirects don't count - self.assertEqual(3, len(session.reqplug)) - - paths_expected = ['/item999.html', '/item2.html', '/item1.html'] - - urls_requested = set([rq[0].url for rq in session.reqplug]) - urls_expected = set([session.geturl(p) for p in paths_expected]) - assert urls_expected <= urls_requested - - def test_responses_downloaded(self): - """ - Check responses downloaded - """ - # response tests - self.assertEqual(6, len(session.respplug)) - - for response, spider in session.respplug: - if session.getpath(response.url) == '/item999.html': - self.assertEqual('404', response.status) - if session.getpath(response.url) == '/redirect': - self.assertEqual('302', response.status) - self.assertEqual(response.domain, spider.domain_name) - - def test_item_data(self): - """ - Check item data - """ - # item tests - self.assertEqual(2, len(session.itemresp)) - for item, response in session.itemresp: - self.assertEqual(item.url, response.url) - if 'item1.html' in item.url: - self.assertEqual('Item 1 name', item.name) - self.assertEqual('100', item.price) - if 'item2.html' in item.url: - self.assertEqual('Item 2 name', item.name) - self.assertEqual('200', item.price) - - def test_signals(self): - """ - Check signals were sent properly - """ - from scrapy.core import signals - - assert signals.engine_started in session.signals_catched - assert signals.engine_stopped in session.signals_catched - assert signals.domain_opened in session.signals_catched - assert signals.domain_idle in session.signals_catched - assert signals.domain_closed in session.signals_catched - - self.assertEqual({'domain': session.domain, 'spider': session.spider}, - session.signals_catched[signals.domain_opened]) - self.assertEqual({'domain': session.domain, 'spider': session.spider}, - session.signals_catched[signals.domain_idle]) - self.assertEqual({'domain': session.domain, 'spider': session.spider, 'status': 'finished'}, - session.signals_catched[signals.domain_closed]) - -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == 'runserver': - port = start_test_site() - print "Test server running at http://localhost:%d/ - hit Ctrl-C to finish." % port.getHost().port - reactor.run() - else: - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_http_request.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_http_request.py deleted file mode 100644 index 511ca9797..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_http_request.py +++ /dev/null @@ -1,230 +0,0 @@ -import unittest -from scrapy.http import Request, Headers -from scrapy.core.scheduler import GroupFilter - -class RequestTest(unittest.TestCase): - """ c14n comparison functions """ - - def test_groupfilter(self): - k1 = Request(url="http://www.scrapy.org/path?key=val").fingerprint() - k2 = Request(url="http://www.scrapy.org/path?key=val&key=val").fingerprint() - self.assertEqual(k1, k2) - - f = GroupFilter() - f.open("mygroup") - self.assertTrue(f.add("mygroup", k1)) - self.assertFalse(f.add("mygroup", k1)) - self.assertFalse(f.add("mygroup", k2)) - - f.open('anothergroup') - self.assertTrue(f.add("anothergroup", k1)) - self.assertFalse(f.add("anothergroup", k1)) - self.assertFalse(f.add("anothergroup", k2)) - - f.close('mygroup') - f.open('mygroup') - self.assertTrue(f.add("mygroup", k2)) - self.assertFalse(f.add("mygroup", k1)) - - def test_headers(self): - # Different ways of setting headers attribute - url = 'http://www.scrapy.org' - headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'} - r = Request(url=url, headers=headers) - p = Request(url=url, headers=r.headers) - - self.assertEqual(r.headers, p.headers) - self.assertFalse(r.headers is headers) - self.assertFalse(p.headers is r.headers) - - # headers must not be unicode - h = Headers({'key1': u'val1', u'key2': 'val2'}) - h[u'newkey'] = u'newval' - for k,v in h.iteritems(): - self.assert_(isinstance(k, str)) - self.assert_(isinstance(v, str)) - - r = Request(url="http://www.example.org/1", referer=u"http://www.example.org") - self.assert_(isinstance(r.headers['referer'], str)) - - def test_eq(self): - url = 'http://www.scrapy.org' - r1 = Request(url=url) - r2 = Request(url=url) - self.assertNotEqual(r1, r2) - - set_ = set() - set_.add(r1) - set_.add(r2) - self.assertEqual(len(set_), 2) - - def test_fingerprint(self): - url = 'http://www.scrapy.org' - r = Request(url=url) - urlhash = r.fingerprint() - - # fingerprint including all initials headers - r.fingerprint_params['exclude_headers'] = [] - fullhash = r.fingerprint() - del r.fingerprint_params['exclude_headers'] - - # all headers are excluded from fingerprint by default - r.headers['Accept'] = 'application/json' - accepthash = r.fingerprint() - r.headers['User-Agent'] = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20071004 Iceweasel/2.0.0.8 (Debian-2.0.0.8-1)' - useragenthash = r.fingerprint() - self.assertEqual(urlhash, accepthash) - self.assertEqual(urlhash, useragenthash) - self.assertEqual(accepthash, useragenthash) - - # take User-Agent header in count as specified in `include_headers` - r.fingerprint_params['include_headers'] = ['user-agent'] - useragenthash = r.fingerprint() - self.assertNotEqual(useragenthash, urlhash) - - # include_headers has precedence over exclude_headers - r.fingerprint_params['exclude_headers'] = ['user-agent', 'Accept'] - self.assertEqual(useragenthash, r.fingerprint()) - - del r.fingerprint_params['include_headers'] - self.assertNotEqual(useragenthash, r.fingerprint()) # exclude_headers is excluding 'User-Agent' from hash - self.assertEqual(fullhash, r.fingerprint()) # all headers previously seted was excluded - - # set more headers - r.headers['Accept-Language'] = 'en-us,en;q=0.5' - r.headers['SESSIONID'] = 'an ugly session id header' - self.assertNotEqual(urlhash, r.fingerprint()) - - # force emtpy include_headers (ignore exclude_headers) - r.fingerprint_params['include_headers'] = [] - self.assertEqual(urlhash, r.fingerprint()) - - # Tamper Function - r.fingerprint_params['tamperfunc'] = lambda req: Request(url=req.url) - self.assertEqual(urlhash, r.fingerprint()) - - # Compare request with None vs {} header - r = Request(url=url, headers=None) - o = Request(url=url, headers={}) - self.assertEqual(o.fingerprint(), r.fingerprint()) - self.assertNotEqual(r, o) - - # Different ways of setting headers attribute - headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'} - r = Request(url=url, headers=headers) - p = Request(url=url, headers=r.headers) - o = Request(url=url) - o.headers = headers - self.assertEqual(r.fingerprint(), o.fingerprint()) - self.assertEqual(r.fingerprint(), p.fingerprint()) - self.assertEqual(o.fingerprint(), p.fingerprint()) - self.assertNotEqual(r, o) - self.assertNotEqual(r, p) - self.assertNotEqual(o, p) - - # same url, implicit method - r1 = Request(url=url) - r2 = Request(url=url, method='GET') - self.assertEqual(r1.fingerprint(), r2.fingerprint()) - - # same url, different method - r3 = Request(url=url, method='POST') - self.assertNotEqual(r1.fingerprint(), r3.fingerprint()) - - # implicit POST method - r3.body = '' - r4 = Request(url=url, body='') - self.assertEqual(r3.fingerprint(), r4.fingerprint()) - - # body is not important in GET or DELETE - r1 = Request(url=url, method='get', body='data') - r2 = Request(url=url, method='get') - self.assertEqual(r1.fingerprint(), r2.fingerprint()) - - r1 = Request(url=url, method='delete', body='data') - r2 = Request(url=url, method='delete') - self.assertEqual(r1.fingerprint(), r2.fingerprint()) - - # no body by default - r1 = Request(url=url, method='POST') - r2 = Request(url=url, method='POST', body=None) - self.assertEqual(r1.fingerprint(), r2.fingerprint()) - - # empty body is equal to None body - r3 = Request(url=url, method='POST', body='') - self.assertEqual(r1.fingerprint(), r3.fingerprint()) - - def test_insensitive_request_fingerprints(self): - url = 'http://www.scrapy.org' - fp = {'include_headers':['Accept','Content-Type']} - r1a = Request(url=url.lower()) - r1b = Request(url=url.upper()) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - - r1a = Request(url=url.lower(), method='get') - r1b = Request(url=url.upper(), method='GET') - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - - r1c = Request(url=url.upper(), method='GET', body='this is not important') - self.assertEqual(r1b.fingerprint(), r1c.fingerprint()) - - r1a = Request(url=url.lower(), method='get', fingerprint_params=fp) - r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - - r1a = Request(url=url.lower(), method='get', headers={'ACCEPT':'Black'}) - r1b = Request(url=url.upper(), method='GET', headers={'ACCEPT':'Black'}) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - - r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'}) - r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'ACCEPT':'Black'}) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - - r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'}) - r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black'}) - r1c = Request(url=url.upper(), method='gEt', fingerprint_params=fp, headers={'accept':'Black'}) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - self.assertEqual(r1a.fingerprint(), r1c.fingerprint()) - - r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'}) - r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'}) - r1c = Request(url=url.upper(), method='Get', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'}) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - self.assertEqual(r1a.fingerprint(), r1c.fingerprint()) - - r1a = Request(url=url.lower(), method='Post', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'}) - r1b = Request(url=url.upper(), method='POST', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'}) - r1c = Request(url=url.upper(), method='posT', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'}) - self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) - self.assertEqual(r1a.fingerprint(), r1c.fingerprint()) - - def test_url(self): - """Request url tests""" - r = Request(url="http://www.scrapy.org/path") - self.assertEqual(r.url, "http://www.scrapy.org/path") - - # url quoting on attribute assign - r.url = "http://www.scrapy.org/blank%20space" - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") - r.url = "http://www.scrapy.org/blank space" - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") - - # url quoting on creation - r = Request(url="http://www.scrapy.org/blank%20space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") - r = Request(url="http://www.scrapy.org/blank space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") - - # url coercion to string - r.url = u"http://www.scrapy.org/test" - self.assert_(isinstance(r.url, str)) - - # url encoding - r = Request(url=u"http://www.scrapy.org/price/\xa3", url_encoding="utf-8") - self.assert_(isinstance(r.url, str)) - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") - -if __name__ == "__main__": - unittest.main() - - diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_http_response.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_http_response.py deleted file mode 100644 index 5c3a7692f..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_http_response.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from unittest import TestCase, main -import libxml2 -from scrapy.http import ResponseBody - -class ResponseTest(TestCase): - def test_responsebodyencoding(self): - string = 'кириллический текст' - unicode_string = unicode(string, 'utf-8') - body_cp1251 = unicode_string.encode('cp1251') - body = ResponseBody(body_cp1251, 'cp1251') - self.assertEqual(body.to_unicode(), unicode_string) - self.assertEqual(isinstance(body.to_unicode(), unicode), True) - self.assertEqual(body.to_string('utf-8'), string) - self.assertEqual(isinstance(body.to_string('utf-8'), str), True) - self.assertEqual(body.to_string(), body_cp1251) - self.assertEqual(isinstance(body.to_string('utf-8'), str), True) - -if __name__ == "__main__": - main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_http_url.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_http_url.py deleted file mode 100644 index 08266a262..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_http_url.py +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -from scrapy.http import Url - -class UrlClassTest(unittest.TestCase): - - def test_url_attributes(self): - u = Url("http://scrapy.org/wiki/info") - self.assertEqual("scrapy.org", u.hostname) - self.assertEqual("/wiki/info", u.path) - self.assertEqual(None, u.username) - self.assertEqual(None, u.password) - - u = Url("http://someuser:somepass@example.com/ticket/query?owner=pablo") - self.assertEqual("someuser", u.username) - self.assertEqual("somepass", u.password) - self.assertEqual("example.com", u.hostname) - self.assertEqual("/ticket/query", u.path) - self.assertEqual("owner=pablo", u.query) - - u = Url("http://example.com/somepage.html#fragment-1") - self.assertEqual("fragment-1", u.fragment) - - u = Url("file:///home/pablo/file.txt") - self.assertEqual("/home/pablo/file.txt", u.path) - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_libxml2.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_libxml2.py deleted file mode 100644 index 35b0cade4..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_libxml2.py +++ /dev/null @@ -1,43 +0,0 @@ -from unittest import TestCase, main -import libxml2 -from scrapy.http import ResponseBody, Response - -class Libxml2Test(TestCase): - def setUp(self): - libxml2.debugMemory(1) - - def tearDown(self): - libxml2.cleanupParser() - leaked_bytes = libxml2.debugMemory(0) - assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes - - def test_xpath(self): - #this test will fail in version 2.6.27 but passes on 2.6.29+ - html = "123" - node = libxml2.htmlParseDoc(html, 'utf-8') - result = [str(r) for r in node.xpathEval('//text()')] - self.assertEquals(result, ['1', '2', '3']) - node.freeDoc() - -class ResponseLibxml2DocTest(TestCase): - def setUp(self): - libxml2.debugMemory(1) - - def tearDown(self): - libxml2.cleanupParser() - leaked_bytes = libxml2.debugMemory(0) - assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes - - def test_getlibxml2doc(self): - # test to simulate '\x00' char in body of html page - #this method don't should raise TypeError Exception - from scrapy.core.manager import scrapymanager - scrapymanager.configure() - - self.body_content = 'test problematic \x00 body' - self.problematic_body = ResponseBody(self.body_content, 'utf-8') - response = Response('example.com', 'http://example.com/catalog/product/blabla-123', body=self.problematic_body) - response.getlibxml2doc() - -if __name__ == "__main__": - main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_link.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_link.py deleted file mode 100644 index e4cf3ee58..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_link.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest - -from scrapy.http import Response -from scrapy.link import LinkExtractor - -class LinkExtractorTestCase(unittest.TestCase): - def test_basic(self): - html = """Page title<title> - <body><p><a href="item/12.html">Item 12</a></p> - <p><a href="/about.html">About us</a></p> - <img src="/logo.png" alt="Company logo (not a link)" /> - <p><a href="../othercat.html">Other category</a></p> - <p><a href="/" /></p> - </body></html>""" - response = Response("example.org", "http://example.org/somepage/index.html", body=html) - - lx = LinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_urls(response), - {'http://example.org/somepage/item/12.html': 'Item 12', - 'http://example.org/about.html': 'About us', - 'http://example.org/othercat.html': 'Other category', - 'http://example.org/': ''}) - - def test_base_url(self): - html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> - <body><p><a href="item/12.html">Item 12</a></p> - </body></html>""" - response = Response("example.org", "http://example.org/somepage/index.html", body=html) - - lx = LinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_urls(response), - {'http://otherdomain.com/base/item/12.html': 'Item 12'}) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_pipeline_images.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_pipeline_images.py deleted file mode 100644 index b3b5d09a6..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_pipeline_images.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -from scrapy.contrib.pipeline.images import thumbnail_name, image_path - -class ImagesPipelineTestCase(unittest.TestCase): - def test_image_path(self): - self.assertEqual(image_path("https://dev.mydeco.com/mydeco.gif"), - 'dev.mydeco.com/mydeco.gif') - self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"), - 'www.maddiebrown.co.uk/catalogue-items/image_54642_12175_95307.jpg') - self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif"), - 'dev.mydeco.com/two/dirs/with spaces+signs.gif') - self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"), - 'www.dfsonline.co.uk/4507be485f38b0da8a0be9eb2e1dfab8a19223f2') - self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"), - 'www.dorma.co.uk/images/product_details/2532.jpg') - self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"), - 'www.dorma.co.uk/images/product_details/2532') - - def test_thumbnail_name(self): - name = '50' - self.assertEqual(thumbnail_name("/tmp/foo.jpg", name), - '/tmp/foo_50.jpg') - self.assertEqual(thumbnail_name("foo.png", name), - 'foo_50.jpg') - self.assertEqual(thumbnail_name("/tmp/foo", name), - '/tmp/foo_50.jpg') - self.assertEqual(thumbnail_name("/tmp/some.name/foo", name), - '/tmp/some.name/foo_50.jpg') - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_serialization.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_serialization.py deleted file mode 100644 index 1c7d3f18c..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_serialization.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest -from scrapy.utils.serialization import serialize, unserialize, serialize_funcs - -class SerializationTest(unittest.TestCase): - def test_string_serialization(self): - """Test simple string serialization""" - - s = "string to serialize" - for format in serialize_funcs.iterkeys(): - self.assertEqual(s, unserialize(serialize(s))) - - def test_dict_serialization(self): - """Test dict serialization""" - - # using only string keys/values since it's the only data type that - # works with all serializers - d = {'one': 'item one', 'two': 'item two'} - for format in serialize_funcs.iterkeys(): - self.assertEqual(d, unserialize(serialize(d))) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/testplugin.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/testplugin.py deleted file mode 100644 index 0454db00e..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/testplugin.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -This is a spider for the unittest sample site. - -See scrapy/tests/test_engine.py for more info. -""" -import re - -from scrapy.spider import BaseSpider -from scrapy.item import ScrapedItem -from scrapy.link import LinkExtractor -from scrapy.http import Request - -class TestSpider(BaseSpider): - domain_name = "scrapytest.org" - extra_domain_names = ["localhost"] - start_urls = ['http://localhost'] - - itemurl_re = re.compile("item\d+.html") - name_re = re.compile("<h1>(.*?)</h1>", re.M) - price_re = re.compile(">Price: \$(.*?)<", re.M) - - def parse(self, response): - xlink = LinkExtractor() - itemre = re.compile(self.itemurl_re) - for url in xlink.extract_urls(response): - if itemre.search(url): - yield Request(url=url, callback=self.parse_item) - - def parse_item(self, response): - item = ScrapedItem() - m = self.name_re.search(response.body.to_string()) - if m: - item.name = m.group(1) - item.url = response.url - m = self.price_re.search(response.body.to_string()) - if m: - item.price = m.group(1) - return [item] - - -SPIDER = TestSpider() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_storedb.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_storedb.py deleted file mode 100644 index 4e2d920b3..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_storedb.py +++ /dev/null @@ -1,76 +0,0 @@ -from datetime import datetime, timedelta -from time import sleep - -from twisted.trial import unittest -from scrapy.conf import settings - - -class ProductComparisonTestCase(unittest.TestCase): - """ Test product comparison functions """ - - def setUp(self): - self.test_db = settings.get('TEST_SCRAPING_DB') - if not self.test_db: - raise unittest.SkipTest("Missing TEST_SCRAPING_DB setting") - - try: - import MySQLdb - from scrapy.utils.db import mysql_connect - mysql_connect(self.test_db) - except ImportError: - raise unittest.SkipTest("MySQLdb module not available") - except MySQLdb.OperationalError: - raise unittest.SkipTest("Test database not available at: %s" % self.test_db) - - def test_domaindatahistory(self): - from scrapy.store.db import DomainDataHistory - from time import sleep - - ddh = DomainDataHistory(self.test_db, 'domain_data_history') - c = ddh.mysql_conn.cursor() - c.execute("DELETE FROM domain_data_history") - ddh.mysql_conn.commit() - - def now_nomicro(): - now = datetime.now() - return now - timedelta(microseconds=now.microsecond) - - assert hasattr(ddh.get('scrapy.org'), '__iter__') - self.assertEqual(list(ddh.get('scrapy.org')), []) - - self.assertEqual(ddh.domain_count(), 0) - - now = now_nomicro() - ddh.put('scrapy.org', 'value', timestamp=now) - self.assertEqual(list(ddh.get('scrapy.org')), [(now, 'value')]) - self.assertEqual(list(ddh.get('scrapy2.org')), []) - - sleep(1) - now2 = now_nomicro() - ddh.put('scrapy.org', 'newvalue', timestamp=now2) - self.assertEqual(list(ddh.getall('scrapy.org')), [(now2, 'newvalue'), (now, 'value')]) - - self.assertEqual(ddh.getlast('scrapy.org'), (now2, 'newvalue')) - self.assertEqual(ddh.getlast('scrapy.org', offset=1), (now, 'value')) - self.assertEqual(ddh.getlast('scrapy2.org'), None) - - ddh.remove('scrapy.org') - self.assertEqual(list(ddh.get('scrapy.org')), []) - - now3 = now_nomicro() - d1 = {'name': 'John', 'surname': 'Doe'} - ddh.put('scrapy.org', d1, timestamp=now3) - self.assertEqual(list(ddh.getall('scrapy.org')), [(now3, d1)]) - - # get path support - self.assertEqual(ddh.getlast('scrapy.org', path='name'), (now3, 'John')) - # behaviour for non existant paths - self.assertEqual(ddh.getlast('scrapy.org', path='name2'), (now3, None)) - - self.assertEqual(ddh.domain_count(), 1) - - self.assertEqual(list(ddh.getlast_alldomains()), - [('scrapy.org', now3, {'surname': 'Doe', 'name': 'John'})]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_datatypes.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_datatypes.py deleted file mode 100644 index 629eaf2ff..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_datatypes.py +++ /dev/null @@ -1,26 +0,0 @@ -import unittest -from scrapy.utils.datatypes import PriorityQueue - -class DatatypesTestCase(unittest.TestCase): - - def test_priority_queue(self): - pq = PriorityQueue() - - pq.put('b', priority=1) - pq.put('a', priority=1) - pq.put('c', priority=1) - pq.put('z', priority=0) - pq.put('d', priority=2) - - v = [] - p = [] - while not pq.empty(): - priority, value = pq.get() - v.append(value) - p.append(priority) - - self.assertEqual(v, ['z', 'b', 'a', 'c', 'd']) - self.assertEqual(p, [0, 1, 1, 1, 2]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_markup.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_markup.py deleted file mode 100644 index 0624560d8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_markup.py +++ /dev/null @@ -1,113 +0,0 @@ -import unittest - -from scrapy.utils.markup import remove_entities, replace_tags, remove_comments -from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags - -class UtilsMarkupTest(unittest.TestCase): - - def test_remove_entities(self): - # make sure it always return uncode - assert isinstance(remove_entities('no entities'), unicode) - assert isinstance(remove_entities('Price: £100!'), unicode) - - # regular conversions - self.assertEqual(remove_entities(u'As low as £100!'), - u'As low as \xa3100!') - self.assertEqual(remove_entities('As low as £100!'), - u'As low as \xa3100!') - self.assertEqual(remove_entities('redirectTo=search&searchtext=MR0221Y&aff=buyat&affsrc=d_data&cm_mmc=buyat-_-ELECTRICAL & SEASONAL-_-MR0221Y-_-9-carat gold ½oz solid crucifix pendant'), - u'redirectTo=search&searchtext=MR0221Y&aff=buyat&affsrc=d_data&cm_mmc=buyat-_-ELECTRICAL & SEASONAL-_-MR0221Y-_-9-carat gold \xbdoz solid crucifix pendant') - # keep some entities - self.assertEqual(remove_entities('<b>Low < High & Medium £ six</b>', keep=['lt', 'amp']), - u'<b>Low < High & Medium \xa3 six</b>') - - # illegal entities - self.assertEqual(remove_entities('a < b &illegal; c � six', remove_illegal=False), - u'a < b &illegal; c � six') - self.assertEqual(remove_entities('a < b &illegal; c � six', remove_illegal=True), - u'a < b c six') - - - - def test_replace_tags(self): - # make sure it always return uncode - assert isinstance(replace_tags('no entities'), unicode) - - self.assertEqual(replace_tags(u'This text contains <a>some tag</a>'), - u'This text contains some tag') - - self.assertEqual(replace_tags('This text is very im<b>port</b>ant', ' '), - u'This text is very im port ant') - - # multiline tags - self.assertEqual(replace_tags('Click <a class="one"\r\n href="url">here</a>'), - u'Click here') - - def test_remove_comments(self): - # make sure it always return unicode - assert isinstance(remove_comments('without comments'), unicode) - assert isinstance(remove_comments('<!-- with comments -->'), unicode) - - # text without comments - self.assertEqual(remove_comments(u'text without comments'), u'text without comments') - - # text with comments - self.assertEqual(remove_comments(u'<!--text with comments-->'), u'') - self.assertEqual(remove_comments(u'Hello<!--World-->'),u'Hello') - - def test_remove_tags(self): - # make sure it always return unicode - assert isinstance(remove_tags('no tags'), unicode) - assert isinstance(remove_tags('no tags', which_ones=('p',)), unicode) - assert isinstance(remove_tags('<p>one tag</p>'), unicode) - assert isinstance(remove_tags('<p>one tag</p>', which_ones=('p')), unicode) - assert isinstance(remove_tags('<a>link</a>', which_ones=('b',)), unicode) - - # text without tags - self.assertEqual(remove_tags(u'no tags'), u'no tags') - self.assertEqual(remove_tags(u'no tags', which_ones=('p','b',)), u'no tags') - - # text with tags - self.assertEqual(remove_tags(u'<p>one p tag</p>'), u'one p tag') - self.assertEqual(remove_tags(u'<p>one p tag</p>', which_ones=('b',)), u'<p>one p tag</p>') - - self.assertEqual(remove_tags(u'<b>not will removed</b><i>i will removed</i>', which_ones=('i',)), - u'<b>not will removed</b>i will removed') - - # text with tags and attributes - self.assertEqual(remove_tags(u'<p align="center" class="one">texty</p>'), u'texty') - self.assertEqual(remove_tags(u'<p align="center" class="one">texty</p>', which_ones=('b',)), - u'<p align="center" class="one">texty</p>') - - def test_remove_tags_with_content(self): - # make sure it always return unicode - assert isinstance(remove_tags_with_content('no tags'), unicode) - assert isinstance(remove_tags_with_content('no tags', which_ones=('p',)), unicode) - assert isinstance(remove_tags_with_content('<p>one tag</p>', which_ones=('p',)), unicode) - assert isinstance(remove_tags_with_content('<a>link</a>', which_ones=('b',)), unicode) - - # text without tags - self.assertEqual(remove_tags_with_content(u'no tags'), u'no tags') - self.assertEqual(remove_tags_with_content(u'no tags', which_ones=('p','b',)), u'no tags') - - # text with tags - self.assertEqual(remove_tags_with_content(u'<p>one p tag</p>'), u'<p>one p tag</p>') - self.assertEqual(remove_tags_with_content(u'<p>one p tag</p>', which_ones=('p',)), u'') - - self.assertEqual(remove_tags_with_content(u'<b>not will removed</b><i>i will removed</i>', which_ones=('i',)), - u'<b>not will removed</b>') - - def test_remove_escape_chars(self): - # make sure it always return unicode - assert isinstance(remove_escape_chars('no ec'), unicode) - assert isinstance(remove_escape_chars('no ec', which_ones=('\n','\t',)), unicode) - - # text without escape chars - self.assertEqual(remove_escape_chars(u'no ec'), u'no ec') - self.assertEqual(remove_escape_chars(u'no ec', which_ones=('\n',)), u'no ec') - - # text with escape chars - self.assertEqual(remove_escape_chars(u'escape\n\n'), u'escape') - self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n') - self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n') - diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_url.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_url.py deleted file mode 100644 index bc7dae523..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_url.py +++ /dev/null @@ -1,151 +0,0 @@ -import unittest -from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url - -class UrlUtilsTest(unittest.TestCase): - - def test_url_is_from_any_domain(self): - url = 'http://www.wheele-bin-art.co.uk/get/product/123' - self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) - self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) - - url = 'http://wheele-bin-art.co.uk/get/product/123' - self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) - self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) - - url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' - self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) - self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) - - def test_safe_url_string(self): - # Motoko Kusanagi (Cyborg from Ghost in the Shell) - motoko = u'\u8349\u8599 \u7d20\u5b50' - self.assertEqual(safe_url_string(motoko), # note the %20 for space - '%E8%8D%89%E8%96%99%20%E7%B4%A0%E5%AD%90') - self.assertEqual(safe_url_string(motoko), - safe_url_string(safe_url_string(motoko))) - self.assertEqual(safe_url_string(u'\xa9'), # copyright symbol - '%C2%A9') - self.assertEqual(safe_url_string(u'\xa9', 'iso-8859-1'), - '%A9') - self.assertEqual(safe_url_string("http://www.scrapy.org/"), - 'http://www.scrapy.org/') - - alessi = u'/ecommerce/oggetto/Te \xf2/tea-strainer/1273' - - self.assertEqual(safe_url_string(alessi), - '/ecommerce/oggetto/Te%20%C3%B2/tea-strainer/1273') - - self.assertEqual(safe_url_string("http://www.example.com/test?p(29)url(http://www.another.net/page)"), - "http://www.example.com/test?p(29)url(http://www.another.net/page)") - self.assertEqual(safe_url_string("http://www.example.com/Brochures_&_Paint_Cards&PageSize=200"), - "http://www.example.com/Brochures_&_Paint_Cards&PageSize=200") - - def test_safe_download_url(self): - self.assertEqual(safe_download_url('http://www.scrapy.org/../'), - 'http://www.scrapy.org/') - self.assertEqual(safe_download_url('http://www.scrapy.org/../../images/../image'), - 'http://www.scrapy.org/image') - self.assertEqual(safe_download_url('http://www.scrapy.org/dir/'), - 'http://www.scrapy.org/dir/') - - def test_url_query_parameter(self): - self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "id"), - '200') - self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault"), - 'mydefault') - self.assertEqual(url_query_parameter("product.html?id=", "id"), - None) - self.assertEqual(url_query_parameter("product.html?id=", "id", keep_blank_values=1), - '') - - def test_add_or_replace_parameter(self): - url = 'http://domain/test' - self.assertEqual(add_or_replace_parameter(url, 'arg', 'v'), - 'http://domain/test?arg=v') - url = 'http://domain/test?arg1=v1&arg2=v2&arg3=v3' - self.assertEqual(add_or_replace_parameter(url, 'arg4', 'v4'), - 'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4') - self.assertEqual(add_or_replace_parameter(url, 'arg3', 'nv3'), - 'http://domain/test?arg1=v1&arg2=v2&arg3=nv3') - url = 'http://domain/test?arg1=v1' - self.assertEqual(add_or_replace_parameter(url, 'arg2', 'v2', sep=';'), - 'http://domain/test?arg1=v1;arg2=v2') - self.assertEqual(add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20'), - 'http://domain/moreInfo.asp?prodID=20') - - def test_url_query_cleaner(self): - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id'), - 'product.html?id=200') - self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']), - 'product.html?id=200&name=wired') - - def test_canonicalize_url(self): - # no query arguments - self.assertEqual(canonicalize_url("http://www.example.com"), - "http://www.example.com") - - # typical usage - self.assertEqual(canonicalize_url("http://www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3") - self.assertEqual(canonicalize_url("http://www.example.com/do?c=1&b=2&a=3"), - "http://www.example.com/do?a=3&b=2&c=1") - self.assertEqual(canonicalize_url("http://www.example.com/do?&a=1"), - "http://www.example.com/do?a=1") - - # sorting by argument values - self.assertEqual(canonicalize_url("http://www.example.com/do?c=3&b=5&b=2&a=50"), - "http://www.example.com/do?a=50&b=2&b=5&c=3") - - # using keep_blank_values - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"), - "http://www.example.com/do?a=2") - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=True), - "http://www.example.com/do?a=2&b=") - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"), - "http://www.example.com/do?a=2") - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=True), - "http://www.example.com/do?a=2&b=&c=") - - # spaces - self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"), - "http://www.example.com/do?a=1&q=a+space") - self.assertEqual(canonicalize_url("http://www.example.com/do?q=a+space&a=1"), - "http://www.example.com/do?a=1&q=a+space") - self.assertEqual(canonicalize_url("http://www.example.com/do?q=a%20space&a=1"), - "http://www.example.com/do?a=1&q=a+space") - - # normalize percent-encoding case (in paths) - self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"), - "http://www.example.com/a%A3do"), - # normalize percent-encoding case (in query arguments) - self.assertEqual(canonicalize_url("http://www.example.com/do?k=b%a3"), - "http://www.example.com/do?k=b%A3") - - # non-ASCII percent-encoding in paths - self.assertEqual(canonicalize_url("http://www.example.com/a do?a=1"), - "http://www.example.com/a%20do?a=1"), - self.assertEqual(canonicalize_url("http://www.example.com/a %20do?a=1"), - "http://www.example.com/a%20%20do?a=1"), - self.assertEqual(canonicalize_url("http://www.example.com/a do\xc2\xa3.html?a=1"), - "http://www.example.com/a%20do%C2%A3.html?a=1") - # non-ASCII percent-encoding in query arguments - self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=\xa3500&a=5&z=3"), - u"http://www.example.com/do?a=5&price=%C2%A3500&z=3") - self.assertEqual(canonicalize_url("http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"), - "http://www.example.com/do?a=5&price=%C2%A3500&z=3") - self.assertEqual(canonicalize_url("http://www.example.com/do?price(\xc2\xa3)=500&a=1"), - "http://www.example.com/do?a=1&price%28%C2%A3%29=500") - - # urls containing auth and ports - self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com:81/do?now=1"), - u"http://user:pass@www.example.com:81/do?now=1") - - # remove fragments - self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag"), - u"http://user:pass@www.example.com/do?a=1") - self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", keep_fragments=True), - u"http://user:pass@www.example.com/do?a=1#frag") - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_xml.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_xml.py deleted file mode 100644 index fd59a446b..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_utils_xml.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import unittest - -from scrapy.utils.xml import xpathselector_iternodes -from scrapy.http import Response - -class UtilsXmlTestCase(unittest.TestCase): - - def test_iterator(self): - body = """<?xml version="1.0" encoding="UTF-8"?> -<products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd"> - <product id="001"> - <type>Type 1</type> - <name>Name 1</name> - </product> - <product id="002"> - <type>Type 2</type> - <name>Name 2</name> - </product> -</products> - """ - response = Response(domain="example.com", url="http://example.com", body=body) - attrs = [] - for x in xpathselector_iternodes(response, 'product'): - attrs.append((x.x("@id").extract(), x.x("name/text()").extract(), x.x("./type/text()").extract())) - - self.assertEqual(attrs, - [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) - - def test_iterator_text(self): - body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" - - self.assertEqual([x.x("text()").extract() for x in xpathselector_iternodes(body, 'product')], - [[u'one'], [u'two']]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_xpath.py b/scrapy/branches/cluster-refactor/scrapy/tests/test_xpath.py deleted file mode 100644 index 99c06119b..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/tests/test_xpath.py +++ /dev/null @@ -1,225 +0,0 @@ -import re -import unittest - -import libxml2 - -from scrapy.http import Response -from scrapy.xpath.selector import XmlXPathSelector, HtmlXPathSelector - -class XPathTestCase(unittest.TestCase): - - def setUp(self): - libxml2.debugMemory(1) - - def tearDown(self): - libxml2.cleanupParser() - leaked_bytes = libxml2.debugMemory(0) - assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes - - def test_selector_simple(self): - """Simple selector tests""" - body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>" - response = Response(domain="example.com", url="http://example.com", body=body) - xpath = HtmlXPathSelector(response) - - xl = xpath.x('//input') - self.assertEqual(2, len(xl)) - for x in xl: - assert isinstance(x, HtmlXPathSelector) - - self.assertEqual(xpath.x('//input').extract(), - [x.extract() for x in xpath.x('//input')]) - - self.assertEqual([x.extract() for x in xpath.x("//input[@name='a']/@name")], - [u'a']) - self.assertEqual([x.extract() for x in xpath.x("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0']) - - self.assertEqual(xpath.x("concat('xpath', 'rules')").extract(), - [u'xpathrules']) - self.assertEqual([x.extract() for x in xpath.x("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12']) - - def test_selector_same_type(self): - """Test XPathSelector returning the same type in x() method""" - text = '<p>test<p>' - assert isinstance(XmlXPathSelector(text=text).x("//p")[0], - XmlXPathSelector) - assert isinstance(HtmlXPathSelector(text=text).x("//p")[0], - HtmlXPathSelector) - - def test_selector_xml_html(self): - """Test that XML and HTML XPathSelector's behave differently""" - - # some text which is parsed differently by XML and HTML flavors - text = '<div><img src="a.jpg"><p>Hello</div>' - - self.assertEqual(XmlXPathSelector(text=text).x("//div").extract(), - [u'<div><img src="a.jpg"><p>Hello</p></img></div>']) - - self.assertEqual(HtmlXPathSelector(text=text).x("//div").extract(), - [u'<div><img src="a.jpg"><p>Hello</p></div>']) - - def test_selector_nested(self): - """Nested selector tests""" - body = """<body> - <div class='one'> - <ul> - <li>one</li><li>two</li> - </ul> - </div> - <div class='two'> - <ul> - <li>four</li><li>five</li><li>six</li> - </ul> - </div> - </body>""" - - response = Response(domain="example.com", url="http://example.com", body=body) - x = HtmlXPathSelector(response) - - divtwo = x.x('//div[@class="two"]') - self.assertEqual(divtwo.x("//li").extract(), - ["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"]) - self.assertEqual(divtwo.x("./ul/li").extract(), - ["<li>four</li>", "<li>five</li>", "<li>six</li>"]) - self.assertEqual(divtwo.x(".//li").extract(), - ["<li>four</li>", "<li>five</li>", "<li>six</li>"]) - self.assertEqual(divtwo.x("./li").extract(), - []) - - def test_selector_re(self): - body = """<div>Name: Mary - <ul> - <li>Name: John</li> - <li>Age: 10</li> - <li>Name: Paul</li> - <li>Age: 20</li> - </ul> - Age: 20 - </div> - - """ - response = Response(domain="example.com", url="http://example.com", body=body) - x = HtmlXPathSelector(response) - - name_re = re.compile("Name: (\w+)") - self.assertEqual(x.x("//ul/li").re(name_re), - ["John", "Paul"]) - self.assertEqual(x.x("//ul/li").re("Age: (\d+)"), - ["10", "20"]) - - def test_selector_over_text(self): - hxs = HtmlXPathSelector(text='<root>lala</root>') - self.assertEqual(hxs.extract(), - u'<html><body><root>lala</root></body></html>') - - xxs = XmlXPathSelector(text='<root>lala</root>') - self.assertEqual(xxs.extract(), - u'<root>lala</root>') - - xxs = XmlXPathSelector(text='<root>lala</root>') - self.assertEqual(xxs.x('.').extract(), - [u'<root>lala</root>']) - - - def test_selector_namespaces_simple(self): - body = """ - <test xmlns:somens="http://scrapy.org"> - <somens:a id="foo"/> - <a id="bar">found</a> - </test> - """ - - response = Response(domain="example.com", url="http://example.com", body=body) - x = XmlXPathSelector(response) - - x.register_namespace("somens", "http://scrapy.org") - self.assertEqual(x.x("//somens:a").extract(), - ['<somens:a id="foo"/>']) - - - def test_selector_namespaces_multiple(self): - body = """<?xml version="1.0" encoding="UTF-8"?> -<BrowseNode xmlns="http://webservices.amazon.com/AWSECommerceService/2005-10-05" - xmlns:b="http://somens.com" - xmlns:p="http://www.scrapy.org/product" > - <b:Operation>hello</b:Operation> - <TestTag b:att="value"><Other>value</Other></TestTag> - <p:SecondTestTag><material/><price>90</price><p:name>Dried Rose</p:name></p:SecondTestTag> -</BrowseNode> - """ - response = Response(domain="example.com", url="http://example.com", body=body) - x = XmlXPathSelector(response) - - x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") - x.register_namespace("p", "http://www.scrapy.org/product") - x.register_namespace("b", "http://somens.com") - self.assertEqual(len(x.x("//xmlns:TestTag")), 1) - self.assertEqual(x.x("//b:Operation/text()").extract()[0], 'hello') - self.assertEqual(x.x("//xmlns:TestTag/@b:att").extract()[0], 'value') - self.assertEqual(x.x("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') - self.assertEqual(x.x("//p:SecondTestTag").x("./xmlns:price/text()")[0].extract(), '90') - self.assertEqual(x.x("//p:SecondTestTag/xmlns:material").extract()[0], '<material/>') - - def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) - - meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' - head = u'<head>' + meta + u'</head>' - body_content = u'<span id="blank">\xa3</span>' - body = u'<body>' + body_content + u'</body>' - html = u'<html>' + head + body + u'</html>' - encoding = 'utf-8' - html_utf8 = html.encode(encoding) - - headers = {'Content-Type': ['text/html; charset=utf-8']} - response = Response(domain="example.com", url="http://example.com", headers=headers, body=html_utf8) - x = HtmlXPathSelector(response) - self.assertEquals(x.x("//span[@id='blank']/text()").extract(), - [u'\xa3']) - - def test_null_bytes(self): - hxs = HtmlXPathSelector(text='<root>la\x00la</root>') - self.assertEqual(hxs.extract(), - u'<html><body><root>lala</root></body></html>') - - xxs = XmlXPathSelector(text='<root>la\x00la</root>') - self.assertEqual(xxs.extract(), - u'<root>lala</root>') - - def test_unquote(self): - xmldoc = '\n'.join(( - '<root>', - ' lala', - ' <node>', - ' blabla&more<!--comment-->a<b>test</b>oh', - ' <![CDATA[lalalal&ppppp<b>PPPP</b>ppp&la]]>', - ' </node>', - ' pff', - '</root>')) - xxs = XmlXPathSelector(text=xmldoc) - - self.assertEqual(xxs.extract_unquoted(), u'') - - self.assertEqual(xxs.x('/root').extract_unquoted(), [u'']) - self.assertEqual(xxs.x('/root/text()').extract_unquoted(), [ - u'\n lala\n ', - u'\n pff\n']) - - self.assertEqual(xxs.x('//*').extract_unquoted(), [u'', u'', u'']) - self.assertEqual(xxs.x('//text()').extract_unquoted(), [ - u'\n lala\n ', - u'\n blabla&more', - u'a', - u'test', - u'oh\n ', - u'lalalal&ppppp<b>PPPP</b>ppp&la', - u'\n ', - u'\n pff\n']) - - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/datatypes.py b/scrapy/branches/cluster-refactor/scrapy/utils/datatypes.py deleted file mode 100644 index ab32a82ce..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/datatypes.py +++ /dev/null @@ -1,458 +0,0 @@ -""" -This module contains data types used by Scrapy which are not included in the -Python Standard Library. - -This module must not depend on any module outside the Standard Library. -""" - -import copy -import gzip -import Queue -import bisect -from cStringIO import StringIO - -class MergeDict(object): - """ - A simple class for creating new "virtual" dictionaries that actualy look - up values in more than one dictionary, passed in the constructor. - """ - def __init__(self, *dicts): - self.dicts = dicts - - def __getitem__(self, key): - for dict_ in self.dicts: - try: - return dict_[key] - except KeyError: - pass - raise KeyError - - def __copy__(self): - return self.__class__(*self.dicts) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def getlist(self, key): - for dict in self.dicts: - try: - return dict.getlist(key) - except KeyError: - pass - raise KeyError - - def items(self): - item_list = [] - for dict in self.dicts: - item_list.extend(dict.items()) - return item_list - - def has_key(self, key): - for dict in self.dicts: - if key in dict: - return True - return False - - __contains__ = has_key - - def copy(self): - """ returns a copy of this object""" - return self.__copy__() - -class SortedDict(dict): - "A dictionary that keeps its keys in the order in which they're inserted." - def __init__(self, data=None): - if data is None: - data = {} - dict.__init__(self, data) - self.keyOrder = data.keys() - - def __setitem__(self, key, value): - dict.__setitem__(self, key, value) - if key not in self.keyOrder: - self.keyOrder.append(key) - - def __delitem__(self, key): - dict.__delitem__(self, key) - self.keyOrder.remove(key) - - def __iter__(self): - for k in self.keyOrder: - yield k - - def items(self): - return zip(self.keyOrder, self.values()) - - def iteritems(self): - for key in self.keyOrder: - yield key, dict.__getitem__(self, key) - - def keys(self): - return self.keyOrder[:] - - def iterkeys(self): - return iter(self.keyOrder) - - def values(self): - return [dict.__getitem__(self, k) for k in self.keyOrder] - - def itervalues(self): - for key in self.keyOrder: - yield dict.__getitem__(self, key) - - def update(self, dict): - for k, v in dict.items(): - self.__setitem__(k, v) - - def setdefault(self, key, default): - if key not in self.keyOrder: - self.keyOrder.append(key) - return dict.setdefault(self, key, default) - - def value_for_index(self, index): - "Returns the value of the item at the given zero-based index." - return self[self.keyOrder[index]] - - def insert(self, index, key, value): - "Inserts the key, value pair before the item with the given index." - if key in self.keyOrder: - n = self.keyOrder.index(key) - del self.keyOrder[n] - if n < index: - index -= 1 - self.keyOrder.insert(index, key) - dict.__setitem__(self, key, value) - - def copy(self): - "Returns a copy of this object." - # This way of initializing the copy means it works for subclasses, too. - obj = self.__class__(self) - obj.keyOrder = self.keyOrder - return obj - - def __repr__(self): - """ - Replaces the normal dict.__repr__ with a version that returns the keys - in their sorted order. - """ - return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) - -class MultiValueDictKeyError(KeyError): - pass - -class MultiValueDict(dict): - """ - A subclass of dictionary customized to handle multiple values for the same key. - - >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']}) - >>> d['name'] - 'Simon' - >>> d.getlist('name') - ['Adrian', 'Simon'] - >>> d.get('lastname', 'nonexistent') - 'nonexistent' - >>> d.setlist('lastname', ['Holovaty', 'Willison']) - - This class exists to solve the irritating problem raised by cgi.parse_qs, - which returns a list for every key, even though most Web forms submit - single name-value pairs. - """ - def __init__(self, key_to_list_mapping=()): - dict.__init__(self, key_to_list_mapping) - - def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self)) - - def __getitem__(self, key): - """ - Returns the last data value for this key, or [] if it's an empty list; - raises KeyError if not found. - """ - try: - list_ = dict.__getitem__(self, key) - except KeyError: - raise MultiValueDictKeyError, "Key %r not found in %r" % (key, self) - try: - return list_[-1] - except IndexError: - return [] - - def __setitem__(self, key, value): - dict.__setitem__(self, key, [value]) - - def __copy__(self): - return self.__class__(dict.items(self)) - - def __deepcopy__(self, memo=None): - if memo is None: - memo = {} - result = self.__class__() - memo[id(self)] = result - for key, value in dict.items(self): - dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo)) - return result - - def get(self, key, default=None): - "Returns the default value if the requested data doesn't exist" - try: - val = self[key] - except KeyError: - return default - if val == []: - return default - return val - - def getlist(self, key): - "Returns an empty list if the requested data doesn't exist" - try: - return dict.__getitem__(self, key) - except KeyError: - return [] - - def setlist(self, key, list_): - dict.__setitem__(self, key, list_) - - def setdefault(self, key, default=None): - if key not in self: - self[key] = default - return self[key] - - def setlistdefault(self, key, default_list=()): - if key not in self: - self.setlist(key, default_list) - return self.getlist(key) - - def appendlist(self, key, value): - "Appends an item to the internal list associated with key" - self.setlistdefault(key, []) - dict.__setitem__(self, key, self.getlist(key) + [value]) - - def items(self): - """ - Returns a list of (key, value) pairs, where value is the last item in - the list associated with the key. - """ - return [(key, self[key]) for key in self.keys()] - - def lists(self): - "Returns a list of (key, list) pairs." - return dict.items(self) - - def values(self): - "Returns a list of the last value on every key list." - return [self[key] for key in self.keys()] - - def copy(self): - "Returns a copy of this object." - return self.__deepcopy__() - - def update(self, *args, **kwargs): - "update() extends rather than replaces existing key lists. Also accepts keyword args." - if len(args) > 1: - raise TypeError, "update expected at most 1 arguments, got %d" % len(args) - if args: - other_dict = args[0] - if isinstance(other_dict, MultiValueDict): - for key, value_list in other_dict.lists(): - self.setlistdefault(key, []).extend(value_list) - else: - try: - for key, value in other_dict.items(): - self.setlistdefault(key, []).append(value) - except TypeError: - raise ValueError, "MultiValueDict.update() takes either a MultiValueDict or dictionary" - for key, value in kwargs.iteritems(): - self.setlistdefault(key, []).append(value) - -class DotExpandedDict(dict): - """ - A special dictionary constructor that takes a dictionary in which the keys - may contain dots to specify inner dictionaries. It's confusing, but this - example should make sense. - - >>> d = DotExpandedDict({'person.1.firstname': ['Simon'], \ - 'person.1.lastname': ['Willison'], \ - 'person.2.firstname': ['Adrian'], \ - 'person.2.lastname': ['Holovaty']}) - >>> d - {'person': {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}} - >>> d['person'] - {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}} - >>> d['person']['1'] - {'lastname': ['Willison'], 'firstname': ['Simon']} - - # Gotcha: Results are unpredictable if the dots are "uneven": - >>> DotExpandedDict({'c.1': 2, 'c.2': 3, 'c': 1}) - {'c': 1} - """ - def __init__(self, key_to_list_mapping): - for k, v in key_to_list_mapping.items(): - current = self - bits = k.split('.') - for bit in bits[:-1]: - current = current.setdefault(bit, {}) - # Now assign value to current position - try: - current[bits[-1]] = v - except TypeError: # Special-case if current isn't a dict. - current = {bits[-1] : v} - -class FileDict(dict): - """ - A dictionary used to hold uploaded file contents. The only special feature - here is that repr() of this object won't dump the entire contents of the - file to the output. A handy safeguard for a large file upload. - """ - def __repr__(self): - if 'content' in self: - d = dict(self, content='<omitted>') - return dict.__repr__(d) - return dict.__repr__(self) - -class Sitemap(object): - """Sitemap class is used to build a map of the traversed pages""" - - def __init__(self): - self._nodes = {} - self._roots = [] - - def add_node(self, url, parent_url): - if not url in self._nodes: - parent = self._nodes.get(parent_url, None) - node = SiteNode(url) - self._nodes[url] = node - if parent: - parent.add_child(node) - else: - self._roots.append(node) - - def add_item(self, url, item): - if url in self._nodes: - self._nodes[url].itemnames.append(str(item)) - - def to_string(self): - s = ''.join([n.to_string(0) for n in self._roots]) - return s - -class SiteNode(object): - """Class to represent a site node (page, image or any other file)""" - - def __init__(self, url): - self.url = url - self.itemnames = [] - self.children = [] - self.parent = None - - def add_child(self, node): - self.children.append(node) - node.parent = self - - def to_string(self, level=0): - s = "%s%s\n" % (' '*level, self.url) - if self.itemnames: - for n in self.itemnames: - s += "%sScraped: %s\n" % (' '*(level+1), n) - for node in self.children: - s += node.to_string(level+1) - return s - -class CaselessDict(dict): - def __init__(self, other=None): - if other: - # Doesn't do keyword args - if isinstance(other, dict): - for k, v in other.items(): - dict.__setitem__(self, self.normkey(k), v) - else: - for k, v in other: - dict.__setitem__(self, self.normkey(k), v) - - def __getitem__(self, key): - return dict.__getitem__(self, self.normkey(key)) - - def __setitem__(self, key, value): - dict.__setitem__(self, self.normkey(key), value) - - def __delitem__(self, key): - dict.__delitem__(self, self.normkey(key)) - - def __contains__(self, key): - return dict.__contains__(self, self.normkey(key)) - - def normkey(self, key): - return key.lower() - - def has_key(self, key): - return dict.has_key(self, self.normkey(key)) - - def get(self, key, def_val=None): - return dict.get(self, self.normkey(key), def_val) - - def setdefault(self, key, def_val=None): - return dict.setdefault(self, self.normkey(key), def_val) - - def update(self, other): - for k, v in other.items(): - dict.__setitem__(self, self.normkey(k), v) - - def fromkeys(self, iterable, value=None): - d = CaselessDict() - for k in iterable: - dict.__setitem__(d, self.normkey(k), value) - return d - - def pop(self, key, def_val=None): - return dict.pop(self, self.normkey(key), def_val) - - -class PriorityQueue(Queue.Queue): - - def __len__(self): - return self.qsize() - - def _init(self, maxsize): - self.maxsize = maxsize - # Python 2.5 uses collections.deque, but we can't because - # we need insert(pos, item) for our priority stuff - self.queue = [] - - def put(self, item, priority=0, block=True, timeout=None): - """Puts an item onto the queue with a numeric priority (default is zero). - - Note that we are "shadowing" the original Queue.Queue put() method here. - """ - Queue.Queue.put(self, (priority, item), block, timeout) - - def _put(self, item): - """Override of the Queue._put to support prioritisation.""" - # Priorities must be integers! - priority = int(item[0]) - - # Using a tuple (priority+1,) finds us the correct insertion - # position to maintain the existing ordering. - self.queue.insert(bisect.bisect_left(self.queue, (priority+1,)), item) - - def _get(self): - """Override of Queue._get(). Strips the priority.""" - return self.queue.pop(0) - - -class PriorityStack(PriorityQueue): - def _put(self, item): - priority = int(item[0]) - self.queue.insert(bisect.bisect_left(self.queue, (priority,)), item) - -class gzStringIO: - """a file like object, similar to StringIO, but gzip-compressed.""" - def __init__(self, data, compress_level = 9, filename = ""): - self._s = StringIO() - g = gzip.GzipFile(filename, "wb", compress_level, self._s) - g.write(data) - g.flush() - g.close() - def read(self): - return self._s.getvalue() - diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/db.py b/scrapy/branches/cluster-refactor/scrapy/utils/db.py deleted file mode 100644 index a3aa5a672..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/db.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Function for dealing with databases -""" -import re -from scrapy.conf import settings -from scrapy import log -from scrapy.core.engine import scrapyengine - -def mysql_connect(db_uri, **kwargs): - """ - Connects to a MySQL DB given a mysql URI - """ - import MySQLdb - - if not db_uri or not db_uri.startswith('mysql://'): - raise Exception("Incorrect MySQL URI: %s" % db_uri) - m = re.search(r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/]+)/(?P<db>.*)$", db_uri) - if m: - d = m.groupdict() - if d['passwd'] is None: - del(d['passwd']) - - d.update(settings.get("MYSQL_CONNECTION_SETTINGS")) - d.update(kwargs) - - dcopy = d.copy() - if dcopy.get("passwd"): - dcopy["passwd"] = "********" - log.msg("Connecting db with settings %s" % dcopy ) - - conn = MySQLdb.connect(**d) - - #this is to maintain active the connection - def _ping(): - log.msg("Pinging connection to %s/%s" % (d.get('host'), d.get('db')) ) - conn.ping() - scrapyengine.addtask(_ping, settings.getint("MYSQL_CONNECTION_PING_PERIOD", 600)) - - return conn diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/decompressor.py b/scrapy/branches/cluster-refactor/scrapy/utils/decompressor.py deleted file mode 100644 index 67849ef48..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/decompressor.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Utility for autodetecting and decompressing responses -""" - -import zipfile -import tarfile -import gzip -import bz2 -from scrapy.http import ResponseBody -try: - from cStringIO import StringIO -except: - from StringIO import StringIO - -class Decompressor(object): - class ArchiveIsEmpty(Exception): - pass - - def __init__(self): - self.decompressors = {'tar': self.is_tar, 'zip': self.is_zip, - 'gz': self.is_gzip, 'bz2': self.is_bzip2} - - def is_tar(self, response): - try: - tar_file = tarfile.open(name='tar.tmp', fileobj=self.archive) - except tarfile.ReadError: - return False - if tar_file.members: - return response.replace(body=ResponseBody(tar_file.extractfile(tar_file.members[0]).read())) - else: - raise self.ArchiveIsEmpty - - def is_zip(self, response): - try: - zip_file = zipfile.ZipFile(self.archive) - except zipfile.BadZipfile: - return False - namelist = zip_file.namelist() - if namelist: - return response.replace(body=ResponseBody(zip_file.read(namelist[0]))) - else: - raise self.ArchiveIsEmpty - - def is_gzip(self, response): - try: - gzip_file = gzip.GzipFile(fileobj=self.archive) - decompressed_body = gzip_file.read() - except IOError: - return False - return response.replace(body=decompressed_body) - - def is_bzip2(self, response): - try: - decompressed_body = bz2.decompress(self.body) - except IOError: - return False - return response.replace(body=ResponseBody(decompressed_body)) - - def extract_winfo(self, response): - self.body = response.body.to_string() - self.archive = StringIO() - self.archive.write(self.body) - - for decompressor in self.decompressors.keys(): - self.archive.seek(0) - new_response = self.decompressors[decompressor](response) - if new_response: - return new_response, decompressor - return response, '' - - def extract(self, response): - return self.extract_winfo(response)[0] diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/defer.py b/scrapy/branches/cluster-refactor/scrapy/utils/defer.py deleted file mode 100644 index 453d7320d..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/defer.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Helper functions for dealing with Twisted deferreds -""" - -from twisted.internet import defer, reactor -from twisted.python import failure - -def defer_fail(_failure): - """same as twsited.internet.defer.fail, but delay calling errback """ - d = defer.Deferred() - reactor.callLater(0, d.errback, _failure) - return d - -def defer_succeed(result): - """same as twsited.internet.defer.succed, but delay calling callback""" - d = defer.Deferred() - reactor.callLater(0, d.callback, result) - return d - -def defer_result(result): - if isinstance(result, defer.Deferred): - return result - elif isinstance(result, failure.Failure): - return defer_fail(result) - else: - return defer_succeed(result) - -def mustbe_deferred(f, *args, **kw): - """same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback""" - try: - result = f(*args, **kw) - except: - return defer_fail(failure.Failure()) - else: - return defer_result(result) - -def chain_deferred(d1, d2): - if callable(d2): - d2 = lambda_deferred(d2) - - def _pause(_): - d2.pause() - reactor.callLater(0, d2.unpause) - return _ - - def _reclaim(_): - return d2 - - #d1.addBoth(_pause) ## needs more debugging before reenable it - d1.chainDeferred(d2) - d1.addBoth(_reclaim) - return d1 - -def lambda_deferred(func): - deferred = defer.Deferred() - def _success(res): - d = func() - d.callback(res) - return d - def _fail(res): - d = func() - d.errback(res) - return d - return deferred.addCallbacks(_success, _fail) - -def deferred_degenerate(generator, container=None, next_delay=0): - generator = iter(generator or []) - deferred = defer.Deferred() - container = container or [] - def _next(): - try: - container.append(generator.next()) - except StopIteration: - reactor.callLater(0, deferred.callback, container) - except: - reactor.callLater(0, deferred.errback, failure.Failure()) - else: - reactor.callLater(next_delay, _next) - _next() - return deferred - diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/iterators.py b/scrapy/branches/cluster-refactor/scrapy/utils/iterators.py deleted file mode 100644 index 5f989d0a3..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/iterators.py +++ /dev/null @@ -1,23 +0,0 @@ -import csv - -from scrapy.http import Response -from scrapy import log - -def csv_iter(response, delimiter=None, headers=None): - if delimiter: - csv_r = csv.reader(response.body.to_unicode().split('\n'), delimiter=delimiter) - else: - csv_r = csv.reader(response.body.to_unicode().split('\n')) - - if not headers: - headers = csv_r.next() - - while True: - node = csv_r.next() - - if len(node) != len(headers): - log.msg("ignoring node %d (length: %d, should be: %d)" % (csv_r.line_num, len(node), len(headers)), log.WARNING) - continue - - yield dict(zip(headers, node)) - diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/markup.py b/scrapy/branches/cluster-refactor/scrapy/utils/markup.py deleted file mode 100644 index 5ec5b1719..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/markup.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -Functions for dealing with markup text -""" - -import re -import htmlentitydefs - -_ent_re = re.compile(r'&(#?)([^&;]+);') -_tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL) - -def remove_entities(text, keep=(), remove_illegal=True): - """Remove entities from the given text. - - 'text' can be a unicode string or a regular string encoded as 'utf-8' - - If 'keep' is passed (with a list of entity names) those entities will - be kept (they won't be removed). - - It supports both numeric (&#nnnn;) and named (  >) entities. - - If remove_illegal is True, entities that can't be converted are removed. - If remove_illegal is False, entities that can't be converted are kept "as - is". For more information see the tests. - - Always returns a unicode string (with the entities removed). - """ - - def convert_entity(m): - if m.group(1) == '#': - try: - return unichr(int(m.group(2))) - except ValueError: - if remove_illegal: - return u'' - else: - return u'&#%s;' % m.group(2) - try: - if m.group(2) in keep: - return '&%s;' % m.group(2) - else: - return unichr(htmlentitydefs.name2codepoint[m.group(2)]) - except KeyError: - if remove_illegal: - return u'' - else: - return u'&%s;' % m.group(2) - - return _ent_re.sub(convert_entity, text.decode('utf-8')) - -def has_entities(text): - return bool(_ent_re.search(text)) - -def replace_tags(text, token=''): - """Replace all markup tags found in the given text by the given token. By - default token is a null string so it just remove all tags. - - 'text' can be a unicode string or a regular string encoded as 'utf-8' - - Always returns a unicode string. - """ - return _tag_re.sub(token, text.decode('utf-8')) - - -def remove_comments(text): - """ Remove HTML Comments. """ - return re.sub('<!--.*?-->', u'', text.decode('utf-8'), re.DOTALL) - -def remove_tags(text, which_ones=()): - """ Remove HTML Tags only. - - which_ones -- is a tuple of which tags we want to remove. - if is empty remove all tags. - """ - if len(which_ones) > 0: - tags = [ '<%s>|<%s .*?>|</%s>' % (tag,tag,tag) for tag in which_ones ] - reg_exp_remove_tags = '|'.join(tags) - else: - reg_exp_remove_tags = '<.*?>' - re_tags = re.compile(reg_exp_remove_tags, re.DOTALL) - return re_tags.sub(u'', text.decode('utf-8')) - -def remove_tags_with_content(text, which_ones=()): - """ Remove tags and its content. - - which_ones -- is a tuple of which tags with its content we want to remove. - if is empty do nothing. - """ - tags = [ '<%s.*?</%s>' % (tag,tag) for tag in which_ones ] - re_tags_remove = re.compile('|'.join(tags), re.DOTALL) - return re_tags_remove.sub(u'', text.decode('utf-8')) - -def remove_escape_chars(text, which_ones=('\n','\t','\r')): - """ Remove escape chars. Default : \\n, \\t, \\r - - which_ones -- is a tuple of which escape chars we want to remove. - By default removes \n, \t, \r. - """ - re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) - return re_escape_chars.sub(u'', text.decode('utf-8')) - diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/misc.py b/scrapy/branches/cluster-refactor/scrapy/utils/misc.py deleted file mode 100644 index f3cc378f8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/misc.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Auxiliary functions which doesn't fit anywhere else -""" -import re - -from twisted.internet import defer - -from scrapy.core.exceptions import UsageError -from scrapy.utils.python import flatten -from scrapy.utils.markup import remove_entities -from scrapy.utils.defer import defer_succeed - -def dict_updatedefault(D, E, **F): - """ - updatedefault(D, E, **F) -> None. - - Update D from E and F: for k in E: D.setdefault(k, E[k]) - (if E has keys else: for (k, v) in E: D.setdefault(k, v)) - then: for k in F: D.setdefault(k, F[k]) - """ - for k in E: - if isinstance(k, tuple): - k, v = k - else: - v = E[k] - D.setdefault(k, v) - - for k in F: - D.setdefault(k, F[k]) - -def memoize(cache, hash): - def decorator(func): - def wrapper(*args, **kwargs): - key = hash(*args, **kwargs) - if key in cache: - return defer_succeed(cache[key]) - - def _store(_): - cache[key] = _ - return _ - - result = func(*args, **kwargs) - if isinstance(result, defer.Deferred): - return result.addBoth(_store) - cache[key] = result - return result - return wrapper - return decorator - -def stats_getpath(dict_, path, default=None): - for key in path.split('/'): - if key in dict_: - dict_ = dict_[key] - else: - return default - return dict_ - -def load_class(class_path): - """Load a class given its absolute class path, and return it without - instantiating it""" - try: - dot = class_path.rindex('.') - except ValueError: - raise UsageError, '%s isn\'t a module' % class_path - module, classname = class_path[:dot], class_path[dot+1:] - try: - mod = __import__(module, {}, {}, ['']) - except ImportError, e: - raise UsageError, 'Error importing %s: "%s"' % (module, e) - try: - cls = getattr(mod, classname) - except AttributeError: - raise UsageError, 'module "%s" does not define a "%s" class' % (module, classname) - - return cls - -def extract_regex(regex, text, encoding): - """Extract a list of unicode strings from the given text/encoding using the following policies: - - * if the regex contains a named group called "extract" that will be returned - * if the regex contains multiple numbered groups, all those will be returned (flattened) - * if the regex doesn't contain any group the entire regex matching is returned - """ - - if isinstance(regex, basestring): - regex = re.compile(regex) - - try: - strings = [regex.search(text).group('extract')] # named group - except: - strings = regex.findall(text) # full regex or numbered groups - strings = flatten(strings) - - if isinstance(text, unicode): - return [remove_entities(s, keep=['lt', 'amp']) for s in strings] - else: - return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings] diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/python.py b/scrapy/branches/cluster-refactor/scrapy/utils/python.py deleted file mode 100644 index 23e08075a..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/python.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -This module contains essential stuff that should've come with Python itself ;) -""" - -from sgmllib import SGMLParser - -class FixedSGMLParser(SGMLParser): - """The SGMLParser that comes with Python has a bug in the convert_charref() - method. This is the same class with the bug fixed""" - - def convert_charref(self, name): - """This method fixes a bug in Python's SGMLParser.""" - try: - n = int(name) - except ValueError: - return - if not 0 <= n <= 127 : # ASCII ends at 127, not 255 - return - return self.convert_codepoint(n) - - -def flatten(x): - """flatten(sequence) -> list - - Returns a single, flat list which contains all elements retrieved - from the sequence and all recursively contained sub-sequences - (iterables). - - Examples: - >>> [1, 2, [3,4], (5,6)] - [1, 2, [3, 4], (5, 6)] - >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) - [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]""" - - result = [] - for el in x: - if hasattr(el, "__iter__"): - result.extend(flatten(el)) - else: - result.append(el) - return result - - -def unique(list_): - """efficient function to uniquify a list preserving item order""" - seen = {} - result = [] - for item in list_: - if item in seen: - continue - seen[item] = 1 - result.append(item) - return result - - diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/response.py b/scrapy/branches/cluster-refactor/scrapy/utils/response.py deleted file mode 100644 index 63fa3f686..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/response.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -This module provides some useful functions for working with -scrapy.http.Response objects -""" - -from scrapy.xpath import XPathSelector - -def new_response_from_xpaths(response, xpaths): - """Return a new response constructed by applying the given xpaths to the - original response body - """ - xs = XPathSelector(response) - parts = [''.join([n for n in xs.x(x).extract()]) for x in xpaths] - new_body = ''.join(parts) - return response.replace(body=new_body) diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/url.py b/scrapy/branches/cluster-refactor/scrapy/utils/url.py deleted file mode 100644 index 091827760..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/url.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -This module contains general purpose URL functions not found in the standard -library. -""" - -import re -import urlparse -import urllib -import posixpath -import cgi - -def url_is_from_any_domain(url, domains): - """Return True if the url belongs to the given domain""" - host = urlparse.urlparse(url).hostname - - if host: - return any(((host == d) or (host.endswith('.%s' % d)) for d in domains)) - else: - return False - -def url_is_from_spider(url, spider): - """Return True if the url belongs to the given spider""" - domains = [spider.domain_name] + spider.extra_domain_names - return url_is_from_any_domain(url, domains) - -def urljoin_rfc(base, ref): - """ - Fixed urlparse.urljoin version that handles - relative query string as RFC states. - """ - if ref.startswith('?'): - fpart = urlparse.urlsplit(str(base))[2].rsplit('/', 1)[-1] - ref = ''.join([fpart, ref]) - # convert ref to a string. This should already - # be the case, however, many spiders do not convert. - return urlparse.urljoin(base, str(ref)) - - -_reserved = ';/?:@&=+$|,#' # RFC 2396 (Generic Syntax) -_unreserved_marks = "-_.!~*'()" #RFC 2396 sec 2.3 -_safe_chars = urllib.always_safe + '%' + _reserved + _unreserved_marks - -def safe_url_string(url, use_encoding='utf8'): - """Convert a unicode (or utf8 string) object into a legal URL. - - Illegal characters are escaped (RFC-3986) - - It is safe to call this function multiple times. Do not pass this - function strings in encodings other than utf8. - - The use_encoding argument is the encoding to use to determine the numerical - values in the escaping. For urls on html pages, you should use the original - encoding of that page. - """ - s = url.encode(use_encoding) - return urllib.quote(s, _safe_chars) - - -_parent_dirs = re.compile(r'/?(\.\./)+') - -def safe_download_url(url): - """ Make a url for download. This will call safe_url_string - and then strip the fragment, if one exists. The path will - be normalised. - - If the path is outside the document root, it will be changed - to be within the document root. - """ - safe_url = safe_url_string(url) - scheme, netloc, path, query, _ = urlparse.urlsplit(safe_url) - if path: - path = _parent_dirs.sub('', posixpath.normpath(path)) - if url.endswith('/') and not path.endswith('/'): - path += '/' - else: - path = '/' - return urlparse.urlunsplit((scheme, netloc, path, query, '')) - -def is_url(text): - return text.partition("://")[0] in ('file', 'http', 'https') - -def url_query_parameter(url, parameter, default=None, keep_blank_values=0): - """Return the value of a url parameter, given the url and parameter name""" - queryparams = cgi.parse_qs(urlparse.urlsplit(str(url))[3], keep_blank_values=keep_blank_values) - return queryparams.get(parameter, [default])[0] - -def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='): - """Clean url arguments leaving only those passed in the parameterlist""" - try: - base, query = url.split('?', 1) - parameters = [pair.split(kvsep, 1) for pair in query.split(sep)] - except: - base = url - query = "" - parameters = [] - - # unique parameters while keeping order - unique = {} - querylist = [] - for pair in parameters: - k = pair[0] - if not unique.get(k): - querylist += [pair] - unique[k] = 1 - - query = sep.join([kvsep.join(pair) for pair in querylist if pair[0] in parameterlist]) - return '?'.join([base, query]) - -def add_or_replace_parameter(url, name, new_value, sep='&'): - """Add or remove a parameter to a given url""" - def has_querystring(url): - _, _, _, query, _ = urlparse.urlsplit(url) - return bool(query) - - parameter = url_query_parameter(url, name, keep_blank_values=1) - if parameter is None: - if has_querystring(url): - next_url = url + sep + name + '=' + new_value - else: - next_url = url + '?' + name + '=' + new_value - else: - next_url = url.replace(name+'='+parameter, - name+'='+new_value) - return next_url - -def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): - """Canonicalize url by applying the following procedures: - - - sort query arguments, first by key, then by value - - percent encode paths and query arguments. non-ASCII characters are - percent-encoded using UTF-8 (RFC-3986) - - normalize all spaces (in query arguments) '+' (plus symbol) - - normalize percent encodings case (%2f -> %2F) - - remove query arguments with blank values (unless keep_blank_values is True) - - remove fragments (unless keep_fragments is True) - - For examples see the tests in scrapy.tests.test_utils_url - """ - - parts = list(urlparse.urlparse(url)) - keyvals = cgi.parse_qsl(parts[4], keep_blank_values) - keyvals.sort() - parts[2] = urllib.quote(urllib.unquote(parts[2])) - parts[4] = urllib.urlencode(keyvals) - if not keep_fragments: - parts[5] = '' - return urlparse.urlunparse(parts) diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/xml.py b/scrapy/branches/cluster-refactor/scrapy/utils/xml.py deleted file mode 100644 index acfa52363..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/utils/xml.py +++ /dev/null @@ -1,28 +0,0 @@ -import re - -from scrapy.xpath import XmlXPathSelector -from scrapy.http import Response - -def xpathselector_iternodes(obj, nodename): - """Return a iterator of XPathSelector's over all nodes of a XML document, - given tha name of the node to iterate. Useful for parsing XML feeds. - - obj can be: - - a Response object - - a unicode string - - a string encoded as utf-8 - """ - - assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ - - if isinstance(obj, Response): - text = obj.body.to_unicode() - elif isinstance(obj, str): - text = obj.decode('utf-8') - else: - text = obj - - r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL) - for match in r.finditer(text): - nodetext = match.group() - yield XmlXPathSelector(text=nodetext).x('/' + nodename)[0] diff --git a/scrapy/branches/cluster-refactor/scrapy/xpath/constructors.py b/scrapy/branches/cluster-refactor/scrapy/xpath/constructors.py deleted file mode 100644 index e6ca7fdcc..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/xpath/constructors.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -This module provides functions for generating libxml2 documents (xmlDoc). - -Constructors must receive a Response object and return a xmlDoc object. -""" - -import libxml2 - -xml_parser_options = libxml2.XML_PARSE_RECOVER + \ - libxml2.XML_PARSE_NOERROR + \ - libxml2.XML_PARSE_NOWARNING - -html_parser_options = libxml2.HTML_PARSE_RECOVER + \ - libxml2.HTML_PARSE_NOERROR + \ - libxml2.HTML_PARSE_NOWARNING - -def xmlDoc_from_html(response): - """Return libxml2 doc for HTMLs""" - try: - lxdoc = libxml2.htmlReadDoc(response.body.to_string('utf-8'), response.url, 'utf-8', html_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.htmlReadDoc(response.body.to_string('utf-8').replace("\x00", ""), response.url, 'utf-8', html_parser_options) - return lxdoc - -def xmlDoc_from_xml(response): - """Return libxml2 doc for XMLs""" - try: - lxdoc = libxml2.readDoc(response.body.to_string('utf-8'), response.url, 'utf-8', xml_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.readDoc(response.body.to_string('utf-8').replace("\x00", ""), response.url, 'utf-8', xml_parser_options) - return lxdoc diff --git a/scrapy/branches/cluster-refactor/scrapy/xpath/extension.py b/scrapy/branches/cluster-refactor/scrapy/xpath/extension.py deleted file mode 100644 index 0642e0cd6..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/xpath/extension.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -The ResponseLibxml2 extension causes the Response objects to grow a new method -("getlibxml2doc") which returns a (cached) libxml2 document of itself. -""" - -from scrapy.http import Response -from scrapy.xpath.document import Libxml2Document -from scrapy.xpath.constructors import xmlDoc_from_html - -class ResponseLibxml2(object): - def __init__(self): - setattr(Response, 'getlibxml2doc', getlibxml2doc) - -def getlibxml2doc(response, constructor=xmlDoc_from_html): - attr = '_lx2doc_%s' % constructor.__name__ - if not hasattr(response, attr): - lx2doc = Libxml2Document(response, constructor=constructor) - setattr(response, attr, lx2doc) - return getattr(response, attr) - diff --git a/scrapy/branches/cluster-refactor/scrapy/xpath/selector.py b/scrapy/branches/cluster-refactor/scrapy/xpath/selector.py deleted file mode 100644 index cb97f7da8..000000000 --- a/scrapy/branches/cluster-refactor/scrapy/xpath/selector.py +++ /dev/null @@ -1,133 +0,0 @@ -import libxml2 - -from scrapy.http import Response -from scrapy.xpath.extension import Libxml2Document -from scrapy.xpath.constructors import xmlDoc_from_html, xmlDoc_from_xml -from scrapy.utils.python import flatten -from scrapy.utils.misc import extract_regex - -class XPathSelector(object): - """The XPathSelector class provides a convenient way for selecting document - parts using XPaths and regexs, with support for nested queries. - - Although this is not an abstract class, you usually instantiate one of its - children: - - - XmlXPathSelector (for XML content) - - HtmlXPathSelector (for HTML content) - """ - - def __init__(self, response=None, text=None, node=None, parent=None, expr=None, constructor=xmlDoc_from_html): - if parent: - self.doc = parent.doc - self.xmlNode = node - elif response: - try: - self.doc = response.getlibxml2doc(constructor=constructor) # try with cached version first - except AttributeError: - self.doc = Libxml2Document(response, constructor=constructor) - self.xmlNode = self.doc.xmlDoc - elif text: - response = Response(domain=None, url=None, body=str(text)) - self.doc = Libxml2Document(response, constructor=constructor) - self.xmlNode = self.doc.xmlDoc - self.expr = expr - self.response = response - - def x(self, xpath): - """Perform the given XPath query on the current XPathSelector and - return a XPathSelectorList of the result""" - if hasattr(self.xmlNode, 'xpathEval'): - self.doc.xpathContext.setContextNode(self.xmlNode) - xpath_result = self.doc.xpathContext.xpathEval(xpath) - cls = type(self) - if hasattr(xpath_result, '__iter__'): - return XPathSelectorList([cls(node=node, parent=self, expr=xpath, response=self.response) - for node in xpath_result]) - else: - return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath, response=self.response)]) - else: - return XPathSelectorList([]) - - def re(self, regex): - """Return a list of unicode strings by applying the regex over all - current XPath selections, and flattening the results""" - return extract_regex(regex, self.extract(), 'utf-8') - - def extract(self): - """Return a unicode string of the content referenced by the XPathSelector""" - if isinstance(self.xmlNode, basestring): - text = unicode(self.xmlNode, 'utf-8', errors='ignore') - elif hasattr(self.xmlNode, 'serialize'): - if isinstance(self.xmlNode, libxml2.xmlDoc): - data = self.xmlNode.getRootElement().serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - elif isinstance(self.xmlNode, libxml2.xmlAttr): - # serialization doesn't work sometimes for xmlAttr types - text = unicode(self.xmlNode.content, errors='ignore') - else: - data = self.xmlNode.serialize('utf-8') - text = unicode(data, 'utf-8', errors='ignore') if data else u'' - else: - try: - text = unicode(self.xmlNode, errors='ignore') - except TypeError: # catched when self.xmlNode is a float - see tests - text = unicode(self.xmlNode) - return text - - def extract_unquoted(self): - """Get unescaped contents from the text node (no entities, no CDATA)""" - if self.x('self::text()'): - return unicode(self.xmlNode.getContent(), errors='ignore') - else: - return u'' - - def register_namespace(self, prefix, uri): - """Register namespace so that it can be used in XPath queries""" - self.doc.xpathContext.xpathRegisterNs(prefix, uri) - - def __nonzero__(self): - return bool(self.extract()) - - def __str__(self): - return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, 'name', type(self.xmlNode).__name__), self.expr) - - __repr__ = __str__ - - -class XPathSelectorList(list): - """List of XPathSelector objects""" - - def __getslice__(self, i, j): - return XPathSelectorList(list.__getslice__(self, i, j)) - - def x(self, xpath): - """Perform the given XPath query on each XPathSelector of the list and - return a new (flattened) XPathSelectorList of the results""" - return XPathSelectorList(flatten([x.x(xpath) for x in self])) - - def re(self, regex): - """Perform the re() method on each XPathSelector of the list, and - return the result as a flattened list of unicode strings""" - return flatten([x.re(regex) for x in self]) - - def extract(self): - """Return a list of unicode strings with the content referenced by each - XPathSelector of the list""" - return [x.extract() if isinstance(x, XPathSelector) else x for x in self] - - def extract_unquoted(self): - return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self] - - -class XmlXPathSelector(XPathSelector): - """XPathSelector for XML content""" - def __init__(self, *args, **kwargs): - kwargs['constructor'] = xmlDoc_from_xml - XPathSelector.__init__(self, *args, **kwargs) - -class HtmlXPathSelector(XPathSelector): - """XPathSelector for HTML content""" - def __init__(self, *args, **kwargs): - kwargs['constructor'] = xmlDoc_from_html - XPathSelector.__init__(self, *args, **kwargs) diff --git a/scrapy/branches/cluster-refactor/setup.cfg b/scrapy/branches/cluster-refactor/setup.cfg deleted file mode 100644 index 2b1e9475f..000000000 --- a/scrapy/branches/cluster-refactor/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[bdist_rpm] -release = 1 -install-script = scripts/rpm-install.sh - -[install] -optimize = 1 - -[aliases] -rpm = bdist_rpm --binary-only clean -a diff --git a/scrapy/branches/cluster-refactor/setup.py b/scrapy/branches/cluster-refactor/setup.py deleted file mode 100644 index da54c4a35..000000000 --- a/scrapy/branches/cluster-refactor/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -from setuptools import setup, find_packages -import os, os.path, glob - -def findfiles(pattern, base='.'): - matches = [] - for root, _, _ in os.walk(base): - matches.extend(glob.glob(os.path.join(root, pattern))) - return matches - - -setup( - name = 'scrapy', - version = '0.8', - description = '', - long_description = '', - author = '', - author_email = '', - license = '', - url = 'http://scrapy.org', - - packages = find_packages(), - package_data = { - 'scrapy': ['templates/*.tmpl'], - }, - scripts = ['scrapy/bin/scrapy-admin.py'], -) diff --git a/scrapy/branches/cluster-refactor/scrapy/command/__init__.py b/scrapy/command/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/__init__.py rename to scrapy/command/__init__.py diff --git a/scrapy/trunk/scrapy/command/cmdline.py b/scrapy/command/cmdline.py similarity index 100% rename from scrapy/trunk/scrapy/command/cmdline.py rename to scrapy/command/cmdline.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/__init__.py b/scrapy/command/commands/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/__init__.py rename to scrapy/command/commands/__init__.py diff --git a/scrapy/trunk/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py similarity index 100% rename from scrapy/trunk/scrapy/command/commands/crawl.py rename to scrapy/command/commands/crawl.py diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/download.py b/scrapy/command/commands/download.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/commands/download.py rename to scrapy/command/commands/download.py diff --git a/scrapy/trunk/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py similarity index 100% rename from scrapy/trunk/scrapy/command/commands/genspider.py rename to scrapy/command/commands/genspider.py diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/help.py b/scrapy/command/commands/help.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/commands/help.py rename to scrapy/command/commands/help.py diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/list.py b/scrapy/command/commands/list.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/commands/list.py rename to scrapy/command/commands/list.py diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/log.py b/scrapy/command/commands/log.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/commands/log.py rename to scrapy/command/commands/log.py diff --git a/scrapy/trunk/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py similarity index 100% rename from scrapy/trunk/scrapy/command/commands/parse.py rename to scrapy/command/commands/parse.py diff --git a/scrapy/trunk/scrapy/command/commands/shell.py b/scrapy/command/commands/shell.py similarity index 100% rename from scrapy/trunk/scrapy/command/commands/shell.py rename to scrapy/command/commands/shell.py diff --git a/scrapy/branches/cluster-refactor/scrapy/command/commands/start.py b/scrapy/command/commands/start.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/command/commands/start.py rename to scrapy/command/commands/start.py diff --git a/scrapy/trunk/scrapy/command/commands/stats.py b/scrapy/command/commands/stats.py similarity index 100% rename from scrapy/trunk/scrapy/command/commands/stats.py rename to scrapy/command/commands/stats.py diff --git a/scrapy/trunk/scrapy/command/models.py b/scrapy/command/models.py similarity index 100% rename from scrapy/trunk/scrapy/command/models.py rename to scrapy/command/models.py diff --git a/scrapy/trunk/scrapy/conf/__init__.py b/scrapy/conf/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/conf/__init__.py rename to scrapy/conf/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/conf/commands/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/__init__.py rename to scrapy/conf/commands/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/crawl.py b/scrapy/conf/commands/crawl.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/crawl.py rename to scrapy/conf/commands/crawl.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/help.py b/scrapy/conf/commands/help.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/help.py rename to scrapy/conf/commands/help.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/list.py b/scrapy/conf/commands/list.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/list.py rename to scrapy/conf/commands/list.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/log.py b/scrapy/conf/commands/log.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/log.py rename to scrapy/conf/commands/log.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/scrape.py b/scrapy/conf/commands/scrape.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/scrape.py rename to scrapy/conf/commands/scrape.py diff --git a/scrapy/trunk/scrapy/conf/commands/shell.py b/scrapy/conf/commands/shell.py similarity index 100% rename from scrapy/trunk/scrapy/conf/commands/shell.py rename to scrapy/conf/commands/shell.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/stats.py b/scrapy/conf/commands/stats.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/stats.py rename to scrapy/conf/commands/stats.py diff --git a/scrapy/branches/cluster-refactor/scrapy/conf/commands/test.py b/scrapy/conf/commands/test.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/conf/commands/test.py rename to scrapy/conf/commands/test.py diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py similarity index 100% rename from scrapy/trunk/scrapy/conf/default_settings.py rename to scrapy/conf/default_settings.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/__init__.py rename to scrapy/contrib/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/aws.py b/scrapy/contrib/aws.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/aws.py rename to scrapy/contrib/aws.py diff --git a/scrapy/trunk/scrapy/contrib/closedomain.py b/scrapy/contrib/closedomain.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/closedomain.py rename to scrapy/contrib/closedomain.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/__init__.py b/scrapy/contrib/cluster/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/__init__.py rename to scrapy/contrib/cluster/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/response/__init__.py b/scrapy/contrib/cluster/crawler/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/response/__init__.py rename to scrapy/contrib/cluster/crawler/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/crawler/manager.py b/scrapy/contrib/cluster/crawler/manager.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/crawler/manager.py rename to scrapy/contrib/cluster/crawler/manager.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/hooks/__init__.py b/scrapy/contrib/cluster/hooks/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/hooks/__init__.py rename to scrapy/contrib/cluster/hooks/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/hooks/svn.py b/scrapy/contrib/cluster/hooks/svn.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/hooks/svn.py rename to scrapy/contrib/cluster/hooks/svn.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spider/__init__.py b/scrapy/contrib/cluster/master/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/spider/__init__.py rename to scrapy/contrib/cluster/master/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/master/manager.py b/scrapy/contrib/cluster/master/manager.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/master/manager.py rename to scrapy/contrib/cluster/master/manager.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/master/web.py b/scrapy/contrib/cluster/master/web.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/master/web.py rename to scrapy/contrib/cluster/master/web.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/master/ws_api.txt b/scrapy/contrib/cluster/master/ws_api.txt similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/master/ws_api.txt rename to scrapy/contrib/cluster/master/ws_api.txt diff --git a/scrapy/trunk/scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py b/scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py rename to scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/tools/test-worker.py b/scrapy/contrib/cluster/tools/test-worker.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/tools/test-worker.py rename to scrapy/contrib/cluster/tools/test-worker.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/cluster/worker/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/__init__.py rename to scrapy/contrib/cluster/worker/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/worker/manager.py b/scrapy/contrib/cluster/worker/manager.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/worker/manager.py rename to scrapy/contrib/cluster/worker/manager.py diff --git a/scrapy/trunk/scrapy/contrib/codecs/__init__.py b/scrapy/contrib/codecs/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/codecs/__init__.py rename to scrapy/contrib/codecs/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/codecs/x_mac_roman.py b/scrapy/contrib/codecs/x_mac_roman.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/codecs/x_mac_roman.py rename to scrapy/contrib/codecs/x_mac_roman.py diff --git a/scrapy/trunk/scrapy/contrib/debug.py b/scrapy/contrib/debug.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/debug.py rename to scrapy/contrib/debug.py diff --git a/scrapy/trunk/scrapy/contrib/delayedclosedomain.py b/scrapy/contrib/delayedclosedomain.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/delayedclosedomain.py rename to scrapy/contrib/delayedclosedomain.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/web/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/web/__init__.py rename to scrapy/contrib/downloadermiddleware/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/contrib/downloadermiddleware/cache.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py rename to scrapy/contrib/downloadermiddleware/cache.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py b/scrapy/contrib/downloadermiddleware/common.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py rename to scrapy/contrib/downloadermiddleware/common.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py rename to scrapy/contrib/downloadermiddleware/cookies.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py b/scrapy/contrib/downloadermiddleware/debug.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py rename to scrapy/contrib/downloadermiddleware/debug.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/errorpages.py b/scrapy/contrib/downloadermiddleware/errorpages.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/errorpages.py rename to scrapy/contrib/downloadermiddleware/errorpages.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py rename to scrapy/contrib/downloadermiddleware/httpauth.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py rename to scrapy/contrib/downloadermiddleware/httpcompression.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py rename to scrapy/contrib/downloadermiddleware/redirect.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py rename to scrapy/contrib/downloadermiddleware/retry.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py rename to scrapy/contrib/downloadermiddleware/robotstxt.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py rename to scrapy/contrib/downloadermiddleware/stats.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py rename to scrapy/contrib/downloadermiddleware/useragent.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/groupsettings.py b/scrapy/contrib/groupsettings.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/groupsettings.py rename to scrapy/contrib/groupsettings.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/item/__init__.py b/scrapy/contrib/item/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/item/__init__.py rename to scrapy/contrib/item/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/contrib/item/models.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/item/models.py rename to scrapy/contrib/item/models.py diff --git a/scrapy/trunk/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/itemsampler.py rename to scrapy/contrib/itemsampler.py diff --git a/scrapy/trunk/scrapy/contrib/link_extractors.py b/scrapy/contrib/link_extractors.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/link_extractors.py rename to scrapy/contrib/link_extractors.py diff --git a/scrapy/trunk/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/memdebug.py rename to scrapy/contrib/memdebug.py diff --git a/scrapy/trunk/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/memusage.py rename to scrapy/contrib/memusage.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/__init__.py b/scrapy/contrib/pipeline/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/__init__.py rename to scrapy/contrib/pipeline/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/pipeline/images.py rename to scrapy/contrib/pipeline/images.py diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/pipeline/media.py rename to scrapy/contrib/pipeline/media.py diff --git a/scrapy/trunk/scrapy/contrib/pipeline/s3images.py b/scrapy/contrib/pipeline/s3images.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/pipeline/s3images.py rename to scrapy/contrib/pipeline/s3images.py diff --git a/scrapy/trunk/scrapy/contrib/pipeline/show.py b/scrapy/contrib/pipeline/show.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/pipeline/show.py rename to scrapy/contrib/pipeline/show.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/prioritizers.py b/scrapy/contrib/prioritizers.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/prioritizers.py rename to scrapy/contrib/prioritizers.py diff --git a/scrapy/branches/cluster-refactor/scrapy/management/__init__.py b/scrapy/contrib/response/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/management/__init__.py rename to scrapy/contrib/response/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/response/soup.py b/scrapy/contrib/response/soup.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/response/soup.py rename to scrapy/contrib/response/soup.py diff --git a/scrapy/branches/cluster-refactor/scrapy/patches/__init__.py b/scrapy/contrib/schedulermiddleware/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/patches/__init__.py rename to scrapy/contrib/schedulermiddleware/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py rename to scrapy/contrib/schedulermiddleware/duplicatesfilter.py diff --git a/scrapy/branches/cluster-refactor/scrapy/store/__init__.py b/scrapy/contrib/spider/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/store/__init__.py rename to scrapy/contrib/spider/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spider/profiler.py b/scrapy/contrib/spider/profiler.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/spider/profiler.py rename to scrapy/contrib/spider/profiler.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/spider/reloader.py b/scrapy/contrib/spider/reloader.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/spider/reloader.py rename to scrapy/contrib/spider/reloader.py diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/__init__.py rename to scrapy/contrib/spidermiddleware/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py rename to scrapy/contrib/spidermiddleware/depth.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py b/scrapy/contrib/spidermiddleware/duplicatesfilter.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py rename to scrapy/contrib/spidermiddleware/duplicatesfilter.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py b/scrapy/contrib/spidermiddleware/limit.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py rename to scrapy/contrib/spidermiddleware/limit.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py rename to scrapy/contrib/spidermiddleware/offsite.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py rename to scrapy/contrib/spidermiddleware/referer.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/contrib/spidermiddleware/restrict.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py rename to scrapy/contrib/spidermiddleware/restrict.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py b/scrapy/contrib/spidermiddleware/urlfilter.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py rename to scrapy/contrib/spidermiddleware/urlfilter.py diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py rename to scrapy/contrib/spidermiddleware/urllength.py diff --git a/scrapy/trunk/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spiders/__init__.py rename to scrapy/contrib/spiders/__init__.py diff --git a/scrapy/trunk/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spiders/crawl.py rename to scrapy/contrib/spiders/crawl.py diff --git a/scrapy/trunk/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spiders/feed.py rename to scrapy/contrib/spiders/feed.py diff --git a/scrapy/trunk/scrapy/contrib/spiders/generic.py b/scrapy/contrib/spiders/generic.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/spiders/generic.py rename to scrapy/contrib/spiders/generic.py diff --git a/scrapy/branches/cluster-refactor/scrapy/utils/__init__.py b/scrapy/contrib/web/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/utils/__init__.py rename to scrapy/contrib/web/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/web/http.py b/scrapy/contrib/web/http.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/web/http.py rename to scrapy/contrib/web/http.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/web/json.py b/scrapy/contrib/web/json.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/web/json.py rename to scrapy/contrib/web/json.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/web/service.py b/scrapy/contrib/web/service.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/web/service.py rename to scrapy/contrib/web/service.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/web/site.py b/scrapy/contrib/web/site.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/web/site.py rename to scrapy/contrib/web/site.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/web/stats.py b/scrapy/contrib/web/stats.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/web/stats.py rename to scrapy/contrib/web/stats.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/__init__.py b/scrapy/contrib/webconsole/__init__.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/__init__.py rename to scrapy/contrib/webconsole/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/enginestatus.py b/scrapy/contrib/webconsole/enginestatus.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/enginestatus.py rename to scrapy/contrib/webconsole/enginestatus.py diff --git a/scrapy/trunk/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/webconsole/livestats.py rename to scrapy/contrib/webconsole/livestats.py diff --git a/scrapy/trunk/scrapy/contrib/webconsole/scheduler.py b/scrapy/contrib/webconsole/scheduler.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/webconsole/scheduler.py rename to scrapy/contrib/webconsole/scheduler.py diff --git a/scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py rename to scrapy/contrib/webconsole/spiderctl.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderstats.py b/scrapy/contrib/webconsole/spiderstats.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderstats.py rename to scrapy/contrib/webconsole/spiderstats.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/stats.py rename to scrapy/contrib/webconsole/stats.py diff --git a/scrapy/trunk/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/__init__.py rename to scrapy/contrib_exp/__init__.py diff --git a/scrapy/trunk/scrapy/contrib_exp/adaptors/__init__.py b/scrapy/contrib_exp/adaptors/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/adaptors/__init__.py rename to scrapy/contrib_exp/adaptors/__init__.py diff --git a/scrapy/trunk/scrapy/contrib_exp/adaptors/date.py b/scrapy/contrib_exp/adaptors/date.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/adaptors/date.py rename to scrapy/contrib_exp/adaptors/date.py diff --git a/scrapy/trunk/scrapy/contrib_exp/adaptors/extraction.py b/scrapy/contrib_exp/adaptors/extraction.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/adaptors/extraction.py rename to scrapy/contrib_exp/adaptors/extraction.py diff --git a/scrapy/trunk/scrapy/contrib_exp/adaptors/markup.py b/scrapy/contrib_exp/adaptors/markup.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/adaptors/markup.py rename to scrapy/contrib_exp/adaptors/markup.py diff --git a/scrapy/trunk/scrapy/contrib_exp/adaptors/misc.py b/scrapy/contrib_exp/adaptors/misc.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/adaptors/misc.py rename to scrapy/contrib_exp/adaptors/misc.py diff --git a/scrapy/trunk/examples/experimental/googledir/googledir/spiders/__init__.py b/scrapy/contrib_exp/downloadermiddleware/__init__.py similarity index 100% rename from scrapy/trunk/examples/experimental/googledir/googledir/spiders/__init__.py rename to scrapy/contrib_exp/downloadermiddleware/__init__.py diff --git a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/decompression.py rename to scrapy/contrib_exp/downloadermiddleware/decompression.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/history/__init__.py b/scrapy/contrib_exp/history/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/history/__init__.py rename to scrapy/contrib_exp/history/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/history/history.py b/scrapy/contrib_exp/history/history.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/history/history.py rename to scrapy/contrib_exp/history/history.py diff --git a/scrapy/trunk/scrapy/contrib_exp/history/middleware.py b/scrapy/contrib_exp/history/middleware.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/history/middleware.py rename to scrapy/contrib_exp/history/middleware.py diff --git a/scrapy/trunk/scrapy/contrib_exp/history/scheduler.py b/scrapy/contrib_exp/history/scheduler.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/history/scheduler.py rename to scrapy/contrib_exp/history/scheduler.py diff --git a/scrapy/branches/cluster-refactor/scrapy/contrib/history/store.py b/scrapy/contrib_exp/history/store.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/contrib/history/store.py rename to scrapy/contrib_exp/history/store.py diff --git a/scrapy/trunk/scrapy/contrib_exp/link/__init__.py b/scrapy/contrib_exp/link/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/link/__init__.py rename to scrapy/contrib_exp/link/__init__.py diff --git a/scrapy/trunk/scrapy/contrib_exp/newitem/__init__.py b/scrapy/contrib_exp/newitem/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/newitem/__init__.py rename to scrapy/contrib_exp/newitem/__init__.py diff --git a/scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py b/scrapy/contrib_exp/newitem/adaptors.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py rename to scrapy/contrib_exp/newitem/adaptors.py diff --git a/scrapy/trunk/scrapy/contrib_exp/newitem/fields.py b/scrapy/contrib_exp/newitem/fields.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/newitem/fields.py rename to scrapy/contrib_exp/newitem/fields.py diff --git a/scrapy/trunk/scrapy/contrib_exp/newitem/models.py b/scrapy/contrib_exp/newitem/models.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/newitem/models.py rename to scrapy/contrib_exp/newitem/models.py diff --git a/scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py similarity index 100% rename from scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py rename to scrapy/contrib_exp/pipeline/shoveitem.py diff --git a/scrapy/branches/cluster-refactor/scrapy/core/__init__.py b/scrapy/core/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/core/__init__.py rename to scrapy/core/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/core/downloader/__init__.py rename to scrapy/core/downloader/__init__.py diff --git a/scrapy/trunk/scrapy/core/downloader/dnscache.py b/scrapy/core/downloader/dnscache.py similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/dnscache.py rename to scrapy/core/downloader/dnscache.py diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/handlers.py rename to scrapy/core/downloader/handlers.py diff --git a/scrapy/trunk/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/manager.py rename to scrapy/core/downloader/manager.py diff --git a/scrapy/trunk/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/middleware.py rename to scrapy/core/downloader/middleware.py diff --git a/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py b/scrapy/core/downloader/responsetypes/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py rename to scrapy/core/downloader/responsetypes/__init__.py diff --git a/scrapy/trunk/scrapy/core/downloader/responsetypes/mime.types b/scrapy/core/downloader/responsetypes/mime.types similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/responsetypes/mime.types rename to scrapy/core/downloader/responsetypes/mime.types diff --git a/scrapy/trunk/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py similarity index 100% rename from scrapy/trunk/scrapy/core/downloader/webclient.py rename to scrapy/core/downloader/webclient.py diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/core/engine.py similarity index 100% rename from scrapy/trunk/scrapy/core/engine.py rename to scrapy/core/engine.py diff --git a/scrapy/trunk/scrapy/core/exceptions.py b/scrapy/core/exceptions.py similarity index 100% rename from scrapy/trunk/scrapy/core/exceptions.py rename to scrapy/core/exceptions.py diff --git a/scrapy/trunk/scrapy/core/manager.py b/scrapy/core/manager.py similarity index 100% rename from scrapy/trunk/scrapy/core/manager.py rename to scrapy/core/manager.py diff --git a/scrapy/branches/cluster-refactor/scrapy/core/prioritizers.py b/scrapy/core/prioritizers.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/core/prioritizers.py rename to scrapy/core/prioritizers.py diff --git a/scrapy/trunk/scrapy/core/scheduler/__init__.py b/scrapy/core/scheduler/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/core/scheduler/__init__.py rename to scrapy/core/scheduler/__init__.py diff --git a/scrapy/trunk/scrapy/core/scheduler/middleware.py b/scrapy/core/scheduler/middleware.py similarity index 100% rename from scrapy/trunk/scrapy/core/scheduler/middleware.py rename to scrapy/core/scheduler/middleware.py diff --git a/scrapy/trunk/scrapy/core/scheduler/schedulers.py b/scrapy/core/scheduler/schedulers.py similarity index 100% rename from scrapy/trunk/scrapy/core/scheduler/schedulers.py rename to scrapy/core/scheduler/schedulers.py diff --git a/scrapy/branches/cluster-refactor/scrapy/core/scheduler/store.py b/scrapy/core/scheduler/store.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/core/scheduler/store.py rename to scrapy/core/scheduler/store.py diff --git a/scrapy/trunk/scrapy/core/signals.py b/scrapy/core/signals.py similarity index 100% rename from scrapy/trunk/scrapy/core/signals.py rename to scrapy/core/signals.py diff --git a/scrapy/trunk/scrapy/dupefilter/__init__.py b/scrapy/dupefilter/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/dupefilter/__init__.py rename to scrapy/dupefilter/__init__.py diff --git a/scrapy/trunk/scrapy/extension/__init__.py b/scrapy/extension/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/extension/__init__.py rename to scrapy/extension/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/fetcher/__init__.py b/scrapy/fetcher/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/fetcher/__init__.py rename to scrapy/fetcher/__init__.py diff --git a/scrapy/trunk/scrapy/http/__init__.py b/scrapy/http/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/http/__init__.py rename to scrapy/http/__init__.py diff --git a/scrapy/trunk/scrapy/http/cookies.py b/scrapy/http/cookies.py similarity index 100% rename from scrapy/trunk/scrapy/http/cookies.py rename to scrapy/http/cookies.py diff --git a/scrapy/trunk/scrapy/http/headers.py b/scrapy/http/headers.py similarity index 100% rename from scrapy/trunk/scrapy/http/headers.py rename to scrapy/http/headers.py diff --git a/scrapy/trunk/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/http/request/__init__.py rename to scrapy/http/request/__init__.py diff --git a/scrapy/trunk/scrapy/http/request/form.py b/scrapy/http/request/form.py similarity index 100% rename from scrapy/trunk/scrapy/http/request/form.py rename to scrapy/http/request/form.py diff --git a/scrapy/trunk/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py similarity index 100% rename from scrapy/trunk/scrapy/http/request/rpc.py rename to scrapy/http/request/rpc.py diff --git a/scrapy/trunk/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/http/response/__init__.py rename to scrapy/http/response/__init__.py diff --git a/scrapy/trunk/scrapy/http/response/html.py b/scrapy/http/response/html.py similarity index 100% rename from scrapy/trunk/scrapy/http/response/html.py rename to scrapy/http/response/html.py diff --git a/scrapy/trunk/scrapy/http/response/text.py b/scrapy/http/response/text.py similarity index 100% rename from scrapy/trunk/scrapy/http/response/text.py rename to scrapy/http/response/text.py diff --git a/scrapy/trunk/scrapy/http/response/xml.py b/scrapy/http/response/xml.py similarity index 100% rename from scrapy/trunk/scrapy/http/response/xml.py rename to scrapy/http/response/xml.py diff --git a/scrapy/branches/cluster-refactor/scrapy/http/url.py b/scrapy/http/url.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/http/url.py rename to scrapy/http/url.py diff --git a/scrapy/trunk/scrapy/item/__init__.py b/scrapy/item/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/item/__init__.py rename to scrapy/item/__init__.py diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/item/adaptors.py similarity index 100% rename from scrapy/trunk/scrapy/item/adaptors.py rename to scrapy/item/adaptors.py diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/item/models.py similarity index 100% rename from scrapy/trunk/scrapy/item/models.py rename to scrapy/item/models.py diff --git a/scrapy/trunk/scrapy/item/pipeline.py b/scrapy/item/pipeline.py similarity index 100% rename from scrapy/trunk/scrapy/item/pipeline.py rename to scrapy/item/pipeline.py diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/link/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/link/__init__.py rename to scrapy/link/__init__.py diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/link/extractors.py similarity index 100% rename from scrapy/trunk/scrapy/link/extractors.py rename to scrapy/link/extractors.py diff --git a/scrapy/trunk/scrapy/log/__init__.py b/scrapy/log/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/log/__init__.py rename to scrapy/log/__init__.py diff --git a/scrapy/trunk/scrapy/mail/__init__.py b/scrapy/mail/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/mail/__init__.py rename to scrapy/mail/__init__.py diff --git a/scrapy/trunk/examples/googledir/googledir/__init__.py b/scrapy/management/__init__.py similarity index 100% rename from scrapy/trunk/examples/googledir/googledir/__init__.py rename to scrapy/management/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/management/telnet.py b/scrapy/management/telnet.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/management/telnet.py rename to scrapy/management/telnet.py diff --git a/scrapy/trunk/scrapy/management/web.py b/scrapy/management/web.py similarity index 100% rename from scrapy/trunk/scrapy/management/web.py rename to scrapy/management/web.py diff --git a/scrapy/trunk/scrapy/command/commands/__init__.py b/scrapy/patches/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/command/commands/__init__.py rename to scrapy/patches/__init__.py diff --git a/scrapy/trunk/scrapy/patches/monkeypatches.py b/scrapy/patches/monkeypatches.py similarity index 100% rename from scrapy/trunk/scrapy/patches/monkeypatches.py rename to scrapy/patches/monkeypatches.py diff --git a/scrapy/branches/cluster-refactor/scrapy/spider/__init__.py b/scrapy/spider/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/spider/__init__.py rename to scrapy/spider/__init__.py diff --git a/scrapy/trunk/scrapy/spider/manager.py b/scrapy/spider/manager.py similarity index 100% rename from scrapy/trunk/scrapy/spider/manager.py rename to scrapy/spider/manager.py diff --git a/scrapy/trunk/scrapy/spider/middleware.py b/scrapy/spider/middleware.py similarity index 100% rename from scrapy/trunk/scrapy/spider/middleware.py rename to scrapy/spider/middleware.py diff --git a/scrapy/trunk/scrapy/spider/models.py b/scrapy/spider/models.py similarity index 100% rename from scrapy/trunk/scrapy/spider/models.py rename to scrapy/spider/models.py diff --git a/scrapy/branches/cluster-refactor/scrapy/stats/__init__.py b/scrapy/stats/__init__.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/stats/__init__.py rename to scrapy/stats/__init__.py diff --git a/scrapy/trunk/scrapy/stats/corestats.py b/scrapy/stats/corestats.py similarity index 100% rename from scrapy/trunk/scrapy/stats/corestats.py rename to scrapy/stats/corestats.py diff --git a/scrapy/branches/cluster-refactor/scrapy/stats/statscollector.py b/scrapy/stats/statscollector.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/stats/statscollector.py rename to scrapy/stats/statscollector.py diff --git a/scrapy/trunk/scrapy/conf/commands/__init__.py b/scrapy/store/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/conf/commands/__init__.py rename to scrapy/store/__init__.py diff --git a/scrapy/branches/cluster-refactor/scrapy/store/db.py b/scrapy/store/db.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/store/db.py rename to scrapy/store/db.py diff --git a/scrapy/trunk/scrapy/contrib/__init__.py b/scrapy/templates/project/module/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/__init__.py rename to scrapy/templates/project/module/__init__.py diff --git a/scrapy/trunk/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/items.py.tmpl rename to scrapy/templates/project/module/items.py.tmpl diff --git a/scrapy/trunk/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/pipelines.py.tmpl rename to scrapy/templates/project/module/pipelines.py.tmpl diff --git a/scrapy/trunk/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/settings.py.tmpl rename to scrapy/templates/project/module/settings.py.tmpl diff --git a/scrapy/trunk/examples/googledir/googledir/spiders/__init__.py b/scrapy/templates/project/module/spiders/__init__.py similarity index 100% rename from scrapy/trunk/examples/googledir/googledir/spiders/__init__.py rename to scrapy/templates/project/module/spiders/__init__.py diff --git a/scrapy/trunk/scrapy/templates/project/module/templates/spider_basic.tmpl b/scrapy/templates/project/module/templates/spider_basic.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/templates/spider_basic.tmpl rename to scrapy/templates/project/module/templates/spider_basic.tmpl diff --git a/scrapy/trunk/scrapy/templates/project/module/templates/spider_crawl.tmpl b/scrapy/templates/project/module/templates/spider_crawl.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/templates/spider_crawl.tmpl rename to scrapy/templates/project/module/templates/spider_crawl.tmpl diff --git a/scrapy/trunk/scrapy/templates/project/module/templates/spider_csvfeed.tmpl b/scrapy/templates/project/module/templates/spider_csvfeed.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/templates/spider_csvfeed.tmpl rename to scrapy/templates/project/module/templates/spider_csvfeed.tmpl diff --git a/scrapy/trunk/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl b/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl similarity index 100% rename from scrapy/trunk/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl rename to scrapy/templates/project/module/templates/spider_xmlfeed.tmpl diff --git a/scrapy/trunk/scrapy/templates/project/root/scrapy-ctl.py b/scrapy/templates/project/root/scrapy-ctl.py similarity index 100% rename from scrapy/trunk/scrapy/templates/project/root/scrapy-ctl.py rename to scrapy/templates/project/root/scrapy-ctl.py diff --git a/scrapy/trunk/scrapy/tests/__init__.py b/scrapy/tests/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/tests/__init__.py rename to scrapy/tests/__init__.py diff --git a/scrapy/trunk/scrapy/tests/run.py b/scrapy/tests/run.py similarity index 100% rename from scrapy/trunk/scrapy/tests/run.py rename to scrapy/tests/run.py diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html b/scrapy/tests/sample_data/adaptors/enc-ascii.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html rename to scrapy/tests/sample_data/adaptors/enc-ascii.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html b/scrapy/tests/sample_data/adaptors/enc-cp1252.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html rename to scrapy/tests/sample_data/adaptors/enc-cp1252.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html b/scrapy/tests/sample_data/adaptors/enc-latin1.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html rename to scrapy/tests/sample_data/adaptors/enc-latin1.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html b/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html rename to scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html b/scrapy/tests/sample_data/adaptors/enc-utf8.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html rename to scrapy/tests/sample_data/adaptors/enc-utf8.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml b/scrapy/tests/sample_data/adaptors/extr_unquoted.xml similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml rename to scrapy/tests/sample_data/adaptors/extr_unquoted.xml diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.tar b/scrapy/tests/sample_data/compressed/feed-sample1.tar similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.tar rename to scrapy/tests/sample_data/compressed/feed-sample1.tar diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml b/scrapy/tests/sample_data/compressed/feed-sample1.xml similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml rename to scrapy/tests/sample_data/compressed/feed-sample1.xml diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 b/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 rename to scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz b/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz rename to scrapy/tests/sample_data/compressed/feed-sample1.xml.gz diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.zip b/scrapy/tests/sample_data/compressed/feed-sample1.zip similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.zip rename to scrapy/tests/sample_data/compressed/feed-sample1.zip diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin b/scrapy/tests/sample_data/compressed/html-gzip.bin similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin rename to scrapy/tests/sample_data/compressed/html-gzip.bin diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin b/scrapy/tests/sample_data/compressed/html-rawdeflate.bin similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin rename to scrapy/tests/sample_data/compressed/html-rawdeflate.bin diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin b/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin rename to scrapy/tests/sample_data/compressed/html-zlibdeflate.bin diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/feeds/feed-sample1.xml b/scrapy/tests/sample_data/feeds/feed-sample1.xml similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/feeds/feed-sample1.xml rename to scrapy/tests/sample_data/feeds/feed-sample1.xml diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/feeds/feed-sample2.xml b/scrapy/tests/sample_data/feeds/feed-sample2.xml similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/feeds/feed-sample2.xml rename to scrapy/tests/sample_data/feeds/feed-sample2.xml diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.csv b/scrapy/tests/sample_data/feeds/feed-sample3.csv similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.csv rename to scrapy/tests/sample_data/feeds/feed-sample3.csv diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv b/scrapy/tests/sample_data/feeds/feed-sample4.csv similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv rename to scrapy/tests/sample_data/feeds/feed-sample4.csv diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv b/scrapy/tests/sample_data/feeds/feed-sample5.csv similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv rename to scrapy/tests/sample_data/feeds/feed-sample5.csv diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html b/scrapy/tests/sample_data/link_extractor/image_linkextractor.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html rename to scrapy/tests/sample_data/link_extractor/image_linkextractor.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html b/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html rename to scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html b/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html rename to scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html b/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html similarity index 100% rename from scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html rename to scrapy/tests/sample_data/link_extractor/regex_linkextractor.html diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/index.html b/scrapy/tests/sample_data/test_site/index.html similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/index.html rename to scrapy/tests/sample_data/test_site/index.html diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/item1.html b/scrapy/tests/sample_data/test_site/item1.html similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/item1.html rename to scrapy/tests/sample_data/test_site/item1.html diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/item2.html b/scrapy/tests/sample_data/test_site/item2.html similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/item2.html rename to scrapy/tests/sample_data/test_site/item2.html diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/tests/test_adaptors.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_adaptors.py rename to scrapy/tests/test_adaptors.py diff --git a/scrapy/trunk/scrapy/tests/test_aws.py b/scrapy/tests/test_aws.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_aws.py rename to scrapy/tests/test_aws.py diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_c14nurls.py b/scrapy/tests/test_c14nurls.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/test_c14nurls.py rename to scrapy/tests/test_c14nurls.py diff --git a/scrapy/trunk/scrapy/tests/test_contrib_response_soup.py b/scrapy/tests/test_contrib_response_soup.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_contrib_response_soup.py rename to scrapy/tests/test_contrib_response_soup.py diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_dependencies.py b/scrapy/tests/test_dependencies.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/test_dependencies.py rename to scrapy/tests/test_dependencies.py diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py rename to scrapy/tests/test_downloadermiddleware_cookies.py diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_downloadermiddleware_decompression.py rename to scrapy/tests/test_downloadermiddleware_decompression.py diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py rename to scrapy/tests/test_downloadermiddleware_httpcompression.py diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py rename to scrapy/tests/test_downloadermiddleware_redirect.py diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_downloadermiddleware_retry.py rename to scrapy/tests/test_downloadermiddleware_retry.py diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py rename to scrapy/tests/test_downloadermiddleware_useragent.py diff --git a/scrapy/trunk/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_dupefilter.py rename to scrapy/tests/test_dupefilter.py diff --git a/scrapy/trunk/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_engine.py rename to scrapy/tests/test_engine.py diff --git a/scrapy/trunk/scrapy/tests/test_http_cookies.py b/scrapy/tests/test_http_cookies.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_http_cookies.py rename to scrapy/tests/test_http_cookies.py diff --git a/scrapy/trunk/scrapy/tests/test_http_headers.py b/scrapy/tests/test_http_headers.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_http_headers.py rename to scrapy/tests/test_http_headers.py diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_http_request.py rename to scrapy/tests/test_http_request.py diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_http_response.py rename to scrapy/tests/test_http_response.py diff --git a/scrapy/trunk/scrapy/tests/test_http_url.py b/scrapy/tests/test_http_url.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_http_url.py rename to scrapy/tests/test_http_url.py diff --git a/scrapy/trunk/scrapy/tests/test_item.py b/scrapy/tests/test_item.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_item.py rename to scrapy/tests/test_item.py diff --git a/scrapy/trunk/scrapy/tests/test_itemadaptor.py b/scrapy/tests/test_itemadaptor.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_itemadaptor.py rename to scrapy/tests/test_itemadaptor.py diff --git a/scrapy/trunk/scrapy/tests/test_libxml2.py b/scrapy/tests/test_libxml2.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_libxml2.py rename to scrapy/tests/test_libxml2.py diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/tests/test_link.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_link.py rename to scrapy/tests/test_link.py diff --git a/scrapy/trunk/scrapy/tests/test_newitem.py b/scrapy/tests/test_newitem.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_newitem.py rename to scrapy/tests/test_newitem.py diff --git a/scrapy/trunk/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_pipeline_images.py rename to scrapy/tests/test_pipeline_images.py diff --git a/scrapy/trunk/scrapy/tests/test_responsetypes.py b/scrapy/tests/test_responsetypes.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_responsetypes.py rename to scrapy/tests/test_responsetypes.py diff --git a/scrapy/trunk/scrapy/tests/test_robustscrapeditem.py b/scrapy/tests/test_robustscrapeditem.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_robustscrapeditem.py rename to scrapy/tests/test_robustscrapeditem.py diff --git a/scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py rename to scrapy/tests/test_schedulermiddleware_duplicatesfilter.py diff --git a/scrapy/trunk/scrapy/tests/test_serialization.py b/scrapy/tests/test_serialization.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_serialization.py rename to scrapy/tests/test_serialization.py diff --git a/scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py b/scrapy/tests/test_spidermiddleware_duplicatesfilter.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py rename to scrapy/tests/test_spidermiddleware_duplicatesfilter.py diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_spidermonkey.py b/scrapy/tests/test_spidermonkey.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/test_spidermonkey.py rename to scrapy/tests/test_spidermonkey.py diff --git a/scrapy/trunk/scrapy/contrib/cluster/crawler/__init__.py b/scrapy/tests/test_spiders/__init__.py similarity index 100% rename from scrapy/trunk/scrapy/contrib/cluster/crawler/__init__.py rename to scrapy/tests/test_spiders/__init__.py diff --git a/scrapy/trunk/scrapy/tests/test_spiders/testspider.py b/scrapy/tests/test_spiders/testspider.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_spiders/testspider.py rename to scrapy/tests/test_spiders/testspider.py diff --git a/scrapy/branches/cluster-refactor/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py similarity index 100% rename from scrapy/branches/cluster-refactor/scrapy/tests/test_stats.py rename to scrapy/tests/test_stats.py diff --git a/scrapy/trunk/scrapy/tests/test_storedb.py b/scrapy/tests/test_storedb.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_storedb.py rename to scrapy/tests/test_storedb.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_datatypes.py b/scrapy/tests/test_utils_datatypes.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_datatypes.py rename to scrapy/tests/test_utils_datatypes.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_defer.py b/scrapy/tests/test_utils_defer.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_defer.py rename to scrapy/tests/test_utils_defer.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_iterators.py rename to scrapy/tests/test_utils_iterators.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_markup.py b/scrapy/tests/test_utils_markup.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_markup.py rename to scrapy/tests/test_utils_markup.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_misc.py b/scrapy/tests/test_utils_misc.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_misc.py rename to scrapy/tests/test_utils_misc.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_python.py rename to scrapy/tests/test_utils_python.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_request.py b/scrapy/tests/test_utils_request.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_request.py rename to scrapy/tests/test_utils_request.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_response.py rename to scrapy/tests/test_utils_response.py diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_utils_url.py rename to scrapy/tests/test_utils_url.py diff --git a/scrapy/trunk/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_webclient.py rename to scrapy/tests/test_webclient.py diff --git a/scrapy/trunk/scrapy/tests/test_xpath.py b/scrapy/tests/test_xpath.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_xpath.py rename to scrapy/tests/test_xpath.py diff --git a/scrapy/trunk/scrapy/tests/test_xpath_extension.py b/scrapy/tests/test_xpath_extension.py similarity index 100% rename from scrapy/trunk/scrapy/tests/test_xpath_extension.py rename to scrapy/tests/test_xpath_extension.py diff --git a/scrapy/trunk/README b/scrapy/trunk/README deleted file mode 100644 index e59d381f7..000000000 --- a/scrapy/trunk/README +++ /dev/null @@ -1,3 +0,0 @@ -This is Scrapy, an opensource screen scraping framework written in Python. - -For more visit the project home page at http://scrapy.org diff --git a/scrapy/trunk/docs/media/scrapy-architecture.dia b/scrapy/trunk/docs/media/scrapy-architecture.dia deleted file mode 100644 index d23cba1ac5461e6cdfc27241c0e1236ef77b2cf7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2751 zcmV;w3PANAiwFP!000021MOW)Z{s);zW1*%+`ybN4BwCOqz5~P#V*i`UF^=B1I1Px z?Z}c>Ryv)_{Pra&<zZX06;V{;ga*3NQqwGzT>QSOVv+p$m+z}|@+~elNuFIy5rET4 zoXzt^k}WT$zyJ2vALaCyFQ0y1B+*a$&$5VC6a9^BW3DcyH)Xm0>HPfe?hd5)o2bkS zkS4ca6QBPXrD=4o8=X(Te40$2cd&@c$lR;%MrBzfSGQ$6$)Z(!F};fBe=LjqHd{=q zR&~31p612mTa;c*KVRF=^t|roe5;=WeXpZsd{xBJAMIU3{ZXn<*Xy|0?RK@!H;Hai z-mmwXwDe>Ce%7XLwb6~T<(JQ2<Fbs4&(&kqH=p`zde~M$<ti$cNwzObzn`>v85l64 z%!eb%2*iRsyqA}YyBb$qI<B~4Tye?fa-A1t5hdlm$g4b0<0z}7mc?z{U(RM8rP{;} zsfGIYew~zMe&qSDqjb|nf$H|-h<oYiT^7mW@ZjBQ(bCIZvM6sZzk7OJRqywndjFPe zlB+asdio?Q2kZWRnC^Ff9gE;kUw5>Is<&2(Sjd>g?Bx`(yiFGI=7^bXw`u9=rfznA z?5*#%>-u7<TRVhpMWoSvT-4qF`b5=}`UkEN=h<7xul^g)%UZqP|M7M5<K!>%GWnL2 z_tPiOL;D@c;$r$wxcz!|71Et)<lB2c>-^D(Fb0&fGXw!ayL#`VMw^VQt6dR=RwBxl zX}m9HhIw^?0qQ9;PUF?(JTJ0+2Vqd%tQ-p@`}#eA2zBWCCQg<&WqaQU$bP+R8j3H? zAiUjsLA%}?g#AQZCu!P2#PxN=`idyy@8$Fel<l->V`l67ZMEBR72`}hByo$YcmQA9 z-cFvCP2$Y`$#t|!(tG_>QMQ>*Hs!qm0;$Jz_1(Y4>9@E{=FyM8=IP=&YQ6k9_AIQy z${>(%`$OG?ZbMGv(4^JSM1U~N+@x*OPe)7dBS<@U%WM@l|FCg(yK3O&L(JSnE^P#P z{=^Nv$pQ^P77Vd741wT&$>LFrAq#t<?{Uv$;iy|6ivh^ud7|R&lEr4N{lwxG$zq=? z-YZSmL8(m}2AUX&Ca^0_AOaXMZZ{SlS>l<L84~v27(n5ZBb*fr6!BpwVx6qx^mQYO z4?+@%*-@!O2^2VDB#s~hgc3yQ01!~(H<EaeVgd=S4djdfOgsk?Zklx>34oEH>gf|$ z<6UH-HuIZ!ahtwsZ1K?&Q4A7!h5;3UJw{>=>dGF30*b67t~|2GGbuBow9}96qy<zy z>BCvIZuB8Jeoh}U(8qhAk41i$rFpan{4o}P5Dp|o*oGe@5XdkD@*j=Di>O#=7Wt|% zpn*Y{bB;>XK_l{-qY~P!P~6HviqEP^25PP`hdq7vd7fpLakjA9yjaKfX>$EcJ*UaF z!OYjY57Jbpa;Q_ImVSHsXWs0g<-`9>ZnR<4sf{wXxys4Y|8C{Z+P{nH@51fxlKrxe zod+7mbh&frT)V9O1KDn|vvEYI&|;8KLdVZqtT9~(2#3V8={D=wK9AhL#N+jBmu%Y~ zcd%VFlI;p-+vToU)6I6dv0Y`ck0|ZE8#95Z#$DvwcpH^$b@lZ#s#CLF=zzQQb#PY> zfV+-mYcIlG6KOIfEbem6<5+vQXbj+OP_lhywtu<An=#%O0WW$T;8mwm=c$8zMVWOP z7_<(rRE*VL1M_XXjY_tr==(cRgV=6ky2HlMyFEWeWPQ{8+2S%Piu|s<XXM!PWg2J8 zLpdi1n5&2*S*^|zhCmJY64cmn_1H+Lf#3lCn&YHdH`ri6aqSBk*pP<vxV~n}j%49I zPHtAR-LHG)U?bqhN#VxM*cT3A9v%xf&`@!nZ+D}P^AymqA!QZkAzPv))M#<uF?Yjx zN+-a^hj%xyLxc^-f(<;-yl($F*6yf5gvq>c_&hs=wIAReH5zqn2bccYhJYF$4>f3O zpz&g7LyUzQWGH9DcVbQlDNxC$#GDRN5Tqst#yJxsIU7e4a|YZrn3<>As)r5G5uoD* z4#(ib|2I?P@@(&%hv82L$jd8Z@kjy0`&xY0)O;mL0*+@I53sD{_FbAdW^E7(D-lL0 zCT+IF_fQN3Jro1A?ZaUosU6FYr6LDHCg&m}j5IQ6dr$y-2Bun&1VZBy0*Y9v`~+YL z2UzkBOD$#?Q0Wv=$-RC|Xm--(KmqiON?H&lX9O^a6`f&Ij%ejlMG<-X&_n_>d55MJ zTMW>2iqIqmfF_fH)l4OkZ)mCoVPSwt*+35Vu1XvemS}*b5n!p&4g)NmA}l#qeiEWX zDFYA$K)<;3D9NlxH4{^iRC4JF022$q<bAnWn<WNhIz?ns15ABMR3n7vVD3Q@1_~RO z3CT*XJb@$=56I-*Woom;fJ`TdOc)N3`xd;KhbG8(@>@`1CS3%>$}BlRq!KLXDY#7{ zfYS&W@MdES;B<=MG(>r_R7{f<2gJA2WaThZk4$F-0xr$kJ`RLLC*U~!ke0K5$D4JY zZQ>u6<t#we7=BcPX$GKr%Rtpq+KAl?q9LF}RBH@;rkx;wwHV9{N=bXAJt^NyHk%bZ z&`v+o8>(sClu=P0D?OCXeRn86q&q-`-Gv@4M;u6Ihw4q~zj||MWxTz<Ue%olIV@jA z_;nVPVCT|sVwPDjJGDok@S3?tfY*{R6B<M@p!V8;sPGP>ZF(MPsvoTPD+yp4!1T1i zbRdMHL!7B&GwBpa<~L8KC1GVHQHE)CvJSGrDdw2_-<zXZ&z7YwbyJQ5LIs2h2-P2< z*brL>DU*+|imVs=<Idx1c^Uq7uAA(AjP!q!3iN;YE?5!|L{IKly&0yc$Ev|A|p z7)_E_iYDDF=cUO9*IRk`_0^>LNh(Pcvl&(Z!>alMQS5ymwKhEuG&NCW@YMny24Bqq zp~Mi;BsRm*l(hSfCMg$I5|bEJ(Il~Jqbb58xNFUN9*8ETL-B4vsDMx(3ZdFcFWh%g zRsd2R)_L9Fsg|Lkh(bzjE*@fhrgZFvI60NFi~rrmn=({gj8b*6Z<V3q;uK56oGa>) z0^F{-;bxC$pJ&5pL1wE;X3|^qan}Zu?!x>;O2htM#;X8Oqa1SEVwM4=POvVFI9Jyr z{<vkO=S>_;Iv&dEI0^*S44GZ;5HssMxRAbQ4O{hWZdxxud1D*jbqG<^N7$Wipuyu@ z{f*4oCACrcU7DgLK_;cyX;yJG(_;y8MC-BRd!yM_HJh9AFm~Sm*l{jM)+TJu6x1dq z{E`lYsyb~KDQZYZlaH|W$))e)Vxx}D%>v=P*X>Yo>mgBIw?ie)C9h_4QYiSDSz_e1 z;`K=o#zIvQET`g30E`q0S+)?wb~Viq#GYWPjnDyh`+^i(b0EHV`>F*YWmY#4sV@Ww z7!@$8vrz_&ddnE~uROhPi+P4RE8cg^W6p|a_RM0&>$BpW&wX9%_j1|wp!j0}&%^PW zM~|Suu4lt~JSw77(FGc#=z@(<WHDld<Z`fGr*mp>^AWsotjKJb@=;SeP5>vubNT3l zBq)S31rSULY*y4ju29%_BKSR>jWaktuj=?z*J*Sg7hgVA7y8e#h*n=d{U3-@Fei_$ F006OyQr-Xn diff --git a/scrapy/trunk/extras/sql/scraping.sql b/scrapy/trunk/extras/sql/scraping.sql deleted file mode 100644 index c3314f5a1..000000000 --- a/scrapy/trunk/extras/sql/scraping.sql +++ /dev/null @@ -1,72 +0,0 @@ -DROP TABLE IF EXISTS `url_history`; -DROP TABLE IF EXISTS `version`; -DROP TABLE IF EXISTS `url_status`; -DROP TABLE IF EXISTS `ticket`; -DROP TABLE IF EXISTS `domain_stats`; -DROP TABLE IF EXISTS `domain_stats_history`; -DROP TABLE IF EXISTS `domain_data_history`; - -CREATE TABLE `ticket` ( - `guid` char(40) NOT NULL, - `domain` varchar(255) default NULL, - `url` varchar(2048) default NULL, - `url_hash` char(40) default NULL, -- so we can join to url_status - PRIMARY KEY (`guid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `version` ( - `id` bigint(20) NOT NULL auto_increment, - `guid` char(40) NOT NULL, - `version` char(40) NOT NULL, - `seen` datetime NOT NULL, - PRIMARY KEY (`id`), - FOREIGN KEY (`guid`) REFERENCES ticket(guid) ON UPDATE CASCADE ON DELETE CASCADE, - UNIQUE KEY (`version`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_status` ( - -- see http://support.microsoft.com/kb/q208427/ for explanation of 2048 - `url_hash` char(40) NOT NULL, -- for faster searches - `url` varchar(2048) NOT NULL, - `parent_hash` char(40) default NULL, -- the url that was followed to this one - for reporting - `last_version` char(40) default NULL, -- can be null if it generated an error the last time is was checked - `last_checked` datetime NOT NULL, - PRIMARY KEY (`url_hash`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_history` ( - `url_hash` char(40) NOT NULL, - `version` char(40) NOT NULL, - `postdata_hash` char(40) default NULL, - `created` datetime NOT NULL, - PRIMARY KEY (`version`), - FOREIGN KEY (`url_hash`) REFERENCES url_status(url_hash) ON UPDATE CASCADE ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats` ( - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` text, - PRIMARY KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats_history` ( - `id` bigint(20) NOT NULL auto_increment, - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` varchar(2048) NOT NULL, - `stored` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`), - KEY `stored` (`stored`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_data_history` ( - `domain` varchar(255) NOT NULL, - `stored` datetime NOT NULL, - `data` text, - KEY `domain_stored` (`domain`, `stored`), - KEY `domain` (`domain`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/scrapy/trunk/scrapy/command/__init__.py b/scrapy/trunk/scrapy/command/__init__.py deleted file mode 100644 index 0692578bb..000000000 --- a/scrapy/trunk/scrapy/command/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from scrapy.command.models import ScrapyCommand diff --git a/scrapy/trunk/scrapy/command/commands/download.py b/scrapy/trunk/scrapy/command/commands/download.py deleted file mode 100644 index a8a09a0f0..000000000 --- a/scrapy/trunk/scrapy/command/commands/download.py +++ /dev/null @@ -1,28 +0,0 @@ -import sys, pprint - -from scrapy.command import ScrapyCommand -from scrapy.fetcher import fetch - -class Command(ScrapyCommand): - def syntax(self): - return "[options] <url>" - - def short_desc(self): - return "Download a URL using the Scrapy downloader" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-s", "--source", dest="source", action="store_true", help="output HTTP body only") - parser.add_option("--headers", dest="headers", action="store_true", help="output HTTP headers only") - - def run(self, args, opts): - if not args: - print "A URL is required" - return - - responses = fetch(args) - if responses: - if opts.headers: - pprint.pprint(responses[0].headers) - else: - sys.stdout.write(str(responses[0].body)) diff --git a/scrapy/trunk/scrapy/command/commands/help.py b/scrapy/trunk/scrapy/command/commands/help.py deleted file mode 100644 index 8a7358763..000000000 --- a/scrapy/trunk/scrapy/command/commands/help.py +++ /dev/null @@ -1,26 +0,0 @@ -from scrapy.command import ScrapyCommand, cmdline - -class Command(ScrapyCommand): - def syntax(self): - return "<command>" - - def short_desc(self): - return "Provides extended help for the given command" - - def run(self, args, opts): - if not args: - return False - - commands = cmdline.builtin_commands_dict() - commands.update(cmdline.custom_commands_dict()) - - cmdname = args[0] - if cmdname in commands: - cmd = commands[cmdname] - help = getattr(cmd, 'help', None) or getattr(cmd, 'long_desc', None) - print "%s: %s" % (cmdname, cmd.short_desc()) - print "usage: %s %s" % (cmdname, cmd.syntax()) - print - print help() - else: - print "Unknown command: %s" % cmdname diff --git a/scrapy/trunk/scrapy/command/commands/list.py b/scrapy/trunk/scrapy/command/commands/list.py deleted file mode 100644 index 70f6f43c4..000000000 --- a/scrapy/trunk/scrapy/command/commands/list.py +++ /dev/null @@ -1,16 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.spider import spiders - -class Command(ScrapyCommand): - def syntax(self): - return "" - - def short_desc(self): - return "List available spiders (both enabled and disabled)" - - def run(self, args, opts): - spiders_dict = spiders.asdict() - for n, p in spiders_dict.items(): - disabled = "disabled" if getattr(p, 'disabled', False) else "enabled" - print "%-30s %-30s %s" % (n, p.__class__.__name__, disabled) - print "Total spiders: %d" % len(spiders_dict) diff --git a/scrapy/trunk/scrapy/command/commands/log.py b/scrapy/trunk/scrapy/command/commands/log.py deleted file mode 100644 index fff7ebbd8..000000000 --- a/scrapy/trunk/scrapy/command/commands/log.py +++ /dev/null @@ -1,46 +0,0 @@ -import re - -from scrapy.command import ScrapyCommand - -log_crawled_re = re.compile(r'^(.*?) \[\w+/(.*?)\].*Crawled <(.*?)> from <(.*?)>$') -log_scraped_re = re.compile('^(.*?) \[\w+/(.*?)\].*Scraped (.*) in <(.*?)>$') -log_opendomain_re = re.compile('^(.*?) \[\w+.*? Started scraping (.*)$') -log_closedomain_re = re.compile('^(.*?) \[\w+.*? Finished scraping (.*)$') -log_debug_re = re.compile(r'\[\w+/(.*?)\] DEBUG') -log_info_re = re.compile(r'\[\w+/(.*?)\] INFO') -log_warning_re = re.compile(r'\[\w+/(.*?)\] WARNING') -log_error_re = re.compile(r'\[\w+/(.*?)\] ERROR') - -class Command(ScrapyCommand): - def syntax(self): - return "[options] <logfile>" - - def short_desc(self): - return "Several tools to perform scrapy log analysis" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("--sitemap", dest="sitemap", action="store_true", help="show sitemap based on given log file") - - def sitemap(self, logfile): - from scrapy.utils.datatypes import Sitemap - - sm = Sitemap() - for l in open(logfile).readlines(): - m = log_crawled_re.search(l.strip()) - if m: - sm.add_node(m.group(3), m.group(4)) - m = log_scraped_re.search(l.strip()) - if m: - sm.add_item(m.group(4), m.group(3)) - print sm.to_string() - - def run(self, args, opts): - if not args: - print "A log file is required" - return - - if opts.sitemap: - self.sitemap(args[0]) - else: - print "No analysis method specified" diff --git a/scrapy/trunk/scrapy/command/commands/start.py b/scrapy/trunk/scrapy/command/commands/start.py deleted file mode 100644 index fb8786df2..000000000 --- a/scrapy/trunk/scrapy/command/commands/start.py +++ /dev/null @@ -1,16 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.core.manager import scrapymanager - - -class Command(ScrapyCommand): - def syntax(self): - return "[options]" - - def short_desc(self): - return "Start the Scrapy server" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - - def run(self, args, opts): - scrapymanager.start(*args, **opts.__dict__) diff --git a/scrapy/trunk/scrapy/conf/commands/crawl.py b/scrapy/trunk/scrapy/conf/commands/crawl.py deleted file mode 100644 index 357422ea3..000000000 --- a/scrapy/trunk/scrapy/conf/commands/crawl.py +++ /dev/null @@ -1 +0,0 @@ -LOG_STDOUT = True diff --git a/scrapy/trunk/scrapy/conf/commands/help.py b/scrapy/trunk/scrapy/conf/commands/help.py deleted file mode 100644 index bcee80b55..000000000 --- a/scrapy/trunk/scrapy/conf/commands/help.py +++ /dev/null @@ -1 +0,0 @@ -LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/list.py b/scrapy/trunk/scrapy/conf/commands/list.py deleted file mode 100644 index bcee80b55..000000000 --- a/scrapy/trunk/scrapy/conf/commands/list.py +++ /dev/null @@ -1 +0,0 @@ -LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/log.py b/scrapy/trunk/scrapy/conf/commands/log.py deleted file mode 100644 index bcee80b55..000000000 --- a/scrapy/trunk/scrapy/conf/commands/log.py +++ /dev/null @@ -1 +0,0 @@ -LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/scrape.py b/scrapy/trunk/scrapy/conf/commands/scrape.py deleted file mode 100644 index 96afeec1a..000000000 --- a/scrapy/trunk/scrapy/conf/commands/scrape.py +++ /dev/null @@ -1,2 +0,0 @@ -LOG_ENABLED = False - diff --git a/scrapy/trunk/scrapy/conf/commands/stats.py b/scrapy/trunk/scrapy/conf/commands/stats.py deleted file mode 100644 index bcee80b55..000000000 --- a/scrapy/trunk/scrapy/conf/commands/stats.py +++ /dev/null @@ -1 +0,0 @@ -LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/test.py b/scrapy/trunk/scrapy/conf/commands/test.py deleted file mode 100644 index bcee80b55..000000000 --- a/scrapy/trunk/scrapy/conf/commands/test.py +++ /dev/null @@ -1 +0,0 @@ -LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py b/scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py deleted file mode 100644 index d9d0931ad..000000000 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py +++ /dev/null @@ -1,13 +0,0 @@ -from scrapy.core.exceptions import HttpException - -class ErrorPagesMiddleware(object): - """This middleware allows the spiders to receive error (non 200) responses, - the same way the receive normal responses""" - - def process_exception(self, request, exception, spider): - if isinstance(exception, HttpException): - statuses = getattr(spider, 'handle_httpstatus_list', None) - httpstatus = exception.response.status - if statuses and httpstatus in statuses: - return exception.response - diff --git a/scrapy/trunk/scrapy/contrib/groupsettings.py b/scrapy/trunk/scrapy/contrib/groupsettings.py deleted file mode 100644 index 4bad4100b..000000000 --- a/scrapy/trunk/scrapy/contrib/groupsettings.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Extensions to override scrapy settings with per-group settings according to the -group the spider belongs to. It only overrides the settings when running the -crawl command with *only one domain as argument*. -""" - -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured -from scrapy.command.cmdline import command_executed - -class GroupSettings(object): - - def __init__(self): - if not settings.getbool("GROUPSETTINGS_ENABLED"): - raise NotConfigured - - if command_executed and command_executed['name'] == 'crawl': - mod = __import__(settings['GROUPSETTINGS_MODULE'], {}, {}, ['']) - args = command_executed['args'] - if len(args) == 1 and not args[0].startswith('http://'): - domain = args[0] - settings.overrides.update(mod.default_settings) - for group, domains in mod.group_spiders.iteritems(): - if domain in domains: - settings.overrides.update(mod.group_settings.get(group, {})) - diff --git a/scrapy/trunk/scrapy/contrib/item/__init__.py b/scrapy/trunk/scrapy/contrib/item/__init__.py deleted file mode 100644 index a6226fec9..000000000 --- a/scrapy/trunk/scrapy/contrib/item/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline diff --git a/scrapy/trunk/scrapy/contrib/pipeline/__init__.py b/scrapy/trunk/scrapy/contrib/pipeline/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/prioritizers.py b/scrapy/trunk/scrapy/contrib/prioritizers.py deleted file mode 100644 index e23a6c74a..000000000 --- a/scrapy/trunk/scrapy/contrib/prioritizers.py +++ /dev/null @@ -1,34 +0,0 @@ -import time - -from scrapy.core.exceptions import NotConfigured -from scrapy.store.db import DomainDataHistory -from scrapy.conf import settings - -class LessScrapedPrioritizer(object): - """ - A spider prioritizer based on these few simple rules: - 1. if spider was never scraped, it has top priority - 2. if spider was scraped before, then the less recently the spider - has been scraped, the more priority it has - """ - def __init__(self, elements): - if not settings['SCRAPING_DB']: - raise NotConfigured("SCRAPING_DB setting is required") - - self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - domains_to_scrape = set(elements) - - self.priorities = {} - - for domain in domains_to_scrape: - stat = self.ddh.getlast(domain, path="start_time") - if stat and stat[1]: - last_started = stat[1] - # spider is the timestamp of last start time - self.priorities[domain] = time.mktime(last_started.timetuple()) - else: - # if domain was never scraped, it has top priority - self.priorities[domain] = 1 - - def get_priority(self, element): - return self.priorities[element] diff --git a/scrapy/trunk/scrapy/contrib/response/__init__.py b/scrapy/trunk/scrapy/contrib/response/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/schedulermiddleware/__init__.py b/scrapy/trunk/scrapy/contrib/schedulermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/spider/__init__.py b/scrapy/trunk/scrapy/contrib/spider/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/spider/profiler.py b/scrapy/trunk/scrapy/contrib/spider/profiler.py deleted file mode 100644 index 2c8638bd7..000000000 --- a/scrapy/trunk/scrapy/contrib/spider/profiler.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -SpiderProfiler is an extension that hooks itself into every Request callback -returned from spiders to measure the processing time and memory allocation -caused by spiders code. - -The results are collected using the StatsCollector. - -This extension introduces a big impact on crawling performance, so enable only -when needed. -""" - -import datetime - -from pydispatch import dispatcher - -from scrapy.extension import extensions -from scrapy.core import signals -from scrapy.core.exceptions import NotConfigured -from scrapy.stats import stats -from scrapy.conf import settings - -class SpiderProfiler(object): - - def __init__(self): - if not settings.getbool('SPIDERPROFILER_ENABLED'): - raise NotConfigured - dispatcher.connect(self._request_received, signals.request_received) - dispatcher.connect(self._engine_started, signals.engine_started) - - def _engine_started(self): - self.memusage = extensions.enabled.get('MemoryUsage', None) - - def _request_received(self, request, spider): - old_cbs = request.deferred.callbacks[0] - new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], old_cbs[0][2]), old_cbs[1]) - request.deferred.callbacks[0] = new_cbs - - def _profiled_callback(self, function, spider): - def new_callback(*args, **kwargs): - tbefore = datetime.datetime.now() - mbefore = self._memusage() - r = function(*args, **kwargs) - tafter = datetime.datetime.now() - mafter = self._memusage() - ct = tafter-tbefore - tcc = stats.getpath('%s/profiling/total_callback_time' % spider.domain_name, datetime.timedelta(0)) - sct = stats.getpath('%s/profiling/slowest_callback_time' % spider.domain_name, datetime.timedelta(0)) - stats.setpath('%s/profiling/total_callback_time' % spider.domain_name, tcc+ct) - if ct > sct: - stats.setpath('%s/profiling/slowest_callback_time' % spider.domain_name, ct) - stats.setpath('%s/profiling/slowest_callback_name' % spider.domain_name, function.__name__) - stats.setpath('%s/profiling/slowest_callback_url' % spider.domain_name, args[0].url) - if self.memusage: - tma = stats.getpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, 0) - stats.setpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, tma+mafter-mbefore) - return r - return new_callback - - def _memusage(self): - return self.memusage.virtual if self.memusage else 0.0 diff --git a/scrapy/trunk/scrapy/contrib/spider/reloader.py b/scrapy/trunk/scrapy/contrib/spider/reloader.py deleted file mode 100644 index 1aa116f12..000000000 --- a/scrapy/trunk/scrapy/contrib/spider/reloader.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Reload spider modules once they are finished scraping - -This is to release any resources held on to by scraping spiders. -""" -import sys -from pydispatch import dispatcher -from scrapy.core import signals -from scrapy import log - -class SpiderReloader(object): - - def __init__(self): - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) - - def domain_closed(self, domain, spider): - module = spider.__module__ - log.msg("reloading module %s" % module, domain=domain) - reload(sys.modules[module]) diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/web/__init__.py b/scrapy/trunk/scrapy/contrib/web/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/web/http.py b/scrapy/trunk/scrapy/contrib/web/http.py deleted file mode 100644 index dcc7732b0..000000000 --- a/scrapy/trunk/scrapy/contrib/web/http.py +++ /dev/null @@ -1,115 +0,0 @@ -"""A django alike request-response model - -most of this code is borrowed from django -""" - -from Cookie import SimpleCookie -from scrapy.utils.datatypes import MultiValueDict, CaselessDict - - -def build_httprequest(twistedrequest): - """Translate twisted request object to a django request approach""" - request = HttpRequest() - request.path = twistedrequest.path - request.method = twistedrequest.method.upper() - request.COOKIES = SimpleCookie(twistedrequest.received_cookies) - request.HEADERS = Headers(twistedrequest.received_headers) - request.ARGS = MultiValueDict(twistedrequest.args) - request.FILES = {} # not yet supported - request.content = twistedrequest.content - request.twistedrequest = twistedrequest - return request - - -class HttpRequest(object): - def __init__(self): - self.path = '' - self.method = None - self.COOKIES = {} - self.HEADERS = {} - self.ARGS = {} - self.FILES = {} - - -class HttpResponse(object): - status_code = 200 - - def __init__(self, content='', status=None, content_type=None): - content_type = content_type or "text/html; charset=utf-8" - self._headers = {'content-type': content_type} - self.content = content - self.cookies = SimpleCookie() - self.status_code = status - - def __str__(self): - "Full HTTP message, including headers" - return '\n'.join(['%s: %s' % (key, value) - for key, value in self._headers.items()]) \ - + '\n\n' + self.content - - def __setitem__(self, header, value): - self._headers[header.lower()] = value - - def __delitem__(self, header): - try: - del self._headers[header.lower()] - except KeyError: - pass - - def __getitem__(self, header): - return self._headers[header.lower()] - - def has_header(self, header): - "Case-insensitive check for a header" - return self._headers.has_key(header.lower()) - - __contains__ = has_header - - def items(self): - return self._headers.items() - - def get(self, header, alternate): - return self._headers.get(header, alternate) - - def set_cookie(self, key, value='', max_age=None, expires=None, path='/', domain=None, secure=None): - self.cookies[key] = value - for var in ('max_age', 'path', 'domain', 'secure', 'expires'): - val = locals()[var] - if val is not None: - self.cookies[key][var.replace('_', '-')] = val - - def delete_cookie(self, key, path='/', domain=None): - self.cookies[key] = '' - if path is not None: - self.cookies[key]['path'] = path - if domain is not None: - self.cookies[key]['domain'] = domain - self.cookies[key]['expires'] = 0 - self.cookies[key]['max-age'] = 0 - - -class Headers(CaselessDict): - def __init__(self, source=None, encoding='utf-8'): - self.encoding = encoding - - if getattr(source, 'iteritems', None): - d = source.iteritems() - else: - d = source # best effort - - # can't use CaselessDict.__init__(self, d) because it doesn't call __setitem__ - for k,v in d: - self.__setitem__(k.lower(), v) - - def normkey(self, key): - return key.title() # 'Content-Type' styles headers - - def __setitem__(self, key, value): - """Headers must not be unicode""" - if isinstance(key, unicode): - key = key.encode(self.encoding) - if isinstance(value, unicode): - value = value.encode(self.encoding) - super(Headers, self).__setitem__(key, value) - - diff --git a/scrapy/trunk/scrapy/contrib/web/json.py b/scrapy/trunk/scrapy/contrib/web/json.py deleted file mode 100644 index 1987b1dc9..000000000 --- a/scrapy/trunk/scrapy/contrib/web/json.py +++ /dev/null @@ -1,99 +0,0 @@ -from functools import wraps - -from scrapy.utils.defer import mustbe_deferred -from scrapy.utils.serialization import serialize as _serialize, unserialize as _unserialize - -from .http import HttpResponse - -JSON_CONTENT_TYPES = ('application/json',) - -serialize = lambda x: _serialize(x, 'json') -unserialize = lambda x: _unserialize(x, 'json') - -class JsonException(Exception): - pass - -class JsonResponse(HttpResponse): - def __init__(self, content=None, callback=None, serialize=serialize, *args, **kwargs): - content = serialize(content) - if callback: # JSONP support - status, content = 200, '%s(%s)' % (callback, content) - kwargs.setdefault('content_type', 'application/x-javascript') - HttpResponse.__init__(self, content=content, *args, **kwargs) - -class JsonResponseAccepted(JsonResponse): - status_code = 202 - -class JsonResponseNoContent(JsonResponse): - status_code = 204 - -class JsonResponseNotModified(JsonResponse): - status_code = 304 - -class JsonResponseBadRequest(JsonResponse): - status_code = 400 - -class JsonResponseUnauthorized(JsonResponse): - status_code = 401 - -class JsonResponseForbidden(JsonResponse): - status_code = 403 - -class JsonResponseNotFound(JsonResponse): - status_code = 404 - -class JsonResponseInternalServerError(JsonResponse): - status_code = 500 - -class JsonResponseNotImplemented(JsonResponse): - status_code = 501 - - -def json(func): - """ Decorator to wrap a json prepared view and return a JsonResponse - - if content-type is application/json, sets request.JSON to unserialized request body. - in case of unserialization failure, returns JsonResponseBadRequest() - - if returned data from func is a dictionary, serialize it and returns JsonResponse() - """ - if not hasattr(func, '__call__'): - raise TypeError('The argument should be a callable') - - @wraps(func) - def wrapper(request, *args, **kwargs): - json_callback = request.ARGS.get('callback') # JSONP support - request.method = method = _x_http_method_override(request) - request.content_type = ct = content_type(request) - request.JSON = None - - if method in ('POST', 'PUT'): - if ct in JSON_CONTENT_TYPES: - body = request.content.read() - try: - request.JSON = unserialize(body) - except Exception, e: - return JsonResponseBadRequest('Invalid json: %s' % e ) - - def _onsuccess(response): - if not isinstance(response, HttpResponse): - return JsonResponse(response, json_callback) # best effort - return response - - ret = mustbe_deferred(func, request, *args, **kwargs) - ret.addCallback(_onsuccess) - return ret - return wrapper - -def content_type(request): - ct = request.HEADERS.get('content-type','') - return ct.split(';')[0].strip() - -def _x_http_method_override(request): - """ support for X-Http-Method-Override hack - - some clients does not support methods others than GET and POST, that clients - has a chance to set an extra header to indicate intended method. - """ - return request.HEADERS.get('x-http-method-override', request.method).upper() - diff --git a/scrapy/trunk/scrapy/contrib/web/service.py b/scrapy/trunk/scrapy/contrib/web/service.py deleted file mode 100644 index 8db4cc1ac..000000000 --- a/scrapy/trunk/scrapy/contrib/web/service.py +++ /dev/null @@ -1,89 +0,0 @@ -import re -import urllib -import hashlib -from twisted.internet import defer - -from scrapy.core.engine import scrapyengine -from scrapy.spider import spiders -from scrapy.http import Request -from scrapy.item import ScrapedItem -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings -from scrapy.utils.misc import memoize -from lrucache import LRUCache - -from .site import WebSite, WebResource -from .http import HttpResponse -from .json import JsonResponse - -JSONCALLBACK_RE = '^[a-zA-Z][a-zA-Z_.-]*$' -CACHESIZE = settings.get('WS_CACHESIZE', 20) - - -def _urlhash(request): - h = hashlib.sha1() - for a in sorted(request.ARGS): - h.update(request.ARGS[a]) - return h.hexdigest() - - -@memoize(cache=LRUCache(CACHESIZE), hash=_urlhash) -def url_to_guid(httprequest): - url = httprequest.ARGS.get('url') - if not url: - return HttpResponse('Bad Request', 400) - url = urllib.unquote(url) - - jsoncb = httprequest.ARGS.get('callback') - if jsoncb and not re.match(JSONCALLBACK_RE, jsoncb): - return HttpResponse('Bad callback argument', 400) - - def _response(guids=(), message=None): - content = { - 'guids': list(guids), - 'domain': getattr(spider, 'domain_name', None), - 'message': message, - } - return JsonResponse(content=content, callback=jsoncb) - - spider = spiders.fromurl(url) - if not spider: - return _response(message='No crawler found for site') - - if httprequest.ARGS.get('dontcrawl'): - return _response() - - - def _on_error(_failure): - return _response(message='Error downloading url from site') - - def _on_success(pagedata): - try: - items = spider.identify(pagedata) - except Exception, ex: - return _response(message='Error processing url') - - guids = [i.guid for i in items if isinstance(i, ScrapedItem)] - return _response(guids=guids) - - deferred = defer.Deferred().addCallbacks(_on_success, _on_error) - request = Request(url=url, callback=deferred, dont_filter=True) - schd = scrapyengine.schedule(request, spider) - schd.chainDeferred(deferred) - return deferred - - -urlmapping = ( - ('^ws/tools/url_to_guid/$', url_to_guid), -) - - -class UrlToGuidService(WebSite): - def __init__(self): - if not settings.getbool('WS_ENABLED'): - raise NotConfigured - - port = settings.getint('WS_PORT') or 8088 - timeout = settings.getint('WS_TIMEOUT') or 15 # seconds - resource = WebResource(urlmapping, timeout=timeout) - WebSite.__init__(self, port=port, resource=resource) diff --git a/scrapy/trunk/scrapy/contrib/web/site.py b/scrapy/trunk/scrapy/contrib/web/site.py deleted file mode 100644 index 40c23772b..000000000 --- a/scrapy/trunk/scrapy/contrib/web/site.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Twisted website object as django - -################################################################################ -## Simple Usage example: - -from twisted.internet import reactor -from scrapy.contrib.web.http import WebSite, HttpResponse - -def helloword(request): - return HttpResponse('Hello World!') - -def hello(request, name): - return HttpResponse('Hello %s' % name) - -urls = ( - ('^hello/(?P<name>\w+)/$', hello), - ('^$', helloword), - ) - - -resource = WebResource(urls) -site = WebSite(port=8081, resource=resource) -reactor.run() - -# now go to http://localhost:8081/ - - - -################################################################################ -## Complex usage example: - -from twisted.internet import reactor, defer -from scrapy.contrib.web.http import WebSite, HttpResponse - -def delayed(request): - def _callback(result): - return HttpResponse('Heavy task completed: %s' % result) - - def _errback(_failure): - return HttpResponse('Internal Server Error: %s' % _failure, status=500) - - def heavytask(_): - import random - assert random.randint(0,1), "Exception found processing request" - return _ - - d = defer.Deferred().addCallback(heavytask) - d.addCallbacks(_callback, _errback) - reactor.callLater(1, d.callback, "Well done") - return d - -urls = (('^delayed/$', delayed),) - -resource = WebResource(urls) -site = WebSite(port=8081, resource=resource) -reactor.run() - -""" - -import re - -from twisted.web import server, resource -from twisted.internet import reactor -from scrapy.utils.defer import mustbe_deferred - -from .http import HttpResponse, build_httprequest - - -def urlresolver(urls, path): - """Simple path to view mapper""" - path = path.lstrip('/') - for pathre, view in urls: - m = re.search(pathre, path) - if m: - kwargs = m.groupdict() - args = () if kwargs else m.groups() - return view, args, kwargs - return None, (), {} - - -class WebSite(server.Site): - def __init__(self, port=None, *args, **kwargs): - server.Site.__init__(self, *args, **kwargs) - if port: - self.bind(port) - - def bind(self, port): - from scrapy.core.engine import scrapyengine - scrapyengine.listenTCP(port, self) - - -class WebResource(resource.Resource): - """Translate twisted web approach to django alike way""" - isLeaf = True - debug = True - - def __init__(self, urls, timeout=3, urlresolver=urlresolver): - resource.Resource.__init__(self) - self.urlresolver = urlresolver - self.timeout = timeout - self.urls = urls - - def render(self, twistedrequest): - httprequest = build_httprequest(twistedrequest) - - def _send_response(response): - assert isinstance(response, HttpResponse), 'view should return a HttpResponse object' - twistedrequest.setResponseCode(response.status_code or 200) - for key, val in response.items(): - twistedrequest.setHeader(key, response[key]) - twistedrequest.write(response.content) - twistedrequest.finish() - - def _on_error(_failure): - content = _failure.getTraceback() if self.debug else 'Internal Error' - response = HttpResponse(content=str(_failure), status=500) - return _send_response(response) - - view, args, kwargs = self.urlresolver(self.urls, httprequest.path) - if not view: - response = HttpResponse(content='Not Found', status=404) - _send_response(response) - return server.NOT_DONE_YET - - deferred = mustbe_deferred(view, httprequest, *args, **kwargs) - deferred.addCallback(_send_response) - deferred.addErrback(_on_error) - if not deferred.timeoutCall: - deferred.setTimeout(self.timeout) - - return server.NOT_DONE_YET - - diff --git a/scrapy/trunk/scrapy/contrib/web/stats.py b/scrapy/trunk/scrapy/contrib/web/stats.py deleted file mode 100644 index b86961482..000000000 --- a/scrapy/trunk/scrapy/contrib/web/stats.py +++ /dev/null @@ -1,55 +0,0 @@ -from functools import partial -from scrapy.core.exceptions import NotConfigured -from scrapy.utils.serialization import serialize -from scrapy.conf import settings - -from .site import WebSite, WebResource -from .json import JsonResponse - -STATS_WSPORT = settings.getint('STATS_WSPORT', 8089) -STATS_WSTIMEOUT = settings.getint('STATS_WSTIMEOUT', 15) - -jsonserialize = partial(serialize, format='json') - - -def stats(request): - service = request.ARGS.get('service', 'getstats') - if service == 'getstats': - domain = request.ARGS.get('domain') - count = request.ARGS.get('count', 1) - if not domain: - path = request.ARGS.get('path') - offset = request.ARGS.get('offset') - order = request.ARGS.get('order', 'domain') - olist = request.ARGS.get('olist', 'ASC') - - from scrapy.store.db import DomainDataHistory - ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - - if service == 'countdomains': - stats = [ddh.domain_count()] - elif domain: - stats = list(ddh.get(domain, count=int(count))) - else: - stats = list(ddh.getlast_alldomains(count, offset, order, olist, path=path)) - - content = {'data': stats if stats else "No stats found"} - return JsonResponse(content=content, serialize=jsonserialize) - - -urlmapping = ( - ('^ws/tools/stats/$', stats), - ) - - -class StatsService(WebSite): - def __init__(self): - if not settings['WS_ENABLED']: - raise NotConfigured - - if not settings['SCRAPING_DB']: - print "SCRAPING_DB setting is required for the stats web service" - raise NotConfigured - - resource = WebResource(urlmapping, timeout=STATS_WSTIMEOUT) - WebSite.__init__(self, port=STATS_WSPORT, resource=resource) diff --git a/scrapy/trunk/scrapy/contrib/webconsole/__init__.py b/scrapy/trunk/scrapy/contrib/webconsole/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py b/scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py deleted file mode 100644 index e90b9ab30..000000000 --- a/scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Scheduler information module for Scrapy webconsole -""" -from pydispatch import dispatcher -from scrapy.core.engine import scrapyengine -from scrapy.management.web import banner - -class EngineStatus(object): - webconsole_id = 'enginestatus' - webconsole_name = 'Engine status' - - def __init__(self): - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - s = banner(self) - s += "<pre><code>\n" - s += scrapyengine.getstatus() - s += "</pre></code>\n" - s += "</body>\n" - s += "</html>\n" - - return s - - def webconsole_discover_module(self): - return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py b/scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py deleted file mode 100644 index df8d56515..000000000 --- a/scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Extension for persistent statistics by domain -""" -import re -import pprint - -from pydispatch import dispatcher - -from scrapy.spider import spiders -from scrapy.core.exceptions import NotConfigured -from scrapy.management.web import banner -from scrapy.conf import settings - -class SpiderStats(object): - webconsole_id = 'spiderstats' - webconsole_name = 'Spider stats (all-time)' - - stats_mainpage = [ - ('item_scraped_count', "Items scraped <br>(last run)"), - ('item_passed_count', "Items passed <br>(last run)"), - ('response_count', "Pages crawled <br>(last run)"), - ('start_time', "Start time <br>(last run)"), - ('finish_time', "Finish time <br>(last run)"), - ('finish_status', "Finish status<br>(last run)"), - ] - - PATH_RE = re.compile("/spiderstats/([^/]+)") - - def __init__(self): - if not settings['SCRAPING_DB']: - raise NotConfigured("SpiderStats: Requires SCRAPING_DB setting") - - from scrapy.store.db import DomainDataHistory - self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - m = self.PATH_RE.search(wc_request.path) - if m: - return self.render_domain(m.group(1)) - else: - return self.render_main() - - def render_main(self): - s = banner(self) - s += "<table border='1'>\n" - s += "<tr>\n" - s += "<th>Domain</th>" - for path, title in self.stats_mainpage: - s += "<th>%s</th>" % title - s += "</tr>\n" - for domain in spiders.asdict().keys(): - s += "<tr>\n" - s += "<td><a href='%s'>%s</a></td>" % (domain, domain) - for path, title in self.stats_mainpage: - stat = self.ddh.getlast(domain, path=path) - value = stat[1] if stat else "None" - s += "<td>%s</td>" % value - s += "</tr>\n" - s += "</table>\n" - s += "</body>\n" - s += "</html>\n" - - return str(s) - - def render_domain(self, domain): - s = banner(self) - stats = self.ddh.getall(domain) - s += "<pre>\n" - s += pprint.pformat(list(stats)) - s += "</pre>\n" - s += "</body>\n" - s += "</html>\n" - - return str(s) - - def webconsole_discover_module(self): - return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/stats.py b/scrapy/trunk/scrapy/contrib/webconsole/stats.py deleted file mode 100644 index f58cb50e8..000000000 --- a/scrapy/trunk/scrapy/contrib/webconsole/stats.py +++ /dev/null @@ -1,26 +0,0 @@ -import pprint - -from pydispatch import dispatcher - -from scrapy.stats import stats -from scrapy.management.web import banner, webconsole_discover_module - -class StatsDump(object): - webconsole_id = 'stats' - webconsole_name = 'StatsCollector dump' - - def __init__(self): - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - s = banner(self) - s += "<pre><code>\n" - s += pprint.pformat(stats) - s += "</pre></code>\n" - s += "</body>\n" - s += "</html>\n" - - return str(s) - - def webconsole_discover_module(self): - return self diff --git a/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/__init__.py b/scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/contrib_exp/history/__init__.py b/scrapy/trunk/scrapy/contrib_exp/history/__init__.py deleted file mode 100644 index 81eb201d6..000000000 --- a/scrapy/trunk/scrapy/contrib_exp/history/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.contrib.history.history import ItemHistory -from scrapy.contrib.history.scheduler import RulesScheduler -from scrapy.contrib.history.store import SQLHistoryStore diff --git a/scrapy/trunk/scrapy/contrib_exp/history/history.py b/scrapy/trunk/scrapy/contrib_exp/history/history.py deleted file mode 100644 index 6767ef260..000000000 --- a/scrapy/trunk/scrapy/contrib_exp/history/history.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -History management -""" -import re -from datetime import datetime - -import MySQLdb - -class History(object): - """Base class for tracking different kinds of histories""" - - def __init__(self, db_uri): - self.db_uri = db_uri - self._mysql_conn = None - - def connect(self): - """ - Connect to PDB and open mysql connect to PRODUCT_DB - """ - m = re.search(r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/]+)/(?P<db>.*)$", self.db_uri) - if m: - d = m.groupdict() - if d['passwd'] is None: - del(d['passwd']) - - d['charset'] = "utf8" - self._mysql_conn = MySQLdb.connect(**d) - - def get_mysql_conn(self): - if self._mysql_conn is None: - self.connect() - return self._mysql_conn - mysql_conn = property(get_mysql_conn) - - -class ItemHistory(History): - """ - Class instance registers item guids, and versions with check_item method. - It also gives access to stored ItemTicket and ItemVersion objects. - """ - NEW = 0 - UPDATE = 1 - DUPLICATE = 2 - - def check_item(self, domain, item): - """ - Store item's guid and version to the database along - with date and time of last occurence. - Return: - * ItemHistory.NEW - if guid hasn't been met - * ItemHistory.UPDATE - if history already contains guid, but versions doesn't match - * ItemHistory.DUPLICATE - both guid and version are not new - """ - version = item.version - - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - - def add_version(version): - insert = "INSERT INTO version (guid, version, seen) VALUES (%s,%s,%s)" - c.execute(insert, (item.guid, version, datetime.now())) - - select = "SELECT * FROM ticket WHERE guid=%s" - c.execute(select, item.guid) - r = c.fetchone() - if r: - select = "SELECT * FROM version WHERE version=%s" - if c.execute(select, version): - update = "UPDATE version SET seen=%s WHERE version=%s" - c.execute(update, (datetime.now(), version)) - self.mysql_conn.commit() - return ItemHistory.DUPLICATE - else: - add_version(version) - self.mysql_conn.commit() - return ItemHistory.UPDATE - else: - insert = "INSERT INTO ticket (guid, domain, url, url_hash) VALUES (%s,%s,%s,%s)" - c.execute(insert, (item.guid, domain, item.url, hash(item.url))) - add_version(version) - self.mysql_conn.commit() - return ItemHistory.NEW - - def get_ticket(self, guid): - """ - Return ItemTicket object for guid. - ItemVersion objects can be accessed via 'versions' list. - """ - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM ticket WHERE guid=%s" - c.execute(select, guid) - ticket = c.fetchone() - if not ticket: - raise Exception("Item ticket with guid = '%s' not found" % guid) - ticket['versions'] = [] - select = "SELECT * FROM version WHERE guid=%s" - c.execute(select, guid) - for version in c.fetchall(): - ticket['versions'].append(version) - return ticket - - def delete_ticket(self, guid): - """Delete item ticket and associated versions from DB""" - c = self.mysql_conn.cursor() - delete = "DELETE FROM ticket WHERE guid=%s" - c.execute(delete, guid) - self.mysql_conn.commit() - - -class URLHistory(History): - """ - Access URL status and history for the scraping engine - - This is degsigned to have an instance per domain where typically - a call will be made to get_url_status, followed by either - update_checked or record_version. - """ - - def get_url_status(self, urlkey): - """ - Get the url status (url, last_version, last_checked), - or None if the url data has not been seen before - """ - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM url_status WHERE url_hash=%s" - c.execute(select, urlkey) - r = c.fetchone() - return (r['url'], r['last_version'], r['last_checked']) if r else None - - def record_version(self, urlkey, url, parent_key, version, postdata_hash=None): - """ - Record a version of a page and update the last checked time. - - If the same version (or None) is passed, the last checked time is still updated. - """ - now = datetime.now() - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM url_status WHERE url_hash=%s" - c.execute(select, urlkey) - r = c.fetchone() - - if not r: - insert = "INSERT INTO url_status (url_hash, url, parent_hash, last_version, last_checked) VALUES (%s,%s,%s,%s,%s)" - c.execute(insert, (urlkey, url, parent_key, version, now)) - else: - update = "UPDATE url_status SET last_version=%s, last_checked=%s WHERE url_hash=%s" - c.execute(update, (version, now, urlkey)) - self.mysql_conn.commit() - - last_version = r['last_version'] if r else None - if version and version != last_version: - if not c.execute("SELECT url_hash FROM url_history WHERE version=%s", version): - insert = "INSERT INTO url_history (url_hash, version, postdata_hash, created) VALUES (%s,%s,%s,%s)" - c.execute(insert, (urlkey, version, postdata_hash, now)) - self.mysql_conn.commit() - - def get_version_info(self, version): - """Simple accessor method""" - c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) - select = "SELECT * FROM url_history WHERE version=%s" - c.execute(select, version) - r = c.fetchone() - return (r['url_hash'], r['created']) if r else None diff --git a/scrapy/trunk/scrapy/contrib_exp/history/store.py b/scrapy/trunk/scrapy/contrib_exp/history/store.py deleted file mode 100644 index bd9d01681..000000000 --- a/scrapy/trunk/scrapy/contrib_exp/history/store.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -SQLHistoryStore - -Persistent history storage using relational database -""" -from scrapy.contrib.history.history import URLHistory -from scrapy import log -from scrapy.conf import settings - -class SQLHistoryStore(object) : - """Implementation of a data store that stores information in a relation - database - - This maintains a URLHistory object per site. That means each domain - has it's own session and is isolated from the others. - """ - - def __init__(self): - self._store = {} - self._dbinfo = settings['SCRAPING_DB'] - self._debug = settings['DEBUG_SQL_HISTORY_STORE'] - - def open(self, site): - self._store[site] = URLHistory(self._dbinfo) - - def close_site(self, site): - self._store[site].close() - del self._store[site] - - def store(self, site, key, url, parent=None, version=None, post_version=None): - history = self._store[site] - if self._debug: - log.msg("record_version(key=%s, url=%s, parent=%s, version=%s, post_version=%s)" % - (key, url, parent, version, post_version), domain=site, level=log.DEBUG) - history.record_version(key, url, parent, version, post_version) - - def has_site(self, site): - return site in self._store - - def status(self, site, key): - if site in self._store: - history = self._store[site] - return history.get_url_status(key) - - def version_info(self, site, version): - history = self._store[site] - return history.get_version_info(version) diff --git a/scrapy/trunk/scrapy/core/__init__.py b/scrapy/trunk/scrapy/core/__init__.py deleted file mode 100644 index 59e07e643..000000000 --- a/scrapy/trunk/scrapy/core/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Scrapy core library classes and functions. -""" diff --git a/scrapy/trunk/scrapy/core/downloader/__init__.py b/scrapy/trunk/scrapy/core/downloader/__init__.py deleted file mode 100644 index cecc64c8e..000000000 --- a/scrapy/trunk/scrapy/core/downloader/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from scrapy.core.downloader.manager import Downloader diff --git a/scrapy/trunk/scrapy/core/prioritizers.py b/scrapy/trunk/scrapy/core/prioritizers.py deleted file mode 100644 index 214a475a3..000000000 --- a/scrapy/trunk/scrapy/core/prioritizers.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -The Prioritizer is a class which receives a list of elements and prioritizes -it. It's used for defining the order in which domains are to be scraped. - -A Prioritizer basically consists of a class which receives a list of elements -in its constructs and contains only one method: get_priority() which returns -the priority of the given element. The element passed to get_priority() must -exists in the list of elements passed in the constructor. - -This module contains several basic prioritizers. - -For more advanced prioritizers see: scrapy.contrib.prioritizers -""" -import random - -class NullPrioritizer(object): - """ - This prioritizer always return the same priority (1) - """ - def __init__(self, elements): - pass - - def get_priority(self, element): - return 1 - -class RandomPrioritizer(object): - """ - This prioritizer always return a random priority - """ - def __init__(self, elements): - self.count = len(elements) - - def get_priority(self, element): - return random.randrange(0, self.count) - -class AlphabeticPrioritizer(object): - """ - This prioritizer priotizes based on the alphabetic order of the element - """ - def __init__(self, elements): - self.elements = elements[:] - self.elements.sort() - - def get_priority(self, element): - return self.elements.index(element) diff --git a/scrapy/trunk/scrapy/core/scheduler/store.py b/scrapy/trunk/scrapy/core/scheduler/store.py deleted file mode 100644 index b7eddfcf4..000000000 --- a/scrapy/trunk/scrapy/core/scheduler/store.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -The datastore module contains implementations of data storage engines for the -crawling process. These are used to track metrics on each site and on each -page visited. -""" -from datetime import datetime - -class MemoryStore(object) : - """Simple implementation of a data store. This is useful if no persistent - history is required (e.g. unit testing or development) and provides a - simpler reference implementation. - """ - def __init__(self) : - """The store will just be a dict with an entry for each site, that - entry will contain dicts and lists of data. - """ - self._store = {} - - def open(self, site): - self._store[site] = {} - - def close_site(self, site): - del self._store[site] - - def store(self, site, key, url, parent=None, version=None, post_version=None): - checked = datetime.now() - self._store[key] = (version, checked) - - def status(self, site, key): - """Get the version and last checked time for a key. - - this will be changed later to support checking (lazily) the last modified - time and perhaps other statistics needed by the scheduling algorithms - """ - return self._store.get(key) diff --git a/scrapy/trunk/scrapy/fetcher/__init__.py b/scrapy/trunk/scrapy/fetcher/__init__.py deleted file mode 100644 index 29fbde74a..000000000 --- a/scrapy/trunk/scrapy/fetcher/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -import urlparse - -from scrapy.spider import spiders -from scrapy.http import Request -from scrapy.core.manager import scrapymanager -from scrapy.spider import BaseSpider - -def fetch(urls, perdomain=False): - """Download a set of urls. This is starts a reactor and is suitable - for calling from a main method - not for calling from within the - framework. - - It will return a list of Response objects for all pages successfully - downloaded. - """ - bigbucket = {} # big bucket to store response objects (per domain) - - def _add(response, domain): - if not domain in bigbucket: - bigbucket[domain] = [] - bigbucket[domain] += [response] - - # url clasification by domain - requestdata = set() - for url in urls: - domain = get_or_create_spider(url).domain_name - request = Request(url=url, callback=lambda r: _add(r, domain), dont_filter=True) - requestdata.add(request) - - scrapymanager.runonce(*requestdata) - - # returns bigbucket as dict or flatted - if perdomain: - return bigbucket - - flatbucket = [] - for domain, responses in bigbucket.iteritems(): - flatbucket.extend(responses) - return flatbucket - - -def get_or_create_spider(url): - # dirty hack to allow downloading pages from unknown domains - spider = spiders.fromurl(url) - if not spider: - domain = str(urlparse.urlparse(url).hostname or spiders.default_domain) - spider = BaseSpider() - spider.domain_name = domain - spiders._alldict[domain] = spider - return spider - diff --git a/scrapy/trunk/scrapy/http/url.py b/scrapy/trunk/scrapy/http/url.py deleted file mode 100644 index 098150be1..000000000 --- a/scrapy/trunk/scrapy/http/url.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -The Url class is similar to the urlparse.ParseResult class (it has the same -attributes) with the following differences: - -- it inherits from str -- it's lazy, so it only parses the url when needed -""" - -import urlparse - -class Url(str): - - @property - def parsedurl(self): - if not hasattr(self, '_parsedurl'): - self._parsedurl = urlparse.urlparse(self) - return self._parsedurl - - @property - def scheme(self): - return self.parsedurl.scheme - - @property - def netloc(self): - return self.parsedurl.netloc - - @property - def path(self): - return self.parsedurl.path - - @property - def params(self): - return self.parsedurl.params - - @property - def query(self): - return self.parsedurl.query - - @property - def fragment(self): - return self.parsedurl.fragment - - @property - def username(self): - return self.parsedurl.username - - @property - def password(self): - return self.parsedurl.password - - @property - def hostname(self): - return self.parsedurl.hostname - - @property - def port(self): - return self.parsedurl.port diff --git a/scrapy/trunk/scrapy/management/__init__.py b/scrapy/trunk/scrapy/management/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/management/telnet.py b/scrapy/trunk/scrapy/management/telnet.py deleted file mode 100644 index 52178785c..000000000 --- a/scrapy/trunk/scrapy/management/telnet.py +++ /dev/null @@ -1,42 +0,0 @@ -import pprint - -from twisted.internet import reactor -from twisted.conch import manhole, telnet -from twisted.conch.insults import insults -from twisted.internet import protocol - -from scrapy.extension import extensions -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.stats import stats -from scrapy.conf import settings - -try: - import guppy - hpy = guppy.hpy() -except ImportError: - hpy = None - -telnet_namespace = {'ee': scrapyengine, - 'em': scrapymanager, - 'ex': extensions.enabled, - 'st': stats, - 'h': hpy, - 'p': pprint.pprint} # useful shortcut for debugging - -def makeProtocol(): - return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, - insults.ServerProtocol, - manhole.Manhole, telnet_namespace) - -class TelnetConsole(protocol.ServerFactory): - - def __init__(self): - if not settings.getbool('TELNETCONSOLE_ENABLED'): - return - - #protocol.ServerFactory.__init__(self) - self.protocol = makeProtocol - self.noisy = False - port = settings.getint('TELNETCONSOLE_PORT') - scrapyengine.listenTCP(port, self) diff --git a/scrapy/trunk/scrapy/patches/__init__.py b/scrapy/trunk/scrapy/patches/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/spider/__init__.py b/scrapy/trunk/scrapy/spider/__init__.py deleted file mode 100644 index 2a88816de..000000000 --- a/scrapy/trunk/scrapy/spider/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from scrapy.spider.models import BaseSpider -from scrapy.spider.manager import SpiderManager - -spiders = SpiderManager() diff --git a/scrapy/trunk/scrapy/stats/__init__.py b/scrapy/trunk/scrapy/stats/__init__.py deleted file mode 100644 index 2b2b2fb61..000000000 --- a/scrapy/trunk/scrapy/stats/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.stats.statscollector import StatsCollector - -stats = StatsCollector() diff --git a/scrapy/trunk/scrapy/stats/statscollector.py b/scrapy/trunk/scrapy/stats/statscollector.py deleted file mode 100644 index 2ff1f3e37..000000000 --- a/scrapy/trunk/scrapy/stats/statscollector.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Scrapy extension for collecting scraping stats -""" -import pprint - -from pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.utils.misc import stats_getpath -from scrapy.conf import settings - -class StatsCollector(dict): - # signal sent after a domain is opened for stats collection and its resorces have been allocated. args: domain - domain_open = object() - # signal sent before a domain is closed, and before the stats are persisted. it can be catched to add additional stats. args: domain - domain_closing = object() - # signal sent after a domain is closed and its resources have been freed. args: domain - domain_closed = object() - - def __init__(self, enabled=None): - self.db = None - self.debug = settings.getbool('STATS_DEBUG') - self.enabled = enabled if enabled is not None else settings.getbool('STATS_ENABLED') - self.cleanup = settings.getbool('STATS_CLEANUP') - - if self.enabled: - if settings['SCRAPING_DB']: - from scrapy.store.db import DomainDataHistory - self.db = DomainDataHistory(settings['SCRAPING_DB'], table_name='domain_data_history') - - dispatcher.connect(self._domain_open, signal=signals.domain_open) - dispatcher.connect(self._domain_closed, signal=signals.domain_closed) - else: - self.setpath = lambda *args, **kwargs: None - self.getpath = lambda path, default=None: default - self.incpath = lambda *args, **kwargs: None - self.delpath = lambda *args, **kwargs: None - - def setpath(self, path, value): - d = self - keys = path.split('/') - for key in keys[:-1]: - if key not in d: - d[key] = {} - d = d[key] - d[keys[-1]] = value - - def getpath(self, path, default=None): - return stats_getpath(self, path, default) - - def delpath(self, path): - d = self - keys = path.split('/') - for key in keys[:-1]: - if key in d: - d = d[key] - else: - return - del d[keys[-1]] - - def incpath(self, path, value_diff=1): - curvalue = self.getpath(path) or 0 - self.setpath(path, curvalue + value_diff) - - def _domain_open(self, domain, spider): - dispatcher.send(signal=self.domain_open, sender=self.__class__, domain=domain, spider=spider) - - def _domain_closed(self, domain, spider, status): - dispatcher.send(signal=self.domain_closing, sender=self.__class__, domain=domain, spider=spider, status=status) - if self.debug: - log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG) - if self.db: - self.db.put(domain, self[domain]) - if self.cleanup: - del self[domain] - dispatcher.send(signal=self.domain_closed, sender=self.__class__, domain=domain, spider=spider, status=status) diff --git a/scrapy/trunk/scrapy/store/__init__.py b/scrapy/trunk/scrapy/store/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/store/db.py b/scrapy/trunk/scrapy/store/db.py deleted file mode 100644 index f38e6edda..000000000 --- a/scrapy/trunk/scrapy/store/db.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -Store implementations using a MySQL DB -""" -import cPickle as pickle -from datetime import datetime - -from scrapy.utils.misc import stats_getpath -from scrapy.utils.db import mysql_connect - -class DomainDataHistory(object): - """ - This is a store for domain data, with history. - """ - - def __init__(self, db_uri, table_name): - self.db_uri = db_uri - self._mysql_conn = None - self._table = table_name - - def get_mysql_conn(self): - if self._mysql_conn is None: - self._mysql_conn = mysql_connect(self.db_uri, use_unicode=False) - return self._mysql_conn - mysql_conn = property(get_mysql_conn) - - def get(self, domain, count=1, offset=0, stored_after=None, stored_before=None, path=None): - """ - Get the last records stored for the given domain. - If count is given that number of records is retunred, otherwise all records. - If stored_{after,before} is given the results are restricted to that time interval. - The result is a list of tuples: (timestamp, data) - If path is given it only returns stats of that given path (only for dict objects like stats) - """ - sqlsuf = "" - if count is not None: - sqlsuf += "LIMIT %d " % count - if offset: - sqlsuf += "OFFSET %d " % offset - filter = "" - if stored_after: - filter += "AND stored>='%s'" % stored_after.strftime("%Y-%m-%d %H:%M:%S") - if stored_before: - filter += "AND stored<='%s'" % stored_before.strftime("%Y-%m-%d %H:%M:%S") - - c = self.mysql_conn.cursor() - select = "SELECT stored,data FROM %s WHERE domain=%%s %s ORDER BY stored DESC %s" % (self._table, filter, sqlsuf) - c.execute(select, domain) - for stored, datas in c: - obj = pickle.loads(datas) - if path and isinstance(obj, dict): - yield stored, stats_getpath(obj, path) - else: - yield stored, obj - - def getlast(self, domain, offset=0, path=None): - """ - Get the Nth last data stored for the given domain - If offset=0 get the last data stored - If offset=1 get the previous data stored - ...and so on - """ - datal = list(self.get(domain, count=1, offset=offset, path=path)) - if datal: - return datal[0] - - def getall(self, domain, path=None): - """ - Get all data stored for the given domain as a list of tuples: - (stored, timestamp) - """ - return self.get(domain, count=None, path=path) - - def getlast_alldomains(self, count=None, offset=None, order='domain', olist='ASC', path=None): - """ - Get all last data for all domains as a list of tuples, unless - data is sliced by count and offset values. - Ordered by domain is the default. - TODO: This could be merged with: get(domain=None, ..., +order) - - Return a list of tuples (domain, timestamp, data) - """ - sqlsuf = "" - if count is not None: - sqlsuf += "LIMIT %s " % count - if offset: - sqlsuf += "OFFSET %s " % offset - - c = self.mysql_conn.cursor() - select = "SELECT domain, MAX(stored) as stored, data \ - FROM ( \ - SELECT * FROM %s ORDER BY stored DESC \ - ) AS inner_ordered \ - GROUP BY domain \ - ORDER BY %s %s %s" % (self._table, order, olist, sqlsuf) - c.execute(select) - for domain, stored, datas in c: - obj = pickle.loads(datas) - if path and isinstance(obj, dict): - yield domain, stored, stats_getpath(obj, path) - else: - yield domain, stored, obj - - def domain_count(self): - """ - Return the number of domains stored in the database - """ - c = self.mysql_conn.cursor() - c.execute("SELECT COUNT(DISTINCT(domain)) FROM %s" % self._table) - return c.fetchone()[0] - - def put(self, domain, data, timestamp=None): - """ - Store data in history using the given timestamp (or now if omitted). - data can be any pickable object - """ - stored = timestamp if timestamp else datetime.now() - datas = pickle.dumps(data) - - c = self.mysql_conn.cursor() - c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % self._table, (domain, stored, datas)) - self.mysql_conn.commit() - - def remove(self, domain): - """ - Remove all data stored for the given domain - """ - c = self.mysql_conn.cursor() - c.execute("DELETE FROM %s WHERE domain=%%s" % self._table, domain) - self.mysql_conn.commit() - diff --git a/scrapy/trunk/scrapy/templates/project/module/__init__.py b/scrapy/trunk/scrapy/templates/project/module/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/templates/project/module/spiders/__init__.py b/scrapy/trunk/scrapy/templates/project/module/spiders/__init__.py deleted file mode 100644 index 59d02a55f..000000000 --- a/scrapy/trunk/scrapy/templates/project/module/spiders/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Place here all your scrapy spiders diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.tar b/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.tar deleted file mode 100644 index 361751ec2dff30d0758c85e7ee4e005dad80cf6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeHN+j84B5cRXaVy%ZZov|*I@2;$j;@GuYUs7A{WilBKOhOW32%rJbvikL1f|MvH zO)s&mxRccjC1P=~d$2eH7wmB!S1lVRk_WZ>p<H@`PXj-j>+4H+3AQ#Gvsb*;Tw7b) zTnd85X0W~$Y&C+VMzFrIxxNIAC-CJX$kaM!0G7B+v=maZk1X;V|J448ZqNO8>m^P) z+;L-tR(qA8)~Enik&cDx?^Uk4ht=)M%l0z(Pb)DxP9tXlJ1M(+Z^d3^;9Rm>ua8Ef z+Gwq&O~2m6d9MF<dU6>J_>fhFLW)(yD}eU9N}sVIw+V}Q-hQGZ=KSD2Avq!?%nd@P zLmL^E@Y=nNEA7@wwF=#%<4ZWby6nQM1Gqfs!r}P^{L(o&e%<LFpPxbZ9DY5&_z8~B z;P>;Z3pjp#0M%-H`2+7k#2Dz-=JrNq5*R=$H4;a8yN_6%w=*&Ed%rkpksj7rA6;Yb znZ=N9@Q0M#jOzs{_0Ww(#6xDoVRn7@uHC9<sKxL&hQi3pK~8n59FL)+VmLI|6vG=4 zxzzAo=w?vRv2co4kL3H7P-4jXJp4CKr-{@own(dS3g`I~QEW6Cb@niX+WVyM$1Vpe z9Y_p%)^2Xs8jTiGko_nd*&P#-T?;APxC~fnkM^Lk>|@g;k%F7jeujGAacWU^in$ei zjH>L^@pA+}7Lx=&<g?@frq&MtA|sM40ZWWl(tgER>O?P<tDqCEfX5=Ufxd_&=YRn# z2R);QkPrB+1#LuMC?;XV95<1cIyH6`4EM%3yIMIt+XxBfx5+KCb%jd~&|7c=4!0>o z@&rfNp1(0gPZ$o3rZqs`+vsW+&2xW*g-9#Q&7EISFf1~RVTK76i#rh`tQ><UEr#o{ zhsMMZ*3}BQA_pO23a$|uQ$}M{!t!`!GGs@U(PrQ=m!dy#NEZ^+bE{de9*-O_3k*?W zqme0XKp`c#X0St~jd!vJFAi+NBjFvxzoBg6`p`pTrpD-n1F2D;R=%4IK5@{BKX9c4 zThS2i^GmUcl6)0ml$pU=!yEX^i=g?<ZK~b(S&!mtjyuVZX)=^4f-dmBHZzIbG0xjh zTT0`M3G{Xxj)WVCnVbr$O^Dgp76U!BD*9&&$qVdcDNO`-Dp}1rOIDP|vE-;&d8ULo znPJasQCP6({ry}S+t1-W#$p7l2x#8YHRk!mk#-frM%sn{2LDIxI?_YjSVR%G7XMO; zv|hZBt<71;63NC+4Ea+j*_#4?fl@XJiL{LNVYCl})>NdXk7-|SAGSGX`5G`6I(&$_ zdE8^;5$=||8O#D@YU(=a6zLS{)brA*ow@5&l8i%C;>YR~!oWVuAEOuZWH?}j>!eep zQ>0VRN~d;$IqTGID!2<(R+N04QvI9(!C3~hz&+KB`=nN1rCR+>g*_S=EZ1|DmC3Ve zJCx=2zbda6W&Oj8g98|3#rmF5!VY-63VAI&PjSCiisE?RaGo78VD;T(CkK&CW32xF zhXaCn*XZ-Z4!;8N5a)Rp>38N`q_?TEc$UCJnd15M_4sj$c3GgoNshKaS^F>$(l*kz zho^`0*0yP~7fXi(ZVsoAJM%8yJAtQ=<B?D|_Ho+QErAv&YcuYXqJ5Q$HrejbI}^o= zVqdaWKY5A5wwIs#+jrz2F+dCu1H=F^KnxHA!~iis3=jjv05L!e5Cg;jF+dCu1H=F^ kKnxHA!~iis3=jjv05L!e5Cg;jF+dCu1H=F^@OcdU2{b_$H2?qr diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml b/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml deleted file mode 100644 index 7849e5631..000000000 --- a/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml +++ /dev/null @@ -1,100 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="file://path_to_scrape.xsd"><!-- THIS MUST BE SET FOR VALIDATION TO WORK IN YOUR IDE --> - - <product id="17685"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_twice_as_mice_]]></url> - <name><![CDATA[Mounted And Framed Picture: Twice As Mice]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_twice_as_mice.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[no]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17682"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_cuddles_mounted_and_framed_picture]]></url> - <name><![CDATA[Mounted And Framed Picture: Cuddles]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_cuddles.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Home Accessories]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17683"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_playtime_mounted_and_framed_picture]]></url> - <name><![CDATA[Mounted And Framed Picture: Playtime]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_playtime.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17686"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_hat_trick_]]></url> - <name><![CDATA[Mounted And Framed Picture: Hat Trick]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_hat_trick.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17689"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_happy_cat_]]></url> - <name><![CDATA[Mounted And Framed Picture: Happy Cat]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_happy_cat.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17691"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_quiet_night_]]></url> - <name><![CDATA[Mounted And Framed Picture: Quiet Night]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_quiet_night.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>An original and delightful christening gift , perfect nursery decoration. </p><p>FREE hand finished, "Quiet night" greeting card included!</p><p>Picture can be personalised! </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12"). </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17692"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_sunshine_lullaby_]]></url> - <name><![CDATA[Mounted And Framed Picture: Sunshine Lullaby]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_sunshine_lullaby.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>An original and delightful christening gift , perfect nursery decoration. </p><p>FREE hand finished, "Sunshine lullaby" greeting card included!</p><p>Picture can be personalised! </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12"). </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17693"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_tiddlywinks_]]></url> - <name><![CDATA[Mounted And Framed Picture: Tiddlywinks]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_tiddlywinks.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - </products> diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 b/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 deleted file mode 100644 index 245c7846a73a2bff2d1719f88e86e70278310e92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1430 zcmV;H1!?+1T4*^jL0KkKS!d@zu>b^H-+&Ylap(L0zyJU0zyJU6U<jW800*D|00000 z90Lf#X^3DD41gF$nKEG*2xxE&BMGJ<fIu<;VH#w~gkT||z%Yy^n1%rW$N_|DlO_>> zhJ#f<NwlCe01W^D0LTCU00F=-j3$_d0RYGWglUr|5rBq)Qb-Au(9=K&WK#+HjT)Ow zL|~d}rU8;%f_4)6k|N0nSp?PpM+$z=H?%uN3LFW8a#>+$rG43txLeik&ex0w+An(j z7YzNM#~#6(j=Mcp!nO6^ntlWG8V}(%jT7}8Nr1d$-kG)!cw8|Vk0qHVg7Ic)@wpfb zQAst(TWV8)U^@poVlo&F24fL;$Zbmra9CI<f8mTjVMA!tG~&=|pDmOj@bG#eztQq} z{P)}B)6tyh=Ey!5V5FY<a}ADhnqcM}qrW@&$FBW*M~r*yc?SUK6WkST{8xVW-O-L7 z@*3te%ytcf#ym(Rhs|M-s4o;QgToQx9k&ai#2!z?ci-t9g5g2dIx^%9zbL-`jDqY1 zeB&WA;vd>DsnrSmLHqRe5yb%JaEea#gykNXU>={GNnz3{62kaUd42)iZXR0>5$xm_ z4)O2>!{G~ujB{Qsh+;4oiwr{H`G|B88F{}St`PPJb#w|H0)?cdg@i2n#6IRfXQ}i) zp9I%tsuI-K@3QarVV~XqJy-rz*gnnQ>iur|e=pCBF-O(;>5{X;{!aU{=I7oamKLo= zO*i)3^;R-T@`zL;xI^BY)O-r4p{Vl@0d@aY`x;ix04~6Y15qhQa1dBcevpI_BpYH- zFlI#Y2&(<?U3MXuY#5EnLxG6!Bg0XUUKj_sFc}A&9^s(m572zcc)&dfem{_($mY;$ zq0*$wX!GF7fV@&-=TF#sLp-GP*2M?j8gB<E+A!2}k67^z$+;(pC{aPk#9}*p`ueI2 z=-~KTHm6|1Mo*}H;X>Pk>=QAV^Fc|lQjpk#a^aYvDH*6)6B=MCa+wUJ^1{j+hT%em z3LCK*4e4auHVhDV@@3RGMXWueV6a{obBS4Xe?fr50f3=Gs;*SnyR0xzq-LB$^&U}> zp_rjz&M<9Wz05e%QNqA76DFf8*awL(8ja$G3<BYBzQcet65zyJ;F8vA4ZE*5y2IEx z#}N4U4%H0DAfd>KjdBj4?g|D3$S5Yzvp5vkSUrM=GmT)eXIVJIiuLK`-?%<YT)JjG zp~xsxgm7)Hq0$x^_nQU7o#9xFMB@{jJbp0pk0{c22LPOCvCc1On^qo8T+;zQ-XS=K zLSIl^8jdld@p~}5Xebmi48yo^ON7}k8^sKVO@+fic*6sys-Z5V&|W75xIANEVipSx zg(Nh<cLwCqUPh@kmy87pN(vK3p=!`6QW-ub!DYb2Vjbd<kfB0_2}7)PjBjIzxQmI_ zCqT4`ri_<I>E+|vH<&^wC{l%nV^q*{t0C*;=s}RMSZ0<J;?WV*E*OkbhlwdJ9hh7u z@ZuxPI}yT>#Rp0bltm4j0?^-bZWb4Z#wc`}31saJqGL7<1_9I-7X?=+WiT5b$s7dG zY)KzTb2+dVhSn%KS_R_`LzCaDp)M0(d7A-p4M%cZHVPFFB%4CO*c2t#CgE`Qjt;@| zUtyr~VR-Nri(=gL9&JWrL1l}aqX5Yy#^At94%|M3ti}}B>m7rYj=c$(p+d=Z&WsaL zY%d(dP}?Qf)nv0A(JU5FUMdF&n?q`cIOLlf1>-0>15mtY%LO*LAfy&%AJFGSU-MUU z`e5Y?p7nZM&U7&&q+AycBo_`5(kNax4pBFYlc#u|5PSCrcLT0!7{Rs-MC%jqw#xAB k<4z&Wlg^6?!YELoW(x<!Cixi7K>y<INT&)C4E*QzAc1O;O#lD@ diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz b/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz deleted file mode 100644 index c59ba8f052be412a0ede8cce68a8fe911c39dee3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1131 zcmV-x1eE(9iwFq8bH+#j17>AqWG!=HZE$R5F)ny*Yyj<=ZBH9H5Xay1DTchb>q%sj zfZ9?nq)KThTzTk0xCd3$%Gl1Xi`O=`m$3cxH~WxmKzFA*C5@=Mya|cN|M|~&{8JL8 z|N1_WaL26^TJ5)bLAM25QO$&k_FGru!}f0L^<V>vzrL}WrPMpXP0H=vJF(x2y*GQE z&U88rraOVQQD?i`?REY*J-MVYPe@xR=ZT`c1sK1l^chLGGlcST{X|pZi{uj_IVu_A z<IwBSQA-RD?j36l`kU=GjE{~l;q>Zq3~xqoIU2*^`33woJUM<l93P*b!T210Kfibn z$7k^8`PBs+za2rlJ=l2U9t6WeU%uFV-kKGLi>Z~E<>LWr4L>f_#P3OBBoC-gIwV4C z<TG(t(hWY5lH06P$#z0N6_kg>g-QPS>C>R!$yMtd3M;RZoa$70&S0n*99mpr@Ig?Y zTD}M4TndH`PVuo&D)!b=A|VkE|BLC=NKKf-wpy?7od2PU=iP3HJS-vjY@#A}ovb!s zEJh5rcY|)X-zlHei{y?7Nv?$yKC2BXU8p@+toOLo7%8}|;}@zZHqImJO^iDcsl`ph zqB{7cDi7Y3^%Vm1=m!GS3X>0Dt<g7wH=LwiOj5Z8+HeJ&i97}(K_%ya1uG8|s}m>_ z{^&qk5eY>kOo``~YN=D}w!rct7*AI#uNN1=;C$OMA|D~Va)55Z#~f}`g8dO3Whcc8 zb9F;d4MPLKz7NpqIG*_t4yaa++q>dMK~hZ^!3h(Ji95kiR%TGNfx&fFNaJ7#=YvYF znn9pM!8I!5$Y{(f9A}%eltWef$;BC$B8ok>3mY`?TluaD$E!vh5L97SW2dwQg)PC; zzztN}!eju?M$T|53S)TO`N2gn!C;~?bm2j2<kRL)@W`MOU+|{%UeW~a%ZKU~nFj$o z-6EiPc-Gtg;WpL&=X^)?Z>c+*kNGgvBZ3xqr0v2W53Mia=PQfzBn7(7!m04FSeU6^ zdm9+ZT$Siy)QnIwmOKXT99i2dMixyOlN_&mIVm9?W;iLj7*<_e3S$SQ{2BpP6|`z} zWmORkyBc<V+w3~h3BIwQlskuQsK$EzAlq3n$c!YJ7YYAbB>Pb5uMuUlk{XsZENfWy zwBCokSTVk0;zN(O_hz||jYoJ&<6LHqn3`K}IMr~f;ndUO)T@<q%9t#~I+o=W%D@5f zYvojBz2Q{DsfJTeic_z8E9TT~D!30-et64~>epNd&T^qOvTDJ8L#ywSR{ugG8e0tH zDmR)_32H3;9fm5w!;8@f;#_JXlyEU;TTp_c9Pwh<Du_gu^Sn5ul+8U;nKGp0{{ZQ# zSr&XtFkAJP^e$Bn4e(H=60dBw9MLW-H8?5N)`+#og*0qy*!FeVHXrtC;u^rM0&uJT x^4<%^WHuG*#x41?iCSo#KboksHbna_iPn6!d_uIEz2F8Le*tdgkJ8>I001iODP;fv diff --git a/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.zip b/scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.zip deleted file mode 100644 index f1c35af3473c9c02d99fd5260165766c11c5438d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1260 zcmV<I1QYvEO9KQH0000806S1AId0{T(ntgV0Ny4501yBb0A^)nWG!=HZE$R5F)ny* zY*kbV00Y5u#z@t3#z<9o1OVs*=mYJVZBH9H5Xay1DTchb>q%sjfZ9?nq)KThTzTk0 zxCd3$%Gl1Xi`O=`m$3cxH~WxmKzFA*C5@=Mya|cN|M|~&{8JL8|N1_WaL26^TJ5)b zLAM25QO$&k_FGru!}f0L^<V>vzrL}WrPMpXP0H=vJF(x2y*GQE&U88rraOVQQD?i` z?REY*J-MVYPe@xR=ZT`c1sK1l^chLGGlcST{X|pZi{uj_IVu_A<IwBSQA-RD?j36l z`kU=GjE{~l;q>Zq3~xqoIU2*^`33woJUM<l93P*b!T210Kfibn$7k^8`PBs+za2rl zJ=l2U9t6WeU%uFV-kKGLi>Z~E<>LWr4L>f_#P3OBBoC-gIwV4C<TG(t(hWY5lH06P z$#z0N6_kg>g-QPS>C>R!$yMtd3M;RZoa$70&S0n*99mpr@Ig?YTD}M4TndH`PVuo& zD)!b=A|VkE|BLC=NKKf-wpy?7od2PU=iP3HJS-vjY@#A}ovb!sEJh5rcY|)X-zlHe zi{y?7Nv?$yKC2BXU8p@+toOLo7%8}|;}@zZHqImJO^iDcsl`phqB{7cDi7Y3^%Vm1 z=m!GS3X>0Dt<g7wH=LwiOj5Z8+HeJ&i97}(K_%ya1uG8|s}m>_{^&qk5eY>kOo``~ zYN=D}w!rct7*AI#uNN1=;C$OMA|D~Va)55Z#~f}`g8dO3Whcc8b9F;d4MPLKz7Npq zIG*_t4yaa++q>dMK~hZ^!3h(Ji95kiR%TGNfx&fFNaJ7#=YvYFnn9pM!8I!5$Y{(f z9A}%eltWef$;BC$B8ok>3mY`?TluaD$E!vh5L97SW2dwQg)PC;zztN}!eju?M$T|5 z3S)TO`N2gn!C;~?bm2j2<kRL)@W`MOU+|{%UeW~a%ZKU~nFj$o-6EiPc-Gtg;WpL& z=X^)?Z>c+*kNGgvBZ3xqr0v2W53Mia=PQfzBn7(7!m04FSeU6^dm9+ZT$Siy)QnIw zmOKXT99i2dMixyOlN_&mIVm9?W;iLj7*<_e3S$SQ{2BpP6|`z}WmORkyBc<V+w3~h z3BIwQlskuQsK$EzAlq3n$c!YJ7YYAbB>Pb5uMuUlk{XsZENfWywBCokSTVk0;zN(O z_hz||jYoJ&<6LHqn3`K}IMr~f;ndUO)T@<q%9t#~I+o=W%D@5fYvojBz2Q{DsfJTe zic_z8E9TT~D!30-et64~>epNd&T^qOvTDJ8L#ywSR{ugG8e0tHDmR)_32H3;9fm5w z!;8@f;#_JXlyEU;TTp_c9Pwh<Du_gu^Sn5ul+8U;nKGp0{{ZQ#Sr&XtFkAJP^e$Bn z4e(H=60dBw9MLW-H8?5N)`+#og*0qy*!FeVHXrtC;u^rM0&uJT^4<%^WHuG*#x41? ziCSo#KboksHbna_iPn6!d_uIEz2F8Le*sWS0Rk5T6aWAK2mm`!DLHQCkJ3m4007=5 z000mG4FCWD00031005+c00000W@Tk$EpuUQaBO8UE_iKhRa6B41Hp5~NL6?M08mQ> W1^@s600962080P>0E+|w0002aRYJ1> diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml deleted file mode 100644 index 7849e5631..000000000 --- a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml +++ /dev/null @@ -1,100 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="file://path_to_scrape.xsd"><!-- THIS MUST BE SET FOR VALIDATION TO WORK IN YOUR IDE --> - - <product id="17685"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_twice_as_mice_]]></url> - <name><![CDATA[Mounted And Framed Picture: Twice As Mice]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_twice_as_mice.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[no]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17682"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_cuddles_mounted_and_framed_picture]]></url> - <name><![CDATA[Mounted And Framed Picture: Cuddles]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_cuddles.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Home Accessories]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17683"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_playtime_mounted_and_framed_picture]]></url> - <name><![CDATA[Mounted And Framed Picture: Playtime]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_playtime.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17686"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_hat_trick_]]></url> - <name><![CDATA[Mounted And Framed Picture: Hat Trick]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_hat_trick.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17689"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_happy_cat_]]></url> - <name><![CDATA[Mounted And Framed Picture: Happy Cat]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_happy_cat.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17691"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_quiet_night_]]></url> - <name><![CDATA[Mounted And Framed Picture: Quiet Night]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_quiet_night.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>An original and delightful christening gift , perfect nursery decoration. </p><p>FREE hand finished, "Quiet night" greeting card included!</p><p>Picture can be personalised! </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12"). </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17692"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_sunshine_lullaby_]]></url> - <name><![CDATA[Mounted And Framed Picture: Sunshine Lullaby]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_sunshine_lullaby.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>An original and delightful christening gift , perfect nursery decoration. </p><p>FREE hand finished, "Sunshine lullaby" greeting card included!</p><p>Picture can be personalised! </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12"). </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - <product id="17693"> - <url><![CDATA[http://www.example.com/agathajasinskagallery/product/_tiddlywinks_]]></url> - <name><![CDATA[Mounted And Framed Picture: Tiddlywinks]]></name> - <image_url><![CDATA[http://www.example.com/uploads/partners/agathajasinskagallery/500/a_tiddlywinks.jpg]]></image_url> - <price>28.00</price> - <availability><![CDATA[yes]]></availability> - <supplier><![CDATA[example.com]]></supplier> - <designer><![CDATA[Agatha Jasinska Gallery]]></designer> - <description><![CDATA[<p>Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting. </p><p>The print is conservation mounted and individually signed by the artist.</p><p>Picture can be personalised! </p><p>An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!) </p><p>Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12") </p>]]></description> - <category><![CDATA[Art]]></category> - <delivery_within><![CDATA[1-2 days]]></delivery_within> - </product> - </products> diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml deleted file mode 100644 index 0261ae903..000000000 --- a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml +++ /dev/null @@ -1,100 +0,0 @@ -<?xml version="1.0"?> -<rss version ="2.0" -xmlns:g="http://base.google.com/ns/1.0"> -<channel> -<title>Example.com -Example.com -http://www.example.com/ -Abadeh 7500.924 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1602 -http://www.example.com/uploads/images_products/1602.jpg - -69 - -Abadeh 7500.924 - 83cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1602 -http://www.example.com/uploads/images_products/1602.jpg - -89 - -Abadeh 7500.929 - 83cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1603 -http://www.example.com/uploads/images_products/1603.jpg - -89 - -Abadeh 7500.929 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1603 -http://www.example.com/uploads/images_products/1603.jpg - -69 - -Abadeh 7500.938 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1604 -http://www.example.com/uploads/images_products/1604.jpg - -69 - -Abadeh 7500.938 - 83cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1604 -http://www.example.com/uploads/images_products/1604.jpg - -89 - -Abadeh 7500.944 - 83cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1605 -http://www.example.com/uploads/images_products/1605.jpg - -89 - -Abadeh 7500.944 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1605 -http://www.example.com/uploads/images_products/1605.jpg - -69 - -Abadeh 7501.938 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=939 -http://www.example.com/uploads/images_products/939.jpg - -69 - -Abadeh 7501.938 - 83cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=939 -http://www.example.com/uploads/images_products/939.jpg - -89 - -Abadeh 7501.958 - 83cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1606 -http://www.example.com/uploads/images_products/1606.jpg - -89 - -Abadeh 7501.958 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=1606 -http://www.example.com/uploads/images_products/1606.jpg - -69 - -Abadeh 7502.938 - 67cm Wide -<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> -http://www.example.com/product.asp?P_ID=936 -http://www.example.com/uploads/images_products/936.jpg - -69 - - - diff --git a/scrapy/trunk/scrapy/tests/sample_data/test_site/index.html b/scrapy/trunk/scrapy/tests/sample_data/test_site/index.html deleted file mode 100644 index d268c846a..000000000 --- a/scrapy/trunk/scrapy/tests/sample_data/test_site/index.html +++ /dev/null @@ -1,18 +0,0 @@ - - - -Scrapy test site - - - - -

    Scrapy test site

    - - - - - diff --git a/scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html b/scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html deleted file mode 100644 index ceeb6dc87..000000000 --- a/scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html +++ /dev/null @@ -1,17 +0,0 @@ - - - -Item 1 - Scrapy test site - - - - -

    Item 1 name

    - -
      -
    • Price: $100
      • -
    • Stock: 12
      • -
    - - - diff --git a/scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html b/scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html deleted file mode 100644 index a64c92810..000000000 --- a/scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html +++ /dev/null @@ -1,17 +0,0 @@ - - - -Item 2 - Scrapy test site - - - - -

    Item 2 name

    - -
      -
    • Price: $200
      • -
    • Stock: 5
      • -
    - - - diff --git a/scrapy/trunk/scrapy/tests/test_c14nurls.py b/scrapy/trunk/scrapy/tests/test_c14nurls.py deleted file mode 100644 index e0a8e8166..000000000 --- a/scrapy/trunk/scrapy/tests/test_c14nurls.py +++ /dev/null @@ -1,28 +0,0 @@ -import unittest -from scrapy.utils.c14n import canonicalize - -class C14nTest(unittest.TestCase): - """ c14n comparison functions """ - - def test_canonicalize(self): - """Test URL canonicalization function""" - urls = [ - ('http://www.maddiebrown.co.uk//cgi-bin//html_parser.cgi?web_page_id=12&template_file=catalogue_list_items_template&category-id=1000070&category-type=1&page=3&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2', - 'http://www.maddiebrown.co.uk//cgi-bin//html_parser.cgi?web_page_id=12&template_file=catalogue_list_items_template&category-id=1000070&category-type=1&page=3&page=2'), - - - ('http://www.mfi.co.uk/mfi/productinfo.asp?CT=/1010/YourHomeOffice&CT=/1025/YourHomeOffice/Packages', - 'http://www.mfi.co.uk/mfi/productinfo.asp?CT=/1010/YourHomeOffice&CT=/1025/YourHomeOffice/Packages'), - - ('http://www.test-nice-url.com/index.html', - 'http://www.test-nice-url.com/index.html'), - - ('http://www.homebase.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=20001&langId=-1&catalogId=10701&productId=729482&Trail=C$cip=50704&categoryId=50704', - 'http://www.homebase.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=20001&langId=-1&catalogId=10701&productId=729482&Trail=C$cip=50704&categoryId=50704'), - ] - for origurl, c14nurl in urls: - self.assertEqual(canonicalize(origurl), c14nurl) - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/trunk/scrapy/tests/test_dependencies.py b/scrapy/trunk/scrapy/tests/test_dependencies.py deleted file mode 100644 index 5109edee8..000000000 --- a/scrapy/trunk/scrapy/tests/test_dependencies.py +++ /dev/null @@ -1,16 +0,0 @@ -from unittest import TestCase, main - -class ScrapyUtilsTest(TestCase): - def test_openssl(self): - try: - module = __import__('OpenSSL', {}, {}, ['']) - except ImportError, ex: - return # no openssl installed - - required_version = '0.6' - if hasattr(module, '__version__'): - for cur, req in zip(module.__version__.split('.'), required_version.split('.')): - self.assertFalse(cur < req, "module %s >= %s required" % ('OpenSSL', required_version)) - -if __name__ == "__main__": - main() diff --git a/scrapy/trunk/scrapy/tests/test_spidermonkey.py b/scrapy/trunk/scrapy/tests/test_spidermonkey.py deleted file mode 100644 index 52b753d32..000000000 --- a/scrapy/trunk/scrapy/tests/test_spidermonkey.py +++ /dev/null @@ -1,24 +0,0 @@ -from twisted.trial import unittest - -class SpidermonkeyTest(unittest.TestCase): - - def setUp(self): - try: - from spidermonkey import Runtime - r = Runtime() - self.cx = r.new_context() - except ImportError: - raise unittest.SkipTest("Spidermonkey C library not available") - - def test_spidermonkey(self): - """Spidermonkey basic functionality tests""" - - self.cx.eval_script("var item0={'price':50}") - self.cx.eval_script("var item1={'price':20}") - self.assertEqual(self.cx.eval_script("item0"), {'price': 50}) - self.assertEqual(self.cx.eval_script("item1"), {'price': 20}) - self.cx.eval_script("var itemArray=new Array(item0, item1)") - self.assertEqual(self.cx.eval_script("itemArray"), [{'price': 50}, {'price':20}]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_spiders/__init__.py b/scrapy/trunk/scrapy/tests/test_spiders/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/tests/test_stats.py b/scrapy/trunk/scrapy/tests/test_stats.py deleted file mode 100644 index 3b7c2ac2d..000000000 --- a/scrapy/trunk/scrapy/tests/test_stats.py +++ /dev/null @@ -1,52 +0,0 @@ -from unittest import TestCase, main -from scrapy.conf import settings -from scrapy.stats.statscollector import StatsCollector - -class StatsTest(TestCase): - def test_stats(self): - stats = StatsCollector(enabled=False) - self.assertEqual(stats, {}) - self.assertEqual(stats.getpath('anything'), None) - self.assertEqual(stats.getpath('anything', 'default'), 'default') - stats.setpath('test', 'value') - self.assertEqual(stats, {}) - - stats = StatsCollector(enabled=True) - self.assertEqual(stats, {}) - self.assertEqual(stats.getpath('anything'), None) - self.assertEqual(stats.getpath('anything', 'default'), 'default') - stats.setpath('test', 'value') - self.assertEqual(stats, {'test': 'value'}) - stats.setpath('test2', 23) - self.assertEqual(stats, {'test': 'value', 'test2': 23}) - stats.setpath('one/two', 'val2') - self.assertEqual(stats, {'test': 'value', 'test2': 23, 'one': {'two': 'val2'}}) - self.assertEqual(stats.getpath('one/two'), 'val2') - self.assertEqual(stats.getpath('one/three'), None) - self.assertEqual(stats.getpath('one/three', 'four'), 'four') - self.assertEqual(stats.getpath('one'), {'two': 'val2'}) - - # nodes must contain either data or other nodes, but not both! - self.assertRaises(TypeError, stats.setpath, 'one/two/three', 22) - - stats.delpath('test2') - self.assertEqual(stats, {'test': 'value', 'one': {'two': 'val2'}}) - stats.delpath('one/two') - self.assertEqual(stats, {'test': 'value', 'one': {}}) - stats.delpath('one') - self.assertEqual(stats, {'test': 'value'}) - - stats.setpath('one/other/three', 20) - self.assertEqual(stats.getpath('one/other/three'), 20) - stats.incpath('one/other/three') - self.assertEqual(stats.getpath('one/other/three'), 21) - stats.incpath('one/other/three', 4) - self.assertEqual(stats.getpath('one/other/three'), 25) - - stats.incpath('one/newnode', 1) - self.assertEqual(stats.getpath('one/newnode'), 1) - stats.incpath('one/newnode', -1) - self.assertEqual(stats.getpath('one/newnode'), 0) - -if __name__ == "__main__": - main() diff --git a/scrapy/trunk/scrapy/utils/__init__.py b/scrapy/trunk/scrapy/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/trunk/scrapy/utils/c14n.py b/scrapy/trunk/scrapy/utils/c14n.py deleted file mode 100644 index 88cdd16df..000000000 --- a/scrapy/trunk/scrapy/utils/c14n.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Canonicalization routine for URLs -""" -import re -import urlparse - -_hexmatcher = re.compile("\%[0-9A-Fa-f]{2}") - -def get_canonized_port(port): - if port and port != 80: - cport = ":" + str(port) - else: - cport = "" - return cport - -def create_params_dict(params): - param_dict = {} - for param in params.split("&"): - nvpair = param.split("=") - if len(nvpair) >= 2: - param_dict[nvpair[0]] = '='.join(nvpair[1:]) - else: - param_dict[nvpair[0]] = None - return param_dict - -def create_unique_params_list(params): - """ - This function build a query string from unique params only. - Unique params are params with unique key=>value pair. So param=qwe and param=123 - are diffrent and a query string will be param=qwe¶m=123. - If input params will contain pairs like this param=123 and param=123 a query - string will be just param=123 - We have to do this because some ASP sites uses the same parameters names in the URLs, - but with diffrent values. Such queries are interpreted as arrays in the ASP framework. - """ - param_list = [] - if params: - for param in params.split("&"): - nvpair = param.split("=", 1) - if len(nvpair) >= 2: - param_list.append(nvpair[0] + '=' + nvpair[1]) - else: - param_list.append(nvpair[0] + '=') - if param_list: - p_list = [] - # select only unique pairs, it is possible to use set() for this but change order of pairs... - for param in param_list: - if param not in p_list: - p_list.append(param) - return '&'.join(p_list) - return '' - -def convert_escaped(string_to_convert): - hexvalues = _hexmatcher.findall(string_to_convert) - for hexvalue in hexvalues: - string_to_convert = string_to_convert.replace( - hexvalue, chr(int(hexvalue[1:len(hexvalue)],16))) - return string_to_convert - -def remove_dirs(path): - dirs = path.split("/") - dirs.reverse() - stripped_dirs = [] - ellipsis = 0 - for dir in dirs: - if dir == ".." : - ellipsis += 1 - elif dir == "." : - pass - else: - if ellipsis > 0 : - ellipsis -= 1 - else: - stripped_dirs.append(dir) - stripped_dirs.reverse() - return "/".join(stripped_dirs) - -def canonicalize(url): - """ - This method c14ns the url we are passed by doing the following - 1) lowercasing the hostname and scheme - 2) removing the port if it is 80 - 3) making the query params unique and alphabetized - 4) removing url encoding - 5) Removing any ./ - 6) Removing and ../ and their preceding directories - """ - - # Remove the fragment marker, even if it's url encoded - defragmented_url = urlparse.urldefrag(url)[0] - - # This automatically converts the scheme and hostname to lower case - parsed = urlparse.urlparse(defragmented_url) - - # Get the port in the correct format - cport = get_canonized_port(parsed.port) - - query = create_unique_params_list(parsed.query) - - # Remove the url encoded params from the path and query string - query = convert_escaped(query) - if query : - query = "?" + query - path = convert_escaped(parsed.path) - - # Remove the ../ and the previous directories and remove the ./ - path = remove_dirs(path) - - # Now put it all back together - return "%s://%s%s%s%s" % (parsed.scheme, parsed.hostname, cport, path, query) diff --git a/scrapy/trunk/scrapy/utils/display.py b/scrapy/trunk/scrapy/utils/display.py deleted file mode 100644 index 710e7d743..000000000 --- a/scrapy/trunk/scrapy/utils/display.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Helper functions for formatting and pretty printing some objects -""" -import sys -import pprint as pypprint - -from scrapy.item import ScrapedItem -from scrapy.http import Request, Response - -nocolour = False - -def colorize(text): - if nocolour or not sys.stdout.isatty(): - return text - try: - from pygments import highlight - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer - return highlight(text, PythonLexer(), TerminalFormatter()) - except ImportError: - return text - -def pformat_dictobj(obj): - clsname = obj.__class__.__name__ - return "%s(%s)\n" % (clsname, colorize(pypprint.pformat(obj.__dict__))) - -def pprint_dictobj(obj): - print pformat_dictobj(obj) - -def pformat(obj, *args, **kwargs): - """ - Wrapper which autodetects the object type and uses the proper formatting - function - """ - - if isinstance(obj, (list, tuple)): - return "".join([pformat(i, *args, **kwargs) for i in obj]) - elif isinstance(obj, (ScrapedItem, Request, Response)): - return pformat_dictobj(obj) - else: - return colorize(pypprint.pformat(obj)) - -def pprint(obj, *args, **kwargs): - """ - Wrapper which autodetects the object type and uses the proper printing - function - """ - - print pformat(obj, *args, **kwargs) diff --git a/scrapy/trunk/scrapy/utils/serialization.py b/scrapy/trunk/scrapy/utils/serialization.py deleted file mode 100644 index 82bd01eb3..000000000 --- a/scrapy/trunk/scrapy/utils/serialization.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -This module provides a universal API for different serialization formats. - -Keep in mind that not all formats support all types for serialization, and some -formats (like pprint) may be unsafe for unserialization (like pprint) but it -may still very convenient for serialization. -""" - -import datetime -import decimal -import cPickle as pickle -import pprint - -import simplejson - -class ScrapyJSONEncoder(simplejson.JSONEncoder): - """ - JSONEncoder subclass that knows how to encode date/time and decimal types. - """ - - DATE_FORMAT = "%Y-%m-%d" - TIME_FORMAT = "%H:%M:%S" - - def default(self, o): - if isinstance(o, datetime.datetime): - return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) - elif isinstance(o, datetime.date): - return o.strftime(self.DATE_FORMAT) - elif isinstance(o, datetime.time): - return o.strftime(self.TIME_FORMAT) - elif isinstance(o, decimal.Decimal): - return str(o) - else: - return super(ScrapyJSONEncoder, self).default(o) - -serialize_funcs = { - 'json': lambda obj: simplejson.dumps(obj, cls=ScrapyJSONEncoder), - 'pprint': lambda obj: pprint.pformat(obj), - 'pickle': lambda obj: pickle.dumps(obj), -} - -unserialize_funcs = { - 'json': lambda text: simplejson.loads(text), - 'pprint': lambda text: eval(text), - 'pickle': lambda text: pickle.loads(text), -} - -def serialize(obj, format='pickle'): - """ - Main entrance point for serialization. - - Supported formats: pickle, json, pprint - """ - try: - func = serialize_funcs[format] - return func(obj) - except KeyError: - raise TypeError("Unknown serialization format: %s" % format) - -def unserialize(text, format='pickle'): - """ - Main entrance point for unserialization. - """ - try: - func = unserialize_funcs[format] - return func(text) - except KeyError: - raise TypeError("Unknown serialization format: %s" % format) - -def parse_jsondatetime(text): - if isinstance(text, basestring): - return datetime.datetime.strptime(text, "%s %s" % \ - (ScrapyJSONEncoder.DATE_FORMAT, ScrapyJSONEncoder.TIME_FORMAT)) - diff --git a/scrapy/trunk/scrapy/xlib/BeautifulSoup.py b/scrapy/trunk/scrapy/xlib/BeautifulSoup.py deleted file mode 100644 index 0e55abaf8..000000000 --- a/scrapy/trunk/scrapy/xlib/BeautifulSoup.py +++ /dev/null @@ -1,1931 +0,0 @@ -"""Beautiful Soup -Elixir and Tonic -"The Screen-Scraper's Friend" -http://www.crummy.com/software/BeautifulSoup/ - -Beautiful Soup parses a (possibly invalid) XML or HTML document into a -tree representation. It provides methods and Pythonic idioms that make -it easy to navigate, search, and modify the tree. - -A well-formed XML/HTML document yields a well-formed data -structure. An ill-formed XML/HTML document yields a correspondingly -ill-formed data structure. If your document is only locally -well-formed, you can use this library to find and process the -well-formed part of it. - -Beautiful Soup works with Python 2.2 and up. It has no external -dependencies, but you'll have more success at converting data to UTF-8 -if you also install these three packages: - -* chardet, for auto-detecting character encodings - http://chardet.feedparser.org/ -* cjkcodecs and iconv_codec, which add more encodings to the ones supported - by stock Python. - http://cjkpython.i18n.org/ - -Beautiful Soup defines classes for two main parsing strategies: - - * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific - language that kind of looks like XML. - - * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid - or invalid. This class has web browser-like heuristics for - obtaining a sensible parse tree in the face of common HTML errors. - -Beautiful Soup also defines a class (UnicodeDammit) for autodetecting -the encoding of an HTML or XML document, and converting it to -Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser. - -For more than you ever wanted to know about Beautiful Soup, see the -documentation: -http://www.crummy.com/software/BeautifulSoup/documentation.html - -Here, have some legalese: - -Copyright (c) 2004-2007, Leonard Richardson - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the the Beautiful Soup Consortium and All - Night Kosher Bakery nor the names of its contributors may be - used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. - -""" -from __future__ import generators - -__author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "3.0.6" -__copyright__ = "Copyright (c) 2004-2008 Leonard Richardson" -__license__ = "New-style BSD" - -from sgmllib import SGMLParser, SGMLParseError -import codecs -import types -import re -import sgmllib -try: - from htmlentitydefs import name2codepoint -except ImportError: - name2codepoint = {} - -#This hack makes Beautiful Soup able to parse XML with namespaces -sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') - -DEFAULT_OUTPUT_ENCODING = "utf-8" - -# First, the classes that represent markup elements. - -class PageElement: - """Contains the navigational information for some part of the page - (either a tag or a piece of text)""" - - def setup(self, parent=None, previous=None): - """Sets up the initial relations between this element and - other elements.""" - self.parent = parent - self.previous = previous - self.next = None - self.previousSibling = None - self.nextSibling = None - if self.parent and self.parent.contents: - self.previousSibling = self.parent.contents[-1] - self.previousSibling.nextSibling = self - - def replaceWith(self, replaceWith): - oldParent = self.parent - myIndex = self.parent.contents.index(self) - if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent: - # We're replacing this element with one of its siblings. - index = self.parent.contents.index(replaceWith) - if index and index < myIndex: - # Furthermore, it comes before this element. That - # means that when we extract it, the index of this - # element will change. - myIndex = myIndex - 1 - self.extract() - oldParent.insert(myIndex, replaceWith) - - def extract(self): - """Destructively rips this element out of the tree.""" - if self.parent: - try: - self.parent.contents.remove(self) - except ValueError: - pass - - #Find the two elements that would be next to each other if - #this element (and any children) hadn't been parsed. Connect - #the two. - lastChild = self._lastRecursiveChild() - nextElement = lastChild.next - - if self.previous: - self.previous.next = nextElement - if nextElement: - nextElement.previous = self.previous - self.previous = None - lastChild.next = None - - self.parent = None - if self.previousSibling: - self.previousSibling.nextSibling = self.nextSibling - if self.nextSibling: - self.nextSibling.previousSibling = self.previousSibling - self.previousSibling = self.nextSibling = None - return self - - def _lastRecursiveChild(self): - "Finds the last element beneath this object to be parsed." - lastChild = self - while hasattr(lastChild, 'contents') and lastChild.contents: - lastChild = lastChild.contents[-1] - return lastChild - - def insert(self, position, newChild): - if (isinstance(newChild, basestring) - or isinstance(newChild, unicode)) \ - and not isinstance(newChild, NavigableString): - newChild = NavigableString(newChild) - - position = min(position, len(self.contents)) - if hasattr(newChild, 'parent') and newChild.parent != None: - # We're 'inserting' an element that's already one - # of this object's children. - if newChild.parent == self: - index = self.find(newChild) - if index and index < position: - # Furthermore we're moving it further down the - # list of this object's children. That means that - # when we extract this element, our target index - # will jump down one. - position = position - 1 - newChild.extract() - - newChild.parent = self - previousChild = None - if position == 0: - newChild.previousSibling = None - newChild.previous = self - else: - previousChild = self.contents[position-1] - newChild.previousSibling = previousChild - newChild.previousSibling.nextSibling = newChild - newChild.previous = previousChild._lastRecursiveChild() - if newChild.previous: - newChild.previous.next = newChild - - newChildsLastElement = newChild._lastRecursiveChild() - - if position >= len(self.contents): - newChild.nextSibling = None - - parent = self - parentsNextSibling = None - while not parentsNextSibling: - parentsNextSibling = parent.nextSibling - parent = parent.parent - if not parent: # This is the last element in the document. - break - if parentsNextSibling: - newChildsLastElement.next = parentsNextSibling - else: - newChildsLastElement.next = None - else: - nextChild = self.contents[position] - newChild.nextSibling = nextChild - if newChild.nextSibling: - newChild.nextSibling.previousSibling = newChild - newChildsLastElement.next = nextChild - - if newChildsLastElement.next: - newChildsLastElement.next.previous = newChildsLastElement - self.contents.insert(position, newChild) - - def append(self, tag): - """Appends the given tag to the contents of this tag.""" - self.insert(len(self.contents), tag) - - def findNext(self, name=None, attrs={}, text=None, **kwargs): - """Returns the first item that matches the given criteria and - appears after this Tag in the document.""" - return self._findOne(self.findAllNext, name, attrs, text, **kwargs) - - def findAllNext(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns all items that match the given criteria and appear - after this Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.nextGenerator, - **kwargs) - - def findNextSibling(self, name=None, attrs={}, text=None, **kwargs): - """Returns the closest sibling to this Tag that matches the - given criteria and appears after this Tag in the document.""" - return self._findOne(self.findNextSiblings, name, attrs, text, - **kwargs) - - def findNextSiblings(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns the siblings of this Tag that match the given - criteria and appear after this Tag in the document.""" - return self._findAll(name, attrs, text, limit, - self.nextSiblingGenerator, **kwargs) - fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x - - def findPrevious(self, name=None, attrs={}, text=None, **kwargs): - """Returns the first item that matches the given criteria and - appears before this Tag in the document.""" - return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs) - - def findAllPrevious(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns all items that match the given criteria and appear - before this Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.previousGenerator, - **kwargs) - fetchPrevious = findAllPrevious # Compatibility with pre-3.x - - def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs): - """Returns the closest sibling to this Tag that matches the - given criteria and appears before this Tag in the document.""" - return self._findOne(self.findPreviousSiblings, name, attrs, text, - **kwargs) - - def findPreviousSiblings(self, name=None, attrs={}, text=None, - limit=None, **kwargs): - """Returns the siblings of this Tag that match the given - criteria and appear before this Tag in the document.""" - return self._findAll(name, attrs, text, limit, - self.previousSiblingGenerator, **kwargs) - fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x - - def findParent(self, name=None, attrs={}, **kwargs): - """Returns the closest parent of this Tag that matches the given - criteria.""" - # NOTE: We can't use _findOne because findParents takes a different - # set of arguments. - r = None - l = self.findParents(name, attrs, 1) - if l: - r = l[0] - return r - - def findParents(self, name=None, attrs={}, limit=None, **kwargs): - """Returns the parents of this Tag that match the given - criteria.""" - - return self._findAll(name, attrs, None, limit, self.parentGenerator, - **kwargs) - fetchParents = findParents # Compatibility with pre-3.x - - #These methods do the real heavy lifting. - - def _findOne(self, method, name, attrs, text, **kwargs): - r = None - l = method(name, attrs, text, 1, **kwargs) - if l: - r = l[0] - return r - - def _findAll(self, name, attrs, text, limit, generator, **kwargs): - "Iterates over a generator looking for things that match." - - if isinstance(name, SoupStrainer): - strainer = name - else: - # Build a SoupStrainer - strainer = SoupStrainer(name, attrs, text, **kwargs) - results = ResultSet(strainer) - g = generator() - while True: - try: - i = g.next() - except StopIteration: - break - if i: - found = strainer.search(i) - if found: - results.append(found) - if limit and len(results) >= limit: - break - return results - - #These Generators can be used to navigate starting from both - #NavigableStrings and Tags. - def nextGenerator(self): - i = self - while i: - i = i.next - yield i - - def nextSiblingGenerator(self): - i = self - while i: - i = i.nextSibling - yield i - - def previousGenerator(self): - i = self - while i: - i = i.previous - yield i - - def previousSiblingGenerator(self): - i = self - while i: - i = i.previousSibling - yield i - - def parentGenerator(self): - i = self - while i: - i = i.parent - yield i - - # Utility methods - def substituteEncoding(self, str, encoding=None): - encoding = encoding or "utf-8" - return str.replace("%SOUP-ENCODING%", encoding) - - def toEncoding(self, s, encoding=None): - """Encodes an object to a string in some encoding, or to Unicode. - .""" - if isinstance(s, unicode): - if encoding: - s = s.encode(encoding) - elif isinstance(s, str): - if encoding: - s = s.encode(encoding) - else: - s = unicode(s) - else: - if encoding: - s = self.toEncoding(str(s), encoding) - else: - s = unicode(s) - return s - -class NavigableString(unicode, PageElement): - - def __getnewargs__(self): - return (NavigableString.__str__(self),) - - def __getattr__(self, attr): - """text.string gives you text. This is for backwards - compatibility for Navigable*String, but for CData* it lets you - get the string without the CData wrapper.""" - if attr == 'string': - return self - else: - raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr) - - def __unicode__(self): - return str(self).decode(DEFAULT_OUTPUT_ENCODING) - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - if encoding: - return self.encode(encoding) - else: - return self - -class CData(NavigableString): - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class ProcessingInstruction(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - output = self - if "%SOUP-ENCODING%" in output: - output = self.substituteEncoding(output, encoding) - return "" % self.toEncoding(output, encoding) - -class Comment(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class Declaration(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class Tag(PageElement): - - """Represents a found HTML tag with its attributes and contents.""" - - def _invert(h): - "Cheap function to invert a hash." - i = {} - for k,v in h.items(): - i[v] = k - return i - - XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", - "quot" : '"', - "amp" : "&", - "lt" : "<", - "gt" : ">" } - - XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) - - def _convertEntities(self, match): - """Used in a call to re.sub to replace HTML, XML, and numeric - entities with the appropriate Unicode characters. If HTML - entities are being converted, any unrecognized entities are - escaped.""" - x = match.group(1) - if self.convertHTMLEntities and x in name2codepoint: - return unichr(name2codepoint[x]) - elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS: - if self.convertXMLEntities: - return self.XML_ENTITIES_TO_SPECIAL_CHARS[x] - else: - return u'&%s;' % x - elif len(x) > 0 and x[0] == '#': - # Handle numeric entities - if len(x) > 1 and x[1] == 'x': - return unichr(int(x[2:], 16)) - else: - return unichr(int(x[1:])) - - elif self.escapeUnrecognizedEntities: - return u'&%s;' % x - else: - return u'&%s;' % x - - def __init__(self, parser, name, attrs=None, parent=None, - previous=None): - "Basic constructor." - - # We don't actually store the parser object: that lets extracted - # chunks be garbage-collected - self.parserClass = parser.__class__ - self.isSelfClosing = parser.isSelfClosingTag(name) - self.name = name - if attrs == None: - attrs = [] - self.attrs = attrs - self.contents = [] - self.setup(parent, previous) - self.hidden = False - self.containsSubstitutions = False - self.convertHTMLEntities = parser.convertHTMLEntities - self.convertXMLEntities = parser.convertXMLEntities - self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities - - # Convert any HTML, XML, or numeric entities in the attribute values. - convert = lambda(k, val): (k, - re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);", - self._convertEntities, - val)) - self.attrs = map(convert, self.attrs) - - def get(self, key, default=None): - """Returns the value of the 'key' attribute for the tag, or - the value given for 'default' if it doesn't have that - attribute.""" - return self._getAttrMap().get(key, default) - - def has_key(self, key): - return self._getAttrMap().has_key(key) - - def __getitem__(self, key): - """tag[key] returns the value of the 'key' attribute for the tag, - and throws an exception if it's not there.""" - return self._getAttrMap()[key] - - def __iter__(self): - "Iterating over a tag iterates over its contents." - return iter(self.contents) - - def __len__(self): - "The length of a tag is the length of its list of contents." - return len(self.contents) - - def __contains__(self, x): - return x in self.contents - - def __nonzero__(self): - "A tag is non-None even if it has no contents." - return True - - def __setitem__(self, key, value): - """Setting tag[key] sets the value of the 'key' attribute for the - tag.""" - self._getAttrMap() - self.attrMap[key] = value - found = False - for i in range(0, len(self.attrs)): - if self.attrs[i][0] == key: - self.attrs[i] = (key, value) - found = True - if not found: - self.attrs.append((key, value)) - self._getAttrMap()[key] = value - - def __delitem__(self, key): - "Deleting tag[key] deletes all 'key' attributes for the tag." - for item in self.attrs: - if item[0] == key: - self.attrs.remove(item) - #We don't break because bad HTML can define the same - #attribute multiple times. - self._getAttrMap() - if self.attrMap.has_key(key): - del self.attrMap[key] - - def __call__(self, *args, **kwargs): - """Calling a tag like a function is the same as calling its - findAll() method. Eg. tag('a') returns a list of all the A tags - found within this tag.""" - return apply(self.findAll, args, kwargs) - - def __getattr__(self, tag): - #print "Getattr %s.%s" % (self.__class__, tag) - if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3: - return self.find(tag[:-3]) - elif tag.find('__') != 0: - return self.find(tag) - raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag) - - def __eq__(self, other): - """Returns true iff this tag has the same name, the same attributes, - and the same contents (recursively) as the given tag. - - NOTE: right now this will return false if two tags have the - same attributes in a different order. Should this be fixed?""" - if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): - return False - for i in range(0, len(self.contents)): - if self.contents[i] != other.contents[i]: - return False - return True - - def __ne__(self, other): - """Returns true iff this tag is not identical to the other tag, - as defined in __eq__.""" - return not self == other - - def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): - """Renders this tag as a string.""" - return self.__str__(encoding) - - def __unicode__(self): - return self.__str__(None) - - BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" - + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" - + ")") - - def _sub_entity(self, x): - """Used with a regular expression to substitute the - appropriate XML entity for an XML special character.""" - return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";" - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING, - prettyPrint=False, indentLevel=0): - """Returns a string or Unicode representation of this tag and - its contents. To get Unicode, pass None for encoding. - - NOTE: since Python's HTML parser consumes whitespace, this - method is not certain to reproduce the whitespace present in - the original string.""" - - encodedName = self.toEncoding(self.name, encoding) - - attrs = [] - if self.attrs: - for key, val in self.attrs: - fmt = '%s="%s"' - if isString(val): - if self.containsSubstitutions and '%SOUP-ENCODING%' in val: - val = self.substituteEncoding(val, encoding) - - # The attribute value either: - # - # * Contains no embedded double quotes or single quotes. - # No problem: we enclose it in double quotes. - # * Contains embedded single quotes. No problem: - # double quotes work here too. - # * Contains embedded double quotes. No problem: - # we enclose it in single quotes. - # * Embeds both single _and_ double quotes. This - # can't happen naturally, but it can happen if - # you modify an attribute value after parsing - # the document. Now we have a bit of a - # problem. We solve it by enclosing the - # attribute in single quotes, and escaping any - # embedded single quotes to XML entities. - if '"' in val: - fmt = "%s='%s'" - if "'" in val: - # TODO: replace with apos when - # appropriate. - val = val.replace("'", "&squot;") - - # Now we're okay w/r/t quotes. But the attribute - # value might also contain angle brackets, or - # ampersands that aren't part of entities. We need - # to escape those to XML entities too. - val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val) - - attrs.append(fmt % (self.toEncoding(key, encoding), - self.toEncoding(val, encoding))) - close = '' - closeTag = '' - if self.isSelfClosing: - close = ' /' - else: - closeTag = '' % encodedName - - indentTag, indentContents = 0, 0 - if prettyPrint: - indentTag = indentLevel - space = (' ' * (indentTag-1)) - indentContents = indentTag + 1 - contents = self.renderContents(encoding, prettyPrint, indentContents) - if self.hidden: - s = contents - else: - s = [] - attributeString = '' - if attrs: - attributeString = ' ' + ' '.join(attrs) - if prettyPrint: - s.append(space) - s.append('<%s%s%s>' % (encodedName, attributeString, close)) - if prettyPrint: - s.append("\n") - s.append(contents) - if prettyPrint and contents and contents[-1] != "\n": - s.append("\n") - if prettyPrint and closeTag: - s.append(space) - s.append(closeTag) - if prettyPrint and closeTag and self.nextSibling: - s.append("\n") - s = ''.join(s) - return s - - def decompose(self): - """Recursively destroys the contents of this tree.""" - contents = [i for i in self.contents] - for i in contents: - if isinstance(i, Tag): - i.decompose() - else: - i.extract() - self.extract() - - def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): - return self.__str__(encoding, True) - - def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, - prettyPrint=False, indentLevel=0): - """Renders the contents of this tag as a string in the given - encoding. If encoding is None, returns a Unicode string..""" - s=[] - for c in self: - text = None - if isinstance(c, NavigableString): - text = c.__str__(encoding) - elif isinstance(c, Tag): - s.append(c.__str__(encoding, prettyPrint, indentLevel)) - if text and prettyPrint: - text = text.strip() - if text: - if prettyPrint: - s.append(" " * (indentLevel-1)) - s.append(text) - if prettyPrint: - s.append("\n") - return ''.join(s) - - #Soup methods - - def find(self, name=None, attrs={}, recursive=True, text=None, - **kwargs): - """Return only the first child of this Tag matching the given - criteria.""" - r = None - l = self.findAll(name, attrs, recursive, text, 1, **kwargs) - if l: - r = l[0] - return r - findChild = find - - def findAll(self, name=None, attrs={}, recursive=True, text=None, - limit=None, **kwargs): - """Extracts a list of Tag objects that match the given - criteria. You can specify the name of the Tag and any - attributes you want the Tag to have. - - The value of a key-value pair in the 'attrs' map can be a - string, a list of strings, a regular expression object, or a - callable that takes a string and returns whether or not the - string matches for some custom definition of 'matches'. The - same is true of the tag name.""" - generator = self.recursiveChildGenerator - if not recursive: - generator = self.childGenerator - return self._findAll(name, attrs, text, limit, generator, **kwargs) - findChildren = findAll - - # Pre-3.x compatibility methods - first = find - fetch = findAll - - def fetchText(self, text=None, recursive=True, limit=None): - return self.findAll(text=text, recursive=recursive, limit=limit) - - def firstText(self, text=None, recursive=True): - return self.find(text=text, recursive=recursive) - - #Private methods - - def _getAttrMap(self): - """Initializes a map representation of this tag's attributes, - if not already initialized.""" - if not getattr(self, 'attrMap'): - self.attrMap = {} - for (key, value) in self.attrs: - self.attrMap[key] = value - return self.attrMap - - #Generator methods - def childGenerator(self): - for i in range(0, len(self.contents)): - yield self.contents[i] - raise StopIteration - - def recursiveChildGenerator(self): - stack = [(self, 0)] - while stack: - tag, start = stack.pop() - if isinstance(tag, Tag): - for i in range(start, len(tag.contents)): - a = tag.contents[i] - yield a - if isinstance(a, Tag) and tag.contents: - if i < len(tag.contents) - 1: - stack.append((tag, i+1)) - stack.append((a, 0)) - break - raise StopIteration - -# Next, a couple classes to represent queries and their results. -class SoupStrainer: - """Encapsulates a number of ways of matching a markup element (tag or - text).""" - - def __init__(self, name=None, attrs={}, text=None, **kwargs): - self.name = name - if isString(attrs): - kwargs['class'] = attrs - attrs = None - if kwargs: - if attrs: - attrs = attrs.copy() - attrs.update(kwargs) - else: - attrs = kwargs - self.attrs = attrs - self.text = text - - def __str__(self): - if self.text: - return self.text - else: - return "%s|%s" % (self.name, self.attrs) - - def searchTag(self, markupName=None, markupAttrs={}): - found = None - markup = None - if isinstance(markupName, Tag): - markup = markupName - markupAttrs = markup - callFunctionWithTagData = callable(self.name) \ - and not isinstance(markupName, Tag) - - if (not self.name) \ - or callFunctionWithTagData \ - or (markup and self._matches(markup, self.name)) \ - or (not markup and self._matches(markupName, self.name)): - if callFunctionWithTagData: - match = self.name(markupName, markupAttrs) - else: - match = True - markupAttrMap = None - for attr, matchAgainst in self.attrs.items(): - if not markupAttrMap: - if hasattr(markupAttrs, 'get'): - markupAttrMap = markupAttrs - else: - markupAttrMap = {} - for k,v in markupAttrs: - markupAttrMap[k] = v - attrValue = markupAttrMap.get(attr) - if not self._matches(attrValue, matchAgainst): - match = False - break - if match: - if markup: - found = markup - else: - found = markupName - return found - - def search(self, markup): - #print 'looking for %s in %s' % (self, markup) - found = None - # If given a list of items, scan it for a text element that - # matches. - if isList(markup) and not isinstance(markup, Tag): - for element in markup: - if isinstance(element, NavigableString) \ - and self.search(element): - found = element - break - # If it's a Tag, make sure its name or attributes match. - # Don't bother with Tags if we're searching for text. - elif isinstance(markup, Tag): - if not self.text: - found = self.searchTag(markup) - # If it's text, make sure the text matches. - elif isinstance(markup, NavigableString) or \ - isString(markup): - if self._matches(markup, self.text): - found = markup - else: - raise Exception, "I don't know how to match against a %s" \ - % markup.__class__ - return found - - def _matches(self, markup, matchAgainst): - #print "Matching %s against %s" % (markup, matchAgainst) - result = False - if matchAgainst == True and type(matchAgainst) == types.BooleanType: - result = markup != None - elif callable(matchAgainst): - result = matchAgainst(markup) - else: - #Custom match methods take the tag as an argument, but all - #other ways of matching match the tag name as a string. - if isinstance(markup, Tag): - markup = markup.name - if markup and not isString(markup): - markup = unicode(markup) - #Now we know that chunk is either a string, or None. - if hasattr(matchAgainst, 'match'): - # It's a regexp object. - result = markup and matchAgainst.search(markup) - elif isList(matchAgainst): - result = markup in matchAgainst - elif hasattr(matchAgainst, 'items'): - result = markup.has_key(matchAgainst) - elif matchAgainst and isString(markup): - if isinstance(markup, unicode): - matchAgainst = unicode(matchAgainst) - else: - matchAgainst = str(matchAgainst) - - if not result: - result = matchAgainst == markup - return result - -class ResultSet(list): - """A ResultSet is just a list that keeps track of the SoupStrainer - that created it.""" - def __init__(self, source): - list.__init__([]) - self.source = source - -# Now, some helper functions. - -def isList(l): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is listlike.""" - return hasattr(l, '__iter__') \ - or (type(l) in (types.ListType, types.TupleType)) - -def isString(s): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is stringlike.""" - try: - return isinstance(s, unicode) or isinstance(s, basestring) - except NameError: - return isinstance(s, str) - -def buildTagMap(default, *args): - """Turns a list of maps, lists, or scalars into a single map. - Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and - NESTING_RESET_TAGS maps out of lists and partial maps.""" - built = {} - for portion in args: - if hasattr(portion, 'items'): - #It's a map. Merge it. - for k,v in portion.items(): - built[k] = v - elif isList(portion): - #It's a list. Map each item to the default. - for k in portion: - built[k] = default - else: - #It's a scalar. Map it to the default. - built[portion] = default - return built - -# Now, the parser classes. - -class BeautifulStoneSoup(Tag, SGMLParser): - - """This class contains the basic parser and search code. It defines - a parser that knows nothing about tag behavior except for the - following: - - You can't close a tag without closing all the tags it encloses. - That is, "" actually means - "". - - [Another possible explanation is "", but since - this class defines no SELF_CLOSING_TAGS, it will never use that - explanation.] - - This class is useful for parsing XML or made-up markup languages, - or when BeautifulSoup makes an assumption counter to what you were - expecting.""" - - SELF_CLOSING_TAGS = {} - NESTABLE_TAGS = {} - RESET_NESTING_TAGS = {} - QUOTE_TAGS = {} - - MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'), - lambda x: x.group(1) + ' />'), - (re.compile(']*)>'), - lambda x: '') - ] - - ROOT_TAG_NAME = u'[document]' - - HTML_ENTITIES = "html" - XML_ENTITIES = "xml" - XHTML_ENTITIES = "xhtml" - # TODO: This only exists for backwards-compatibility - ALL_ENTITIES = XHTML_ENTITIES - - # Used when determining whether a text node is all whitespace and - # can be replaced with a single space. A text node that contains - # fancy Unicode spaces (usually non-breaking) should be left - # alone. - STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, } - - def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None, - markupMassage=True, smartQuotesTo=XML_ENTITIES, - convertEntities=None, selfClosingTags=None): - """The Soup object is initialized as the 'root tag', and the - provided markup (which can be a string or a file-like object) - is fed into the underlying parser. - - sgmllib will process most bad HTML, and the BeautifulSoup - class has some tricks for dealing with some HTML that kills - sgmllib, but Beautiful Soup can nonetheless choke or lose data - if your data uses self-closing tags or declarations - incorrectly. - - By default, Beautiful Soup uses regexes to sanitize input, - avoiding the vast majority of these problems. If the problems - don't apply to you, pass in False for markupMassage, and - you'll get better performance. - - The default parser massage techniques fix the two most common - instances of invalid HTML that choke sgmllib: - -
    (No space between name of closing tag and tag close) - (Extraneous whitespace in declaration) - - You can pass in a custom list of (RE object, replace method) - tuples to get Beautiful Soup to scrub your input the way you - want.""" - - self.parseOnlyThese = parseOnlyThese - self.fromEncoding = fromEncoding - self.smartQuotesTo = smartQuotesTo - self.convertEntities = convertEntities - # Set the rules for how we'll deal with the entities we - # encounter - if self.convertEntities: - # It doesn't make sense to convert encoded characters to - # entities even while you're converting entities to Unicode. - # Just convert it all to Unicode. - self.smartQuotesTo = None - if convertEntities == self.HTML_ENTITIES: - self.convertXMLEntities = False - self.convertHTMLEntities = True - self.escapeUnrecognizedEntities = True - elif convertEntities == self.XHTML_ENTITIES: - self.convertXMLEntities = True - self.convertHTMLEntities = True - self.escapeUnrecognizedEntities = False - elif convertEntities == self.XML_ENTITIES: - self.convertXMLEntities = True - self.convertHTMLEntities = False - self.escapeUnrecognizedEntities = False - else: - self.convertXMLEntities = False - self.convertHTMLEntities = False - self.escapeUnrecognizedEntities = False - - self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags) - SGMLParser.__init__(self) - - if hasattr(markup, 'read'): # It's a file-type object. - markup = markup.read() - self.markup = markup - self.markupMassage = markupMassage - try: - self._feed() - except StopParsing: - pass - self.markup = None # The markup can now be GCed - - def convert_charref(self, name): - """This method fixes a bug in Python's SGMLParser.""" - try: - n = int(name) - except ValueError: - return - if not 0 <= n <= 127 : # ASCII ends at 127, not 255 - return - return self.convert_codepoint(n) - - def _feed(self, inDocumentEncoding=None): - # Convert the document to Unicode. - markup = self.markup - if isinstance(markup, unicode): - if not hasattr(self, 'originalEncoding'): - self.originalEncoding = None - else: - dammit = UnicodeDammit\ - (markup, [self.fromEncoding, inDocumentEncoding], - smartQuotesTo=self.smartQuotesTo) - markup = dammit.unicode - self.originalEncoding = dammit.originalEncoding - if markup: - if self.markupMassage: - if not isList(self.markupMassage): - self.markupMassage = self.MARKUP_MASSAGE - for fix, m in self.markupMassage: - markup = fix.sub(m, markup) - # TODO: We get rid of markupMassage so that the - # soup object can be deepcopied later on. Some - # Python installations can't copy regexes. If anyone - # was relying on the existence of markupMassage, this - # might cause problems. - del(self.markupMassage) - self.reset() - - SGMLParser.feed(self, markup) - # Close out any unfinished strings and close all the open tags. - self.endData() - while self.currentTag.name != self.ROOT_TAG_NAME: - self.popTag() - - def __getattr__(self, methodName): - """This method routes method call requests to either the SGMLParser - superclass or the Tag superclass, depending on the method name.""" - #print "__getattr__ called on %s.%s" % (self.__class__, methodName) - - if methodName.find('start_') == 0 or methodName.find('end_') == 0 \ - or methodName.find('do_') == 0: - return SGMLParser.__getattr__(self, methodName) - elif methodName.find('__') != 0: - return Tag.__getattr__(self, methodName) - else: - raise AttributeError - - def isSelfClosingTag(self, name): - """Returns true iff the given string is the name of a - self-closing tag according to this parser.""" - return self.SELF_CLOSING_TAGS.has_key(name) \ - or self.instanceSelfClosingTags.has_key(name) - - def reset(self): - Tag.__init__(self, self, self.ROOT_TAG_NAME) - self.hidden = 1 - SGMLParser.reset(self) - self.currentData = [] - self.currentTag = None - self.tagStack = [] - self.quoteStack = [] - self.pushTag(self) - - def popTag(self): - tag = self.tagStack.pop() - # Tags with just one string-owning child get the child as a - # 'string' property, so that soup.tag.string is shorthand for - # soup.tag.contents[0] - if len(self.currentTag.contents) == 1 and \ - isinstance(self.currentTag.contents[0], NavigableString): - self.currentTag.string = self.currentTag.contents[0] - - #print "Pop", tag.name - if self.tagStack: - self.currentTag = self.tagStack[-1] - return self.currentTag - - def pushTag(self, tag): - #print "Push", tag.name - if self.currentTag: - self.currentTag.contents.append(tag) - self.tagStack.append(tag) - self.currentTag = self.tagStack[-1] - - def endData(self, containerClass=NavigableString): - if self.currentData: - currentData = ''.join(self.currentData) - if not currentData.translate(self.STRIP_ASCII_SPACES): - if '\n' in currentData: - currentData = '\n' - else: - currentData = ' ' - self.currentData = [] - if self.parseOnlyThese and len(self.tagStack) <= 1 and \ - (not self.parseOnlyThese.text or \ - not self.parseOnlyThese.search(currentData)): - return - o = containerClass(currentData) - o.setup(self.currentTag, self.previous) - if self.previous: - self.previous.next = o - self.previous = o - self.currentTag.contents.append(o) - - - def _popToTag(self, name, inclusivePop=True): - """Pops the tag stack up to and including the most recent - instance of the given tag. If inclusivePop is false, pops the tag - stack up to but *not* including the most recent instqance of - the given tag.""" - #print "Popping to %s" % name - if name == self.ROOT_TAG_NAME: - return - - numPops = 0 - mostRecentTag = None - for i in range(len(self.tagStack)-1, 0, -1): - if name == self.tagStack[i].name: - numPops = len(self.tagStack)-i - break - if not inclusivePop: - numPops = numPops - 1 - - for i in range(0, numPops): - mostRecentTag = self.popTag() - return mostRecentTag - - def _smartPop(self, name): - - """We need to pop up to the previous tag of this type, unless - one of this tag's nesting reset triggers comes between this - tag and the previous tag of this type, OR unless this tag is a - generic nesting trigger and another generic nesting trigger - comes between this tag and the previous tag of this type. - - Examples: -

    FooBar *

    * should pop to 'p', not 'b'. -

    FooBar *

    * should pop to 'table', not 'p'. -

    Foo

    Bar *

    * should pop to 'tr', not 'p'. - -

    • *
    • * should pop to 'ul', not the first 'li'. -
    • ** should pop to 'table', not the first 'tr' - tag should - implicitly close the previous tag within the same
    ** should pop to 'tr', not the first 'td' - """ - - nestingResetTriggers = self.NESTABLE_TAGS.get(name) - isNestable = nestingResetTriggers != None - isResetNesting = self.RESET_NESTING_TAGS.has_key(name) - popTo = None - inclusive = True - for i in range(len(self.tagStack)-1, 0, -1): - p = self.tagStack[i] - if (not p or p.name == name) and not isNestable: - #Non-nestable tags get popped to the top or to their - #last occurance. - popTo = name - break - if (nestingResetTriggers != None - and p.name in nestingResetTriggers) \ - or (nestingResetTriggers == None and isResetNesting - and self.RESET_NESTING_TAGS.has_key(p.name)): - - #If we encounter one of the nesting reset triggers - #peculiar to this tag, or we encounter another tag - #that causes nesting to reset, pop up to but not - #including that tag. - popTo = p.name - inclusive = False - break - p = p.parent - if popTo: - self._popToTag(popTo, inclusive) - - def unknown_starttag(self, name, attrs, selfClosing=0): - #print "Start tag %s: %s" % (name, attrs) - if self.quoteStack: - #This is not a real tag. - #print "<%s> is not real!" % name - attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs)) - self.handle_data('<%s%s>' % (name, attrs)) - return - self.endData() - - if not self.isSelfClosingTag(name) and not selfClosing: - self._smartPop(name) - - if self.parseOnlyThese and len(self.tagStack) <= 1 \ - and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)): - return - - tag = Tag(self, name, attrs, self.currentTag, self.previous) - if self.previous: - self.previous.next = tag - self.previous = tag - self.pushTag(tag) - if selfClosing or self.isSelfClosingTag(name): - self.popTag() - if name in self.QUOTE_TAGS: - #print "Beginning quote (%s)" % name - self.quoteStack.append(name) - self.literal = 1 - return tag - - def unknown_endtag(self, name): - #print "End tag %s" % name - if self.quoteStack and self.quoteStack[-1] != name: - #This is not a real end tag. - #print " is not real!" % name - self.handle_data('' % name) - return - self.endData() - self._popToTag(name) - if self.quoteStack and self.quoteStack[-1] == name: - self.quoteStack.pop() - self.literal = (len(self.quoteStack) > 0) - - def handle_data(self, data): - self.currentData.append(data) - - def _toStringSubclass(self, text, subclass): - """Adds a certain piece of text to the tree as a NavigableString - subclass.""" - self.endData() - self.handle_data(text) - self.endData(subclass) - - def handle_pi(self, text): - """Handle a processing instruction as a ProcessingInstruction - object, possibly one with a %SOUP-ENCODING% slot into which an - encoding will be plugged later.""" - if text[:3] == "xml": - text = u"xml version='1.0' encoding='%SOUP-ENCODING%'" - self._toStringSubclass(text, ProcessingInstruction) - - def handle_comment(self, text): - "Handle comments as Comment objects." - self._toStringSubclass(text, Comment) - - def handle_charref(self, ref): - "Handle character references as data." - if self.convertEntities: - data = unichr(int(ref)) - else: - data = '&#%s;' % ref - self.handle_data(data) - - def handle_entityref(self, ref): - """Handle entity references as data, possibly converting known - HTML and/or XML entity references to the corresponding Unicode - characters.""" - data = None - if self.convertHTMLEntities: - try: - data = unichr(name2codepoint[ref]) - except KeyError: - pass - - if not data and self.convertXMLEntities: - data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref) - - if not data and self.convertHTMLEntities and \ - not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref): - # TODO: We've got a problem here. We're told this is - # an entity reference, but it's not an XML entity - # reference or an HTML entity reference. Nonetheless, - # the logical thing to do is to pass it through as an - # unrecognized entity reference. - # - # Except: when the input is "&carol;" this function - # will be called with input "carol". When the input is - # "AT&T", this function will be called with input - # "T". We have no way of knowing whether a semicolon - # was present originally, so we don't know whether - # this is an unknown entity or just a misplaced - # ampersand. - # - # The more common case is a misplaced ampersand, so I - # escape the ampersand and omit the trailing semicolon. - data = "&%s" % ref - if not data: - # This case is different from the one above, because we - # haven't already gone through a supposedly comprehensive - # mapping of entities to Unicode characters. We might not - # have gone through any mapping at all. So the chances are - # very high that this is a real entity, and not a - # misplaced ampersand. - data = "&%s;" % ref - self.handle_data(data) - - def handle_decl(self, data): - "Handle DOCTYPEs and the like as Declaration objects." - self._toStringSubclass(data, Declaration) - - def parse_declaration(self, i): - """Treat a bogus SGML declaration as raw data. Treat a CDATA - declaration as a CData object.""" - j = None - if self.rawdata[i:i+9] == '', i) - if k == -1: - k = len(self.rawdata) - data = self.rawdata[i+9:k] - j = k+3 - self._toStringSubclass(data, CData) - else: - try: - j = SGMLParser.parse_declaration(self, i) - except SGMLParseError: - toHandle = self.rawdata[i:] - self.handle_data(toHandle) - j = i + len(toHandle) - return j - -class BeautifulSoup(BeautifulStoneSoup): - - """This parser knows the following facts about HTML: - - * Some tags have no closing tag and should be interpreted as being - closed as soon as they are encountered. - - * The text inside some tags (ie. 'script') may contain tags which - are not really part of the document and which should be parsed - as text, not tags. If you want to parse the text as tags, you can - always fetch it and parse it explicitly. - - * Tag nesting rules: - - Most tags can't be nested at all. For instance, the occurance of - a

    tag should implicitly close the previous

    tag. - -

    Para1

    Para2 - should be transformed into: -

    Para1

    Para2 - - Some tags can be nested arbitrarily. For instance, the occurance - of a

    tag should _not_ implicitly close the previous -
    tag. - - Alice said:
    Bob said:
    Blah - should NOT be transformed into: - Alice said:
    Bob said:
    Blah - - Some tags can be nested, but the nesting is reset by the - interposition of other tags. For instance, a
    , - but not close a tag in another table. - -
    BlahBlah - should be transformed into: -
    BlahBlah - but, - Blah
    Blah - should NOT be transformed into - Blah
    Blah - - Differing assumptions about tag nesting rules are a major source - of problems with the BeautifulSoup class. If BeautifulSoup is not - treating as nestable a tag your page author treats as nestable, - try ICantBelieveItsBeautifulSoup, MinimalSoup, or - BeautifulStoneSoup before writing your own subclass.""" - - def __init__(self, *args, **kwargs): - if not kwargs.has_key('smartQuotesTo'): - kwargs['smartQuotesTo'] = self.HTML_ENTITIES - BeautifulStoneSoup.__init__(self, *args, **kwargs) - - SELF_CLOSING_TAGS = buildTagMap(None, - ['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) - - QUOTE_TAGS = {'script' : None, 'textarea' : None} - - #According to the HTML standard, each of these inline tags can - #contain another tag of the same type. Furthermore, it's common - #to actually use these tags this way. - NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', - 'center'] - - #According to the HTML standard, these block tags can contain - #another tag of the same type. Furthermore, it's common - #to actually use these tags this way. - NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del'] - - #Lists can contain other lists, but there are restrictions. - NESTABLE_LIST_TAGS = { 'ol' : [], - 'ul' : [], - 'li' : ['ul', 'ol'], - 'dl' : [], - 'dd' : ['dl'], - 'dt' : ['dl'] } - - #Tables can contain other tables, but there are restrictions. - NESTABLE_TABLE_TAGS = {'table' : [], - 'tr' : ['table', 'tbody', 'tfoot', 'thead'], - 'td' : ['tr'], - 'th' : ['tr'], - 'thead' : ['table'], - 'tbody' : ['table'], - 'tfoot' : ['table'], - } - - NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre'] - - #If one of these tags is encountered, all tags up to the next tag of - #this type are popped. - RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript', - NON_NESTABLE_BLOCK_TAGS, - NESTABLE_LIST_TAGS, - NESTABLE_TABLE_TAGS) - - NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS, - NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS) - - # Used to detect the charset in a META tag; see start_meta - CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)") - - def start_meta(self, attrs): - """Beautiful Soup can detect a charset included in a META tag, - try to convert the document to that charset, and re-parse the - document from the beginning.""" - httpEquiv = None - contentType = None - contentTypeIndex = None - tagNeedsEncodingSubstitution = False - - for i in range(0, len(attrs)): - key, value = attrs[i] - key = key.lower() - if key == 'http-equiv': - httpEquiv = value - elif key == 'content': - contentType = value - contentTypeIndex = i - - if httpEquiv and contentType: # It's an interesting meta tag. - match = self.CHARSET_RE.search(contentType) - if match: - if getattr(self, 'declaredHTMLEncoding') or \ - (self.originalEncoding == self.fromEncoding): - # This is our second pass through the document, or - # else an encoding was specified explicitly and it - # worked. Rewrite the meta tag. - newAttr = self.CHARSET_RE.sub\ - (lambda(match):match.group(1) + - "%SOUP-ENCODING%", contentType) - attrs[contentTypeIndex] = (attrs[contentTypeIndex][0], - newAttr) - tagNeedsEncodingSubstitution = True - else: - # This is our first pass through the document. - # Go through it again with the new information. - newCharset = match.group(3) - if newCharset and newCharset != self.originalEncoding: - self.declaredHTMLEncoding = newCharset - self._feed(self.declaredHTMLEncoding) - raise StopParsing - tag = self.unknown_starttag("meta", attrs) - if tag and tagNeedsEncodingSubstitution: - tag.containsSubstitutions = True - -class StopParsing(Exception): - pass - -class ICantBelieveItsBeautifulSoup(BeautifulSoup): - - """The BeautifulSoup class is oriented towards skipping over - common HTML errors like unclosed tags. However, sometimes it makes - errors of its own. For instance, consider this fragment: - - FooBar - - This is perfectly valid (if bizarre) HTML. However, the - BeautifulSoup class will implicitly close the first b tag when it - encounters the second 'b'. It will think the author wrote - "FooBar", and didn't close the first 'b' tag, because - there's no real-world reason to bold something that's already - bold. When it encounters '' it will close two more 'b' - tags, for a grand total of three tags closed instead of two. This - can throw off the rest of your document structure. The same is - true of a number of other tags, listed below. - - It's much more common for someone to forget to close a 'b' tag - than to actually use nested 'b' tags, and the BeautifulSoup class - handles the common case. This class handles the not-co-common - case: where you can't believe someone wrote what they did, but - it's valid HTML and BeautifulSoup screwed up by assuming it - wouldn't be.""" - - I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ - ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', - 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', - 'big'] - - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript'] - - NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, - I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS) - -class MinimalSoup(BeautifulSoup): - """The MinimalSoup class is for parsing HTML that contains - pathologically bad markup. It makes no assumptions about tag - nesting, but it does know which tags are self-closing, that - -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/scrapyorg/blog/templates_backup/inlines/default.html b/sites/scrapy.org/scrapyorg/blog/templates_backup/inlines/default.html deleted file mode 100644 index 5510ba952..000000000 --- a/sites/scrapy.org/scrapyorg/blog/templates_backup/inlines/default.html +++ /dev/null @@ -1,7 +0,0 @@ -{% if object %} - {{ object }} -{% else %} - {% for object in object_list %} - {{ object }} - {% endfor %} -{% endif %} \ No newline at end of file diff --git a/sites/scrapy.org/scrapyorg/blog/templatetags/__init__.py b/sites/scrapy.org/scrapyorg/blog/templatetags/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/sites/scrapy.org/scrapyorg/blog/templatetags/blog.py b/sites/scrapy.org/scrapyorg/blog/templatetags/blog.py deleted file mode 100644 index e80918cb3..000000000 --- a/sites/scrapy.org/scrapyorg/blog/templatetags/blog.py +++ /dev/null @@ -1,103 +0,0 @@ -from django import template -from django.conf import settings -from django.db import models - -import re - -Post = models.get_model('blog', 'post') -Category = models.get_model('blog', 'category') - -register = template.Library() - - -class LatestPosts(template.Node): - def __init__(self, limit, var_name): - self.limit = limit - self.var_name = var_name - - def render(self, context): - posts = Post.objects.published()[:int(self.limit)] - if posts and (int(self.limit) == 1): - context[self.var_name] = posts[0] - else: - context[self.var_name] = posts - return '' - -@register.tag -def get_latest_posts(parser, token): - """ - Gets any number of latest posts and stores them in a varable. - - Syntax:: - - {% get_latest_posts [limit] as [var_name] %} - - Example usage:: - - {% get_latest_posts 10 as latest_post_list %} - """ - try: - tag_name, arg = token.contents.split(None, 1) - except ValueError: - raise template.TemplateSyntaxError, "%s tag requires arguments" % token.contents.split()[0] - m = re.search(r'(.*?) as (\w+)', arg) - if not m: - raise template.TemplateSyntaxError, "%s tag had invalid arguments" % tag_name - format_string, var_name = m.groups() - return LatestPosts(format_string, var_name) - - -class BlogCategories(template.Node): - def __init__(self, var_name): - self.var_name = var_name - - def render(self, context): - categories = Category.objects.all() - context[self.var_name] = categories - return '' - -@register.tag -def get_blog_categories(parser, token): - """ - Gets all blog categories. - - Syntax:: - - {% get_blog_categories as [var_name] %} - - Example usage:: - - {% get_blog_categories as category_list %} - """ - try: - tag_name, arg = token.contents.split(None, 1) - except ValueError: - raise template.TemplateSyntaxError, "%s tag requires arguments" % token.contents.split()[0] - m = re.search(r'as (\w+)', arg) - if not m: - raise template.TemplateSyntaxError, "%s tag had invalid arguments" % tag_name - var_name = m.groups()[0] - return BlogCategories(var_name) - - -@register.filter -def get_links(value): - """ - Extracts links from a ``Post`` body and returns a list. - - Template Syntax:: - - {{ post.body|markdown:"safe"|get_links }} - - """ - try: - try: - from BeautifulSoup import BeautifulSoup - except ImportError: - from beautifulsoup import BeautifulSoup - soup = BeautifulSoup(value) - return soup.findAll('a') - except ImportError: - if settings.DEBUG: - raise template.TemplateSyntaxError, "Error in 'get_links' filter: BeautifulSoup isn't installed." - return value diff --git a/sites/scrapy.org/scrapyorg/blog/tests.py b/sites/scrapy.org/scrapyorg/blog/tests.py deleted file mode 100644 index 1dd87451d..000000000 --- a/sites/scrapy.org/scrapyorg/blog/tests.py +++ /dev/null @@ -1,66 +0,0 @@ -""" ->>> from django.test import Client ->>> from scrapyorg.blog.models import Post, Category ->>> import datetime ->>> from django.core.urlresolvers import reverse ->>> client = Client() - ->>> category = Category(title='Django', slug='django') ->>> category.save() ->>> category2 = Category(title='Rails', slug='rails') ->>> category2.save() - ->>> post = Post(title='DJ Ango', slug='dj-ango', body='Yo DJ! Turn that music up!', status=2, publish=datetime.datetime(2008,5,5,16,20)) ->>> post.save() - ->>> post2 = Post(title='Where my grails at?', slug='where', body='I Can haz Holy plez?', status=2, publish=datetime.datetime(2008,4,2,11,11)) ->>> post2.save() - ->>> post.categories.add(category) ->>> post2.categories.add(category2) - ->>> response = client.get(reverse('blog_index')) ->>> response.context[-1]['object_list'] -[, ] ->>> response.status_code -200 - ->>> response = client.get(reverse('blog_category_list')) ->>> response.context[-1]['object_list'] -[, ] ->>> response.status_code -200 - ->>> response = client.get(category.get_absolute_url()) ->>> response.context[-1]['object_list'] -[] ->>> response.status_code -200 - ->>> response = client.get(post.get_absolute_url()) ->>> response.context[-1]['object'] - ->>> response.status_code -200 - ->>> response = client.get(reverse('blog_search'), {'q': 'DJ'}) ->>> response.context[-1]['object_list'] -[] ->>> response.status_code -200 ->>> response = client.get(reverse('blog_search'), {'q': 'Holy'}) ->>> response.context[-1]['object_list'] -[] ->>> response.status_code -200 ->>> response = client.get(reverse('blog_search'), {'q': ''}) ->>> response.context[-1]['message'] -'Search term was too vague. Please try again.' - ->>> response = client.get(reverse('blog_detail', args=[2008, 'apr', 2, 'where'])) ->>> response.context[-1]['object'] - ->>> response.status_code -200 -""" - diff --git a/sites/scrapy.org/scrapyorg/blog/urls.py b/sites/scrapy.org/scrapyorg/blog/urls.py deleted file mode 100644 index 53aa0f404..000000000 --- a/sites/scrapy.org/scrapyorg/blog/urls.py +++ /dev/null @@ -1,41 +0,0 @@ -from django.conf.urls.defaults import * -from scrapyorg.blog import views as blog_views - - -urlpatterns = patterns('', - url(r'^(?P\d{4})/(?P\w{3})/(?P\d{1,2})/(?P[-\w]+)/$', - view=blog_views.post_detail, - name='blog_detail'), - - url(r'^(?P\d{4})/(?P\w{3})/(?P\d{1,2})/$', - view=blog_views.post_archive_day, - name='blog_archive_day'), - - url(r'^(?P\d{4})/(?P\w{3})/$', - view=blog_views.post_archive_month, - name='blog_archive_month'), - - url(r'^(?P\d{4})/$', - view=blog_views.post_archive_year, - name='blog_archive_year'), - - url(r'^categories/(?P[-\w]+)/$', - view=blog_views.category_detail, - name='blog_category_detail'), - - url (r'^categories/$', - view=blog_views.category_list, - name='blog_category_list'), - - url (r'^search/$', - view=blog_views.search, - name='blog_search'), - - url(r'^page/(?P\w)/$', - view=blog_views.post_list, - name='blog_index_paginated'), - - url(r'^$', - view=blog_views.post_list, - name='blog_index'), -) \ No newline at end of file diff --git a/sites/scrapy.org/scrapyorg/blog/views.py b/sites/scrapy.org/scrapyorg/blog/views.py deleted file mode 100644 index ad65484b5..000000000 --- a/sites/scrapy.org/scrapyorg/blog/views.py +++ /dev/null @@ -1,160 +0,0 @@ -from django.shortcuts import render_to_response, get_object_or_404 -from django.template import RequestContext -from django.http import Http404 -from django.views.generic import date_based, list_detail -from django.db.models import Q -from scrapyorg.blog.models import * - -import datetime -import re - - -def post_list(request, page=0, **kwargs): - return list_detail.object_list( - request, - queryset = Post.objects.published(), - paginate_by = 20, - page = page, - **kwargs - ) -post_list.__doc__ = list_detail.object_list.__doc__ - - -def post_archive_year(request, year, **kwargs): - return date_based.archive_year( - request, - year = year, - date_field = 'publish', - queryset = Post.objects.published(), - make_object_list = True, - **kwargs - ) -post_archive_year.__doc__ = date_based.archive_year.__doc__ - - -def post_archive_month(request, year, month, **kwargs): - return date_based.archive_month( - request, - year = year, - month = month, - date_field = 'publish', - queryset = Post.objects.published(), - **kwargs - ) -post_archive_month.__doc__ = date_based.archive_month.__doc__ - - -def post_archive_day(request, year, month, day, **kwargs): - return date_based.archive_day( - request, - year = year, - month = month, - day = day, - date_field = 'publish', - queryset = Post.objects.published(), - **kwargs - ) -post_archive_day.__doc__ = date_based.archive_day.__doc__ - - -def post_detail(request, slug, year, month, day, **kwargs): - return date_based.object_detail( - request, - year = year, - month = month, - day = day, - date_field = 'publish', - slug = slug, - queryset = Post.objects.published(), - **kwargs - ) -post_detail.__doc__ = date_based.object_detail.__doc__ - - -def category_list(request, template_name = 'blog/category_list.html', **kwargs): - """ - Category list - - Template: ``blog/category_list.html`` - Context: - object_list - List of categories. - """ - return list_detail.object_list( - request, - queryset = Category.objects.all(), - template_name = template_name, - **kwargs - ) - -def category_detail(request, slug, template_name = 'blog/category_detail.html', **kwargs): - """ - Category detail - - Template: ``blog/category_detail.html`` - Context: - object_list - List of posts specific to the given category. - category - Given category. - """ - category = get_object_or_404(Category, slug__iexact=slug) - - return list_detail.object_list( - request, - queryset = category.post_set.published(), - extra_context = {'category': category}, - template_name = template_name, - **kwargs - ) - - -# Stop Words courtesy of http://www.dcs.gla.ac.uk/idom/ir_resources/linguistic_utils/stop_words -STOP_WORDS = r"""\b(a|about|above|across|after|afterwards|again|against|all|almost|alone|along|already|also| -although|always|am|among|amongst|amoungst|amount|an|and|another|any|anyhow|anyone|anything|anyway|anywhere|are| -around|as|at|back|be|became|because|become|becomes|becoming|been|before|beforehand|behind|being|below|beside| -besides|between|beyond|bill|both|bottom|but|by|call|can|cannot|cant|co|computer|con|could|couldnt|cry|de|describe| -detail|do|done|down|due|during|each|eg|eight|either|eleven|else|elsewhere|empty|enough|etc|even|ever|every|everyone| -everything|everywhere|except|few|fifteen|fify|fill|find|fire|first|five|for|former|formerly|forty|found|four|from| -front|full|further|get|give|go|had|has|hasnt|have|he|hence|her|here|hereafter|hereby|herein|hereupon|hers|herself| -him|himself|his|how|however|hundred|i|ie|if|in|inc|indeed|interest|into|is|it|its|itself|keep|last|latter|latterly| -least|less|ltd|made|many|may|me|meanwhile|might|mill|mine|more|moreover|most|mostly|move|much|must|my|myself|name| -namely|neither|never|nevertheless|next|nine|no|nobody|none|noone|nor|not|nothing|now|nowhere|of|off|often|on|once| -one|only|onto|or|other|others|otherwise|our|ours|ourselves|out|over|own|part|per|perhaps|please|put|rather|re|same| -see|seem|seemed|seeming|seems|serious|several|she|should|show|side|since|sincere|six|sixty|so|some|somehow|someone| -something|sometime|sometimes|somewhere|still|such|system|take|ten|than|that|the|their|them|themselves|then|thence| -there|thereafter|thereby|therefore|therein|thereupon|these|they|thick|thin|third|this|those|though|three|through| -throughout|thru|thus|to|together|too|top|toward|towards|twelve|twenty|two|un|under|until|up|upon|us|very|via|was| -we|well|were|what|whatever|when|whence|whenever|where|whereafter|whereas|whereby|wherein|whereupon|wherever|whether| -which|while|whither|who|whoever|whole|whom|whose|why|will|with|within|without|would|yet|you|your|yours|yourself| -yourselves)\b""" - - -def search(request, template_name='blog/post_search.html'): - """ - Search for blog posts. - - This template will allow you to setup a simple search form that will try to return results based on - given search strings. The queries will be put through a stop words filter to remove words like - 'the', 'a', or 'have' to help imporve the result set. - - Template: ``blog/post_search.html`` - Context: - object_list - List of blog posts that match given search term(s). - search_term - Given search term. - """ - context = {} - if request.GET: - stop_word_list = re.compile(STOP_WORDS, re.IGNORECASE) - search_term = '%s' % request.GET['q'] - cleaned_search_term = stop_word_list.sub('', search_term) - cleaned_search_term = cleaned_search_term.strip() - if len(cleaned_search_term) != 0: - post_list = Post.objects.published().filter(Q(body__icontains=cleaned_search_term) | Q(tags__icontains=cleaned_search_term) | Q(categories__title__icontains=cleaned_search_term)) - context = {'object_list': post_list, 'search_term':search_term} - else: - message = 'Search term was too vague. Please try again.' - context = {'message':message} - return render_to_response(template_name, context, context_instance=RequestContext(request)) diff --git a/sites/scrapy.org/scrapyorg/docs/__init__.py b/sites/scrapy.org/scrapyorg/docs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/sites/scrapy.org/scrapyorg/docs/models.py b/sites/scrapy.org/scrapyorg/docs/models.py deleted file mode 100644 index 71a836239..000000000 --- a/sites/scrapy.org/scrapyorg/docs/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/sites/scrapy.org/scrapyorg/docs/urls.py b/sites/scrapy.org/scrapyorg/docs/urls.py deleted file mode 100644 index 164287362..000000000 --- a/sites/scrapy.org/scrapyorg/docs/urls.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.conf import settings -from django.conf.urls.defaults import * - -from scrapyorg.docs.views import document -from scrapyorg.docs.views import images - - -urlpatterns = patterns('', - url(r'^$', document, {'url': ''}), - url(r'^(?P[\w./-]*)/$', document), - url(r'^_images/(?P.*)$', images), -) - -if settings.DEBUG: # devel - urlpatterns += patterns('', - (r'^%s/(?P.*)$' % settings.MEDIA_URL[1:], - 'django.views.static.serve', - {'document_root': settings.MEDIA_ROOT}), - ) - diff --git a/sites/scrapy.org/scrapyorg/docs/views.py b/sites/scrapy.org/scrapyorg/docs/views.py deleted file mode 100644 index 56bb92814..000000000 --- a/sites/scrapy.org/scrapyorg/docs/views.py +++ /dev/null @@ -1,33 +0,0 @@ -import cPickle as pickle -import os - -import django.views.static -from django.conf import settings -from django.http import Http404 -from django.shortcuts import render_to_response -from django.template import RequestContext - - -def document(request, url): - docroot = settings.DOC_PICKLE_ROOT - - if os.path.exists(os.path.join(docroot, url, 'index.fpickle')): - docpath = os.path.join(docroot, url, 'index.fpickle') - elif os.path.exists(os.path.join(docroot, url + '.fpickle')): - docpath = os.path.join(docroot, url + '.fpickle') - else: - raise Http404("'%s' does not exist" % url) - - docfile = open(docpath, 'rb') - doc = pickle.load(docfile) - - return render_to_response('docs/doc.html', {'doc': doc}, - context_instance=RequestContext(request)) - - -def images(request, path): - return django.views.static.serve( - request, - document_root = os.path.join(settings.DOC_PICKLE_ROOT, '_images'), - path = path, - ) diff --git a/sites/scrapy.org/scrapyorg/download/__init__.py b/sites/scrapy.org/scrapyorg/download/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/sites/scrapy.org/scrapyorg/download/admin.py b/sites/scrapy.org/scrapyorg/download/admin.py deleted file mode 100644 index 4c5880561..000000000 --- a/sites/scrapy.org/scrapyorg/download/admin.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.contrib import admin - -from scrapyorg.download.models import DownloadLink - - -class DownloadLinkAdmin(admin.ModelAdmin): - list_display = ("text", "address", "public_link", "created") - list_filter = ("public", "created") - search_fields = ("address", "text") - -admin.site.register(DownloadLink, DownloadLinkAdmin) diff --git a/sites/scrapy.org/scrapyorg/download/models.py b/sites/scrapy.org/scrapyorg/download/models.py deleted file mode 100644 index 95e62d7a4..000000000 --- a/sites/scrapy.org/scrapyorg/download/models.py +++ /dev/null @@ -1,51 +0,0 @@ -from datetime import datetime - -from django.db import models -from django.utils.translation import ugettext_lazy as _ - - -class DownloadLink(models.Model): - description = models.CharField(_("description"), max_length=512, - blank=True, default='') - address = models.CharField(_("address"), max_length=1024, blank=False) - text = models.CharField(_("link text"), max_length=512, blank=False, - default=_("download")) - public = models.BooleanField(_("public"), blank=False, default=True) - - # automatic dates - created = models.DateTimeField(_("created"), editable=False) - updated = models.DateTimeField(_("updated"), editable=False) - - def toggle_public(self): - self.public = not self.public - self.save() - - def save(self): - if not self.id: - self.created = datetime.now() - self.updated = datetime.now() - super(DownloadLink, self).save() - - def __unicode__(self): - return self.address - - # ugly, but django-admin isn't very versatile right now - def public_link(self): - img_url = "/media/img/admin/icon-%s.gif" % \ - (self.public and "yes" or "no") - html = _('%s ' \ - '(Toggle)') - return html % (_(str(self.public)), self.id) - - public_link.short_description = u"public" - public_link.allow_tags = True - -# class Admin: -# list_display = ("text", "address", "public_link", "created") -# list_filter = ("public", "created") -# save_on_top = True - - class Meta: - verbose_name = _("download link") - verbose_name_plural = _("download links") - ordering = ["-created",] diff --git a/sites/scrapy.org/scrapyorg/download/templatetags/__init__.py b/sites/scrapy.org/scrapyorg/download/templatetags/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/sites/scrapy.org/scrapyorg/download/templatetags/download_tags.py b/sites/scrapy.org/scrapyorg/download/templatetags/download_tags.py deleted file mode 100644 index 14cd5a657..000000000 --- a/sites/scrapy.org/scrapyorg/download/templatetags/download_tags.py +++ /dev/null @@ -1,39 +0,0 @@ -from django import template - -from scrapyorg.lib.templatetags import * - -from scrapyorg.download.models import DownloadLink - - -register = template.Library() - - -@register.tag(name="load_download_links") -def do_load_links(parser, token): - syntax_msg = 'Syntax: %s "COUNT" as "VAR_NAME"' - - try: - splited = token.split_contents() - if len(splited) == 4: - tag, count, _as, var_name = splited - elif len(splited) == 3: - tag, _as, var_name = splited - count = None - - if count and not is_string(count) or not is_string(var_name): - raise_syntax(syntax_msg % tag) - count = count and int(unquoute(count)) or count - except: - raise_syntax(syntax_msg % token.split_contents()[0]) - return LoadLinksNode(count, unquoute(var_name)) - - -class LoadLinksNode(template.Node): - def __init__(self, count, var_name): - self.count = count - self.var_name = var_name - - def render(self, context): - links = DownloadLink.objects.filter(public=True)[:self.count] - context[self.var_name] = links - return '' diff --git a/sites/scrapy.org/scrapyorg/download/urls.py b/sites/scrapy.org/scrapyorg/download/urls.py deleted file mode 100644 index 59deb47f4..000000000 --- a/sites/scrapy.org/scrapyorg/download/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.conf.urls.defaults import * - -from scrapyorg.download.views import * - - -urlpatterns = patterns('', - (r"^(?P\d+)/toggle/$", toggle_public), -) diff --git a/sites/scrapy.org/scrapyorg/download/views.py b/sites/scrapy.org/scrapyorg/download/views.py deleted file mode 100644 index bc4b56215..000000000 --- a/sites/scrapy.org/scrapyorg/download/views.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.shortcuts import get_object_or_404 -from django.http import HttpResponseRedirect -from django.contrib.admin.views.decorators import staff_member_required - -from scrapyorg.download.models import DownloadLink - - -@staff_member_required -def toggle_public(request, link_id): - link = get_object_or_404(DownloadLink, pk=link_id) - link.toggle_public() - return HttpResponseRedirect("/admin/download/downloadlink/") diff --git a/sites/scrapy.org/scrapyorg/lib/__init__.py b/sites/scrapy.org/scrapyorg/lib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/sites/scrapy.org/scrapyorg/lib/templatetags.py b/sites/scrapy.org/scrapyorg/lib/templatetags.py deleted file mode 100644 index c664b913f..000000000 --- a/sites/scrapy.org/scrapyorg/lib/templatetags.py +++ /dev/null @@ -1,8 +0,0 @@ -from django import template - - -is_string = lambda val: val[0] in ("'", '"') and val[0] == val[-1] -unquoute = lambda val: val[1:-1] - -def raise_syntax(msg): - raise template.TemplateSyntaxError, msg diff --git a/sites/scrapy.org/scrapyorg/local_settings.py.template b/sites/scrapy.org/scrapyorg/local_settings.py.template deleted file mode 100644 index 5d2d85d7f..000000000 --- a/sites/scrapy.org/scrapyorg/local_settings.py.template +++ /dev/null @@ -1,12 +0,0 @@ -DEBUG = True - -DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. -DATABASE_NAME = '' # Or path to database file if using sqlite3. -DATABASE_USER = '' # Not used with sqlite3. -DATABASE_PASSWORD = '' # Not used with sqlite3. -DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. -DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. - -EMAIL_HOST = '' -EMAIL_HOST_USER = '' -EMAIL_HOST_PASSWORD = '' diff --git a/sites/scrapy.org/scrapyorg/manage.py b/sites/scrapy.org/scrapyorg/manage.py deleted file mode 100755 index 5e78ea979..000000000 --- a/sites/scrapy.org/scrapyorg/manage.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -from django.core.management import execute_manager -try: - import settings # Assumed to be in the same directory. -except ImportError: - import sys - sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) - sys.exit(1) - -if __name__ == "__main__": - execute_manager(settings) diff --git a/sites/scrapy.org/scrapyorg/settings.py b/sites/scrapy.org/scrapyorg/settings.py deleted file mode 100644 index f38c24d05..000000000 --- a/sites/scrapy.org/scrapyorg/settings.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- -# Django settings for scrapy project. - -#from os.path import abspath, dirname, basename, join -import os - -PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -ADMINS = ( - # ('Your Name', 'your_email@domain.com'), -) - -MANAGERS = ADMINS - -DEBUG=False - -DEFAULT_CHARSET = "utf-8" - -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be avilable on all operating systems. -# If running in a Windows environment this must be set to the same as your -# system time zone. -TIME_ZONE = 'America/Montevideo' - -# Language code for this installation. All choices can be found here: -# http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en-us' - -SITE_ID = 1 - -# If you set this to False, Django will make some optimizations so as not -# to load the internationalization machinery. -# USE_I18N = True - -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(PROJECT_ROOT, "static") - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash if there is a path component (optional in other cases). -# Examples: "http://media.lawrence.com", "http://example.com/media/" -MEDIA_URL = '/site-media' - -# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a -# trailing slash. -# Examples: "http://foo.com/media/", "/media/". -ADMIN_MEDIA_PREFIX = '/media/' - -# Make this unique, and don't share it with anybody. -SECRET_KEY = 'fq04ss$#1h=m=39sh4vvph+76i5u716z1-x5$$9xn7sb6y4-di' - -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.load_template_source', - 'django.template.loaders.app_directories.load_template_source', -# 'django.template.loaders.eggs.load_template_source', -) - -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.middleware.doc.XViewMiddleware', -) - -ROOT_URLCONF = 'scrapyorg.urls' - -TEMPLATE_DIRS = ( - os.path.join(PROJECT_ROOT, "templates"), -) - -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.admin', - 'django.contrib.markup', - 'scrapyorg.article', - 'scrapyorg.download', - 'scrapyorg.docs', - 'scrapyorg.blog', -) - -DOC_PICKLE_ROOT = os.path.join(PROJECT_ROOT, 'docs', 'pickle') - -# Override previous settings with values in local_settings.py settings file. -try: - from local_settings import * -except ImportError: - debug_msg ="Can't find local_settings.py, using default settings." - try: - from mod_python import apache - apache.log_error("%s" % debug_msg, apache.APLOG_NOTICE) - except ImportError: - import sys - sys.stderr.write("%s\n" % debug_msg) diff --git a/sites/scrapy.org/scrapyorg/urls.py b/sites/scrapy.org/scrapyorg/urls.py deleted file mode 100644 index 8bfe30671..000000000 --- a/sites/scrapy.org/scrapyorg/urls.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.conf import settings -from django.conf.urls.defaults import * -from django.contrib import admin - - -admin.autodiscover() - - -urlpatterns = patterns('', - # admin - url(r"^admin/download/downloadlink/", include("scrapyorg.download.urls")), - url(r'^admin/(.*)', admin.site.root), - - # pablo: disabled documentation until we improve its usability. for now, - # documentation will be served at http://doc.scrapy.org using static html - # files generated by Sphinx, and a small notice landing page will be shown - # in scrapy.org/doc - # docs -# url(r"^docs/", include("scrapyorg.docs.urls")), - url(r"docs/.*", 'django.views.generic.simple.redirect_to', {'url': '/doc/'}), - # news - url(r"^news/", include("scrapyorg.blog.urls")), -) - - -if settings.DEBUG: # devel - urlpatterns += patterns('', - (r'^%s/(?P.*)$' % settings.MEDIA_URL[1:], - 'django.views.static.serve', - {'document_root': settings.MEDIA_ROOT}), - ) - -# last resort, it's an article -urlpatterns += patterns('', - url(r"", include("scrapyorg.article.urls")), -) - diff --git a/sites/scrapy.org/static/images/box-borders-bottom.gif b/sites/scrapy.org/static/images/box-borders-bottom.gif deleted file mode 100644 index 721647129ef46498b44ada18d45966e043b5c4d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 84 zcmZ?wbhEHb6k=p$XkcXc4+e@qSr{1@7#VaJfB+=Jz@**Nzw-23{>5{)-0I$ZF5x$S j+ar%@$`@w7I<@T`_aaT7>)(3gH_iF}k6-647lSnb8(bq$ diff --git a/sites/scrapy.org/static/images/box-borders-bottom.png b/sites/scrapy.org/static/images/box-borders-bottom.png deleted file mode 100644 index 4cb0dea01561de15aee87243e36ef5484b0c0535..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2852 zcmV+<3)}RGP)KLZ*U+IBfRsybQWXdwQbLP>6pAqfylh#{fb6;Z(vMMVS~$e@S=j*ftg6;Uhf59&ghTmgWD0l;*T zI709Y^p6lP1rIRMx#05C~cW=H_Aw*bJ-5DT&Z2n+x)QHX^p z00esgV8|mQcmRZ%02D^@S3L16t`O%c004NIvOKvYIYoh62rY33S640`D9%Y2D-rV&neh&#Q1i z007~1e$oCcFS8neI|hJl{-P!B1ZZ9hpmq0)X0i`JwE&>$+E?>%_LC6RbVIkUx0b+_+BaR3cnT7Zv!AJxW zizFb)h!jyGOOZ85F;a?DAXP{m@;!0_IfqH8(HlgRxt7s3}k3K`kFu>>-2Q$QMFfPW!La{h336o>X zu_CMttHv6zR;&ZNiS=X8v3CR#fknUxHUxJ0uoBa_M6WNWeqIg~6QE69c9o#eyhGvpiOA@W-aonk<7r1(?fC{oI5N*U!4 zfg=2N-7=cNnjjOr{yriy6mMFgG#l znCF=fnQv8CDz++o6_Lscl}eQ+l^ZHARH>?_s@|##Rr6KLRFA1%Q+=*RRWnoLsR`7U zt5vFIcfW3@?wFpwUVxrVZ>QdQz32KIeJ}k~{cZZE^+ya? z2D1z#2HOnI7(B%_ac?{wFUQ;QQA1tBKtrWrm0_3Rgps+?Jfqb{jYbcQX~taRB;#$y zZN{S}1|}gUOHJxc?wV3fxuz+mJ4`!F$IZ;mqRrNsHJd##*D~ju=bP7?-?v~|cv>vB zsJ6IeNwVZxrdjT`yl#bBIa#GxRa#xMMy;K#CDyyGyQdMSxlWT#tDe?p!?5wT$+oGt z8L;Kp2HUQ-ZMJ=3XJQv;x5ci*?vuTfeY$;({XGW_huIFR9a(?@3)XSs8O^N5RyOM=TTmp(3=8^+zpz2r)C z^>JO{deZfso3oq3?Wo(Y?l$ge?uXo;%ru`Vo>?<<(8I_>;8Eq#KMS9gFl*neeosSB zfoHYnBQIkwkyowPu(zdms`p{<7e4kra-ZWq<2*OsGTvEV%s0Td$hXT+!*8Bnh2KMe zBmZRodjHV?r+_5^X9J0WL4jKW`}lf%A-|44I@@LTvf1rHjG(ze6+w@Jt%Bvjts!X0 z?2xS?_ve_-kiKB_KiJlZ$9G`c^=E@oNG)mWWaNo-3TIW8)$Hg0Ub-~8?KhvJ>$ z3*&nim@mj(aCxE5!t{lw7O5^0EIO7zOo&c6l<+|iDySBWCGrz@C5{St!X3hAA}`T4 z(TLbXTq+(;@<=L8dXnssyft|w#WSTW<++3>sgS%(4NTpeI-VAqb|7ssJvzNHgOZVu zaYCvgO_R1~>SyL=cFU|~g|hy|Zi}}s9+d~lYqOB71z9Z$wnC=pR9Yz4DhIM>Wmjgu z&56o6maCpC&F##y%G;1PobR9i?GnNg;gYtchD%p19a!eQtZF&3JaKv33gZ<8D~47E ztUS1iwkmDaPpj=$m#%)jCVEY4fnLGNg2A-`YwHVD3gv};>)hAvT~AmqS>Lr``i7kw zJ{5_It`yrBmlc25DBO7E8;5VoznR>Ww5hAaxn$2~(q`%A-YuS64wkBy=9dm`4cXeX z4c}I@?e+FW+b@^RDBHV(wnMq2zdX3SWv9u`%{xC-q*U}&`cyXV(%rRT*Z6MH?i+i& z_B8C(+grT%{XWUQ+f@NoP1R=AW&26{v-dx)iK^-Nmiuj8txj!m?Z*Ss1N{dh4z}01 z)YTo*JycSU)+_5r4#yw9{+;i4Ee$peRgIj+;v;ZGdF1K$3E%e~4LaI(jC-u%2h$&R z9cLXcYC@Xwnns&bn)_Q~Te?roKGD|d-g^8;+aC{{G(1^(O7m37Y1-+6)01cN&y1aw zoqc{T`P^XJqPBbIW6s}d4{z_f5Om?vMgNQEJG?v2T=KYd^0M3I6IZxbny)%vZR&LD zJpPl@Psh8QyPB@KTx+@RdcC!KX7}kEo;S|j^u2lU7XQ}Oo;f|;z4Ll+_r>@1-xl3| zawq-H%e&ckC+@AhPrP6BKT#_XdT7&;F71j}Joy zkC~6lh7E@6o;W@^IpRNZ{ptLtL(gQ-CY~4mqW;US7Zxvm_|@yz&e53Bp_lTPlfP|z zrTyx_>lv@x#=^!PzR7qqF<$gm`|ZJZ+;<)Cqu&ot2z=00004XF*Lt006O$eEU(80000WV@Og>004R=004l4008;_004mL004C` z008P>0026e000+nl3&F}0000#Nkl ziOQZcfUXD$@GT@_0<heS|-r+}O-00030{{sN=j1JRrlU;QH00003VtQ&&YGO)d;mK4Rpdz*apAgsoU=UjKH3uldR1)MD z{GZ|Jb^|XUPufftQNQd|;=Mp7Iv9F3&dUN9&eU{Duh V^){cqP!ys*w``CHf-e)GYk*Yt_@F( zf5UpFm82OY=*7$=FhVSpo^T8z(j)(1lc{P;g7M`)2UOeAZqp@_-Vs^*g#}$a&o36s dI~i~8@4U?S?4*9^?LH2-?hpF6qw8*Z`v!atEm!~m diff --git a/sites/scrapy.org/static/images/icon-arrow.gif b/sites/scrapy.org/static/images/icon-arrow.gif deleted file mode 100644 index 6326f028753a186421f7e104c7a1d8e98d22dce2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57 zcmZ?wbhEHbWM|-JXkcL2u5TYLD5&_8g#iR~KmS diff --git a/sites/scrapy.org/static/images/icon-author.gif b/sites/scrapy.org/static/images/icon-author.gif deleted file mode 100644 index d72d4847f0b06c8ef9bc3de1c900ee3df80f67d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57 zcmZ?wbhEHbeVKB0kHDxJl3B?#YgBb~FT1by9Wna@`t1OX_C9Rf- zvM))9%96dZ&hN}nsn7F$evjAp^ZjpbuXEk!T<5y3bDj6O&OP@z&74T>r1#K<16n#- z5R3)}?FWAlsdv4%<|#)z2-4Goc0mxt2rHPx~>~X@BPTkp;m; z52Ic2k$!bQU;@AT0TWx@4}i&vzc4A9pSD?l*aK*h5h0@n!APsIB=g8vjzX-IJ{?!r zgJ?)!AT~+~vOP=D=z;W)==^c2-t)ZeA`fZZ0lf4pz#wa$7F^ zNnz_*Sk|+x-@wMUVIvzG+eR{B+qkU4@ox-}9zz_A&^{=J7A6eQaKLCeV5Dx)ZD!JQ zm@4SkIvC{w4wElh5GZu(=ouK9m{~wL{9XxzXlQ>_Zh&BPFj^Wqnsp5HjO%DwWk4kd zE!{Rw#JYWlt+|9f&LQbHUcHg0wq1nV@QIC#tmpYR>I|Y0M&rE`7(8mz&HM{u8j;4f znon;La4 z&#LZ=i}sZ`ZMin}80LYl=tpH26fv;)@BNI;IZqsRVX&Va+Abt-7a}To;jP-C;t=>m zT${z_S?YA18h_x$IN^WuHj zodsi$b+&$iVx%v8vHvo5W}w6C(N*mupNmd(U%z)yusfG8j|6pCnHbGgwFneoOQmwP zrE79J8_q2Z^)NHd*@>KV#mr!&=~L%kEXrhjXs`7|+$nrnKd3z7=)V0tzt(`=CZgJK zu`-@(l=mWkJCEU5Y8A({B?&t18*4T6VERn-Nx#aeBf;&2x8MAno&19f`J%^XIukc< zK2n@DgUrdfcxEPtm5;dd!?u1emH3#&xQ7HQ64ZUCvTU)=!^*Nr5ob2n-{P7-ymYDC zYpf;9F*&-;IDEvkD2D`be`_67WyuVETm!du+;Apx>#>wCx?ESAwY7}GI2w&^`zViW zJAOt*`AoWIR(js-n7acBLXWr95ku^chriAXm2ZDCT$pjOICNAZuP3#wd!g~e98b%m zyqSbs)-Tsry-Ta`8y3Es-+Pp^Xk7m(8J^krHl|cYGW9S<1Qq5i(0P1pfdm=8 zs!e)9f_NLJ-HGw>D=s>X9?pCgeSEEdNhv}Mo+g{T>i@z%x-2u1 zZ2Q>ZK|tht<#z)E$~+9WXUc9gI#f&MG`ikXuJ0I~Geswi`F0MIpuVq5u}$5{;n82y z8^ykEa@kgqj6L)1OJ~q#;pZnU+@A|tr0H(gjU_`}&LML7@A(bR+J3Px z_+C6byUBbhG-3)vyFMjn62gTVa?X&gL=7cChT8W!uqsl;LdN* z%EEhvtd;EOT4R)s9}R#PVCEi$-)S(!*!7kW@9w!WQ4%UYeP=F(Tji?Xt$SVN{qrn! zDj7{Uqq#19^$%CR4VZh4!bwm=4(4!F=D7;ac1)hIp1VpO(@r&=IvnZEbllP7N6zH* z&b(Dso5&y3xjC*}=Rtz*U@DY@+f0VHs-~J)cC`9>b)C4KO5-I|>OnhojPZdNE6Z)3 zmPn%^VgJ2fW2$D-q~F|guDRmDe1+lhqkVrxHb1)S^=!5Xql%qrD6AAPC;G(5^q3D< zRH_{FROB(pkCe4-m%_zO*WM*Tw`>Mgia+*y!L!htc>`2(U0wZ7`C!!fd*|PcT`wVK zcME^@Klu6mdDq9>zRwVH2Nv+~Iucy+AF8H2c#!jLJDiLA#;iOtCK={; z!SaRC>PGAoWurt~WiuK+N{yHuyTKN0)Nh!LKkRJAy9si$9-cm+(X zXCQ7Hyl;SJU`sl+pe*>UQj7$}%)xU%6)oCl)l}cwbQy^h*qxpptNigS!|uRWB#3wR z;bsQGo7TPoahNPNzwW`#U0p>B^P4KZS9`shE7@}V^%+gq(z$!z*cZ5?B^oBI5bKV9 z5xuqRsLXV^G`nEUyO31f=dj)hEhP`$u%m3dWQ8YHkK~u+*dLakaXQlwb>y{k`_WOs zy(2zSR*6?sM|Slr%IzURO59KV{i~~L=3Y4GjksoGT`)<{wz~@qepxy&<_W19W`*wi z9Cm>O9oczrHX}RVP9Q|=PJIKzl2p)Qgv^py!~1gatO9}Xq^+3>trwKEs^9=*p4^;@ ztmwHg)^5vCw|>q(I$48{Qfev)?>rjF`D#9?>ha)<_Ga46;xG80 zGd1v@D_7!a78;vv2`aRa>=qD+E+avXgr5FA>2q;M$~Dhhs=qSjn9to3XD}mSzr=i> z1ieVn^Doz5x;eOXneYBF%dTc((2O%X2`b{jx0SdLS#E(di?+70&Mk9FC9TITMqSWJ)_JzEK*S?mIEE&;uK3hyNeF3Jw@f>bs?C$p{1CKXq)>R%Inr+%Gy`K^B9mAbm& zGj0C%UM}mqj#;Q&z9cdi<}~!>=G`HG9@(QNp^|=EGbL4Z(l|K-FpSR+v53dul?>&q z7&1Y0u5W#~C@+C5+PbTltEk$%A+=|&Gu(g#ZQ3NiEq7?Mu9u{{)Sc@(?t%$ci;42^ zy#Y7m+XK6ii(@D=p4M`Gcx%etvEgo&GC7MKsJD{ujuks}?e3~2HfGG-pYckjW%{07 z{6VoxS-Y(0L>%jSXHgPVQfR$H|2@Bjt7r6k@$YfZkAIXHpx{nu z$#Gn0Lp}5C#eq+?4&kP+r-xo?d~&g7qn#J+ioL0JC@Qb=SYZ3qhpN-`=RTJ{KEAX& z{MPY~sdL3dMHgGy;U!kthp#$|k{gMbMZLPmGkSwry60VlMIOfN>FhY!T4<)u9BYwF z)NHWdl9$6ZfjN&W;yu6jQ}M?d8Q!{fP6qqx5(W+8v4;8eCxfkyd<;@>6KMaMtoHd6 zzy)1su{ANN!C)%Xy52{Cwz=cMVGukn9&^w6VcQBiio@utb~jsV2JvIJY`^QA*bph8 z&zr!%{gzSsbe&54#X2Q9UqV}zr_P*#Uj@3}R2L?`7Zs7l(oizi+K^=$5*Af|ZN#k8 z()4{>$v%C<2XefGO zlsdLwDynCpWsB=6Jo3UufFim1_cHAirOZU3fWVfum?>qj+#&0LB^&u-rYwKRIZB(f zGvqAr0&ZsqYFq#X3oUTLRya4}6Mk}#6uvp2KV(o8j^r;6rH|i18G_uwV(M=%+E3)_ zzpWzuZ|W$>Fp_OUkU*$H+xDFBcbZBd@yYZE#+0ws->@Ck!a9=ow-N z2v8yqI%Eu1)-E_Vdvcz9>0<2&tIKKxoa;)N;)bEU!;gBhYxH;*M+_Ek>Y|GWCD?E6 ztfSNcHYZ37l!A`ND%%Y9o;WY}-&piGPe*%4H>{hNE6`!pG$LC!pzuI>y|otsr;T;P zdRh~(wtxW{`?zCQgqfFx$sBTxj;p;8xcy)EpVrIM#aQ!*F=%Q9Tr&8-bM)}`*xxwx z)-D8NYy01LY#1!i!=57O;P)Hr>Du6ME@Xp@KiIzsGdbWqefPOI+W$a`eYx}df3Sfj zgDuw1+RKFivP>tjo`k;U7W?nULEIx^IQck$FhAj?VlAu1=7d4k+EhX;M@(o z2zcy|z#!K#yMRFY4=$?>jsODcKlm(Qh#meEr_8z?WMdF3NHEoLN`85UaZvKGmFE$8 zLuomnVH7R0Jj7}V2pPncIRtwScA!84TQuS2$h_h<+*A-_$Nmm2I~v2F_%_BId}5`qpv4j@ke2~Wv^ zjSGnD|05v;sY6E4@`eayEwj9Xz(859gEmOLXaNTT!Ces!cf(6rlmC%i3WIZnpR#s` zBc!C^kdo>tcWaC@mLOz<1#4+#;jz*RVIfCbWnnXvp0u93I@ZB)zpp3O$XDMOJmu={iX~VJopN4x9F zY~+U>4J;nx>F7>y#JLHP?OKB!41%(-Fu9?h*NThQ?s|GZoBCTcTwUQm%>!*EfRdh` zq6QA*<%)G9=xC^Tc{$oDqLEs1XruxfsiCf}p@BfC$;xZ0?~{|4(L$-AH5JfGWORSm z`{mAxO&vEp!P*Uj{b^I?-`QNHw_;NXPVV@wpKJi zQXzlELK1M4=?5WfY~QDPXLY{2dBKJ)EthozLIdAb(9`wT)Ghf7$>6 zXKPKc{#RC3%l=`9(jTys0d`YVDBrgD`|p#i>e7nve|sowF+WGb9c&pq@`dLkR#{3pH zKdTfen?qnwu_`i3@V{fP8l7L&`yKHFoTm@`XO#c9!k^%1_v=CQTW`oU|Je~TR}pMB zf&sB6KYmmYO7Q>St?B}$?ua24k8|;&$jgH{BP|U_km-u*>gqpLkN&gj>gqpLe`%Vy zITBQmE3N&({!>dIbn7Hm!_gj#2RbM@2_y>qBNj>lB(wxl4k>|@mjwy%6l5h(GSVO+ zKpyZFWF(MiISCnQgoF%20VK2p3JH=767YZ>d1;Vnm-W!&9fD^|7(vyEcgQ3-$ymM=0D?r%vC((cna%s zM8gqGPdp_SP+&;^wq_N|zv_?v(v(B3R$Nu_ry^zg7=h1jtcoN6hGJ|L?@#gHFq4r} zkdu{=l}5`+Qz#fGc?5EqA}O3@Dg*c^#IlU61X)X-Lcy4V@d6w`0Y_E=B~77VJV73e zCZJ&ajh ziZwKv+#;G{6HWF6DUFbm0C6h;VqHQ`LS8~a0xbccK_Da$NC||D1VUB<#4!*;$V-5r zM*>+e6~IgYlLQ6k3kY{GGeE%a(Ll)~wD7BF_flGyp3fGztSd%a{ zx~Ooi34t{UQ=^Lt*P0MmlQ1>9sBo9sBodOG87uy7u>jb!lL< zhv~Vj!O6NCkymdpa4#Qh+bgrlW?aPc{Di2wtWm_xd<_06N<%#I)_pl1V_U+r#y&=| z3rx(sn**XWwNeXAn@o1_P4-Wb57)(d1x7C)u=_upudA^Ch{LH%ahb){?VrByMC%-N z^!Y3P_QRTv&$GJ}512do1}EGpdD%HMw|o#U?Yec~a9}z{T3XkiX94e9cNlm~?{V%b z@eINuV;l*N9p>FECa1|KZc8wc zpS*Sd*^Zq}{Zj!^sRh5gthMS;;MHdW|K{^QxctZ2y$hZ}+unMZdAzW!;`Zh@`FGjl zq}OBS>1oz!j^FsgZYXwj9NQM*lXx?8*0kcoAR6`1Q&^%6X(gUTg6efEWAy|=Z}}F! zbPz9zy{sU>aa~FE{p5v|hp)memwhSKAVEoSVF#aIKJvzYxBUs$ulYP!?k#gJw_+Cr zwyXNqYFk`>`#3XaTKHzEU@G*@mHSQ<@t)%ggdauut!6G8w9i zo@tODm>@=8Z8}%sXP2*i0fWayc@I!7&%RU025 zrqvZt6}D+pU^7Y&u6Mb$87GtidPasT>b>44J?CD=pL2;80->AUwIbVZD?SSu`#sq$myvMw{srdWvj^0ao5`%gBP|aigcMSJdGWY{!BhRY?EBj;6vf02%d2AGBP-~RBPuVCR*%;_Tc5)z3^?Yl(=)& z{M)g3jcm6YuGjZ4iCqZ{)e*dJ1i~R?;Hjj0e}63F=~n|Ol0HRKm{(>pmOUJgePfFn zTN*cds%i6^Y;_#hDl?P5#f2P29#F8^nf#7e5AWG0V)$9W`nU-%LPqUP{_AF>tQ$B$ za(1dVETN>h--2BsUvRd%t>WIKR8nhf(S{3tTeEsaurr6RU#&Q`?|Y_AjnjG!TuZKo zlwFl}l+TvQ9&m7MTw(4WryH`P(u?Ner4c#@@?gc90t?1wee-AyA^Wxy1B0*l{Y=NO z3uOy;+D<1A5T!e{Gb`sR?{1Nd8lPn1N|m|?j^J&N(ASIK@Z5&o$Nj!N*OjiusqEcL z9&HT?&0~Ip%36Dvu&Ap8cprB^lj0)wS;=Tt&ONbW=EZFLF3eWdD#i|cOA3rGpN$!+ zZG2epDAZrFMkkm}6TM6KS2GQQ zSr~VS{@Ag?mzWR_j+z9Y`6Z9Z`C2QhNb|hr%CxNslaB&#P9G6_k{TAa5bZnxpB%XV zM6Gg^*UQE;#}pg@T(2q=LLtA8q1O6JHGdM zs`g&3a8>O;eNk5pbH??%Op;xdcx|WttwDoYoRaF1F~Y3>mSK0Ix01ML#%zR!Qa^4+ z5Y?x}IpytH#s7LTBZpY8HB{sO!IH*LTdekQ@xfGO^tK|&&7BqZ#cYSLpSRZ2pBKny zR|v3Q=wF|7@WZI#C-!%>nFYGUnqbCw?KeY(TX#(NBnrGyxu<&bs|q?|_!@YQqU9>4 zZX6+YnZoBTrXXgHAG|lxJKr#Mhy+cAu#h0dYv5#O1Sp6~JNgLvNxc!s`{Fxzi-<{Y zHz4{WpV&KHJUZwW94s%Ej2B7xruhAN1hgeKaQOJt)smL4H(ivHZ?c@PGu$kGR6M~l zY?#jC6gVHeCeE{kt3bUV$R3iiyJj-`*Bzg`rc+#2KG*8nheBDcv+E3)rXMS@Gl*Wq z3|tj&-tn*)QHhvN41JW;Y__!bLBkTQC$Sk;H?`QYbT{qN;tbJXaq1}v`nc6>Nlo|o z6qCo?MH|QP-1noXiN&55)8R?|T9X%xb71Nj-SiA;tn3yIanmXIjAt|9!whwTSMKgu zXnynE?5|_FhSBq?#xc<(2wR1k!w<6!9~126NdvK`?}Kl9LwMyYk`-gL#rdM(*uyFj z5#m*0VvR+G$=2+rBcJC&3=UNluq*NcKLk*h#XQ}z?G$bIUd!t)K2v^DJ7r1p!}iE; z8eVSK*vHRxTj*pg^pT+0PYDfiTQzypLe}@35RDyMtUq{AAZyQoW0%T@!>}b8q~@_} zPE!}p4LBck&USj=^#rfjRr%zgfG*1>A>)ZdDq=6~5qu&KdEF0g6Bd3c)m6v0uH|}J zQL}4WfpxWhaQIPUhg|{94&h{Z&bX_G z;{-bmpS@qgzOwT9%fQV*=jK!EOH3|SDYHcbyFU>{7s|3i+z7G{c!HhBF9tPUA>NBh zxBVE15q#s-DRKQ{ETOStwh{!>XJ4OYrP53b(Y%tYZdq1Z%;Dl2BD9SRLhe*m&F1sz z-tu_Y)#+=qH_1Dr+=;I+I{mq5GkjY?=q1Lc{dvvTl6(Y%AJp{=p%Vyi0@Ci~Pm-Y7 zng=8(neEw9XCARxb~g0xoOXHp$wQ01OSuaN1eZJli64zxDh5@TY}3~ZOnlEDJvP&(Vhhx#u3#IcQhkfQnX(L*=g{Xk|0zlv0hp#s<3l7FU0Be z&D-MrbH#L{$2&fLt!T$DhU98#Esic`t*@HR;+Yyd+%&^pcQ9VUZTrLE9ovF+gilm& z-56t~c6}oEMU)eExT9`lM8k$B4}Q8CQjKU6abu5p7nHs49y*CgZmLOx7+^MUKmRd+ zSg-ubm0?h`B72wS(6I)7Y&Cz9xw=7vYiAqX6);DIw<{sBo zq|Tf&Z|rR6s7_6x%h8(F43B-W&_i$Mdp|MO=FJe*Bs>M5AnL_3>kmRJio)9@9pJ$%w)3av1nf z)7B9tK@pc8CdMco-d@w}IO(pB{KV3~uBg(_xNEvvmqWwm^DPkr!?O{@b>;Rownc6yI@R){Aop=^>)}%Wp*BF0bIKYrN z$PWLUuoR9@b@K3SH9J_Tna#VwDJ$da(IcJ~`5v4beKv*ou!@Ls6`!*Fw$L}2F4Z-} z_mtep#2~GIXfwn>fgj=*dUMZQJU`$n>R=m>^^&S6&3L+Zdycd8vyhhZ zO?OoF0sbz_bpQo3)~-HYnr8BCZ zaXRtX2W58kf~nZSXUC4J@f<8`#y%t*I(1nyI@PVhK8nab_bS1GO)0wZU6DjR|E4oJ zf?shtPoW($)2i=I3q%&GJ_L{3 znXNLOlVRz*X1_HM6IJ7i4rcn;!Qm>&s9GJ@p%l79R{OX*o?fd2R`3 zdE@A}hw*ki6Fp{j+bl*kn$Y28X|BT8dh>?=_R@V2>OHX*KcTa92Rokhg+@=~ z3b}?qz>mfsa8WBHV`j-F`+1Xjlh1oE^X#krM6#4MGf)8JE?@^R3%i@!dmsn^%T9eE zp38SO+Q@s(ko`ah9C+N#l9{Cx%t9*ZQCbZ-l-9K{3~M?nh&@ROu@FkBNb4w#qd0B& zO2ZU&^OutURL)c$wg;o}2xc+}46z~mgjJBkZCN6~x^P@7AONvciT9P}dP?i2s@#;Z z?}(V(FKjT2!R5pB)%DB%`Aq%jVl-?W$cZ-Y&%)Nv$jwUU_V{En-I~va``_bJ&x?0! N{ln(iX>q#t?HB9KIjR5v diff --git a/sites/scrapy.org/static/style/pygments.css b/sites/scrapy.org/static/style/pygments.css deleted file mode 100644 index 1f2d2b618..000000000 --- a/sites/scrapy.org/static/style/pygments.css +++ /dev/null @@ -1,61 +0,0 @@ -.hll { background-color: #ffffcc } -.c { color: #408090; font-style: italic } /* Comment */ -.err { border: 1px solid #FF0000 } /* Error */ -.k { color: #007020; font-weight: bold } /* Keyword */ -.o { color: #666666 } /* Operator */ -.cm { color: #408090; font-style: italic } /* Comment.Multiline */ -.cp { color: #007020 } /* Comment.Preproc */ -.c1 { color: #408090; font-style: italic } /* Comment.Single */ -.cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ -.gd { color: #A00000 } /* Generic.Deleted */ -.ge { font-style: italic } /* Generic.Emph */ -.gr { color: #FF0000 } /* Generic.Error */ -.gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.gi { color: #00A000 } /* Generic.Inserted */ -.go { color: #303030 } /* Generic.Output */ -.gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ -.gs { font-weight: bold } /* Generic.Strong */ -.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.gt { color: #0040D0 } /* Generic.Traceback */ -.kc { color: #007020; font-weight: bold } /* Keyword.Constant */ -.kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ -.kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ -.kp { color: #007020 } /* Keyword.Pseudo */ -.kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ -.kt { color: #902000 } /* Keyword.Type */ -.m { color: #208050 } /* Literal.Number */ -.s { color: #4070a0 } /* Literal.String */ -.na { color: #4070a0 } /* Name.Attribute */ -.nb { color: #007020 } /* Name.Builtin */ -.nc { color: #0e84b5; font-weight: bold } /* Name.Class */ -.no { color: #60add5 } /* Name.Constant */ -.nd { color: #555555; font-weight: bold } /* Name.Decorator */ -.ni { color: #d55537; font-weight: bold } /* Name.Entity */ -.ne { color: #007020 } /* Name.Exception */ -.nf { color: #06287e } /* Name.Function */ -.nl { color: #002070; font-weight: bold } /* Name.Label */ -.nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ -.nt { color: #062873; font-weight: bold } /* Name.Tag */ -.nv { color: #bb60d5 } /* Name.Variable */ -.ow { color: #007020; font-weight: bold } /* Operator.Word */ -.w { color: #bbbbbb } /* Text.Whitespace */ -.mf { color: #208050 } /* Literal.Number.Float */ -.mh { color: #208050 } /* Literal.Number.Hex */ -.mi { color: #208050 } /* Literal.Number.Integer */ -.mo { color: #208050 } /* Literal.Number.Oct */ -.sb { color: #4070a0 } /* Literal.String.Backtick */ -.sc { color: #4070a0 } /* Literal.String.Char */ -.sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ -.s2 { color: #4070a0 } /* Literal.String.Double */ -.se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ -.sh { color: #4070a0 } /* Literal.String.Heredoc */ -.si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ -.sx { color: #c65d09 } /* Literal.String.Other */ -.sr { color: #235388 } /* Literal.String.Regex */ -.s1 { color: #4070a0 } /* Literal.String.Single */ -.ss { color: #517918 } /* Literal.String.Symbol */ -.bp { color: #007020 } /* Name.Builtin.Pseudo */ -.vc { color: #bb60d5 } /* Name.Variable.Class */ -.vg { color: #bb60d5 } /* Name.Variable.Global */ -.vi { color: #bb60d5 } /* Name.Variable.Instance */ -.il { color: #208050 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/sites/scrapy.org/static/style/style.css b/sites/scrapy.org/static/style/style.css deleted file mode 100644 index e09ea3f6c..000000000 --- a/sites/scrapy.org/static/style/style.css +++ /dev/null @@ -1,154 +0,0 @@ -/* Scrapy.org style sheet *******************************/ - -/* main elements */ -html, body { - margin:0; - padding:0; - height: 100%; -} - -body { - background: url('../images/main-bg.jpg') repeat-x left top; - font-family: "Lucida Sans", Verdana, Helvetica, sans-serif; - font-size:small; - color:#333; - background-color:#FFF; -} - -a { color:#6e0909; text-decoration:none; border-bottom:1px solid #d78888;} - -a:hover { color:#f99800; border-bottom:1px solid #f7b7b7; } - -a img { - border: none; -} - -h1, h2 { color:#331f0a; font-weight:normal;} - -h1 { - font-size: 200%; - margin: 0 0 .6em 0; -} - -h2 { - font-size: 150%; - margin: .6em 0; -} - -dt { margin-left: 12px; font-style: italic; } -dd { margin-left: 30px; } - -blockquote { padding:0px 15px; color:#5b1111; font-size:125%; margin:0;} - -p { line-height:1.5em; } - -.clear { - clear:both; - height: 0; -} - -#wrapper { - position: relative; - min-height: 100%; -} - -#header { - padding-top: 42px; - margin: 0 20px; -} - -#body { - padding-bottom: 35px; -} - -#content-wrapper { - width: 100%; - float: left; - margin-right: -300px; -} - -#content { - margin: 0 320px 0 20px; -} - -#sidebar { - width: 300px; - float: right; -} - -#footer { - position: absolute; - bottom: 0; - width: 100%; - height: 35px; - border-top: 1px solid #b5331a; - background: url('../images/footer-bg.jpg') repeat-x left top #5b2a16; -} - -#header h1 { - float: left; - margin: 0; -} - -#header h1 a { - border: none; -} - -#navigation { - float: right; - margin: 0; - padding: 0; - list-style-type: none; -} - -#navigation li { float:left; padding:0 10px; font-size:120%; border-right:1px solid #e57919; } -#navigation li.last { border-right:none; } -#navigation li a { text-decoration:none; border-bottom:1px solid #ffca9b; color:#e57919;} -#navigation li a:hover { color:#c35012; border-bottom:1px solid #ef994c} - -.breadcrumb a { - border-bottom: none; -} - -#footer a:hover { - color: #ffe114; -} - -#footer p { - margin: 10px 20px; - line-height: 15px; - font-size: .85em; - color: #fff; -} - -#footer p a { - color: #fff; -} - -#badge { - float: right; -} - -#badge a { - border-bottom: none; -} - -.box { margin-bottom:15px; background:url('../images/box-borders-bottom.gif') no-repeat left bottom #4d271c; padding-bottom:5px; } -.box.download {background-color:#5b1111; } -.box.download p { color:#d18c59; } -.box.download a.download { color:#d18c59; background:url('../images/icon-arrow.gif') no-repeat 2px 2px; padding-left:15px; border-bottom:1px solid #a0595d } -.box.download a:hover.download { color:#FFF;} - -.box h3 { padding:10px 10px 0; font-weight:normal; font-size:160%; margin:0.3em 0pt; color:#ffebd2; background:url('../images/box-borders-top.png') no-repeat left top ;} -.box p { padding:0 10px; font-size:90%; line-height:1.2em} - -.box .post { color:#FFF; margin:0pt 13px 5px; -padding:0pt 0px 5px; border-bottom:4px solid #462217;} -.box .post.last { border-bottom:none; } -.box .post p.abstract { padding:0; margin:0; font-size:0.8em; height:1%; line-height:1.3em} -.box .post p.author { color:#ffbc97; padding:0 0 0 15px; margin:5px 0px; font-size:0.8em; background:url('../images/icon-author.gif') no-repeat 2px 3px;} -.box .post p.author a { color:#FFF; border-bottom:none } -.box .post h4 { display:inline;font-size:1em; font-weight:bold; } -.box .post h4 a {color:#ffe114; border-bottom:1px solid #957020; font-size:0.9em} -.box .post a.more { font-size:0.8em; color:#ffe114; border-bottom:none;} -.box .post a:hover.more { color:#FFF;} diff --git a/sites/scrapy.org/templates/admin/base_site.html b/sites/scrapy.org/templates/admin/base_site.html deleted file mode 100644 index 3b859a327..000000000 --- a/sites/scrapy.org/templates/admin/base_site.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "admin/base.html" %} -{% load i18n %} - -{% block title %}{{ title|escape }} | {% trans 'Django site admin' %}{% endblock %} - -{% block branding %} -

    {% trans 'Django administration' %}

    -{% endblock %} - -{% block nav-global %}{% endblock %} - -{% block footer %} - - - -{% endblock %} diff --git a/sites/scrapy.org/templates/articles/community.html b/sites/scrapy.org/templates/articles/community.html deleted file mode 100644 index d0a1f0f09..000000000 --- a/sites/scrapy.org/templates/articles/community.html +++ /dev/null @@ -1,28 +0,0 @@ -{% extends "base_home.html" %} - -{% block main-content %} -

    Mailing lists

    -

    - scrapy-users: The main list for help and announcements -

    - -
    -

    Subscribe:

    - - -

    -scrapy-developers: Where discussions about Scrapy development itself takes place -

    - -
    -

    Subscribe:

    - - -

    IRC

    - -

    - Join the #scrapy IRC channel at Freenode to chat with other Scrapy users & developers -

    - - -{% endblock %} diff --git a/sites/scrapy.org/templates/articles/doc.html b/sites/scrapy.org/templates/articles/doc.html deleted file mode 100644 index b5f9c8bb9..000000000 --- a/sites/scrapy.org/templates/articles/doc.html +++ /dev/null @@ -1,29 +0,0 @@ -{% extends "base_home.html" %} - -{% block main-content %} -

    Scrapy Documentation

    - -

    You can browse the Scrapy documentation online at:

    - -

    http://doc.scrapy.org

    - -

    At the moment (January, 2009) our top priority is improving the - documentation so keep in mind that it will change and grow quite a bit - these days.

    - -

    After we finish the documention we'll make it available in another - formats.

    - -

    Getting help

    - -

    Having trouble? We’d like to help!

    - - - -{% endblock %} diff --git a/sites/scrapy.org/templates/articles/download.html b/sites/scrapy.org/templates/articles/download.html deleted file mode 100644 index 771677c8d..000000000 --- a/sites/scrapy.org/templates/articles/download.html +++ /dev/null @@ -1,47 +0,0 @@ -{% extends "base_home.html" %} - -{% block main-content %} -

    How to get Scrapy

    - -

    - Scrapy is available under the BSD license. - It runs on Linux, Windows and Mac. It requires Python 2.5 and some other Python libraries (see Requirements below) -

    - -

    Requirements

    - -
      -
    • Python 2.5 (or 2.6)
      • -
    • Twisted 8.0 or above
      • -
    • libxml2 2.6.28 or above (including Python bidings)
      • -
    - -

    - The following libraries are also recommended: -

    - -
      -
    • pyopenssl - for crawling secure (HTTPS) pages
      • -
    • spidermonkey - for parsing javascript
      • -
    - - -

    Latest release

    - -

    - We're working hard to get the first release of Scrapy out. In the - meantime, you can download the latest development version in a ZIP file (from Trac) or checkout the Subversion repository. -

    - -

    Development version

    - -

    You can get the latest (and more stable) release of Scrapy from the - Subversion repository at: http://svn.scrapy.org/scrapy/trunk/ -

    - -

    After you get it

    - -

    Read the Installation - steps (in the documentation) for further instructions

    - -{% endblock %} diff --git a/sites/scrapy.org/templates/articles/home.html b/sites/scrapy.org/templates/articles/home.html deleted file mode 100644 index f52ae3156..000000000 --- a/sites/scrapy.org/templates/articles/home.html +++ /dev/null @@ -1,56 +0,0 @@ -{% extends "base_home.html" %} - -{% block main-content %} -

    Welcome to Scrapy

    - -

    - Scrapy is a high level scraping and web crawling framework for writing - spiders to crawl and parse web pages for all kinds of purposes, from - information retrieval to monitoring or testing web sites. -

    - -

    Features

    - -
    - -
    Productive
    -
    Just write the rules to extract data from pages and let Scrapy crawl the entire web site for you
    - -
    Scalable
    -
    Scrapy is being used in production to scrape more than 500 sites daily, all in one server
    - -
    Distributed
    -
    If you need more processing/bandwith power Scrapy comes bundled with a master/slave cluster that lets you scrape using as many servers as possible - -
    Extensible
    -
    Scrapy was designed with extensibility in mind and so it provides several mechanisms to plug new code without having to touch the framework core - -
    Portable
    -
    Scrapy runs on Linux, Windows and Mac
    - -
    100% Python
    -
    Scrapy is completely written in Python, which makes it very easy to hack it
    - -
    Well-tested
    -
    Scrapy has an extensive test suite with pretty good code coverage
    - -
    - -

    Project status

    - -

    - We're currently preparing the first official release of Scrapy with a - very stable API. At the moment we consider the API quite stable, and - we're writing documentation, tutorials and examples to make it easier - to start using it. -

    - -

    Where to start?

    - -

    - Please start by reading the documentation and checking - out the community resources where you can ask for further help while we - finish improving the documentation. -

    - -{% endblock %} diff --git a/sites/scrapy.org/templates/base.html b/sites/scrapy.org/templates/base.html deleted file mode 100644 index f316d56d3..000000000 --- a/sites/scrapy.org/templates/base.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - {% block title %}Scrapy | An open source web scraping framework for Python{% endblock %} - - - - {% block extrastyles %}{% endblock %} - - - - {% block extrascripts %}{% endblock %} - - -
    - {% include "header.html" %} -
    -
    -
    - {% block main-content %}{% endblock %} -
    -
    - -
    -
    - {% include "footer.html" %} -
    - - diff --git a/sites/scrapy.org/templates/base_doc.html b/sites/scrapy.org/templates/base_doc.html deleted file mode 100644 index 5efcbdb9f..000000000 --- a/sites/scrapy.org/templates/base_doc.html +++ /dev/null @@ -1,29 +0,0 @@ -{% extends "base.html" %} - -{% block extrastyles %} - - -{% endblock %} diff --git a/sites/scrapy.org/templates/base_home.html b/sites/scrapy.org/templates/base_home.html deleted file mode 100644 index f5a5e5dd6..000000000 --- a/sites/scrapy.org/templates/base_home.html +++ /dev/null @@ -1,36 +0,0 @@ -{% extends "base.html" %} - -{% block extra-content %} - {% load download_tags %} - {% load_download_links as "download_links" %} - - {% if download_links %} -
    -

    Download Scrapy

    - - {% for link in download_links %} -

    {{ link.description }} {{ link.text }}

    - {% endfor %} -
    - {% endif %} - - {% comment %} - {% load lastblogentry %} - {% get_latest_blog_entries 2 as entries %} - - {% if entries %} -
    -

    Scrapy Weblog

    - - {% for entry in entries %} -
    -

    {{ entry.headline }}

    -

    by {{ entry.author }} on {{ entry.pub_date|date:"M d, Y"}}

    -

    {{ entry.summary }}

    - » Read More -
    - {% endfor %} -
    - {% endif %} - {% endcomment %} -{% endblock %} diff --git a/sites/scrapy.org/templates/base_weblog.html b/sites/scrapy.org/templates/base_weblog.html deleted file mode 100644 index d03d640ff..000000000 --- a/sites/scrapy.org/templates/base_weblog.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "base_home.html" %} - -{% block title %}{{ block.super }} - Weblog{% endblock %} - -{% block main-content %} -{% endblock %} diff --git a/sites/scrapy.org/templates/blog/base_blog.html b/sites/scrapy.org/templates/blog/base_blog.html deleted file mode 100644 index 48a039f7d..000000000 --- a/sites/scrapy.org/templates/blog/base_blog.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends "base.html" %} -oaihdasasihjsd - - -{% block body_class %}blog{% endblock %} diff --git a/sites/scrapy.org/templates/blog/category_detail.html b/sites/scrapy.org/templates/blog/category_detail.html deleted file mode 100644 index 880c37515..000000000 --- a/sites/scrapy.org/templates/blog/category_detail.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Posts for {{ category.title }}{% endblock %} -{% block body_class %}{{ block.super }} category_detail{% endblock %} -{% block body_id %}category_{{ category.id }}{% endblock %} - - -{% block main-content_title %} -

    Posts for {{ category.title }}

    -{% endblock %} - - -{% block main-content %} - {% load markup %} -
    - {% for post in object_list %} -
    -

    {{ post.title }}

    -

    {{ post.publish|date:"Y F d" }}

    -

    {{ post.tease }}

    -
    - {% endfor %} -
    -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/templates/blog/category_list.html b/sites/scrapy.org/templates/blog/category_list.html deleted file mode 100644 index d136b4bd1..000000000 --- a/sites/scrapy.org/templates/blog/category_list.html +++ /dev/null @@ -1,20 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Post categories{% endblock %} -{% block body_class %}{{ block.super }} category_list{% endblock %} - - -{% block main-content_title %} -

    Post categories

    -{% endblock %} - - -{% block main-content %} - {% load markup %} -
      - {% for category in object_list %} -
    • {{ category }}
      • - {% endfor %} -
    -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/templates/blog/post_archive_day.html b/sites/scrapy.org/templates/blog/post_archive_day.html deleted file mode 100644 index 4c95f8cf0..000000000 --- a/sites/scrapy.org/templates/blog/post_archive_day.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Post archive for {{ day|date:"d F Y" }}{% endblock %} -{% block body_class %}{{ block.super }} post_archive_day{% endblock %} - - -{% block main-content_title %} -

    Post archive for {{ day|date:"d F Y" }}

    -{% endblock %} - - -{% block main-content %} -
    - {% for post in object_list %} -
    -

    {{ post.title }}

    -

    {{ post.publish|date:"Y F d" }}

    -

    {{ post.tease }}

    -
    - {% endfor %} -
    -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/templates/blog/post_archive_month.html b/sites/scrapy.org/templates/blog/post_archive_month.html deleted file mode 100644 index 45c4c53e4..000000000 --- a/sites/scrapy.org/templates/blog/post_archive_month.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Post archive for {{ month|date:"F Y" }}{% endblock %} -{% block body_class %}{{ block.super }} post_archive_month{% endblock %} - - -{% block main-content_title %} -

    Post archive for {{ month|date:"F Y" }}

    -{% endblock %} - - -{% block main-content %} -
    - {% for post in object_list %} -
    -

    {{ post.title }}

    -

    {{ post.publish|date:"Y F d" }}

    -

    {{ post.tease }}

    -
    - {% endfor %} -
    -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/templates/blog/post_archive_year.html b/sites/scrapy.org/templates/blog/post_archive_year.html deleted file mode 100644 index 1a176ef9d..000000000 --- a/sites/scrapy.org/templates/blog/post_archive_year.html +++ /dev/null @@ -1,21 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Post archive for {{ year }}{% endblock %} -{% block body_class %}{{ block.super }} post_archive_year{% endblock %} - - -{% block main-content_title %} -

    Post archive for {{ year }}

    -{% endblock %} - - -{% block main-content %} - {% load markup %} - - -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/templates/blog/post_detail.html b/sites/scrapy.org/templates/blog/post_detail.html deleted file mode 100644 index 3df6fd644..000000000 --- a/sites/scrapy.org/templates/blog/post_detail.html +++ /dev/null @@ -1,76 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}{{ object.title }}{% endblock %} -{% block body_class %}{{ block.super }} post_detail{% endblock %} -{% block body_id %}post_{{ object.id }}{% endblock %} - - -{% block content_title %} -{% endblock %} - - -{% block main-content %} - {% load blog markup %} -

    {{ object.title }}

    -

    - {% if object.get_previous_by_publish %} - - {% endif %} - {% if object.get_next_by_publish %} - | - {% endif %} -

    - - {% comment %} - {% load comments tagging_tags %} - {% endcomment %} - -

    {{ object.publish|date:"j F Y" }}

    - -
    - {{ object.body|markdown:"safe" }} -
    - - {% comment %} - {% tags_for_object object as tag_list %} - {% if tag_list %} -

    Related tags: - {% for tag in tag_list %} - {{ tag }}{% if not forloop.last %}, {% endif %} - {% endfor %} -

    - {% endif %} - {% endcomment %} - - - {% comment %} - {% get_comment_list for object as comment_list %} - {% if comment_list %} -
    - -

    Comments

    - {% for comment in comment_list %} - {% if comment.is_public %} -
    -
    - {{ forloop.counter }} - {% if comment.user_url %}{{ comment.user_name }}{% else %}{{ comment.user_name }}{% endif %} says... -
    - {{ comment.comment|urlizetrunc:"60"|markdown:"safe" }} -

    Posted at {{ comment.submit_date|date:"P" }} on {{ comment.submit_date|date:"F j, Y" }}

    -
    - {% endif %} - {% endfor %} -
    - {% endif %} - {% if object.allow_comments %} - {% render_comment_form for object %} - {% else %} -
    -

    Comments are closed.

    -

    Comments have been close for this post.

    -
    - {% endif %} - {% endcomment %} -{% endblock %} diff --git a/sites/scrapy.org/templates/blog/post_list.html b/sites/scrapy.org/templates/blog/post_list.html deleted file mode 100644 index 5f8d8c1cd..000000000 --- a/sites/scrapy.org/templates/blog/post_list.html +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Post archive{% endblock %} -{% block body_class %}{{ block.super }} post_list{% endblock %} - - -{% block main-content_title %} -

    Post archive

    -{% endblock %} - - -{% block main-content %} -
    - {% for post in object_list %} -
    -

    {{ post.title }}

    -

    {{ post.publish|date:"Y F d" }}

    -

    {{ post.tease }}

    -
    - {% endfor %} -
    - - {% if is_paginated %} -

    - {% if has_next %} - Older - {% endif %} - {% if has_next and has_previous %} | {% endif %} - {% if has_previous %} - Newer - {% endif %} -

    - {% endif %} -{% endblock %} diff --git a/sites/scrapy.org/templates/blog/post_search.html b/sites/scrapy.org/templates/blog/post_search.html deleted file mode 100644 index 271000351..000000000 --- a/sites/scrapy.org/templates/blog/post_search.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "blog/base_blog.html" %} - - -{% block title %}Post search{% endblock %} -{% block body_class %}{{ block.super }} post_search{% endblock %} - - -{% block main-content_title %} -

    Search

    -{% endblock %} - - -{% block main-content %} -
    -

    - - -

    - - - {% if message %} -

    {{ message }}

    - {% endif %} - - {% if object_list %} -
    - {% for post in object_list %} -
    -

    {{ post.title }}

    -

    {{ post.publish|date:"Y F d" }}

    -

    {{ post.tease }}

    -

    {% if comment_count %}{{ comment_count }} comment{{ comment_count|pluralize }}{% endif %}

    -
    - {% endfor %} -
    - {% endif %} -{% endblock %} \ No newline at end of file diff --git a/sites/scrapy.org/templates/blog_back/entry_archive.html b/sites/scrapy.org/templates/blog_back/entry_archive.html deleted file mode 100644 index 4adeff722..000000000 --- a/sites/scrapy.org/templates/blog_back/entry_archive.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base_weblog.html" %} - -{% block main-content %} - -{% load comments comment_utils %} - -

    Latest entries

    - -{% for object in latest %} -

    {{ object.headline }}

    - {{ object.body }} -

    Posted by {{ object.author }} on {{ object.pub_date|date:"F j, Y" }}

    -{% endfor %} - -{% endblock %} diff --git a/sites/scrapy.org/templates/blog_back/entry_archive_day.html b/sites/scrapy.org/templates/blog_back/entry_archive_day.html deleted file mode 100644 index 5d7ada069..000000000 --- a/sites/scrapy.org/templates/blog_back/entry_archive_day.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "base_weblog.html" %} - -{% block title %}Weblog | {{ day|date:"F j" }}{% endblock %} - -{% block main-content %} - -

    {{ day|date:"F j" }} archive

    - -{% for object in object_list %} -

    {{ object.headline }}

    -

    {{ object.pub_date|date:"F j, Y" }}

    -{{ object.body }} - -{% endfor %} - -{% endblock %} diff --git a/sites/scrapy.org/templates/blog_back/entry_archive_month.html b/sites/scrapy.org/templates/blog_back/entry_archive_month.html deleted file mode 100644 index 388ce67eb..000000000 --- a/sites/scrapy.org/templates/blog_back/entry_archive_month.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "base_weblog.html" %} - -{% block title %}Weblog | {{ month|date:"F" }}{% endblock %} - -{% block main-content %} - -

    {{ month|date:"F" }} archive

    - -{% for object in object_list %} -

    {{ object.headline }}

    -

    {{ object.pub_date|date:"F j, Y" }}

    -{{ object.body }} - -{% endfor %} - -{% endblock %} diff --git a/sites/scrapy.org/templates/blog_back/entry_archive_year.html b/sites/scrapy.org/templates/blog_back/entry_archive_year.html deleted file mode 100644 index d58ff781a..000000000 --- a/sites/scrapy.org/templates/blog_back/entry_archive_year.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base_weblog.html" %} - -{% block title %}Weblog | {{ year }}{% endblock %} - -{% block main-content %} - -

    {{ year }} archive

    - - - -{% endblock %} diff --git a/sites/scrapy.org/templates/blog_back/entry_detail.html b/sites/scrapy.org/templates/blog_back/entry_detail.html deleted file mode 100644 index 99ca4694e..000000000 --- a/sites/scrapy.org/templates/blog_back/entry_detail.html +++ /dev/null @@ -1,34 +0,0 @@ -{% extends "base_weblog.html" %} - -{% block title %}Weblog | {{ object.headline|escape }}{% endblock %} - -{% block main-content %} - -

    {{ object.headline }}

    -{{ object.body }} -

    Posted by {{ object.author }} on {{ object.pub_date|date:"F j, Y" }}

    - -{% comment %} -{% load comments comment_utils %} -{% get_public_free_comment_list for blog.entry object.id as comment_list %} - -
    -

    Comments

    - -{% for comment in comment_list %} -
    -

    {{ comment.person_name|escape }} {{ comment.submit_date|date:"F j, Y" }} at {{ comment.submit_date|date:"P" }}

    - {{ comment.comment|escape|urlizetrunc:"40"|linebreaks }} -
    -{% endfor %} - -{% if object.comments_enabled %} -

    Post a comment

    -{% free_comment_form for blog.entry object.id %} -{% else %} -

    Comments are closed

    -

    To prevent spam, comments are no longer allowed after sixty days.

    -{% endif %} -
    -{% endcomment %} -{% endblock %} diff --git a/sites/scrapy.org/templates/docs/doc.html b/sites/scrapy.org/templates/docs/doc.html deleted file mode 100644 index 806aac810..000000000 --- a/sites/scrapy.org/templates/docs/doc.html +++ /dev/null @@ -1,34 +0,0 @@ -{% extends "base_doc.html" %} - -{% block title %}{{ doc.title|safe }}{% endblock %} - -{% block main-content %} -{% if doc.parents %} -
    -Documentation » -{% for p in doc.parents %} -{{ p.title|safe }} » -{% endfor %} -
    -{% endif %} -{{ doc.body|safe }} -{% endblock %} - -{% block extra-content %} - -{% if doc.display_toc %} -

    Contents:

    -{{ doc.toc|safe }} -{% endif %} - -

    Browse

    - - -{% endblock %} diff --git a/sites/scrapy.org/templates/footer.html b/sites/scrapy.org/templates/footer.html deleted file mode 100644 index 8d8bc7e55..000000000 --- a/sites/scrapy.org/templates/footer.html +++ /dev/null @@ -1,10 +0,0 @@ - diff --git a/sites/scrapy.org/templates/header.html b/sites/scrapy.org/templates/header.html deleted file mode 100644 index c5baf7f96..000000000 --- a/sites/scrapy.org/templates/header.html +++ /dev/null @@ -1,14 +0,0 @@ - diff --git a/sites/static.scrapy.org/asc.png b/sites/static.scrapy.org/asc.png deleted file mode 100644 index 486b7315902f8bd4d7c953f4b6e65e56cb171c50..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 222 zcmeAS@N?(olHy`uVBq!ia0vp^96-#@!VDyjmkZ_sDdu7)&kzm{j@u9Y9{{=h0X`wF zuV25udiCm=GiUbi-@kI@%BvF}%mYfXmIV0)GyDevnG0$lvp5SpB8wRqxITa|qthCb z89+hB64!{5;QX|b^2DN42H(WwqWs*{l*~j0x6Go{^8BLgVg+M8gZG(+KwS(Po-U3d z6?2jk5`q#E5(Elagw)RQ@bu{EG4Y(y(_`b&($n+f@p52bFt%pi>BpeA4ycF0)78&q Iol`;+0Q%iTa{vGU diff --git a/sites/static.scrapy.org/attachment.png b/sites/static.scrapy.org/attachment.png deleted file mode 100644 index a4d212439158aa08cbf451a557a5f987c8f9f32a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 280 zcmeAS@N?(olHy`uVBq!ia0vp^JRr=$3?vg*uel1OSkfJR9T^zbpD<_bdI{u9mbgZg z1m~xflqVLYGB~E>C#5QQ<|d}62BjvZR2H60wE-$J2=EDU_3`lu3JQvdh=`4i&B@6r zD=X{j>Y6ZN!tB|zckbMI^5n^j7cbtrb?e!)XYb#?2Wt9c9la7r36uo+1^-6{3=foJ zC4q85o-U3d6}NoOy%us1~z``8;h%n3^2IkhduF<%B(nCd+l_ z8BIL1SLy72C-HC1@v1R1mYqp&3=!6>U2O2HTqI!GSKc-TW`-;FZ2S>7J>~#SXYh3O Kb6Mw<&;$Szcyo3D diff --git a/sites/static.scrapy.org/changeset.png b/sites/static.scrapy.org/changeset.png deleted file mode 100644 index 31c0356a3eaf6530f6b0a619de10c0435f063fff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 294 zcmeAS@N?(olHy`uVBq!ia0vp^JRr=%3?!FCJ6-`&%*9TgAsieWw;%dH0CIH$d_r9R z|NsB}`}fbEKfiwc`u_dCQJ%$lJ@YOqh8GisJ_kT-2&KSz^iDzCfqFQ+$H@l{dS%G?wQ)VhpiNUHw{cfmfcHzTAeo1w0HN XdU@A;Uma-!G?T&8)z4*}Q$iB}7AA96 diff --git a/sites/static.scrapy.org/closedticket.png b/sites/static.scrapy.org/closedticket.png deleted file mode 100644 index 43f7a844739b0875c3ea281d8c4720585d6ead4f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 297 zcmeAS@N?(olHy`uVBq!ia0vp^JRr=%3?!FCJ6-`&%*9TgAsieWw;%dH0CFV)d_r6q z(ilEGtvtQJ`Tmiv|NsAMPBma?VEFm9RrsysT%a0`k|4iep!k0VAl7XOQ3Z-|7I;J! zGca%qfiUBxyLEqnf{G=s5hcO-X(i=}MX3zFiOEIzxv43ci3)C+MXBZaMcKs)CVB?1 z)YMCXip4;Ror_WvOEUBG6hbm{QyB~m^$pGS4NNZIvWx<%aP@R?45^rtT)@c8mTZt< zpwHeU#8#}jRH-!4g5QPJc*bhBmDAM>W=vO`(r(B+m1Tum!;?cdZuBh($ diff --git a/sites/static.scrapy.org/collapsed.png b/sites/static.scrapy.org/collapsed.png deleted file mode 100644 index 4a2ac198ca926247cafe5781160345335e6ecb9b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 218 zcmeAS@N?(olHy`uVBq!ia0vp^+#t-s3?%0jwTl2LmUKs7M+Sy#H?H_c7y{LCMyhNC3NFBr%L0fu|cdn18jCY~;iAr-fh64KM!8QIF*+{`Ry zH#W9T    '); - var loading_row = tr.next(); - loading_row.children("td").addClass(td_class) - .attr("colspan", tr.children("td").length) - .css("padding-left", depth); - loading_row.find("span.loading").text("Loading " + a.text() + "..."); - - // XHR for getting the rows corresponding to the folder entries - $.ajax({ - type: "GET", - url: a.attr("href"), - data: qargs, - dataType: "html", - success: function(data) { - // Safari 3.1.1 has some trouble reconstructing HTML snippets - // bigger than 50k - splitting in rows before building DOM nodes - var rows = data.replace(/^]+>/, "").split(""); - if (rows.length) { - // insert entry rows - $(rows).each(function() { - row = $(this+""); - row.children("td."+td_class).css("padding-left", depth); - // make all entry rows collapsible but only subdir rows expandable - enableExpandDir(tr, row, qargs); - loading_row.before(row); - }); - // remove "Loading ..." row - loading_row.remove(); - } else { - loading_row.find("span.loading").text("").append("(empty)") - .removeClass("loading"); - // make the (empty) row collapsible - enableExpandDir(tr, loading_row, qargs); - } - }, - error: function(req, err, exc) { - loading_row.find("span.loading").text("").append("(error)") - .removeClass("loading"); - enableExpandDir(tr, loading_row, qargs); - } - }); - } - expander.attr("title", "Fold directory"); - } - -})(jQuery); diff --git a/sites/static.scrapy.org/js/folding.js b/sites/static.scrapy.org/js/folding.js deleted file mode 100644 index 0f16a2de1..000000000 --- a/sites/static.scrapy.org/js/folding.js +++ /dev/null @@ -1,26 +0,0 @@ -(function($){ - - $.fn.enableFolding = function(autofold) { - var fragId = document.location.hash; - if (fragId && /^#no\d+$/.test(fragId)) { - fragId = parseInt(fragId.substr(3)); - } - - var count = 1; - return this.each(function() { - var t = $(this).text(); - $(this).text(""); - var trigger = $(this).append("").children(); - trigger.text(t); - - trigger.click(function() { - if (fragId == count) { fragId = 0; return; } - $(this.parentNode.parentNode).toggleClass("collapsed"); - }); - if ( autofold ) - trigger.click(); - count++; - }).css("cursor", "pointer"); - } - -})(jQuery); \ No newline at end of file diff --git a/sites/static.scrapy.org/js/ie_pre7_hacks.js b/sites/static.scrapy.org/js/ie_pre7_hacks.js deleted file mode 100644 index a2701adba..000000000 --- a/sites/static.scrapy.org/js/ie_pre7_hacks.js +++ /dev/null @@ -1,7 +0,0 @@ -jQuery(function($) { // onload - $('select').bind('focusin', function() { - this.tmpIndex = this.selectedIndex; - }).bind('focus', function() { - this.selectedIndex = this.tmpIndex; - }); -}); diff --git a/sites/static.scrapy.org/js/jquery.js b/sites/static.scrapy.org/js/jquery.js deleted file mode 100644 index 74cdfee17..000000000 --- a/sites/static.scrapy.org/js/jquery.js +++ /dev/null @@ -1,11 +0,0 @@ -/* - * jQuery 1.2.3 - New Wave Javascript - * - * Copyright (c) 2008 John Resig (jquery.com) - * Dual licensed under the MIT (MIT-LICENSE.txt) - * and GPL (GPL-LICENSE.txt) licenses. - * - * $Date: 2008-02-06 00:21:25 -0500 (Wed, 06 Feb 2008) $ - * $Rev: 4663 $ - */ -eval(function(p,a,c,k,e,r){e=function(c){return(c35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)r[e(c)]=k[c]||e(c);k=[function(e){return r[e]}];e=function(){return'\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p}('(J(){7(1e.3N)L w=1e.3N;L E=1e.3N=J(a,b){K 1B E.2l.4T(a,b)};7(1e.$)L D=1e.$;1e.$=E;L u=/^[^<]*(<(.|\\s)+>)[^>]*$|^#(\\w+)$/;L G=/^.[^:#\\[\\.]*$/;E.1n=E.2l={4T:J(d,b){d=d||T;7(d.15){6[0]=d;6.M=1;K 6}N 7(1o d=="25"){L c=u.2O(d);7(c&&(c[1]||!b)){7(c[1])d=E.4a([c[1]],b);N{L a=T.5J(c[3]);7(a)7(a.2w!=c[3])K E().2s(d);N{6[0]=a;6.M=1;K 6}N d=[]}}N K 1B E(b).2s(d)}N 7(E.1q(d))K 1B E(T)[E.1n.21?"21":"3U"](d);K 6.6E(d.1k==1M&&d||(d.5h||d.M&&d!=1e&&!d.15&&d[0]!=10&&d[0].15)&&E.2I(d)||[d])},5h:"1.2.3",87:J(){K 6.M},M:0,22:J(a){K a==10?E.2I(6):6[a]},2F:J(b){L a=E(b);a.54=6;K a},6E:J(a){6.M=0;1M.2l.1g.1i(6,a);K 6},R:J(a,b){K E.R(6,a,b)},4X:J(b){L a=-1;6.R(J(i){7(6==b)a=i});K a},1J:J(c,a,b){L d=c;7(c.1k==4e)7(a==10)K 6.M&&E[b||"1J"](6[0],c)||10;N{d={};d[c]=a}K 6.R(J(i){Q(c 1p d)E.1J(b?6.W:6,c,E.1l(6,d[c],b,i,c))})},1j:J(b,a){7((b==\'27\'||b==\'1R\')&&2M(a)<0)a=10;K 6.1J(b,a,"2o")},1u:J(b){7(1o b!="3V"&&b!=V)K 6.4x().3t((6[0]&&6[0].2i||T).5r(b));L a="";E.R(b||6,J(){E.R(6.3p,J(){7(6.15!=8)a+=6.15!=1?6.6K:E.1n.1u([6])})});K a},5m:J(b){7(6[0])E(b,6[0].2i).5k().3o(6[0]).2c(J(){L a=6;2b(a.1C)a=a.1C;K a}).3t(6);K 6},8w:J(a){K 6.R(J(){E(6).6z().5m(a)})},8p:J(a){K 6.R(J(){E(6).5m(a)})},3t:J(){K 6.3O(18,P,S,J(a){7(6.15==1)6.38(a)})},6q:J(){K 6.3O(18,P,P,J(a){7(6.15==1)6.3o(a,6.1C)})},6o:J(){K 6.3O(18,S,S,J(a){6.1a.3o(a,6)})},5a:J(){K 6.3O(18,S,P,J(a){6.1a.3o(a,6.2B)})},3h:J(){K 6.54||E([])},2s:J(b){L c=E.2c(6,J(a){K E.2s(b,a)});K 6.2F(/[^+>] [^+>]/.17(b)||b.1f("..")>-1?E.57(c):c)},5k:J(e){L f=6.2c(J(){7(E.14.1d&&!E.3E(6)){L a=6.69(P),4Y=T.3s("1x");4Y.38(a);K E.4a([4Y.3d])[0]}N K 6.69(P)});L d=f.2s("*").4R().R(J(){7(6[F]!=10)6[F]=V});7(e===P)6.2s("*").4R().R(J(i){7(6.15==3)K;L c=E.O(6,"2R");Q(L a 1p c)Q(L b 1p c[a])E.16.1b(d[i],a,c[a][b],c[a][b].O)});K f},1E:J(b){K 6.2F(E.1q(b)&&E.3y(6,J(a,i){K b.1P(a,i)})||E.3e(b,6))},56:J(b){7(b.1k==4e)7(G.17(b))K 6.2F(E.3e(b,6,P));N b=E.3e(b,6);L a=b.M&&b[b.M-1]!==10&&!b.15;K 6.1E(J(){K a?E.33(6,b)<0:6!=b})},1b:J(a){K!a?6:6.2F(E.37(6.22(),a.1k==4e?E(a).22():a.M!=10&&(!a.12||E.12(a,"3u"))?a:[a]))},3H:J(a){K a?E.3e(a,6).M>0:S},7j:J(a){K 6.3H("."+a)},5O:J(b){7(b==10){7(6.M){L c=6[0];7(E.12(c,"2k")){L e=c.3T,5I=[],11=c.11,2X=c.U=="2k-2X";7(e<0)K V;Q(L i=2X?e:0,2f=2X?e+1:11.M;i<2f;i++){L d=11[i];7(d.2p){b=E.14.1d&&!d.9J.1A.9y?d.1u:d.1A;7(2X)K b;5I.1g(b)}}K 5I}N K(6[0].1A||"").1r(/\\r/g,"")}K 10}K 6.R(J(){7(6.15!=1)K;7(b.1k==1M&&/5u|5t/.17(6.U))6.3k=(E.33(6.1A,b)>=0||E.33(6.31,b)>=0);N 7(E.12(6,"2k")){L a=b.1k==1M?b:[b];E("98",6).R(J(){6.2p=(E.33(6.1A,a)>=0||E.33(6.1u,a)>=0)});7(!a.M)6.3T=-1}N 6.1A=b})},3q:J(a){K a==10?(6.M?6[0].3d:V):6.4x().3t(a)},6S:J(a){K 6.5a(a).1V()},6Z:J(i){K 6.2K(i,i+1)},2K:J(){K 6.2F(1M.2l.2K.1i(6,18))},2c:J(b){K 6.2F(E.2c(6,J(a,i){K b.1P(a,i,a)}))},4R:J(){K 6.1b(6.54)},O:J(d,b){L a=d.23(".");a[1]=a[1]?"."+a[1]:"";7(b==V){L c=6.5n("8P"+a[1]+"!",[a[0]]);7(c==10&&6.M)c=E.O(6[0],d);K c==V&&a[1]?6.O(a[0]):c}N K 6.1N("8K"+a[1]+"!",[a[0],b]).R(J(){E.O(6,d,b)})},35:J(a){K 6.R(J(){E.35(6,a)})},3O:J(g,f,h,d){L e=6.M>1,3n;K 6.R(J(){7(!3n){3n=E.4a(g,6.2i);7(h)3n.8D()}L b=6;7(f&&E.12(6,"1O")&&E.12(3n[0],"4v"))b=6.3S("1U")[0]||6.38(6.2i.3s("1U"));L c=E([]);E.R(3n,J(){L a=e?E(6).5k(P)[0]:6;7(E.12(a,"1m")){c=c.1b(a)}N{7(a.15==1)c=c.1b(E("1m",a).1V());d.1P(b,a)}});c.R(6A)})}};E.2l.4T.2l=E.2l;J 6A(i,a){7(a.3Q)E.3P({1c:a.3Q,3l:S,1H:"1m"});N E.5g(a.1u||a.6x||a.3d||"");7(a.1a)a.1a.34(a)}E.1s=E.1n.1s=J(){L b=18[0]||{},i=1,M=18.M,5c=S,11;7(b.1k==8d){5c=b;b=18[1]||{};i=2}7(1o b!="3V"&&1o b!="J")b={};7(M==1){b=6;i=0}Q(;i-1}},68:J(b,c,a){L e={};Q(L d 1p c){e[d]=b.W[d];b.W[d]=c[d]}a.1P(b);Q(L d 1p c)b.W[d]=e[d]},1j:J(d,e,c){7(e=="27"||e=="1R"){L b,46={43:"4W",4U:"1Z",19:"3D"},3c=e=="27"?["7O","7M"]:["7J","7I"];J 5E(){b=e=="27"?d.7H:d.7F;L a=0,2N=0;E.R(3c,J(){a+=2M(E.2o(d,"7E"+6,P))||0;2N+=2M(E.2o(d,"2N"+6+"5X",P))||0});b-=24.7C(a+2N)}7(E(d).3H(":4d"))5E();N E.68(d,46,5E);K 24.2f(0,b)}K E.2o(d,e,c)},2o:J(e,k,j){L d;J 3x(b){7(!E.14.2d)K S;L a=T.4c.4K(b,V);K!a||a.4M("3x")==""}7(k=="1w"&&E.14.1d){d=E.1J(e.W,"1w");K d==""?"1":d}7(E.14.2z&&k=="19"){L c=e.W.50;e.W.50="0 7r 7o";e.W.50=c}7(k.1D(/4g/i))k=y;7(!j&&e.W&&e.W[k])d=e.W[k];N 7(T.4c&&T.4c.4K){7(k.1D(/4g/i))k="4g";k=k.1r(/([A-Z])/g,"-$1").2h();L h=T.4c.4K(e,V);7(h&&!3x(e))d=h.4M(k);N{L f=[],2C=[];Q(L a=e;a&&3x(a);a=a.1a)2C.4J(a);Q(L i=0;i<2C.M;i++)7(3x(2C[i])){f[i]=2C[i].W.19;2C[i].W.19="3D"}d=k=="19"&&f[2C.M-1]!=V?"2H":(h&&h.4M(k))||"";Q(L i=0;i]*?)\\/>/g,J(b,a,c){K c.1D(/^(aa|a6|7e|a5|4D|7a|a0|3m|9W|9U|9S)$/i)?b:a+">"});L f=E.3g(d).2h(),1x=h.3s("1x");L e=!f.1f("<9P")&&[1,"<2k 74=\'74\'>",""]||!f.1f("<9M")&&[1,"<73>",""]||f.1D(/^<(9G|1U|9E|9B|9x)/)&&[1,"<1O>",""]||!f.1f("<4v")&&[2,"<1O><1U>",""]||(!f.1f("<9w")||!f.1f("<9v"))&&[3,"<1O><1U><4v>",""]||!f.1f("<7e")&&[2,"<1O><1U><6V>",""]||E.14.1d&&[1,"1x<1x>",""]||[0,"",""];1x.3d=e[1]+d+e[2];2b(e[0]--)1x=1x.5o;7(E.14.1d){L g=!f.1f("<1O")&&f.1f("<1U")<0?1x.1C&&1x.1C.3p:e[1]=="<1O>"&&f.1f("<1U")<0?1x.3p:[];Q(L j=g.M-1;j>=0;--j)7(E.12(g[j],"1U")&&!g[j].3p.M)g[j].1a.34(g[j]);7(/^\\s/.17(d))1x.3o(h.5r(d.1D(/^\\s*/)[0]),1x.1C)}d=E.2I(1x.3p)}7(d.M===0&&(!E.12(d,"3u")&&!E.12(d,"2k")))K;7(d[0]==10||E.12(d,"3u")||d.11)k.1g(d);N k=E.37(k,d)});K k},1J:J(d,e,c){7(!d||d.15==3||d.15==8)K 10;L f=E.3E(d)?{}:E.46;7(e=="2p"&&E.14.2d)d.1a.3T;7(f[e]){7(c!=10)d[f[e]]=c;K d[f[e]]}N 7(E.14.1d&&e=="W")K E.1J(d.W,"9u",c);N 7(c==10&&E.14.1d&&E.12(d,"3u")&&(e=="9r"||e=="9o"))K d.9m(e).6K;N 7(d.28){7(c!=10){7(e=="U"&&E.12(d,"4D")&&d.1a)6Q"U 9i 9h\'t 9g 9e";d.9b(e,""+c)}7(E.14.1d&&/6O|3Q/.17(e)&&!E.3E(d))K d.4z(e,2);K d.4z(e)}N{7(e=="1w"&&E.14.1d){7(c!=10){d.6k=1;d.1E=(d.1E||"").1r(/6M\\([^)]*\\)/,"")+(2M(c).3X()=="96"?"":"6M(1w="+c*6L+")")}K d.1E&&d.1E.1f("1w=")>=0?(2M(d.1E.1D(/1w=([^)]*)/)[1])/6L).3X():""}e=e.1r(/-([a-z])/95,J(a,b){K b.2E()});7(c!=10)d[e]=c;K d[e]}},3g:J(a){K(a||"").1r(/^\\s+|\\s+$/g,"")},2I:J(b){L a=[];7(1o b!="93")Q(L i=0,M=b.M;i*",6).1V();2b(6.1C)6.34(6.1C)}},J(a,b){E.1n[a]=J(){K 6.R(b,18)}});E.R(["8f","5X"],J(i,c){L b=c.2h();E.1n[b]=J(a){K 6[0]==1e?E.14.2z&&T.1h["5e"+c]||E.14.2d&&1e["8e"+c]||T.6F=="79"&&T.1F["5e"+c]||T.1h["5e"+c]:6[0]==T?24.2f(24.2f(T.1h["5d"+c],T.1F["5d"+c]),24.2f(T.1h["5L"+c],T.1F["5L"+c])):a==10?(6.M?E.1j(6[0],b):V):6.1j(b,a.1k==4e?a:a+"2S")}});L C=E.14.2d&&4s(E.14.5K)<8c?"(?:[\\\\w*4r-]|\\\\\\\\.)":"(?:[\\\\w\\8b-\\8a*4r-]|\\\\\\\\.)",6v=1B 4q("^>\\\\s*("+C+"+)"),6u=1B 4q("^("+C+"+)(#)("+C+"+)"),6s=1B 4q("^([#.]?)("+C+"*)");E.1s({6r:{"":J(a,i,m){K m[2]=="*"||E.12(a,m[2])},"#":J(a,i,m){K a.4z("2w")==m[2]},":":{89:J(a,i,m){K im[3]-0},2Z:J(a,i,m){K m[3]-0==i},6Z:J(a,i,m){K m[3]-0==i},3j:J(a,i){K i==0},3J:J(a,i,m,r){K i==r.M-1},6n:J(a,i){K i%2==0},6l:J(a,i){K i%2},"3j-4p":J(a){K a.1a.3S("*")[0]==a},"3J-4p":J(a){K E.2Z(a.1a.5o,1,"4t")==a},"83-4p":J(a){K!E.2Z(a.1a.5o,2,"4t")},6B:J(a){K a.1C},4x:J(a){K!a.1C},82:J(a,i,m){K(a.6x||a.81||E(a).1u()||"").1f(m[3])>=0},4d:J(a){K"1Z"!=a.U&&E.1j(a,"19")!="2H"&&E.1j(a,"4U")!="1Z"},1Z:J(a){K"1Z"==a.U||E.1j(a,"19")=="2H"||E.1j(a,"4U")=="1Z"},80:J(a){K!a.2Y},2Y:J(a){K a.2Y},3k:J(a){K a.3k},2p:J(a){K a.2p||E.1J(a,"2p")},1u:J(a){K"1u"==a.U},5u:J(a){K"5u"==a.U},5t:J(a){K"5t"==a.U},59:J(a){K"59"==a.U},3I:J(a){K"3I"==a.U},58:J(a){K"58"==a.U},6j:J(a){K"6j"==a.U},6i:J(a){K"6i"==a.U},2G:J(a){K"2G"==a.U||E.12(a,"2G")},4D:J(a){K/4D|2k|6h|2G/i.17(a.12)},3Y:J(a,i,m){K E.2s(m[3],a).M},7X:J(a){K/h\\d/i.17(a.12)},7W:J(a){K E.3y(E.3G,J(b){K a==b.Y}).M}}},6g:[/^(\\[) *@?([\\w-]+) *([!*$^~=]*) *(\'?"?)(.*?)\\4 *\\]/,/^(:)([\\w-]+)\\("?\'?(.*?(\\(.*?\\))?[^(]*?)"?\'?\\)/,1B 4q("^([:.#]*)("+C+"+)")],3e:J(a,c,b){L d,2m=[];2b(a&&a!=d){d=a;L f=E.1E(a,c,b);a=f.t.1r(/^\\s*,\\s*/,"");2m=b?c=f.r:E.37(2m,f.r)}K 2m},2s:J(t,p){7(1o t!="25")K[t];7(p&&p.15!=1&&p.15!=9)K[];p=p||T;L d=[p],2r=[],3J,12;2b(t&&3J!=t){L r=[];3J=t;t=E.3g(t);L o=S;L g=6v;L m=g.2O(t);7(m){12=m[1].2E();Q(L i=0;d[i];i++)Q(L c=d[i].1C;c;c=c.2B)7(c.15==1&&(12=="*"||c.12.2E()==12))r.1g(c);d=r;t=t.1r(g,"");7(t.1f(" ")==0)6w;o=P}N{g=/^([>+~])\\s*(\\w*)/i;7((m=g.2O(t))!=V){r=[];L l={};12=m[2].2E();m=m[1];Q(L j=0,3f=d.M;j<3f;j++){L n=m=="~"||m=="+"?d[j].2B:d[j].1C;Q(;n;n=n.2B)7(n.15==1){L h=E.O(n);7(m=="~"&&l[h])1Q;7(!12||n.12.2E()==12){7(m=="~")l[h]=P;r.1g(n)}7(m=="+")1Q}}d=r;t=E.3g(t.1r(g,""));o=P}}7(t&&!o){7(!t.1f(",")){7(p==d[0])d.4l();2r=E.37(2r,d);r=d=[p];t=" "+t.6e(1,t.M)}N{L k=6u;L m=k.2O(t);7(m){m=[0,m[2],m[3],m[1]]}N{k=6s;m=k.2O(t)}m[2]=m[2].1r(/\\\\/g,"");L f=d[d.M-1];7(m[1]=="#"&&f&&f.5J&&!E.3E(f)){L q=f.5J(m[2]);7((E.14.1d||E.14.2z)&&q&&1o q.2w=="25"&&q.2w!=m[2])q=E(\'[@2w="\'+m[2]+\'"]\',f)[0];d=r=q&&(!m[3]||E.12(q,m[3]))?[q]:[]}N{Q(L i=0;d[i];i++){L a=m[1]=="#"&&m[3]?m[3]:m[1]!=""||m[0]==""?"*":m[2];7(a=="*"&&d[i].12.2h()=="3V")a="3m";r=E.37(r,d[i].3S(a))}7(m[1]==".")r=E.55(r,m[2]);7(m[1]=="#"){L e=[];Q(L i=0;r[i];i++)7(r[i].4z("2w")==m[2]){e=[r[i]];1Q}r=e}d=r}t=t.1r(k,"")}}7(t){L b=E.1E(t,r);d=r=b.r;t=E.3g(b.t)}}7(t)d=[];7(d&&p==d[0])d.4l();2r=E.37(2r,d);K 2r},55:J(r,m,a){m=" "+m+" ";L c=[];Q(L i=0;r[i];i++){L b=(" "+r[i].1t+" ").1f(m)>=0;7(!a&&b||a&&!b)c.1g(r[i])}K c},1E:J(t,r,h){L d;2b(t&&t!=d){d=t;L p=E.6g,m;Q(L i=0;p[i];i++){m=p[i].2O(t);7(m){t=t.7V(m[0].M);m[2]=m[2].1r(/\\\\/g,"");1Q}}7(!m)1Q;7(m[1]==":"&&m[2]=="56")r=G.17(m[3])?E.1E(m[3],r,P).r:E(r).56(m[3]);N 7(m[1]==".")r=E.55(r,m[2],h);N 7(m[1]=="["){L g=[],U=m[3];Q(L i=0,3f=r.M;i<3f;i++){L a=r[i],z=a[E.46[m[2]]||m[2]];7(z==V||/6O|3Q|2p/.17(m[2]))z=E.1J(a,m[2])||\'\';7((U==""&&!!z||U=="="&&z==m[5]||U=="!="&&z!=m[5]||U=="^="&&z&&!z.1f(m[5])||U=="$="&&z.6e(z.M-m[5].M)==m[5]||(U=="*="||U=="~=")&&z.1f(m[5])>=0)^h)g.1g(a)}r=g}N 7(m[1]==":"&&m[2]=="2Z-4p"){L e={},g=[],17=/(-?)(\\d*)n((?:\\+|-)?\\d*)/.2O(m[3]=="6n"&&"2n"||m[3]=="6l"&&"2n+1"||!/\\D/.17(m[3])&&"7U+"+m[3]||m[3]),3j=(17[1]+(17[2]||1))-0,d=17[3]-0;Q(L i=0,3f=r.M;i<3f;i++){L j=r[i],1a=j.1a,2w=E.O(1a);7(!e[2w]){L c=1;Q(L n=1a.1C;n;n=n.2B)7(n.15==1)n.4k=c++;e[2w]=P}L b=S;7(3j==0){7(j.4k==d)b=P}N 7((j.4k-d)%3j==0&&(j.4k-d)/3j>=0)b=P;7(b^h)g.1g(j)}r=g}N{L f=E.6r[m[1]];7(1o f=="3V")f=f[m[2]];7(1o f=="25")f=6c("S||J(a,i){K "+f+";}");r=E.3y(r,J(a,i){K f(a,i,m,r)},h)}}K{r:r,t:t}},4u:J(b,c){L d=[];L a=b[c];2b(a&&a!=T){7(a.15==1)d.1g(a);a=a[c]}K d},2Z:J(a,e,c,b){e=e||1;L d=0;Q(;a;a=a[c])7(a.15==1&&++d==e)1Q;K a},5i:J(n,a){L r=[];Q(;n;n=n.2B){7(n.15==1&&(!a||n!=a))r.1g(n)}K r}});E.16={1b:J(f,i,g,e){7(f.15==3||f.15==8)K;7(E.14.1d&&f.53!=10)f=1e;7(!g.2D)g.2D=6.2D++;7(e!=10){L h=g;g=J(){K h.1i(6,18)};g.O=e;g.2D=h.2D}L j=E.O(f,"2R")||E.O(f,"2R",{}),1v=E.O(f,"1v")||E.O(f,"1v",J(){L a;7(1o E=="10"||E.16.5f)K a;a=E.16.1v.1i(18.3R.Y,18);K a});1v.Y=f;E.R(i.23(/\\s+/),J(c,b){L a=b.23(".");b=a[0];g.U=a[1];L d=j[b];7(!d){d=j[b]={};7(!E.16.2y[b]||E.16.2y[b].4j.1P(f)===S){7(f.3F)f.3F(b,1v,S);N 7(f.6b)f.6b("4i"+b,1v)}}d[g.2D]=g;E.16.2a[b]=P});f=V},2D:1,2a:{},1V:J(e,h,f){7(e.15==3||e.15==8)K;L i=E.O(e,"2R"),29,4X;7(i){7(h==10||(1o h=="25"&&h.7T(0)=="."))Q(L g 1p i)6.1V(e,g+(h||""));N{7(h.U){f=h.2q;h=h.U}E.R(h.23(/\\s+/),J(b,a){L c=a.23(".");a=c[0];7(i[a]){7(f)2V i[a][f.2D];N Q(f 1p i[a])7(!c[1]||i[a][f].U==c[1])2V i[a][f];Q(29 1p i[a])1Q;7(!29){7(!E.16.2y[a]||E.16.2y[a].4h.1P(e)===S){7(e.67)e.67(a,E.O(e,"1v"),S);N 7(e.66)e.66("4i"+a,E.O(e,"1v"))}29=V;2V i[a]}}})}Q(29 1p i)1Q;7(!29){L d=E.O(e,"1v");7(d)d.Y=V;E.35(e,"2R");E.35(e,"1v")}}},1N:J(g,c,d,f,h){c=E.2I(c||[]);7(g.1f("!")>=0){g=g.2K(0,-1);L a=P}7(!d){7(6.2a[g])E("*").1b([1e,T]).1N(g,c)}N{7(d.15==3||d.15==8)K 10;L b,29,1n=E.1q(d[g]||V),16=!c[0]||!c[0].36;7(16)c.4J(6.4Z({U:g,2L:d}));c[0].U=g;7(a)c[0].65=P;7(E.1q(E.O(d,"1v")))b=E.O(d,"1v").1i(d,c);7(!1n&&d["4i"+g]&&d["4i"+g].1i(d,c)===S)b=S;7(16)c.4l();7(h&&E.1q(h)){29=h.1i(d,b==V?c:c.71(b));7(29!==10)b=29}7(1n&&f!==S&&b!==S&&!(E.12(d,\'a\')&&g=="4V")){6.5f=P;1S{d[g]()}1X(e){}}6.5f=S}K b},1v:J(c){L a;c=E.16.4Z(c||1e.16||{});L b=c.U.23(".");c.U=b[0];L f=E.O(6,"2R")&&E.O(6,"2R")[c.U],42=1M.2l.2K.1P(18,1);42.4J(c);Q(L j 1p f){L d=f[j];42[0].2q=d;42[0].O=d.O;7(!b[1]&&!c.65||d.U==b[1]){L e=d.1i(6,42);7(a!==S)a=e;7(e===S){c.36();c.44()}}}7(E.14.1d)c.2L=c.36=c.44=c.2q=c.O=V;K a},4Z:J(c){L a=c;c=E.1s({},a);c.36=J(){7(a.36)a.36();a.7S=S};c.44=J(){7(a.44)a.44();a.7R=P};7(!c.2L)c.2L=c.7Q||T;7(c.2L.15==3)c.2L=a.2L.1a;7(!c.4S&&c.5w)c.4S=c.5w==c.2L?c.7P:c.5w;7(c.64==V&&c.63!=V){L b=T.1F,1h=T.1h;c.64=c.63+(b&&b.2v||1h&&1h.2v||0)-(b.62||0);c.7N=c.7L+(b&&b.2x||1h&&1h.2x||0)-(b.60||0)}7(!c.3c&&((c.4f||c.4f===0)?c.4f:c.5Z))c.3c=c.4f||c.5Z;7(!c.7b&&c.5Y)c.7b=c.5Y;7(!c.3c&&c.2G)c.3c=(c.2G&1?1:(c.2G&2?3:(c.2G&4?2:0)));K c},2y:{21:{4j:J(){5M();K},4h:J(){K}},3C:{4j:J(){7(E.14.1d)K S;E(6).2j("4P",E.16.2y.3C.2q);K P},4h:J(){7(E.14.1d)K S;E(6).3w("4P",E.16.2y.3C.2q);K P},2q:J(a){7(I(a,6))K P;18[0].U="3C";K E.16.1v.1i(6,18)}},3B:{4j:J(){7(E.14.1d)K S;E(6).2j("4O",E.16.2y.3B.2q);K P},4h:J(){7(E.14.1d)K S;E(6).3w("4O",E.16.2y.3B.2q);K P},2q:J(a){7(I(a,6))K P;18[0].U="3B";K E.16.1v.1i(6,18)}}}};E.1n.1s({2j:J(c,a,b){K c=="4H"?6.2X(c,a,b):6.R(J(){E.16.1b(6,c,b||a,b&&a)})},2X:J(d,b,c){K 6.R(J(){E.16.1b(6,d,J(a){E(6).3w(a);K(c||b).1i(6,18)},c&&b)})},3w:J(a,b){K 6.R(J(){E.16.1V(6,a,b)})},1N:J(c,a,b){K 6.R(J(){E.16.1N(c,a,6,P,b)})},5n:J(c,a,b){7(6[0])K E.16.1N(c,a,6[0],S,b);K 10},2g:J(){L b=18;K 6.4V(J(a){6.4N=0==6.4N?1:0;a.36();K b[6.4N].1i(6,18)||S})},7D:J(a,b){K 6.2j(\'3C\',a).2j(\'3B\',b)},21:J(a){5M();7(E.2Q)a.1P(T,E);N E.3A.1g(J(){K a.1P(6,E)});K 6}});E.1s({2Q:S,3A:[],21:J(){7(!E.2Q){E.2Q=P;7(E.3A){E.R(E.3A,J(){6.1i(T)});E.3A=V}E(T).5n("21")}}});L x=S;J 5M(){7(x)K;x=P;7(T.3F&&!E.14.2z)T.3F("5W",E.21,S);7(E.14.1d&&1e==3b)(J(){7(E.2Q)K;1S{T.1F.7B("26")}1X(3a){3z(18.3R,0);K}E.21()})();7(E.14.2z)T.3F("5W",J(){7(E.2Q)K;Q(L i=0;i=0){L i=g.2K(e,g.M);g=g.2K(0,e)}c=c||J(){};L f="4Q";7(d)7(E.1q(d)){c=d;d=V}N{d=E.3m(d);f="61"}L h=6;E.3P({1c:g,U:f,1H:"3q",O:d,1y:J(a,b){7(b=="1W"||b=="5U")h.3q(i?E("<1x/>").3t(a.4b.1r(/<1m(.|\\s)*?\\/1m>/g,"")).2s(i):a.4b);h.R(c,[a.4b,b,a])}});K 6},7n:J(){K E.3m(6.5T())},5T:J(){K 6.2c(J(){K E.12(6,"3u")?E.2I(6.7m):6}).1E(J(){K 6.31&&!6.2Y&&(6.3k||/2k|6h/i.17(6.12)||/1u|1Z|3I/i.17(6.U))}).2c(J(i,c){L b=E(6).5O();K b==V?V:b.1k==1M?E.2c(b,J(a,i){K{31:c.31,1A:a}}):{31:c.31,1A:b}}).22()}});E.R("5S,6d,5R,6D,5Q,6m".23(","),J(i,o){E.1n[o]=J(f){K 6.2j(o,f)}});L B=(1B 3v).3L();E.1s({22:J(d,b,a,c){7(E.1q(b)){a=b;b=V}K E.3P({U:"4Q",1c:d,O:b,1W:a,1H:c})},7l:J(b,a){K E.22(b,V,a,"1m")},7k:J(c,b,a){K E.22(c,b,a,"3i")},7i:J(d,b,a,c){7(E.1q(b)){a=b;b={}}K E.3P({U:"61",1c:d,O:b,1W:a,1H:c})},85:J(a){E.1s(E.4I,a)},4I:{2a:P,U:"4Q",2U:0,5P:"4o/x-7h-3u-7g",5N:P,3l:P,O:V,6p:V,3I:V,49:{3M:"4o/3M, 1u/3M",3q:"1u/3q",1m:"1u/4m, 4o/4m",3i:"4o/3i, 1u/4m",1u:"1u/a7",4G:"*/*"}},4F:{},3P:J(s){L f,2W=/=\\?(&|$)/g,1z,O;s=E.1s(P,s,E.1s(P,{},E.4I,s));7(s.O&&s.5N&&1o s.O!="25")s.O=E.3m(s.O);7(s.1H=="4E"){7(s.U.2h()=="22"){7(!s.1c.1D(2W))s.1c+=(s.1c.1D(/\\?/)?"&":"?")+(s.4E||"7d")+"=?"}N 7(!s.O||!s.O.1D(2W))s.O=(s.O?s.O+"&":"")+(s.4E||"7d")+"=?";s.1H="3i"}7(s.1H=="3i"&&(s.O&&s.O.1D(2W)||s.1c.1D(2W))){f="4E"+B++;7(s.O)s.O=(s.O+"").1r(2W,"="+f+"$1");s.1c=s.1c.1r(2W,"="+f+"$1");s.1H="1m";1e[f]=J(a){O=a;1W();1y();1e[f]=10;1S{2V 1e[f]}1X(e){}7(h)h.34(g)}}7(s.1H=="1m"&&s.1T==V)s.1T=S;7(s.1T===S&&s.U.2h()=="22"){L i=(1B 3v()).3L();L j=s.1c.1r(/(\\?|&)4r=.*?(&|$)/,"$a4="+i+"$2");s.1c=j+((j==s.1c)?(s.1c.1D(/\\?/)?"&":"?")+"4r="+i:"")}7(s.O&&s.U.2h()=="22"){s.1c+=(s.1c.1D(/\\?/)?"&":"?")+s.O;s.O=V}7(s.2a&&!E.5H++)E.16.1N("5S");7((!s.1c.1f("a3")||!s.1c.1f("//"))&&s.1H=="1m"&&s.U.2h()=="22"){L h=T.3S("6f")[0];L g=T.3s("1m");g.3Q=s.1c;7(s.7c)g.a2=s.7c;7(!f){L l=S;g.9Z=g.9Y=J(){7(!l&&(!6.39||6.39=="5V"||6.39=="1y")){l=P;1W();1y();h.34(g)}}}h.38(g);K 10}L m=S;L k=1e.78?1B 78("9X.9V"):1B 76();k.9T(s.U,s.1c,s.3l,s.6p,s.3I);1S{7(s.O)k.4C("9R-9Q",s.5P);7(s.5C)k.4C("9O-5A-9N",E.4F[s.1c]||"9L, 9K 9I 9H 5z:5z:5z 9F");k.4C("X-9C-9A","76");k.4C("9z",s.1H&&s.49[s.1H]?s.49[s.1H]+", */*":s.49.4G)}1X(e){}7(s.6Y)s.6Y(k);7(s.2a)E.16.1N("6m",[k,s]);L c=J(a){7(!m&&k&&(k.39==4||a=="2U")){m=P;7(d){6I(d);d=V}1z=a=="2U"&&"2U"||!E.6X(k)&&"3a"||s.5C&&E.6J(k,s.1c)&&"5U"||"1W";7(1z=="1W"){1S{O=E.6W(k,s.1H)}1X(e){1z="5x"}}7(1z=="1W"){L b;1S{b=k.5q("6U-5A")}1X(e){}7(s.5C&&b)E.4F[s.1c]=b;7(!f)1W()}N E.5v(s,k,1z);1y();7(s.3l)k=V}};7(s.3l){L d=53(c,13);7(s.2U>0)3z(J(){7(k){k.9t();7(!m)c("2U")}},s.2U)}1S{k.9s(s.O)}1X(e){E.5v(s,k,V,e)}7(!s.3l)c();J 1W(){7(s.1W)s.1W(O,1z);7(s.2a)E.16.1N("5Q",[k,s])}J 1y(){7(s.1y)s.1y(k,1z);7(s.2a)E.16.1N("5R",[k,s]);7(s.2a&&!--E.5H)E.16.1N("6d")}K k},5v:J(s,a,b,e){7(s.3a)s.3a(a,b,e);7(s.2a)E.16.1N("6D",[a,s,e])},5H:0,6X:J(r){1S{K!r.1z&&9q.9p=="59:"||(r.1z>=6T&&r.1z<9n)||r.1z==6R||r.1z==9l||E.14.2d&&r.1z==10}1X(e){}K S},6J:J(a,c){1S{L b=a.5q("6U-5A");K a.1z==6R||b==E.4F[c]||E.14.2d&&a.1z==10}1X(e){}K S},6W:J(r,b){L c=r.5q("9k-U");L d=b=="3M"||!b&&c&&c.1f("3M")>=0;L a=d?r.9j:r.4b;7(d&&a.1F.28=="5x")6Q"5x";7(b=="1m")E.5g(a);7(b=="3i")a=6c("("+a+")");K a},3m:J(a){L s=[];7(a.1k==1M||a.5h)E.R(a,J(){s.1g(3r(6.31)+"="+3r(6.1A))});N Q(L j 1p a)7(a[j]&&a[j].1k==1M)E.R(a[j],J(){s.1g(3r(j)+"="+3r(6))});N s.1g(3r(j)+"="+3r(a[j]));K s.6a("&").1r(/%20/g,"+")}});E.1n.1s({1G:J(c,b){K c?6.2e({1R:"1G",27:"1G",1w:"1G"},c,b):6.1E(":1Z").R(J(){6.W.19=6.5s||"";7(E.1j(6,"19")=="2H"){L a=E("<"+6.28+" />").6y("1h");6.W.19=a.1j("19");7(6.W.19=="2H")6.W.19="3D";a.1V()}}).3h()},1I:J(b,a){K b?6.2e({1R:"1I",27:"1I",1w:"1I"},b,a):6.1E(":4d").R(J(){6.5s=6.5s||E.1j(6,"19");6.W.19="2H"}).3h()},6N:E.1n.2g,2g:J(a,b){K E.1q(a)&&E.1q(b)?6.6N(a,b):a?6.2e({1R:"2g",27:"2g",1w:"2g"},a,b):6.R(J(){E(6)[E(6).3H(":1Z")?"1G":"1I"]()})},9f:J(b,a){K 6.2e({1R:"1G"},b,a)},9d:J(b,a){K 6.2e({1R:"1I"},b,a)},9c:J(b,a){K 6.2e({1R:"2g"},b,a)},9a:J(b,a){K 6.2e({1w:"1G"},b,a)},99:J(b,a){K 6.2e({1w:"1I"},b,a)},97:J(c,a,b){K 6.2e({1w:a},c,b)},2e:J(l,k,j,h){L i=E.6P(k,j,h);K 6[i.2P===S?"R":"2P"](J(){7(6.15!=1)K S;L g=E.1s({},i);L f=E(6).3H(":1Z"),4A=6;Q(L p 1p l){7(l[p]=="1I"&&f||l[p]=="1G"&&!f)K E.1q(g.1y)&&g.1y.1i(6);7(p=="1R"||p=="27"){g.19=E.1j(6,"19");g.32=6.W.32}}7(g.32!=V)6.W.32="1Z";g.40=E.1s({},l);E.R(l,J(c,a){L e=1B E.2t(4A,g,c);7(/2g|1G|1I/.17(a))e[a=="2g"?f?"1G":"1I":a](l);N{L b=a.3X().1D(/^([+-]=)?([\\d+-.]+)(.*)$/),1Y=e.2m(P)||0;7(b){L d=2M(b[2]),2A=b[3]||"2S";7(2A!="2S"){4A.W[c]=(d||1)+2A;1Y=((d||1)/e.2m(P))*1Y;4A.W[c]=1Y+2A}7(b[1])d=((b[1]=="-="?-1:1)*d)+1Y;e.45(1Y,d,2A)}N e.45(1Y,a,"")}});K P})},2P:J(a,b){7(E.1q(a)||(a&&a.1k==1M)){b=a;a="2t"}7(!a||(1o a=="25"&&!b))K A(6[0],a);K 6.R(J(){7(b.1k==1M)A(6,a,b);N{A(6,a).1g(b);7(A(6,a).M==1)b.1i(6)}})},94:J(b,c){L a=E.3G;7(b)6.2P([]);6.R(J(){Q(L i=a.M-1;i>=0;i--)7(a[i].Y==6){7(c)a[i](P);a.72(i,1)}});7(!c)6.5p();K 6}});L A=J(b,c,a){7(!b)K 10;c=c||"2t";L q=E.O(b,c+"2P");7(!q||a)q=E.O(b,c+"2P",a?E.2I(a):[]);K q};E.1n.5p=J(a){a=a||"2t";K 6.R(J(){L q=A(6,a);q.4l();7(q.M)q[0].1i(6)})};E.1s({6P:J(b,a,c){L d=b&&b.1k==92?b:{1y:c||!c&&a||E.1q(b)&&b,2u:b,3Z:c&&a||a&&a.1k!=91&&a};d.2u=(d.2u&&d.2u.1k==51?d.2u:{90:8Z,9D:6T}[d.2u])||8X;d.5y=d.1y;d.1y=J(){7(d.2P!==S)E(6).5p();7(E.1q(d.5y))d.5y.1i(6)};K d},3Z:{70:J(p,n,b,a){K b+a*p},5j:J(p,n,b,a){K((-24.8V(p*24.8U)/2)+0.5)*a+b}},3G:[],3W:V,2t:J(b,c,a){6.11=c;6.Y=b;6.1l=a;7(!c.47)c.47={}}});E.2t.2l={4y:J(){7(6.11.30)6.11.30.1i(6.Y,[6.2J,6]);(E.2t.30[6.1l]||E.2t.30.4G)(6);7(6.1l=="1R"||6.1l=="27")6.Y.W.19="3D"},2m:J(a){7(6.Y[6.1l]!=V&&6.Y.W[6.1l]==V)K 6.Y[6.1l];L r=2M(E.1j(6.Y,6.1l,a));K r&&r>-8Q?r:2M(E.2o(6.Y,6.1l))||0},45:J(c,b,d){6.5B=(1B 3v()).3L();6.1Y=c;6.3h=b;6.2A=d||6.2A||"2S";6.2J=6.1Y;6.4B=6.4w=0;6.4y();L e=6;J t(a){K e.30(a)}t.Y=6.Y;E.3G.1g(t);7(E.3W==V){E.3W=53(J(){L a=E.3G;Q(L i=0;i6.11.2u+6.5B){6.2J=6.3h;6.4B=6.4w=1;6.4y();6.11.40[6.1l]=P;L b=P;Q(L i 1p 6.11.40)7(6.11.40[i]!==P)b=S;7(b){7(6.11.19!=V){6.Y.W.32=6.11.32;6.Y.W.19=6.11.19;7(E.1j(6.Y,"19")=="2H")6.Y.W.19="3D"}7(6.11.1I)6.Y.W.19="2H";7(6.11.1I||6.11.1G)Q(L p 1p 6.11.40)E.1J(6.Y.W,p,6.11.47[p])}7(b&&E.1q(6.11.1y))6.11.1y.1i(6.Y);K S}N{L n=t-6.5B;6.4w=n/6.11.2u;6.4B=E.3Z[6.11.3Z||(E.3Z.5j?"5j":"70")](6.4w,n,0,1,6.11.2u);6.2J=6.1Y+((6.3h-6.1Y)*6.4B);6.4y()}K P}};E.2t.30={2v:J(a){a.Y.2v=a.2J},2x:J(a){a.Y.2x=a.2J},1w:J(a){E.1J(a.Y.W,"1w",a.2J)},4G:J(a){a.Y.W[a.1l]=a.2J+a.2A}};E.1n.5L=J(){L b=0,3b=0,Y=6[0],5l;7(Y)8M(E.14){L d=Y.1a,41=Y,1K=Y.1K,1L=Y.2i,5D=2d&&4s(5K)<8J&&!/a1/i.17(v),2T=E.1j(Y,"43")=="2T";7(Y.6G){L c=Y.6G();1b(c.26+24.2f(1L.1F.2v,1L.1h.2v),c.3b+24.2f(1L.1F.2x,1L.1h.2x));1b(-1L.1F.62,-1L.1F.60)}N{1b(Y.5G,Y.5F);2b(1K){1b(1K.5G,1K.5F);7(48&&!/^t(8H|d|h)$/i.17(1K.28)||2d&&!5D)2N(1K);7(!2T&&E.1j(1K,"43")=="2T")2T=P;41=/^1h$/i.17(1K.28)?41:1K;1K=1K.1K}2b(d&&d.28&&!/^1h|3q$/i.17(d.28)){7(!/^8G|1O.*$/i.17(E.1j(d,"19")))1b(-d.2v,-d.2x);7(48&&E.1j(d,"32")!="4d")2N(d);d=d.1a}7((5D&&(2T||E.1j(41,"43")=="4W"))||(48&&E.1j(41,"43")!="4W"))1b(-1L.1h.5G,-1L.1h.5F);7(2T)1b(24.2f(1L.1F.2v,1L.1h.2v),24.2f(1L.1F.2x,1L.1h.2x))}5l={3b:3b,26:b}}J 2N(a){1b(E.2o(a,"a8",P),E.2o(a,"a9",P))}J 1b(l,t){b+=4s(l)||0;3b+=4s(t)||0}K 5l}})();',62,631,'||||||this|if||||||||||||||||||||||||||||||||||||||function|return|var|length|else|data|true|for|each|false|document|type|null|style||elem||undefined|options|nodeName||browser|nodeType|event|test|arguments|display|parentNode|add|url|msie|window|indexOf|push|body|apply|css|constructor|prop|script|fn|typeof|in|isFunction|replace|extend|className|text|handle|opacity|div|complete|status|value|new|firstChild|match|filter|documentElement|show|dataType|hide|attr|offsetParent|doc|Array|trigger|table|call|break|height|try|cache|tbody|remove|success|catch|start|hidden||ready|get|split|Math|string|left|width|tagName|ret|global|while|map|safari|animate|max|toggle|toLowerCase|ownerDocument|bind|select|prototype|cur||curCSS|selected|handler|done|find|fx|duration|scrollLeft|id|scrollTop|special|opera|unit|nextSibling|stack|guid|toUpperCase|pushStack|button|none|makeArray|now|slice|target|parseFloat|border|exec|queue|isReady|events|px|fixed|timeout|delete|jsre|one|disabled|nth|step|name|overflow|inArray|removeChild|removeData|preventDefault|merge|appendChild|readyState|error|top|which|innerHTML|multiFilter|rl|trim|end|json|first|checked|async|param|elems|insertBefore|childNodes|html|encodeURIComponent|createElement|append|form|Date|unbind|color|grep|setTimeout|readyList|mouseleave|mouseenter|block|isXMLDoc|addEventListener|timers|is|password|last|runtimeStyle|getTime|xml|jQuery|domManip|ajax|src|callee|getElementsByTagName|selectedIndex|load|object|timerId|toString|has|easing|curAnim|offsetChild|args|position|stopPropagation|custom|props|orig|mozilla|accepts|clean|responseText|defaultView|visible|String|charCode|float|teardown|on|setup|nodeIndex|shift|javascript|currentStyle|application|child|RegExp|_|parseInt|previousSibling|dir|tr|state|empty|update|getAttribute|self|pos|setRequestHeader|input|jsonp|lastModified|_default|unload|ajaxSettings|unshift|getComputedStyle|styleSheets|getPropertyValue|lastToggle|mouseout|mouseover|GET|andSelf|relatedTarget|init|visibility|click|absolute|index|container|fix|outline|Number|removeAttribute|setInterval|prevObject|classFilter|not|unique|submit|file|after|windowData|deep|scroll|client|triggered|globalEval|jquery|sibling|swing|clone|results|wrapAll|triggerHandler|lastChild|dequeue|getResponseHeader|createTextNode|oldblock|checkbox|radio|handleError|fromElement|parsererror|old|00|Modified|startTime|ifModified|safari2|getWH|offsetTop|offsetLeft|active|values|getElementById|version|offset|bindReady|processData|val|contentType|ajaxSuccess|ajaxComplete|ajaxStart|serializeArray|notmodified|loaded|DOMContentLoaded|Width|ctrlKey|keyCode|clientTop|POST|clientLeft|clientX|pageX|exclusive|detachEvent|removeEventListener|swap|cloneNode|join|attachEvent|eval|ajaxStop|substr|head|parse|textarea|reset|image|zoom|odd|ajaxSend|even|before|username|prepend|expr|quickClass|uuid|quickID|quickChild|continue|textContent|appendTo|contents|evalScript|parent|defaultValue|ajaxError|setArray|compatMode|getBoundingClientRect|styleFloat|clearInterval|httpNotModified|nodeValue|100|alpha|_toggle|href|speed|throw|304|replaceWith|200|Last|colgroup|httpData|httpSuccess|beforeSend|eq|linear|concat|splice|fieldset|multiple|cssFloat|XMLHttpRequest|webkit|ActiveXObject|CSS1Compat|link|metaKey|scriptCharset|callback|col|pixelLeft|urlencoded|www|post|hasClass|getJSON|getScript|elements|serialize|black|keyup|keypress|solid|change|mousemove|mouseup|dblclick|resize|focus|blur|stylesheet|rel|doScroll|round|hover|padding|offsetHeight|mousedown|offsetWidth|Bottom|Top|keydown|clientY|Right|pageY|Left|toElement|srcElement|cancelBubble|returnValue|charAt|0n|substring|animated|header|noConflict|line|enabled|innerText|contains|only|weight|ajaxSetup|font|size|gt|lt|uFFFF|u0128|417|Boolean|inner|Height|toggleClass|removeClass|addClass|removeAttr|replaceAll|insertAfter|prependTo|contentWindow|contentDocument|wrap|iframe|children|siblings|prevAll|nextAll|prev|wrapInner|next|parents|maxLength|maxlength|readOnly|readonly|reverse|class|htmlFor|inline|able|boxModel|522|setData|compatible|with|1px|ie|getData|10000|ra|it|rv|PI|cos|userAgent|400|navigator|600|slow|Function|Object|array|stop|ig|NaN|fadeTo|option|fadeOut|fadeIn|setAttribute|slideToggle|slideUp|changed|slideDown|be|can|property|responseXML|content|1223|getAttributeNode|300|method|protocol|location|action|send|abort|cssText|th|td|cap|specified|Accept|With|colg|Requested|fast|tfoot|GMT|thead|1970|Jan|attributes|01|Thu|leg|Since|If|opt|Type|Content|embed|open|area|XMLHTTP|hr|Microsoft|onreadystatechange|onload|meta|adobeair|charset|http|1_|img|br|plain|borderLeftWidth|borderTopWidth|abbr'.split('|'),0,{})) \ No newline at end of file diff --git a/sites/static.scrapy.org/js/keyboard_nav.js b/sites/static.scrapy.org/js/keyboard_nav.js deleted file mode 100644 index 9719888e4..000000000 --- a/sites/static.scrapy.org/js/keyboard_nav.js +++ /dev/null @@ -1,61 +0,0 @@ -(function($){ - var SELECTED_FILE_ELEM = null; - var ENABLE_KEY_NAV = true; - - $(document).keydown(function(event) { - if (!ENABLE_KEY_NAV) - return true; - var selection = SELECTED_FILE_ELEM; - switch (event.keyCode) { - case 74: // j - if (selection == null) { - selection = $('#f0'); - } else { - do { - selection = selection.next(); - } while (selection.length > 0 && selection.css('display') == 'none'); - } - break; - case 75: // k - if (selection == null) { - selection = $('#f0'); - } else { - do { - selection = selection.prev(); - } while (selection.length > 0 && selection.css('display') == 'none'); - } - break; - case 13: // Enter - case 79: // o - if (selection != null) { - var expander = selection.find('.expander'); - if (expander.length > 0) { - expander.click(); - } else { - window.location = selection.find('a.file').attr('href'); - } - } - return false; - break; - default: - return true; - } - if (selection.length > 0) { - if (SELECTED_FILE_ELEM != null) - SELECTED_FILE_ELEM.removeClass('focus'); - selection.addClass('focus'); - SELECTED_FILE_ELEM = selection; - } - return false; - }); - - $(function() { - $('a,input,select,textarea,button') - .focus(function(event) { - ENABLE_KEY_NAV = false; - }) - .blur(function(event) { - ENABLE_KEY_NAV = true; - }); - }); -})(jQuery); diff --git a/sites/static.scrapy.org/js/noconflict.js b/sites/static.scrapy.org/js/noconflict.js deleted file mode 100644 index 8e4a663c2..000000000 --- a/sites/static.scrapy.org/js/noconflict.js +++ /dev/null @@ -1,3 +0,0 @@ -jQuery.noConflict(); // jQuery is now removed from the $ namespace - // to use the $ shorthand, use (function($){ ... })(jQuery); - // and for the onload handler: jQuery(function($){ ... }); \ No newline at end of file diff --git a/sites/static.scrapy.org/js/query.js b/sites/static.scrapy.org/js/query.js deleted file mode 100644 index a8b97d96c..000000000 --- a/sites/static.scrapy.org/js/query.js +++ /dev/null @@ -1,278 +0,0 @@ - -(function($){ - - window.initializeFilters = function() { - - // Bail early for Konqueror and IE5.2/Mac, which don't fully support dynamic - // creation of form controls - try { - var test = document.createElement("input"); - test.type = "button"; - if (test.type != "button") throw Error(); - } catch (e) { - return; - } - - // Removes an existing row from the filters table - function removeRow(button, propertyName) { - var tr = getAncestorByTagName(button, "tr"); - - var mode = null; - var selects = tr.getElementsByTagName("select"); - for (var i = 0; i < selects.length; i++) { - if (selects[i].name == propertyName + "_mode") { - mode = selects[i]; - break; - } - } - if (mode && (getAncestorByTagName(mode, "tr") == tr)) { - // Check whether there are more 'or' rows for this filter - var next = tr.nextSibling; - if (next && (next.className == propertyName)) { - function getChildElementAt(e, idx) { - e = e.firstChild; - var cur = 0; - while (cur <= idx) { - while (e && e.nodeType != 1) e = e.nextSibling; - if (cur++ == idx) break; - e = e.nextSibling; - } - return e; - } - - var thisTh = getChildElementAt(tr, 0); - var nextTh = getChildElementAt(next, 0); - next.insertBefore(thisTh, nextTh); - nextTh.colSpan = 1; - - thisTd = getChildElementAt(tr, 0); - nextTd = getChildElementAt(next, 1); - next.replaceChild(thisTd, nextTd); - } - } - - var tBody = tr.parentNode; - tBody.deleteRow(tr.sectionRowIndex); - if (!tBody.rows.length) { - tBody.parentNode.removeChild(tBody); - } - - if (propertyName) { - var select = document.forms["query"].elements["add_filter"]; - for (var i = 0; i < select.options.length; i++) { - var option = select.options[i]; - if (option.value == propertyName) option.disabled = false; - } - } - } - - // Initializes a filter row, the 'input' parameter is the submit - // button for removing the filter - function initializeFilter(input) { - var removeButton = document.createElement("input"); - removeButton.type = "button"; - removeButton.value = input.value; - if (input.name.substr(0, 10) == "rm_filter_") { - removeButton.onclick = function() { - var endIndex = input.name.search(/_\d+$/); - if (endIndex < 0) endIndex = input.name.length; - removeRow(removeButton, input.name.substring(10, endIndex)); - return false; - } - } else { - removeButton.onclick = function() { - removeRow(removeButton); - return false; - } - } - input.parentNode.replaceChild(removeButton, input); - } - - // Make the submit buttons for removing filters client-side triggers - var filters = document.getElementById("filters"); - var inputs = filters.getElementsByTagName("input"); - for (var i = 0; i < inputs.length; i++) { - var input = inputs[i]; - if (input.type == "submit" && input.name - && input.name.match(/^rm_filter_/)) { - initializeFilter(input); - } - } - - // Make the drop-down menu for adding a filter a client-side trigger - var addButton = document.forms["query"].elements["add"]; - addButton.parentNode.removeChild(addButton); - var select = document.getElementById("add_filter"); - select.onchange = function() { - if (select.selectedIndex < 1) return; - - if (select.options[select.selectedIndex].disabled) { - // Neither IE nor Safari supported disabled options at the time this was - // written, so alert the user - alert("A filter already exists for that property"); - return; - } - - // Convenience function for creating a