mirror of https://github.com/scrapy/scrapy.git
mv scrapy/trunk to root as part of svn2hg migration
--HG-- rename : scrapy/trunk/AUTHORS => AUTHORS rename : scrapy/trunk/INSTALL => INSTALL rename : scrapy/trunk/LICENSE => LICENSE rename : scrapy/trunk/README => README rename : scrapy/trunk/bin/runtests.sh => bin/runtests.sh rename : scrapy/trunk/docs/Makefile => docs/Makefile rename : scrapy/trunk/docs/README => docs/README rename : scrapy/trunk/docs/_ext/scrapydocs.py => docs/_ext/scrapydocs.py rename : scrapy/trunk/docs/_static/items_adaptors-sample1.html => docs/_static/items_adaptors-sample1.html rename : scrapy/trunk/docs/_static/scrapydoc.css => docs/_static/scrapydoc.css rename : scrapy/trunk/docs/_static/selectors-sample1.html => docs/_static/selectors-sample1.html rename : scrapy/trunk/docs/conf.py => docs/conf.py rename : scrapy/trunk/docs/faq.rst => docs/faq.rst rename : scrapy/trunk/docs/index.rst => docs/index.rst rename : scrapy/trunk/docs/intro/index.rst => docs/intro/index.rst rename : scrapy/trunk/docs/intro/install.rst => docs/intro/install.rst rename : scrapy/trunk/docs/intro/overview.rst => docs/intro/overview.rst rename : scrapy/trunk/docs/intro/tutorial.rst => docs/intro/tutorial.rst rename : scrapy/trunk/docs/media/scrapy-architecture.dia => docs/media/scrapy-architecture.dia rename : scrapy/trunk/docs/misc/api-stability.rst => docs/misc/api-stability.rst rename : scrapy/trunk/docs/misc/index.rst => docs/misc/index.rst rename : scrapy/trunk/docs/proposed/_images/scrapy_architecture.odg => docs/proposed/_images/scrapy_architecture.odg rename : scrapy/trunk/docs/proposed/_images/scrapy_architecture.png => docs/proposed/_images/scrapy_architecture.png rename : scrapy/trunk/docs/proposed/index.rst => docs/proposed/index.rst rename : scrapy/trunk/docs/proposed/introduction.rst => docs/proposed/introduction.rst rename : scrapy/trunk/docs/proposed/newitem.rst => docs/proposed/newitem.rst rename : scrapy/trunk/docs/proposed/spiders.rst => docs/proposed/spiders.rst rename : scrapy/trunk/docs/ref/downloader-middleware.rst => docs/ref/downloader-middleware.rst rename : scrapy/trunk/docs/ref/email.rst => docs/ref/email.rst rename : scrapy/trunk/docs/ref/exceptions.rst => docs/ref/exceptions.rst rename : scrapy/trunk/docs/ref/extension-manager.rst => docs/ref/extension-manager.rst rename : scrapy/trunk/docs/ref/extensions.rst => docs/ref/extensions.rst rename : scrapy/trunk/docs/ref/index.rst => docs/ref/index.rst rename : scrapy/trunk/docs/ref/link-extractors.rst => docs/ref/link-extractors.rst rename : scrapy/trunk/docs/ref/logging.rst => docs/ref/logging.rst rename : scrapy/trunk/docs/ref/request-response.rst => docs/ref/request-response.rst rename : scrapy/trunk/docs/ref/selectors.rst => docs/ref/selectors.rst rename : scrapy/trunk/docs/ref/settings.rst => docs/ref/settings.rst rename : scrapy/trunk/docs/ref/signals.rst => docs/ref/signals.rst rename : scrapy/trunk/docs/ref/spiders.rst => docs/ref/spiders.rst rename : scrapy/trunk/docs/topics/_images/adaptors_diagram.png => docs/topics/_images/adaptors_diagram.png rename : scrapy/trunk/docs/topics/_images/adaptors_diagram.svg => docs/topics/_images/adaptors_diagram.svg rename : scrapy/trunk/docs/topics/_images/firebug1.png => docs/topics/_images/firebug1.png rename : scrapy/trunk/docs/topics/_images/firebug2.png => docs/topics/_images/firebug2.png rename : scrapy/trunk/docs/topics/_images/firebug3.png => docs/topics/_images/firebug3.png rename : scrapy/trunk/docs/topics/_images/scrapy_architecture.odg => docs/topics/_images/scrapy_architecture.odg rename : scrapy/trunk/docs/topics/_images/scrapy_architecture.png => docs/topics/_images/scrapy_architecture.png rename : scrapy/trunk/docs/topics/adaptors.rst => docs/topics/adaptors.rst rename : scrapy/trunk/docs/topics/architecture.rst => docs/topics/architecture.rst rename : scrapy/trunk/docs/topics/downloader-middleware.rst => docs/topics/downloader-middleware.rst rename : scrapy/trunk/docs/topics/extensions.rst => docs/topics/extensions.rst rename : scrapy/trunk/docs/topics/firebug.rst => docs/topics/firebug.rst rename : scrapy/trunk/docs/topics/firefox.rst => docs/topics/firefox.rst rename : scrapy/trunk/docs/topics/index.rst => docs/topics/index.rst rename : scrapy/trunk/docs/topics/item-pipeline.rst => docs/topics/item-pipeline.rst rename : scrapy/trunk/docs/topics/items.rst => docs/topics/items.rst rename : scrapy/trunk/docs/topics/link-extractors.rst => docs/topics/link-extractors.rst rename : scrapy/trunk/docs/topics/robotstxt.rst => docs/topics/robotstxt.rst rename : scrapy/trunk/docs/topics/selectors.rst => docs/topics/selectors.rst rename : scrapy/trunk/docs/topics/settings.rst => docs/topics/settings.rst rename : scrapy/trunk/docs/topics/shell.rst => docs/topics/shell.rst rename : scrapy/trunk/docs/topics/spider-middleware.rst => docs/topics/spider-middleware.rst rename : scrapy/trunk/docs/topics/spiders.rst => docs/topics/spiders.rst rename : scrapy/trunk/docs/topics/stats.rst => docs/topics/stats.rst rename : scrapy/trunk/docs/topics/webconsole.rst => docs/topics/webconsole.rst rename : scrapy/trunk/examples/experimental/googledir/googledir/__init__.py => examples/experimental/googledir/googledir/__init__.py rename : scrapy/trunk/examples/experimental/googledir/googledir/items.py => examples/experimental/googledir/googledir/items.py rename : scrapy/trunk/examples/experimental/googledir/googledir/pipelines.py => examples/experimental/googledir/googledir/pipelines.py rename : scrapy/trunk/examples/experimental/googledir/googledir/settings.py => examples/experimental/googledir/googledir/settings.py rename : scrapy/trunk/examples/experimental/googledir/googledir/spiders/__init__.py => examples/experimental/googledir/googledir/spiders/__init__.py rename : scrapy/trunk/examples/experimental/googledir/googledir/spiders/google_directory.py => examples/experimental/googledir/googledir/spiders/google_directory.py rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_basic.tmpl => examples/experimental/googledir/googledir/templates/spider_basic.tmpl rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl => examples/experimental/googledir/googledir/templates/spider_crawl.tmpl rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl => examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl rename : scrapy/trunk/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl => examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl rename : scrapy/trunk/examples/experimental/googledir/scrapy-ctl.py => examples/experimental/googledir/scrapy-ctl.py rename : scrapy/trunk/examples/googledir/googledir/__init__.py => examples/googledir/googledir/__init__.py rename : scrapy/trunk/examples/googledir/googledir/items.py => examples/googledir/googledir/items.py rename : scrapy/trunk/examples/googledir/googledir/pipelines.py => examples/googledir/googledir/pipelines.py rename : scrapy/trunk/examples/googledir/googledir/settings.py => examples/googledir/googledir/settings.py rename : scrapy/trunk/examples/googledir/googledir/spiders/__init__.py => examples/googledir/googledir/spiders/__init__.py rename : scrapy/trunk/examples/googledir/googledir/spiders/google_directory.py => examples/googledir/googledir/spiders/google_directory.py rename : scrapy/trunk/examples/googledir/scrapy-ctl.py => examples/googledir/scrapy-ctl.py rename : scrapy/trunk/extras/sql/scraping.sql => extras/sql/scraping.sql rename : scrapy/trunk/profiling/priorityqueue/pq_classes.py => profiling/priorityqueue/pq_classes.py rename : scrapy/trunk/profiling/priorityqueue/run.py => profiling/priorityqueue/run.py rename : scrapy/trunk/profiling/priorityqueue/test_cases.py => profiling/priorityqueue/test_cases.py rename : scrapy/trunk/scrapy/__init__.py => scrapy/__init__.py rename : scrapy/trunk/scrapy/bin/scrapy-admin.py => scrapy/bin/scrapy-admin.py rename : scrapy/trunk/scrapy/command/__init__.py => scrapy/command/__init__.py rename : scrapy/trunk/scrapy/command/cmdline.py => scrapy/command/cmdline.py rename : scrapy/trunk/scrapy/command/commands/__init__.py => scrapy/command/commands/__init__.py rename : scrapy/trunk/scrapy/command/commands/crawl.py => scrapy/command/commands/crawl.py rename : scrapy/trunk/scrapy/command/commands/download.py => scrapy/command/commands/download.py rename : scrapy/trunk/scrapy/command/commands/genspider.py => scrapy/command/commands/genspider.py rename : scrapy/trunk/scrapy/command/commands/help.py => scrapy/command/commands/help.py rename : scrapy/trunk/scrapy/command/commands/list.py => scrapy/command/commands/list.py rename : scrapy/trunk/scrapy/command/commands/log.py => scrapy/command/commands/log.py rename : scrapy/trunk/scrapy/command/commands/parse.py => scrapy/command/commands/parse.py rename : scrapy/trunk/scrapy/command/commands/shell.py => scrapy/command/commands/shell.py rename : scrapy/trunk/scrapy/command/commands/start.py => scrapy/command/commands/start.py rename : scrapy/trunk/scrapy/command/commands/stats.py => scrapy/command/commands/stats.py rename : scrapy/trunk/scrapy/command/models.py => scrapy/command/models.py rename : scrapy/trunk/scrapy/conf/__init__.py => scrapy/conf/__init__.py rename : scrapy/trunk/scrapy/conf/commands/__init__.py => scrapy/conf/commands/__init__.py rename : scrapy/trunk/scrapy/conf/commands/crawl.py => scrapy/conf/commands/crawl.py rename : scrapy/trunk/scrapy/conf/commands/help.py => scrapy/conf/commands/help.py rename : scrapy/trunk/scrapy/conf/commands/list.py => scrapy/conf/commands/list.py rename : scrapy/trunk/scrapy/conf/commands/log.py => scrapy/conf/commands/log.py rename : scrapy/trunk/scrapy/conf/commands/scrape.py => scrapy/conf/commands/scrape.py rename : scrapy/trunk/scrapy/conf/commands/shell.py => scrapy/conf/commands/shell.py rename : scrapy/trunk/scrapy/conf/commands/stats.py => scrapy/conf/commands/stats.py rename : scrapy/trunk/scrapy/conf/commands/test.py => scrapy/conf/commands/test.py rename : scrapy/trunk/scrapy/conf/default_settings.py => scrapy/conf/default_settings.py rename : scrapy/trunk/scrapy/contrib/__init__.py => scrapy/contrib/__init__.py rename : scrapy/trunk/scrapy/contrib/aws.py => scrapy/contrib/aws.py rename : scrapy/trunk/scrapy/contrib/closedomain.py => scrapy/contrib/closedomain.py rename : scrapy/trunk/scrapy/contrib/cluster/__init__.py => scrapy/contrib/cluster/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/crawler/__init__.py => scrapy/contrib/cluster/crawler/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/crawler/manager.py => scrapy/contrib/cluster/crawler/manager.py rename : scrapy/trunk/scrapy/contrib/cluster/hooks/__init__.py => scrapy/contrib/cluster/hooks/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/hooks/svn.py => scrapy/contrib/cluster/hooks/svn.py rename : scrapy/trunk/scrapy/contrib/cluster/master/__init__.py => scrapy/contrib/cluster/master/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/master/manager.py => scrapy/contrib/cluster/master/manager.py rename : scrapy/trunk/scrapy/contrib/cluster/master/web.py => scrapy/contrib/cluster/master/web.py rename : scrapy/trunk/scrapy/contrib/cluster/master/ws_api.txt => scrapy/contrib/cluster/master/ws_api.txt rename : scrapy/trunk/scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py => scrapy/contrib/cluster/tools/scrapy-cluster-ctl.py rename : scrapy/trunk/scrapy/contrib/cluster/tools/test-worker.py => scrapy/contrib/cluster/tools/test-worker.py rename : scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py => scrapy/contrib/cluster/worker/__init__.py rename : scrapy/trunk/scrapy/contrib/cluster/worker/manager.py => scrapy/contrib/cluster/worker/manager.py rename : scrapy/trunk/scrapy/contrib/codecs/__init__.py => scrapy/contrib/codecs/__init__.py rename : scrapy/trunk/scrapy/contrib/codecs/x_mac_roman.py => scrapy/contrib/codecs/x_mac_roman.py rename : scrapy/trunk/scrapy/contrib/debug.py => scrapy/contrib/debug.py rename : scrapy/trunk/scrapy/contrib/delayedclosedomain.py => scrapy/contrib/delayedclosedomain.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py => scrapy/contrib/downloadermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py => scrapy/contrib/downloadermiddleware/cache.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py => scrapy/contrib/downloadermiddleware/common.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py => scrapy/contrib/downloadermiddleware/cookies.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py => scrapy/contrib/downloadermiddleware/debug.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py => scrapy/contrib/downloadermiddleware/errorpages.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py => scrapy/contrib/downloadermiddleware/httpauth.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/httpcompression.py => scrapy/contrib/downloadermiddleware/httpcompression.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py => scrapy/contrib/downloadermiddleware/redirect.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py => scrapy/contrib/downloadermiddleware/retry.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py => scrapy/contrib/downloadermiddleware/robotstxt.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/stats.py => scrapy/contrib/downloadermiddleware/stats.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py => scrapy/contrib/downloadermiddleware/useragent.py rename : scrapy/trunk/scrapy/contrib/groupsettings.py => scrapy/contrib/groupsettings.py rename : scrapy/trunk/scrapy/contrib/item/__init__.py => scrapy/contrib/item/__init__.py rename : scrapy/trunk/scrapy/contrib/item/models.py => scrapy/contrib/item/models.py rename : scrapy/trunk/scrapy/contrib/itemsampler.py => scrapy/contrib/itemsampler.py rename : scrapy/trunk/scrapy/contrib/link_extractors.py => scrapy/contrib/link_extractors.py rename : scrapy/trunk/scrapy/contrib/memdebug.py => scrapy/contrib/memdebug.py rename : scrapy/trunk/scrapy/contrib/memusage.py => scrapy/contrib/memusage.py rename : scrapy/trunk/scrapy/contrib/pipeline/__init__.py => scrapy/contrib/pipeline/__init__.py rename : scrapy/trunk/scrapy/contrib/pipeline/images.py => scrapy/contrib/pipeline/images.py rename : scrapy/trunk/scrapy/contrib/pipeline/media.py => scrapy/contrib/pipeline/media.py rename : scrapy/trunk/scrapy/contrib/pipeline/s3images.py => scrapy/contrib/pipeline/s3images.py rename : scrapy/trunk/scrapy/contrib/pipeline/show.py => scrapy/contrib/pipeline/show.py rename : scrapy/trunk/scrapy/contrib/prioritizers.py => scrapy/contrib/prioritizers.py rename : scrapy/trunk/scrapy/contrib/response/__init__.py => scrapy/contrib/response/__init__.py rename : scrapy/trunk/scrapy/contrib/response/soup.py => scrapy/contrib/response/soup.py rename : scrapy/trunk/scrapy/contrib/schedulermiddleware/__init__.py => scrapy/contrib/schedulermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/schedulermiddleware/duplicatesfilter.py => scrapy/contrib/schedulermiddleware/duplicatesfilter.py rename : scrapy/trunk/scrapy/contrib/spider/__init__.py => scrapy/contrib/spider/__init__.py rename : scrapy/trunk/scrapy/contrib/spider/profiler.py => scrapy/contrib/spider/profiler.py rename : scrapy/trunk/scrapy/contrib/spider/reloader.py => scrapy/contrib/spider/reloader.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py => scrapy/contrib/spidermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py => scrapy/contrib/spidermiddleware/depth.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py => scrapy/contrib/spidermiddleware/duplicatesfilter.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py => scrapy/contrib/spidermiddleware/limit.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py => scrapy/contrib/spidermiddleware/offsite.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py => scrapy/contrib/spidermiddleware/referer.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py => scrapy/contrib/spidermiddleware/restrict.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py => scrapy/contrib/spidermiddleware/urlfilter.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py => scrapy/contrib/spidermiddleware/urllength.py rename : scrapy/trunk/scrapy/contrib/spiders/__init__.py => scrapy/contrib/spiders/__init__.py rename : scrapy/trunk/scrapy/contrib/spiders/crawl.py => scrapy/contrib/spiders/crawl.py rename : scrapy/trunk/scrapy/contrib/spiders/feed.py => scrapy/contrib/spiders/feed.py rename : scrapy/trunk/scrapy/contrib/spiders/generic.py => scrapy/contrib/spiders/generic.py rename : scrapy/trunk/scrapy/contrib/web/__init__.py => scrapy/contrib/web/__init__.py rename : scrapy/trunk/scrapy/contrib/web/http.py => scrapy/contrib/web/http.py rename : scrapy/trunk/scrapy/contrib/web/json.py => scrapy/contrib/web/json.py rename : scrapy/trunk/scrapy/contrib/web/service.py => scrapy/contrib/web/service.py rename : scrapy/trunk/scrapy/contrib/web/site.py => scrapy/contrib/web/site.py rename : scrapy/trunk/scrapy/contrib/web/stats.py => scrapy/contrib/web/stats.py rename : scrapy/trunk/scrapy/contrib/webconsole/__init__.py => scrapy/contrib/webconsole/__init__.py rename : scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py => scrapy/contrib/webconsole/enginestatus.py rename : scrapy/trunk/scrapy/contrib/webconsole/livestats.py => scrapy/contrib/webconsole/livestats.py rename : scrapy/trunk/scrapy/contrib/webconsole/scheduler.py => scrapy/contrib/webconsole/scheduler.py rename : scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py => scrapy/contrib/webconsole/spiderctl.py rename : scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py => scrapy/contrib/webconsole/spiderstats.py rename : scrapy/trunk/scrapy/contrib/webconsole/stats.py => scrapy/contrib/webconsole/stats.py rename : scrapy/trunk/scrapy/contrib_exp/__init__.py => scrapy/contrib_exp/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/__init__.py => scrapy/contrib_exp/adaptors/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/date.py => scrapy/contrib_exp/adaptors/date.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/extraction.py => scrapy/contrib_exp/adaptors/extraction.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/markup.py => scrapy/contrib_exp/adaptors/markup.py rename : scrapy/trunk/scrapy/contrib_exp/adaptors/misc.py => scrapy/contrib_exp/adaptors/misc.py rename : scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/__init__.py => scrapy/contrib_exp/downloadermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/downloadermiddleware/decompression.py => scrapy/contrib_exp/downloadermiddleware/decompression.py rename : scrapy/trunk/scrapy/contrib_exp/history/__init__.py => scrapy/contrib_exp/history/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/history/history.py => scrapy/contrib_exp/history/history.py rename : scrapy/trunk/scrapy/contrib_exp/history/middleware.py => scrapy/contrib_exp/history/middleware.py rename : scrapy/trunk/scrapy/contrib_exp/history/scheduler.py => scrapy/contrib_exp/history/scheduler.py rename : scrapy/trunk/scrapy/contrib_exp/history/store.py => scrapy/contrib_exp/history/store.py rename : scrapy/trunk/scrapy/contrib_exp/link/__init__.py => scrapy/contrib_exp/link/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/__init__.py => scrapy/contrib_exp/newitem/__init__.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py => scrapy/contrib_exp/newitem/adaptors.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/fields.py => scrapy/contrib_exp/newitem/fields.py rename : scrapy/trunk/scrapy/contrib_exp/newitem/models.py => scrapy/contrib_exp/newitem/models.py rename : scrapy/trunk/scrapy/contrib_exp/pipeline/shoveitem.py => scrapy/contrib_exp/pipeline/shoveitem.py rename : scrapy/trunk/scrapy/core/__init__.py => scrapy/core/__init__.py rename : scrapy/trunk/scrapy/core/downloader/__init__.py => scrapy/core/downloader/__init__.py rename : scrapy/trunk/scrapy/core/downloader/dnscache.py => scrapy/core/downloader/dnscache.py rename : scrapy/trunk/scrapy/core/downloader/handlers.py => scrapy/core/downloader/handlers.py rename : scrapy/trunk/scrapy/core/downloader/manager.py => scrapy/core/downloader/manager.py rename : scrapy/trunk/scrapy/core/downloader/middleware.py => scrapy/core/downloader/middleware.py rename : scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py => scrapy/core/downloader/responsetypes/__init__.py rename : scrapy/trunk/scrapy/core/downloader/responsetypes/mime.types => scrapy/core/downloader/responsetypes/mime.types rename : scrapy/trunk/scrapy/core/downloader/webclient.py => scrapy/core/downloader/webclient.py rename : scrapy/trunk/scrapy/core/engine.py => scrapy/core/engine.py rename : scrapy/trunk/scrapy/core/exceptions.py => scrapy/core/exceptions.py rename : scrapy/trunk/scrapy/core/manager.py => scrapy/core/manager.py rename : scrapy/trunk/scrapy/core/prioritizers.py => scrapy/core/prioritizers.py rename : scrapy/trunk/scrapy/core/scheduler/__init__.py => scrapy/core/scheduler/__init__.py rename : scrapy/trunk/scrapy/core/scheduler/middleware.py => scrapy/core/scheduler/middleware.py rename : scrapy/trunk/scrapy/core/scheduler/schedulers.py => scrapy/core/scheduler/schedulers.py rename : scrapy/trunk/scrapy/core/scheduler/store.py => scrapy/core/scheduler/store.py rename : scrapy/trunk/scrapy/core/signals.py => scrapy/core/signals.py rename : scrapy/trunk/scrapy/dupefilter/__init__.py => scrapy/dupefilter/__init__.py rename : scrapy/trunk/scrapy/extension/__init__.py => scrapy/extension/__init__.py rename : scrapy/trunk/scrapy/fetcher/__init__.py => scrapy/fetcher/__init__.py rename : scrapy/trunk/scrapy/http/__init__.py => scrapy/http/__init__.py rename : scrapy/trunk/scrapy/http/cookies.py => scrapy/http/cookies.py rename : scrapy/trunk/scrapy/http/headers.py => scrapy/http/headers.py rename : scrapy/trunk/scrapy/http/request/__init__.py => scrapy/http/request/__init__.py rename : scrapy/trunk/scrapy/http/request/form.py => scrapy/http/request/form.py rename : scrapy/trunk/scrapy/http/request/rpc.py => scrapy/http/request/rpc.py rename : scrapy/trunk/scrapy/http/response/__init__.py => scrapy/http/response/__init__.py rename : scrapy/trunk/scrapy/http/response/html.py => scrapy/http/response/html.py rename : scrapy/trunk/scrapy/http/response/text.py => scrapy/http/response/text.py rename : scrapy/trunk/scrapy/http/response/xml.py => scrapy/http/response/xml.py rename : scrapy/trunk/scrapy/http/url.py => scrapy/http/url.py rename : scrapy/trunk/scrapy/item/__init__.py => scrapy/item/__init__.py rename : scrapy/trunk/scrapy/item/adaptors.py => scrapy/item/adaptors.py rename : scrapy/trunk/scrapy/item/models.py => scrapy/item/models.py rename : scrapy/trunk/scrapy/item/pipeline.py => scrapy/item/pipeline.py rename : scrapy/trunk/scrapy/link/__init__.py => scrapy/link/__init__.py rename : scrapy/trunk/scrapy/link/extractors.py => scrapy/link/extractors.py rename : scrapy/trunk/scrapy/log/__init__.py => scrapy/log/__init__.py rename : scrapy/trunk/scrapy/mail/__init__.py => scrapy/mail/__init__.py rename : scrapy/trunk/scrapy/management/__init__.py => scrapy/management/__init__.py rename : scrapy/trunk/scrapy/management/telnet.py => scrapy/management/telnet.py rename : scrapy/trunk/scrapy/management/web.py => scrapy/management/web.py rename : scrapy/trunk/scrapy/patches/__init__.py => scrapy/patches/__init__.py rename : scrapy/trunk/scrapy/patches/monkeypatches.py => scrapy/patches/monkeypatches.py rename : scrapy/trunk/scrapy/spider/__init__.py => scrapy/spider/__init__.py rename : scrapy/trunk/scrapy/spider/manager.py => scrapy/spider/manager.py rename : scrapy/trunk/scrapy/spider/middleware.py => scrapy/spider/middleware.py rename : scrapy/trunk/scrapy/spider/models.py => scrapy/spider/models.py rename : scrapy/trunk/scrapy/stats/__init__.py => scrapy/stats/__init__.py rename : scrapy/trunk/scrapy/stats/corestats.py => scrapy/stats/corestats.py rename : scrapy/trunk/scrapy/stats/statscollector.py => scrapy/stats/statscollector.py rename : scrapy/trunk/scrapy/store/__init__.py => scrapy/store/__init__.py rename : scrapy/trunk/scrapy/store/db.py => scrapy/store/db.py rename : scrapy/trunk/scrapy/templates/project/module/__init__.py => scrapy/templates/project/module/__init__.py rename : scrapy/trunk/scrapy/templates/project/module/items.py.tmpl => scrapy/templates/project/module/items.py.tmpl rename : scrapy/trunk/scrapy/templates/project/module/pipelines.py.tmpl => scrapy/templates/project/module/pipelines.py.tmpl rename : scrapy/trunk/scrapy/templates/project/module/settings.py.tmpl => scrapy/templates/project/module/settings.py.tmpl rename : scrapy/trunk/scrapy/templates/project/module/spiders/__init__.py => scrapy/templates/project/module/spiders/__init__.py rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_basic.tmpl => scrapy/templates/project/module/templates/spider_basic.tmpl rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_crawl.tmpl => scrapy/templates/project/module/templates/spider_crawl.tmpl rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_csvfeed.tmpl => scrapy/templates/project/module/templates/spider_csvfeed.tmpl rename : scrapy/trunk/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl => scrapy/templates/project/module/templates/spider_xmlfeed.tmpl rename : scrapy/trunk/scrapy/templates/project/root/scrapy-ctl.py => scrapy/templates/project/root/scrapy-ctl.py rename : scrapy/trunk/scrapy/tests/__init__.py => scrapy/tests/__init__.py rename : scrapy/trunk/scrapy/tests/run.py => scrapy/tests/run.py rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-ascii.html => scrapy/tests/sample_data/adaptors/enc-ascii.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-cp1252.html => scrapy/tests/sample_data/adaptors/enc-cp1252.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-latin1.html => scrapy/tests/sample_data/adaptors/enc-latin1.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html => scrapy/tests/sample_data/adaptors/enc-utf8-meta-latin1.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/enc-utf8.html => scrapy/tests/sample_data/adaptors/enc-utf8.html rename : scrapy/trunk/scrapy/tests/sample_data/adaptors/extr_unquoted.xml => scrapy/tests/sample_data/adaptors/extr_unquoted.xml rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.tar => scrapy/tests/sample_data/compressed/feed-sample1.tar rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml => scrapy/tests/sample_data/compressed/feed-sample1.xml rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 => scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz => scrapy/tests/sample_data/compressed/feed-sample1.xml.gz rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.zip => scrapy/tests/sample_data/compressed/feed-sample1.zip rename : scrapy/trunk/scrapy/tests/sample_data/compressed/html-gzip.bin => scrapy/tests/sample_data/compressed/html-gzip.bin rename : scrapy/trunk/scrapy/tests/sample_data/compressed/html-rawdeflate.bin => scrapy/tests/sample_data/compressed/html-rawdeflate.bin rename : scrapy/trunk/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin => scrapy/tests/sample_data/compressed/html-zlibdeflate.bin rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml => scrapy/tests/sample_data/feeds/feed-sample1.xml rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml => scrapy/tests/sample_data/feeds/feed-sample2.xml rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.csv => scrapy/tests/sample_data/feeds/feed-sample3.csv rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv => scrapy/tests/sample_data/feeds/feed-sample4.csv rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv => scrapy/tests/sample_data/feeds/feed-sample5.csv rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html => scrapy/tests/sample_data/link_extractor/image_linkextractor.html rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html => scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html => scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html rename : scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html => scrapy/tests/sample_data/link_extractor/regex_linkextractor.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/index.html => scrapy/tests/sample_data/test_site/index.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html => scrapy/tests/sample_data/test_site/item1.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html => scrapy/tests/sample_data/test_site/item2.html rename : scrapy/trunk/scrapy/tests/test_adaptors.py => scrapy/tests/test_adaptors.py rename : scrapy/trunk/scrapy/tests/test_aws.py => scrapy/tests/test_aws.py rename : scrapy/trunk/scrapy/tests/test_c14nurls.py => scrapy/tests/test_c14nurls.py rename : scrapy/trunk/scrapy/tests/test_contrib_response_soup.py => scrapy/tests/test_contrib_response_soup.py rename : scrapy/trunk/scrapy/tests/test_dependencies.py => scrapy/tests/test_dependencies.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_cookies.py => scrapy/tests/test_downloadermiddleware_cookies.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_decompression.py => scrapy/tests/test_downloadermiddleware_decompression.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_httpcompression.py => scrapy/tests/test_downloadermiddleware_httpcompression.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py => scrapy/tests/test_downloadermiddleware_redirect.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_retry.py => scrapy/tests/test_downloadermiddleware_retry.py rename : scrapy/trunk/scrapy/tests/test_downloadermiddleware_useragent.py => scrapy/tests/test_downloadermiddleware_useragent.py rename : scrapy/trunk/scrapy/tests/test_dupefilter.py => scrapy/tests/test_dupefilter.py rename : scrapy/trunk/scrapy/tests/test_engine.py => scrapy/tests/test_engine.py rename : scrapy/trunk/scrapy/tests/test_http_cookies.py => scrapy/tests/test_http_cookies.py rename : scrapy/trunk/scrapy/tests/test_http_headers.py => scrapy/tests/test_http_headers.py rename : scrapy/trunk/scrapy/tests/test_http_request.py => scrapy/tests/test_http_request.py rename : scrapy/trunk/scrapy/tests/test_http_response.py => scrapy/tests/test_http_response.py rename : scrapy/trunk/scrapy/tests/test_http_url.py => scrapy/tests/test_http_url.py rename : scrapy/trunk/scrapy/tests/test_item.py => scrapy/tests/test_item.py rename : scrapy/trunk/scrapy/tests/test_itemadaptor.py => scrapy/tests/test_itemadaptor.py rename : scrapy/trunk/scrapy/tests/test_libxml2.py => scrapy/tests/test_libxml2.py rename : scrapy/trunk/scrapy/tests/test_link.py => scrapy/tests/test_link.py rename : scrapy/trunk/scrapy/tests/test_newitem.py => scrapy/tests/test_newitem.py rename : scrapy/trunk/scrapy/tests/test_pipeline_images.py => scrapy/tests/test_pipeline_images.py rename : scrapy/trunk/scrapy/tests/test_responsetypes.py => scrapy/tests/test_responsetypes.py rename : scrapy/trunk/scrapy/tests/test_robustscrapeditem.py => scrapy/tests/test_robustscrapeditem.py rename : scrapy/trunk/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py => scrapy/tests/test_schedulermiddleware_duplicatesfilter.py rename : scrapy/trunk/scrapy/tests/test_serialization.py => scrapy/tests/test_serialization.py rename : scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py => scrapy/tests/test_spidermiddleware_duplicatesfilter.py rename : scrapy/trunk/scrapy/tests/test_spidermonkey.py => scrapy/tests/test_spidermonkey.py rename : scrapy/trunk/scrapy/tests/test_spiders/__init__.py => scrapy/tests/test_spiders/__init__.py rename : scrapy/trunk/scrapy/tests/test_spiders/testspider.py => scrapy/tests/test_spiders/testspider.py rename : scrapy/trunk/scrapy/tests/test_stats.py => scrapy/tests/test_stats.py rename : scrapy/trunk/scrapy/tests/test_storedb.py => scrapy/tests/test_storedb.py rename : scrapy/trunk/scrapy/tests/test_utils_datatypes.py => scrapy/tests/test_utils_datatypes.py rename : scrapy/trunk/scrapy/tests/test_utils_defer.py => scrapy/tests/test_utils_defer.py rename : scrapy/trunk/scrapy/tests/test_utils_iterators.py => scrapy/tests/test_utils_iterators.py rename : scrapy/trunk/scrapy/tests/test_utils_markup.py => scrapy/tests/test_utils_markup.py rename : scrapy/trunk/scrapy/tests/test_utils_misc.py => scrapy/tests/test_utils_misc.py rename : scrapy/trunk/scrapy/tests/test_utils_python.py => scrapy/tests/test_utils_python.py rename : scrapy/trunk/scrapy/tests/test_utils_request.py => scrapy/tests/test_utils_request.py rename : scrapy/trunk/scrapy/tests/test_utils_response.py => scrapy/tests/test_utils_response.py rename : scrapy/trunk/scrapy/tests/test_utils_url.py => scrapy/tests/test_utils_url.py rename : scrapy/trunk/scrapy/tests/test_webclient.py => scrapy/tests/test_webclient.py rename : scrapy/trunk/scrapy/tests/test_xpath.py => scrapy/tests/test_xpath.py rename : scrapy/trunk/scrapy/tests/test_xpath_extension.py => scrapy/tests/test_xpath_extension.py rename : scrapy/trunk/scrapy/utils/__init__.py => scrapy/utils/__init__.py rename : scrapy/trunk/scrapy/utils/c14n.py => scrapy/utils/c14n.py rename : scrapy/trunk/scrapy/utils/datatypes.py => scrapy/utils/datatypes.py rename : scrapy/trunk/scrapy/utils/db.py => scrapy/utils/db.py rename : scrapy/trunk/scrapy/utils/defer.py => scrapy/utils/defer.py rename : scrapy/trunk/scrapy/utils/display.py => scrapy/utils/display.py rename : scrapy/trunk/scrapy/utils/http.py => scrapy/utils/http.py rename : scrapy/trunk/scrapy/utils/iterators.py => scrapy/utils/iterators.py rename : scrapy/trunk/scrapy/utils/markup.py => scrapy/utils/markup.py rename : scrapy/trunk/scrapy/utils/misc.py => scrapy/utils/misc.py rename : scrapy/trunk/scrapy/utils/python.py => scrapy/utils/python.py rename : scrapy/trunk/scrapy/utils/request.py => scrapy/utils/request.py rename : scrapy/trunk/scrapy/utils/response.py => scrapy/utils/response.py rename : scrapy/trunk/scrapy/utils/serialization.py => scrapy/utils/serialization.py rename : scrapy/trunk/scrapy/utils/test.py => scrapy/utils/test.py rename : scrapy/trunk/scrapy/utils/url.py => scrapy/utils/url.py rename : scrapy/trunk/scrapy/xlib/BeautifulSoup.py => scrapy/xlib/BeautifulSoup.py rename : scrapy/trunk/scrapy/xlib/ClientForm.py => scrapy/xlib/ClientForm.py rename : scrapy/trunk/scrapy/xlib/__init__.py => scrapy/xlib/__init__.py rename : scrapy/trunk/scrapy/xlib/lrucache.py => scrapy/xlib/lrucache.py rename : scrapy/trunk/scrapy/xlib/lsprofcalltree.py => scrapy/xlib/lsprofcalltree.py rename : scrapy/trunk/scrapy/xlib/pydispatch/__init__.py => scrapy/xlib/pydispatch/__init__.py rename : scrapy/trunk/scrapy/xlib/pydispatch/dispatcher.py => scrapy/xlib/pydispatch/dispatcher.py rename : scrapy/trunk/scrapy/xlib/pydispatch/errors.py => scrapy/xlib/pydispatch/errors.py rename : scrapy/trunk/scrapy/xlib/pydispatch/license.txt => scrapy/xlib/pydispatch/license.txt rename : scrapy/trunk/scrapy/xlib/pydispatch/robust.py => scrapy/xlib/pydispatch/robust.py rename : scrapy/trunk/scrapy/xlib/pydispatch/robustapply.py => scrapy/xlib/pydispatch/robustapply.py rename : scrapy/trunk/scrapy/xlib/pydispatch/saferef.py => scrapy/xlib/pydispatch/saferef.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/INSTALL.scrapy => scrapy/xlib/spidermonkey/INSTALL.scrapy rename : scrapy/trunk/scrapy/xlib/spidermonkey/__init__.py => scrapy/xlib/spidermonkey/__init__.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/sm_settings.py => scrapy/xlib/spidermonkey/sm_settings.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/spidermonkey.py => scrapy/xlib/spidermonkey/spidermonkey.py rename : scrapy/trunk/scrapy/xpath/__init__.py => scrapy/xpath/__init__.py rename : scrapy/trunk/scrapy/xpath/constructors.py => scrapy/xpath/constructors.py rename : scrapy/trunk/scrapy/xpath/document.py => scrapy/xpath/document.py rename : scrapy/trunk/scrapy/xpath/extension.py => scrapy/xpath/extension.py rename : scrapy/trunk/scrapy/xpath/selector.py => scrapy/xpath/selector.py rename : scrapy/trunk/scrapy/xpath/types.py => scrapy/xpath/types.py rename : scrapy/trunk/scripts/rpm-install.sh => scripts/rpm-install.sh rename : scrapy/trunk/setup.cfg => setup.cfg rename : scrapy/trunk/setup.py => setup.py
This commit is contained in:
Daniel Grana
parent
2de03b5e2a
commit
ba69c29e37
|
Before Width: | Height: | Size: 62 KiB After Width: | Height: | Size: 62 KiB |
|
Before Width: | Height: | Size: 66 KiB After Width: | Height: | Size: 66 KiB |
|
Before Width: | Height: | Size: 39 KiB After Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 43 KiB After Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 68 KiB After Width: | Height: | Size: 68 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 70 KiB After Width: | Height: | Size: 70 KiB |
|
|
@ -1,32 +0,0 @@
|
|||
You need:
|
||||
* Python 2.5
|
||||
* Twisted
|
||||
* libxml2 python bindings. version 2.6.28 or above is highly recommended.
|
||||
* pyopenssl for HTTPS support
|
||||
* for win32: http://webcleaner.sourceforge.net/pyOpenSSL-0.6.win32-py2.5.exe
|
||||
* spidermonkey (optional) - required for JSParser
|
||||
* simplejson (optional) - required for url2guid web service
|
||||
|
||||
The python interpreter should have sys.getdefaultencoding() == 'utf-8'
|
||||
If this is not the case, you can add the following lines to a file
|
||||
[PYTHON_INSTALL]/site-packages/sitecustomize.py:
|
||||
import sys
|
||||
sys.setdefaultencoding('utf-8')
|
||||
|
||||
|
||||
INSTALLING LIBRARIES
|
||||
--------------------
|
||||
|
||||
The procedure for installing the required third party libraries (twisted,
|
||||
libxml2 and pyopenssl) depends on the platform and OS you use.
|
||||
|
||||
Visit the project homepages for more information about it:
|
||||
|
||||
* Twisted: http://twistedmatrix.com/
|
||||
* libxml2: http://xmlsoft.org/
|
||||
* releases available here: ftp://xmlsoft.org/libxml2/
|
||||
* pyopenssl: http://pyopenssl.sourceforge.net/
|
||||
|
||||
|
||||
In Debian/Ubuntu Linux it's as easy as:
|
||||
apt-get install python-twisted python-libxml2 python-pyopenssl
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
"""
|
||||
Scrapy - a screen scraping framework written in Python
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
import sys, os
|
||||
|
||||
if sys.version_info < (2,5):
|
||||
print "Scrapy %s requires Python 2.5 or above" % __version__
|
||||
sys.exit(1)
|
||||
|
||||
# add external python libraries bundled into scrapy
|
||||
sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "xlib"))
|
||||
|
||||
# monkey patches to fix external library issues
|
||||
from scrapy.patches import monkeypatches
|
||||
monkeypatches.apply_patches()
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
"""Scrapy admin script is used to create new scrapy projects and similar
|
||||
tasks"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from optparse import OptionParser
|
||||
|
||||
import scrapy
|
||||
|
||||
usage = """
|
||||
scrapy-admin.py [options] [command]
|
||||
|
||||
Available commands:
|
||||
|
||||
startproject <project_name>
|
||||
Starts a new project with name 'project_name'
|
||||
"""
|
||||
|
||||
def main():
|
||||
parser = OptionParser(usage=usage)
|
||||
opts, args = parser.parse_args()
|
||||
|
||||
if not args:
|
||||
parser.print_help()
|
||||
|
||||
cmd = args[0]
|
||||
if cmd == "startproject":
|
||||
if len(args) >= 2:
|
||||
project_name = args[1]
|
||||
project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template")
|
||||
shutil.copytree(project_tplpath, project_name)
|
||||
else:
|
||||
print "scrapy-admin.py: missing project name"
|
||||
else:
|
||||
print "scrapy-admin.py: unknown command: %s" % cmd
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
import scrapy
|
||||
from scrapy import log
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.conf import settings
|
||||
|
||||
def find_commands(dir):
|
||||
try:
|
||||
return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and f.endswith('.py')]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def builtin_commands_dict():
|
||||
d = {}
|
||||
scrapy_dir = scrapy.__path__[0]
|
||||
commands_dir = os.path.join(scrapy_dir, 'command', 'commands')
|
||||
for cmdname in find_commands(commands_dir):
|
||||
modname = 'scrapy.command.commands.%s' % cmdname
|
||||
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
|
||||
if callable(command):
|
||||
d[cmdname] = command()
|
||||
else:
|
||||
print 'WARNING: Builtin command module %s exists but Command class not found' % modname
|
||||
return d
|
||||
|
||||
def custom_commands_dict():
|
||||
d = {}
|
||||
cmdsmod = settings['COMMANDS_MODULE']
|
||||
if cmdsmod:
|
||||
mod = __import__(cmdsmod, {}, {}, [''])
|
||||
for cmdname in find_commands(mod.__path__[0]):
|
||||
modname = '%s.%s' % (cmdsmod, cmdname)
|
||||
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
|
||||
if callable(command):
|
||||
d[cmdname] = command()
|
||||
else:
|
||||
print 'WARNING: Custom command module %s exists but Command class not found' % modname
|
||||
return d
|
||||
|
||||
def getcmdname(argv):
|
||||
for arg in argv[1:]:
|
||||
if not arg.startswith('-'):
|
||||
return arg
|
||||
|
||||
def usage(argv):
|
||||
s = "usage: %s <subcommand> [options] [args]\n" % argv[0]
|
||||
s += " %s <subcommand> -h\n\n" % argv[0]
|
||||
s += "Built-in subcommands:\n"
|
||||
|
||||
builtin_cmds = builtin_commands_dict()
|
||||
custom_cmds = custom_commands_dict()
|
||||
|
||||
filtered_builtin_cmds = [(name, cls) for name, cls in builtin_cmds.iteritems() if name not in custom_cmds]
|
||||
|
||||
for cmdname, cmdclass in filtered_builtin_cmds:
|
||||
s += " %s %s\n" % (cmdname, cmdclass.syntax())
|
||||
s += " %s\n" % cmdclass.short_desc()
|
||||
|
||||
if custom_cmds:
|
||||
s += "\n"
|
||||
s += "Custom (or overloaded) subcommands:\n"
|
||||
for cmdname, cmdclass in custom_cmds.iteritems():
|
||||
s += " %s %s\n" % (cmdname, cmdclass.syntax())
|
||||
s += " %s\n" % cmdclass.short_desc()
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def update_defaults(defaults, module):
|
||||
settingsdict = vars(module)
|
||||
for k, v in settingsdict.iteritems():
|
||||
if not k.startswith("_"):
|
||||
defaults[k] = v
|
||||
|
||||
def command_settings(cmdname):
|
||||
try:
|
||||
module = __import__('%s.%s' % ('scrapy.conf.commands', cmdname), {}, {}, [''])
|
||||
update_defaults(settings.defaults, module)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
basepath = settings['COMMANDS_SETTINGS_MODULE']
|
||||
if basepath:
|
||||
try:
|
||||
module = __import__('%s.%s' % (basepath, cmdname), {}, {}, [''])
|
||||
update_defaults(settings.defaults, module)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# This dict holds information about the executed command for later use
|
||||
command_executed = {}
|
||||
|
||||
def execute():
|
||||
execute_with_args(sys.argv)
|
||||
|
||||
def execute_with_args(argv):
|
||||
spiders.load()
|
||||
cmds = builtin_commands_dict()
|
||||
cmds.update(custom_commands_dict())
|
||||
|
||||
cmdname = getcmdname(argv)
|
||||
command_settings(cmdname)
|
||||
|
||||
if not cmdname:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print usage(argv)
|
||||
sys.exit()
|
||||
|
||||
parser = optparse.OptionParser()
|
||||
|
||||
if cmdname in cmds:
|
||||
cmd = cmds[cmdname]
|
||||
cmd.add_options(parser)
|
||||
parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
|
||||
parser.description = cmd.long_desc()
|
||||
else:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print "Unknown command: %s\n" % cmdname
|
||||
print 'Type "%s -h" for help' % argv[0]
|
||||
sys.exit()
|
||||
|
||||
(opts, args) = parser.parse_args(args=argv[1:])
|
||||
del args[0] # args[0] is cmdname
|
||||
|
||||
# storing command executed info for later reference
|
||||
command_executed['name'] = cmdname
|
||||
command_executed['class'] = cmd
|
||||
command_executed['args'] = args[:]
|
||||
command_executed['opts'] = opts.__dict__.copy()
|
||||
|
||||
cmd.process_options(args, opts)
|
||||
log.start() # start logging
|
||||
if opts.profile:
|
||||
log.msg("Profiling enabled. Analyze later with: python -m pstats %s" % opts.profile)
|
||||
import cProfile
|
||||
loc = locals()
|
||||
p = cProfile.Profile()
|
||||
p.runctx('ret = cmd.run(args, opts)', globals(), loc)
|
||||
p.dump_stats(opts.profile)
|
||||
try:
|
||||
import lsprofcalltree
|
||||
fn = opts.profile + ".cachegrind"
|
||||
k = lsprofcalltree.KCacheGrind(p)
|
||||
with open(fn, 'w') as f:
|
||||
k.output(f)
|
||||
except ImportError:
|
||||
pass
|
||||
ret = loc['ret']
|
||||
else:
|
||||
ret = cmd.run(args, opts)
|
||||
if ret is False:
|
||||
parser.print_help()
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.replay import Replay
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] [domain|url] ..."
|
||||
|
||||
def short_desc(self):
|
||||
return "Run the web scraping engine from the command line"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--nocache", dest="nocache", action="store_true", help="disable HTTP cache")
|
||||
parser.add_option("--nopipeline", dest="nopipeline", action="store_true", help="disable scraped item pipeline")
|
||||
parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls")
|
||||
parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE")
|
||||
parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
if opts.nopipeline:
|
||||
settings.overrides['ITEM_PIPELINES'] = []
|
||||
|
||||
if opts.nocache:
|
||||
settings.overrides['CACHE2_DIR'] = None
|
||||
|
||||
if opts.restrict:
|
||||
settings.overrides['RESTRICT_TO_URLS'] = args
|
||||
|
||||
if opts.record or opts.recorddir:
|
||||
# self.replay is used for preventing Replay signals handler from
|
||||
# disconnecting since pydispatcher uses weak references
|
||||
self.replay = Replay(opts.record or opts.recorddir, mode='record', usedir=bool(opts.recorddir))
|
||||
self.replay.record(args=args, opts=opts.__dict__)
|
||||
|
||||
def run(self, args, opts):
|
||||
scrapymanager.runonce(*args, **opts.__dict__)
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
import os
|
||||
import string
|
||||
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
""" Default template file name """
|
||||
template_name = 'spider.tmpl'
|
||||
|
||||
""" Childs can define custom tvars """
|
||||
custom_tvars = {}
|
||||
|
||||
def syntax(self):
|
||||
return "<spider_name> <spider_domain_name>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Generate new spider based on predefined template"
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 2:
|
||||
return False
|
||||
|
||||
name = args[0]
|
||||
site = args[1]
|
||||
spiders_dict = spiders.asdict()
|
||||
if not name in spiders_dict.keys():
|
||||
self._genspider(name, site)
|
||||
else:
|
||||
print "Spider '%s' exist" % name
|
||||
|
||||
def _genspider(self, name, site):
|
||||
""" Generate spider """
|
||||
tvars = {
|
||||
'name': name,
|
||||
'site': site,
|
||||
'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('-')])
|
||||
}
|
||||
tvars.update(self.custom_tvars)
|
||||
|
||||
spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, [''])
|
||||
spidersdir = os.path.abspath(os.path.dirname(spiders_module.__file__))
|
||||
if name[0] not in string.letters: # must start with a letter, for valid python modules
|
||||
name = "a" + name
|
||||
name = name.replace('-', '_') # - are replaced by _, for valid python modules
|
||||
self._genfiles(self.template_name, '%s/%s.py' % (spidersdir, name), tvars)
|
||||
|
||||
def _genfiles(self, template_name, source_name, tvars):
|
||||
""" Generate source from template, substitute variables """
|
||||
template_file = os.path.join(settings['TEMPLATES_DIR'], template_name)
|
||||
tmpl = open(template_file)
|
||||
clines = []
|
||||
for l in tmpl.readlines():
|
||||
for key, val in tvars.items():
|
||||
l = l.replace('@%s@' % key, val)
|
||||
clines.append(l)
|
||||
tmpl.close()
|
||||
source = ''.join(clines)
|
||||
if not os.path.exists(source_name):
|
||||
sfile = open(source_name, "w")
|
||||
sfile.write(source)
|
||||
sfile.close()
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.fetcher import fetch
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
def get_item_attr(pagedata, attr="guid"):
|
||||
spider = spiders.fromurl(pagedata.url)
|
||||
items = spider.parse(pagedata)
|
||||
attrs = [getattr(i, attr) for i in items if isinstance(i, ScrapedItem)]
|
||||
return attrs
|
||||
|
||||
def get_attr(url, attr="guid"):
|
||||
pagedatas = fetch([url])
|
||||
if pagedatas:
|
||||
return get_item_attr(pagedatas[0], attr)
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<url> <attribute>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Print an attribute from the item scraped in the given URL"
|
||||
|
||||
def run(self, args, opts):
|
||||
print get_attr(args[0], args[1])
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.fetcher import fetch
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.utils import display
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <url>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Parse the URL and print their results"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items")
|
||||
parser.add_option("--identify", dest="identify", action="store_true", help="try to use identify instead of parse")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output")
|
||||
|
||||
def pipeline_process(self, item, opts):
|
||||
return item
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A URL is required"
|
||||
return
|
||||
|
||||
responses = fetch([args[0]])
|
||||
if responses:
|
||||
response = responses[0]
|
||||
spider = spiders.fromurl(response.url)
|
||||
result = spider.parse(response) if not opts.identify else spider.identify(response)
|
||||
|
||||
items = [self.pipeline_process(i, opts) for i in result if isinstance(i, ScrapedItem)]
|
||||
links = [i for i in result if isinstance(i, Request)]
|
||||
|
||||
display.nocolour = opts.nocolour
|
||||
if not opts.noitems:
|
||||
print "# Scraped Items", "-"*60
|
||||
display.pprint(items)
|
||||
|
||||
if not opts.nolinks:
|
||||
print "# Links", "-"*68
|
||||
display.pprint(links)
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
import os
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.replay import Replay
|
||||
from scrapy.utils import display
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command import cmdline
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <replay_file> [action]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Replay a session previously recorded with crawl --record"
|
||||
|
||||
def help(self):
|
||||
s = "Replay a session previously recorded with crawl --record\n"
|
||||
s += "\n"
|
||||
s += "Available actions:\n"
|
||||
s += " crawl: just replay the crawl (default if action omitted)\n"
|
||||
s += " diff: replay the crawl and show differences in items scraped/passed\n"
|
||||
s += " update: replay the crawl and update both scraped and passed items\n"
|
||||
s += " showitems: show items stored\n"
|
||||
s += " showpages: show all responses downloaded (not only HTML pages)\n"
|
||||
return s
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show verbose output (full items/responses)")
|
||||
parser.add_option("-t", "--item-type", dest="itype", help="item type (scraped, passed). default: scraped", metavar="TYPE")
|
||||
parser.add_option("--output", dest="outfile", help="write output to FILE. if omitted uses stdout", metavar="FILE")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true", help="disable colorized output (for console only)")
|
||||
parser.add_option("-i", "--ignore", dest="ignores", action="append", help="item attribute to ignore. can be passed multiple times", metavar="ATTR")
|
||||
parser.add_option("--target", dest="targets", action="append", help="crawl TARGET instead of recorded urls/domains. can be passed multiple times")
|
||||
# adding option to update
|
||||
parser.add_option("--pages", dest="pages", action="store_true", help="update all the pages in the replay file, recording it again.")
|
||||
parser.add_option("-q", "--quiet", dest="quiet", action="store_true", help="no verbose mode in option diff.")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
if args:
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
self.opts = opts
|
||||
self.action = args[1] if len(args) > 1 else 'crawl'
|
||||
mode = 'update' if self.action == 'update' else 'play'
|
||||
usedir = args and os.path.isdir(args[0])
|
||||
self.replay = Replay(args[0], mode=mode, usedir=usedir)
|
||||
if self.action not in ['crawl', 'diff', 'update']:
|
||||
settings.overrides['LOG_ENABLED'] = False
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A <replay_dir> is required"
|
||||
return
|
||||
|
||||
display.nocolour = opts.nocolour
|
||||
|
||||
if opts.itype == 'passed':
|
||||
self.before_db = self.replay.passed_old
|
||||
self.now_db = self.replay.passed_new
|
||||
else: # default is 'scraped'
|
||||
opts.itype = 'scraped'
|
||||
self.before_db = self.replay.scraped_old
|
||||
self.now_db = self.replay.scraped_new
|
||||
|
||||
actionfunc = getattr(self, 'action_%s' % self.action, None)
|
||||
if actionfunc:
|
||||
rep = actionfunc(opts)
|
||||
if rep:
|
||||
if opts.outfile:
|
||||
f = open(opts.outfile, "w")
|
||||
f.write(rep)
|
||||
f.close()
|
||||
else:
|
||||
print rep,
|
||||
self.replay.cleanup()
|
||||
else:
|
||||
print "Unknown replay action: %s" % self.action
|
||||
|
||||
def action_crawl(self, opts):
|
||||
self.replay.play(args=opts.targets)
|
||||
|
||||
def action_update(self, opts):
|
||||
self.replay.update(args=opts.targets, opts=opts.__dict__)
|
||||
if (opts.pages):
|
||||
args = ['scrapy-crawl', 'crawl']
|
||||
args.extend(self.replay.options['args'])
|
||||
for k in self.replay.options['opts']:
|
||||
if self.replay.options['opts'][k]:
|
||||
args.append("--%s" % k)
|
||||
if self.replay.options['opts'][k] != True:
|
||||
args.append(self.replay.options['opts'][k])
|
||||
cmdline.execute_with_args(args)
|
||||
|
||||
def action_showitems(self, opts):
|
||||
s = ""
|
||||
s += self._format_items(self.before_db.values())
|
||||
s += ">>> Total: %d items %s\n" % (len(self.before_db), opts.itype)
|
||||
return s
|
||||
|
||||
def action_showpages(self, opts):
|
||||
s = ""
|
||||
if self.opts.verbose:
|
||||
for r in self.replay.responses_old.values():
|
||||
s += ">>> %s\n" % str(r)
|
||||
s += display.pformat(r)
|
||||
else:
|
||||
s += "\n".join([str(r) for r in self.replay.responses_old.values()]) + "\n"
|
||||
s += ">>> Total: %d responses received\n" % len(self.replay.responses_old)
|
||||
return s
|
||||
|
||||
def action_diff(self, opts):
|
||||
self.action_crawl(opts)
|
||||
|
||||
guids_before = set(self.before_db.keys())
|
||||
guids_now = set(map(str, self.now_db.keys()))
|
||||
|
||||
guids_new = guids_now - guids_before
|
||||
guids_missing = guids_before - guids_now
|
||||
guids_both = guids_now & guids_before
|
||||
|
||||
changed_items, chreport = self._report_differences(self.before_db, self.now_db, guids_both)
|
||||
|
||||
ok_items = len(guids_both) - changed_items
|
||||
new_items = len(guids_new)
|
||||
missing_items = len(guids_missing)
|
||||
|
||||
if (new_items - missing_items - changed_items) == 0 and opts.quiet:
|
||||
s = ""
|
||||
else:
|
||||
s = "CRAWLING DIFFERENCES REPORT\n\n"
|
||||
|
||||
s += "Total items : %d\n" % (len(guids_both) + new_items + missing_items)
|
||||
s += " Items OK : %d\n" % ok_items
|
||||
s += " New items : %d\n" % new_items
|
||||
s += " Missing items : %d\n" % missing_items
|
||||
s += " Changed items : %d\n" % changed_items
|
||||
|
||||
s += "\n"
|
||||
s += "- NEW ITEMS (%d) -----------------------------------------\n" % new_items
|
||||
s += self._format_items([self.now_db[g] for g in guids_new])
|
||||
|
||||
s += "\n"
|
||||
s += "- MISSING ITEMS (%d) -------------------------------------\n" % missing_items
|
||||
s += self._format_items([self.before_db[g] for g in guids_missing])
|
||||
|
||||
s += "\n"
|
||||
s += "- CHANGED ITEMS (%d) -------------------------------------\n" % changed_items
|
||||
|
||||
s += chreport
|
||||
|
||||
return s
|
||||
|
||||
def _report_differences(self, old_items, new_items, guids):
|
||||
items_old = [old_items[g] for g in guids]
|
||||
items_new = [new_items[g] for g in guids]
|
||||
|
||||
c = 0
|
||||
s = ""
|
||||
for old, new in zip(items_old, items_new):
|
||||
d = self._item_diff(old, new)
|
||||
if d:
|
||||
c += 1
|
||||
s += d
|
||||
return c, s
|
||||
|
||||
def _item_diff(self, old, new):
|
||||
delta = new - old
|
||||
|
||||
s = ""
|
||||
if delta.diff:
|
||||
s += ">>> Item guid=%s name=%s\n" % (old.guid, old.name)
|
||||
s += display.pformat(delta.diff) + "\n"
|
||||
return s
|
||||
|
||||
def _format_items(self, items):
|
||||
if self.opts.verbose:
|
||||
s = display.pformat(items)
|
||||
else:
|
||||
s = ""
|
||||
for i in items:
|
||||
s += "%s\n" % str(i)
|
||||
s += " <%s>\n" % i.url
|
||||
return s
|
||||
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
from twisted.internet import reactor
|
||||
|
||||
import scrapy
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector
|
||||
from scrapy.utils.misc import load_class
|
||||
from scrapy.extension import extensions
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.core.downloader.handlers import download_any
|
||||
from scrapy.fetcher import get_or_create_spider
|
||||
from scrapy.utils.decompressor import Decompressor
|
||||
|
||||
#This code comes from twisted 8. We define here while
|
||||
#using old twisted version.
|
||||
def blockingCallFromThread(reactor, f, *a, **kw):
|
||||
"""
|
||||
Run a function in the reactor from a thread, and wait for the result
|
||||
synchronously, i.e. until the callback chain returned by the function
|
||||
get a result.
|
||||
|
||||
@param reactor: The L{IReactorThreads} provider which will be used to
|
||||
schedule the function call.
|
||||
@param f: the callable to run in the reactor thread
|
||||
@type f: any callable.
|
||||
@param a: the arguments to pass to C{f}.
|
||||
@param kw: the keyword arguments to pass to C{f}.
|
||||
|
||||
@return: the result of the callback chain.
|
||||
@raise: any error raised during the callback chain.
|
||||
"""
|
||||
import Queue
|
||||
from twisted.python import failure
|
||||
from twisted.internet import defer
|
||||
queue = Queue.Queue()
|
||||
def _callFromThread():
|
||||
result = defer.maybeDeferred(f, *a, **kw)
|
||||
result.addBoth(queue.put)
|
||||
reactor.callFromThread(_callFromThread)
|
||||
result = queue.get()
|
||||
if isinstance(result, failure.Failure):
|
||||
result.raiseException()
|
||||
return result
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[url]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Interactive scraping console"
|
||||
|
||||
def long_desc(self):
|
||||
return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html"
|
||||
|
||||
def update_vars(self):
|
||||
""" You can use this function to update the local variables that will be available in the scrape console """
|
||||
pass
|
||||
|
||||
def get_url(self, url, decompress=False):
|
||||
|
||||
print "Downloading URL... ",
|
||||
r = Request(url)
|
||||
spider = get_or_create_spider(url)
|
||||
try:
|
||||
result = blockingCallFromThread(reactor, download_any, r, spider)
|
||||
if isinstance(result, Response):
|
||||
print "Done."
|
||||
if decompress:
|
||||
print "Decompressing response...",
|
||||
d = Decompressor()
|
||||
result = d.extract(result)
|
||||
print "Done."
|
||||
result.request = r
|
||||
self.generate_vars(url, result)
|
||||
return True
|
||||
except Exception, e:
|
||||
print "Error: %s" % e
|
||||
|
||||
def generate_vars(self, url, response):
|
||||
itemcls = load_class(settings['DEFAULT_ITEM_CLASS'])
|
||||
item = itemcls()
|
||||
self.vars['item'] = item
|
||||
if url:
|
||||
self.vars['xxs'] = XmlXPathSelector(response)
|
||||
self.vars['hxs'] = HtmlXPathSelector(response)
|
||||
self.vars['url'] = url
|
||||
self.vars['response'] = response
|
||||
self.vars['spider'] = spiders.fromurl(url)
|
||||
self.update_vars()
|
||||
self.user_ns.update(self.vars)
|
||||
self.print_vars()
|
||||
|
||||
def print_vars(self):
|
||||
print '-' * 78
|
||||
print "Available local variables:"
|
||||
for key, val in self.vars.iteritems():
|
||||
if isinstance(val, basestring):
|
||||
print " %s: %s" % (key, val)
|
||||
else:
|
||||
print " %s: %s" % (key, val.__class__)
|
||||
print "Available commands:"
|
||||
print " get <url>: Fetches an url and updates all variables."
|
||||
print " getd <url>: Similar to get, but filter with decompress."
|
||||
print " scrapehelp: Prints this help."
|
||||
print '-' * 78
|
||||
|
||||
def run(self, args, opts):
|
||||
self.vars = {}
|
||||
self.user_ns = {}
|
||||
url = None
|
||||
if args:
|
||||
url = args[0]
|
||||
|
||||
print "Scrapy %s - Interactive scraping console\n" % scrapy.__version__
|
||||
|
||||
print "Enabling Scrapy extensions...",
|
||||
extensions.load()
|
||||
print "done"
|
||||
|
||||
def _console_thread():
|
||||
|
||||
def _get_magic(shell, arg):
|
||||
self.get_url(arg.strip())
|
||||
def _help_magic(shell, _):
|
||||
self.print_vars()
|
||||
def _getd_magic(shell, arg):
|
||||
self.get_url(arg.strip(), decompress=True)
|
||||
|
||||
if url:
|
||||
result = self.get_url(url)
|
||||
if not result:
|
||||
self.generate_vars(None, None)
|
||||
else:
|
||||
self.generate_vars(None, None)
|
||||
try: # use IPython if available
|
||||
import IPython
|
||||
shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns)
|
||||
ip = shell.IP.getapi()
|
||||
ip.expose_magic("get", _get_magic)
|
||||
ip.expose_magic("getd", _getd_magic)
|
||||
ip.expose_magic("scrapehelp", _help_magic)
|
||||
shell.mainloop()
|
||||
reactor.callFromThread(scrapymanager.stop)
|
||||
except ImportError:
|
||||
import code
|
||||
try: # readline module is only available on unix systems
|
||||
import readline
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
import rlcompleter
|
||||
readline.parse_and_bind("tab:complete")
|
||||
code.interact(local=self.vars)
|
||||
|
||||
reactor.callInThread(_console_thread)
|
||||
scrapymanager.start()
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
import pprint
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<domain> [domain ...]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Show all stats history stored for the given domain(s)"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-p", "--path", dest="path", help="restrict stats to PATH", metavar="PATH")
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A domain is required"
|
||||
return
|
||||
if not settings['SCRAPING_DB']:
|
||||
print "SCRAPING_DB setting is required for this command"
|
||||
return
|
||||
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history')
|
||||
|
||||
for domain in args:
|
||||
print "# %s" % domain
|
||||
pprint.pprint(list(ddh.getall(domain, opts.path)))
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue