mirror of https://github.com/scrapy/scrapy.git
creating cluster-refactor branch to fix several deficiences that the current
cluster code has --HG-- rename : scrapy/trunk/INSTALL => scrapy/branches/cluster-refactor/INSTALL rename : scrapy/trunk/README => scrapy/branches/cluster-refactor/README rename : scrapy/trunk/docs/scrapy-architecture.dia => scrapy/branches/cluster-refactor/docs/scrapy-architecture.dia rename : scrapy/trunk/extras/sql/scraping.sql => scrapy/branches/cluster-refactor/extras/sql/scraping.sql rename : scrapy/trunk/scrapy/__init__.py => scrapy/branches/cluster-refactor/scrapy/__init__.py rename : scrapy/trunk/scrapy/bin/scrapy-admin.py => scrapy/branches/cluster-refactor/scrapy/bin/scrapy-admin.py rename : scrapy/trunk/scrapy/command/__init__.py => scrapy/branches/cluster-refactor/scrapy/command/__init__.py rename : scrapy/trunk/scrapy/command/cmdline.py => scrapy/branches/cluster-refactor/scrapy/command/cmdline.py rename : scrapy/trunk/scrapy/command/commands/__init__.py => scrapy/branches/cluster-refactor/scrapy/command/commands/__init__.py rename : scrapy/trunk/scrapy/command/commands/crawl.py => scrapy/branches/cluster-refactor/scrapy/command/commands/crawl.py rename : scrapy/trunk/scrapy/command/commands/download.py => scrapy/branches/cluster-refactor/scrapy/command/commands/download.py rename : scrapy/trunk/scrapy/command/commands/genspider.py => scrapy/branches/cluster-refactor/scrapy/command/commands/genspider.py rename : scrapy/trunk/scrapy/command/commands/getattr.py => scrapy/branches/cluster-refactor/scrapy/command/commands/getattr.py rename : scrapy/trunk/scrapy/command/commands/help.py => scrapy/branches/cluster-refactor/scrapy/command/commands/help.py rename : scrapy/trunk/scrapy/command/commands/list.py => scrapy/branches/cluster-refactor/scrapy/command/commands/list.py rename : scrapy/trunk/scrapy/command/commands/log.py => scrapy/branches/cluster-refactor/scrapy/command/commands/log.py rename : scrapy/trunk/scrapy/command/commands/parse.py => scrapy/branches/cluster-refactor/scrapy/command/commands/parse.py rename : scrapy/trunk/scrapy/command/commands/replay.py => scrapy/branches/cluster-refactor/scrapy/command/commands/replay.py rename : scrapy/trunk/scrapy/command/commands/shell.py => scrapy/branches/cluster-refactor/scrapy/command/commands/shell.py rename : scrapy/trunk/scrapy/command/commands/start.py => scrapy/branches/cluster-refactor/scrapy/command/commands/start.py rename : scrapy/trunk/scrapy/command/commands/stats.py => scrapy/branches/cluster-refactor/scrapy/command/commands/stats.py rename : scrapy/trunk/scrapy/command/models.py => scrapy/branches/cluster-refactor/scrapy/command/models.py rename : scrapy/trunk/scrapy/conf/__init__.py => scrapy/branches/cluster-refactor/scrapy/conf/__init__.py rename : scrapy/trunk/scrapy/conf/commands/__init__.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/__init__.py rename : scrapy/trunk/scrapy/conf/commands/crawl.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/crawl.py rename : scrapy/trunk/scrapy/conf/commands/help.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/help.py rename : scrapy/trunk/scrapy/conf/commands/list.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/list.py rename : scrapy/trunk/scrapy/conf/commands/log.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/log.py rename : scrapy/trunk/scrapy/conf/commands/scrape.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/scrape.py rename : scrapy/trunk/scrapy/conf/commands/stats.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/stats.py rename : scrapy/trunk/scrapy/conf/commands/test.py => scrapy/branches/cluster-refactor/scrapy/conf/commands/test.py rename : scrapy/trunk/scrapy/conf/core_settings.py => scrapy/branches/cluster-refactor/scrapy/conf/core_settings.py rename : scrapy/trunk/scrapy/conf/project_template/__init__.py => scrapy/branches/cluster-refactor/scrapy/conf/project_template/__init__.py rename : scrapy/trunk/scrapy/conf/project_template/items.py => scrapy/branches/cluster-refactor/scrapy/conf/project_template/items.py rename : scrapy/trunk/scrapy/conf/project_template/scrapy-ctl.py => scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy-ctl.py rename : scrapy/trunk/scrapy/conf/project_template/scrapy_settings.py => scrapy/branches/cluster-refactor/scrapy/conf/project_template/scrapy_settings.py rename : scrapy/trunk/scrapy/conf/project_template/spiders/__init__.py => scrapy/branches/cluster-refactor/scrapy/conf/project_template/spiders/__init__.py rename : scrapy/trunk/scrapy/contrib/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/__init__.py rename : scrapy/trunk/scrapy/contrib/adaptorpipeline.py => scrapy/branches/cluster-refactor/scrapy/contrib/adaptorpipeline.py rename : scrapy/trunk/scrapy/contrib/closedomain.py => scrapy/branches/cluster-refactor/scrapy/contrib/closedomain.py rename : scrapy/trunk/scrapy/contrib/debug.py => scrapy/branches/cluster-refactor/scrapy/contrib/debug.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cache.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/common.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/compression.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/compression.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/cookies.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/debug.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/errorpages.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/httpauth.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/redirect.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/retry.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/robots.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/robots.py rename : scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py => scrapy/branches/cluster-refactor/scrapy/contrib/downloadermiddleware/useragent.py rename : scrapy/trunk/scrapy/contrib/groupsettings.py => scrapy/branches/cluster-refactor/scrapy/contrib/groupsettings.py rename : scrapy/trunk/scrapy/contrib/history/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/history/__init__.py rename : scrapy/trunk/scrapy/contrib/history/history.py => scrapy/branches/cluster-refactor/scrapy/contrib/history/history.py rename : scrapy/trunk/scrapy/contrib/history/middleware.py => scrapy/branches/cluster-refactor/scrapy/contrib/history/middleware.py rename : scrapy/trunk/scrapy/contrib/history/scheduler.py => scrapy/branches/cluster-refactor/scrapy/contrib/history/scheduler.py rename : scrapy/trunk/scrapy/contrib/history/store.py => scrapy/branches/cluster-refactor/scrapy/contrib/history/store.py rename : scrapy/trunk/scrapy/contrib/item/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/item/__init__.py rename : scrapy/trunk/scrapy/contrib/item/models.py => scrapy/branches/cluster-refactor/scrapy/contrib/item/models.py rename : scrapy/trunk/scrapy/contrib/memdebug.py => scrapy/branches/cluster-refactor/scrapy/contrib/memdebug.py rename : scrapy/trunk/scrapy/contrib/memusage.py => scrapy/branches/cluster-refactor/scrapy/contrib/memusage.py rename : scrapy/trunk/scrapy/contrib/pbcluster/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/__init__.py rename : scrapy/trunk/scrapy/contrib/pbcluster/crawler/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/crawler/__init__.py rename : scrapy/trunk/scrapy/contrib/pbcluster/crawler/manager.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/crawler/manager.py rename : scrapy/trunk/scrapy/contrib/pbcluster/master/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/master/__init__.py rename : scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/master/manager.py rename : scrapy/trunk/scrapy/contrib/pbcluster/master/web.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/master/web.py rename : scrapy/trunk/scrapy/contrib/pbcluster/master/ws_api.txt => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/master/ws_api.txt rename : scrapy/trunk/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/tools/scrapy-cluster-ctl.py rename : scrapy/trunk/scrapy/contrib/pbcluster/worker/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/worker/__init__.py rename : scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/worker/manager.py rename : scrapy/trunk/scrapy/contrib/pbcluster/worker/testworker.py => scrapy/branches/cluster-refactor/scrapy/contrib/pbcluster/worker/testworker.py rename : scrapy/trunk/scrapy/contrib/pipeline/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/__init__.py rename : scrapy/trunk/scrapy/contrib/pipeline/images.py => scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/images.py rename : scrapy/trunk/scrapy/contrib/pipeline/media.py => scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/media.py rename : scrapy/trunk/scrapy/contrib/pipeline/s3images.py => scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/s3images.py rename : scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py => scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/shoveitem.py rename : scrapy/trunk/scrapy/contrib/pipeline/show.py => scrapy/branches/cluster-refactor/scrapy/contrib/pipeline/show.py rename : scrapy/trunk/scrapy/contrib/prioritizers.py => scrapy/branches/cluster-refactor/scrapy/contrib/prioritizers.py rename : scrapy/trunk/scrapy/contrib/response/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/response/__init__.py rename : scrapy/trunk/scrapy/contrib/response/soup.py => scrapy/branches/cluster-refactor/scrapy/contrib/response/soup.py rename : scrapy/trunk/scrapy/contrib/spider/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/spider/__init__.py rename : scrapy/trunk/scrapy/contrib/spider/profiler.py => scrapy/branches/cluster-refactor/scrapy/contrib/spider/profiler.py rename : scrapy/trunk/scrapy/contrib/spider/reloader.py => scrapy/branches/cluster-refactor/scrapy/contrib/spider/reloader.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/__init__.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/depth.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/limit.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/offsite.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/referer.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/restrict.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urlfilter.py rename : scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py => scrapy/branches/cluster-refactor/scrapy/contrib/spidermiddleware/urllength.py rename : scrapy/trunk/scrapy/contrib/spiders.py => scrapy/branches/cluster-refactor/scrapy/contrib/spiders.py rename : scrapy/trunk/scrapy/contrib/web/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/web/__init__.py rename : scrapy/trunk/scrapy/contrib/web/http.py => scrapy/branches/cluster-refactor/scrapy/contrib/web/http.py rename : scrapy/trunk/scrapy/contrib/web/json.py => scrapy/branches/cluster-refactor/scrapy/contrib/web/json.py rename : scrapy/trunk/scrapy/contrib/web/service.py => scrapy/branches/cluster-refactor/scrapy/contrib/web/service.py rename : scrapy/trunk/scrapy/contrib/web/site.py => scrapy/branches/cluster-refactor/scrapy/contrib/web/site.py rename : scrapy/trunk/scrapy/contrib/web/stats.py => scrapy/branches/cluster-refactor/scrapy/contrib/web/stats.py rename : scrapy/trunk/scrapy/contrib/webconsole/__init__.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/__init__.py rename : scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/enginestatus.py rename : scrapy/trunk/scrapy/contrib/webconsole/livestats.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/livestats.py rename : scrapy/trunk/scrapy/contrib/webconsole/schedstats.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/schedstats.py rename : scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderctl.py rename : scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/spiderstats.py rename : scrapy/trunk/scrapy/contrib/webconsole/stats.py => scrapy/branches/cluster-refactor/scrapy/contrib/webconsole/stats.py rename : scrapy/trunk/scrapy/core/__init__.py => scrapy/branches/cluster-refactor/scrapy/core/__init__.py rename : scrapy/trunk/scrapy/core/downloader/__init__.py => scrapy/branches/cluster-refactor/scrapy/core/downloader/__init__.py rename : scrapy/trunk/scrapy/core/downloader/handlers.py => scrapy/branches/cluster-refactor/scrapy/core/downloader/handlers.py rename : scrapy/trunk/scrapy/core/downloader/manager.py => scrapy/branches/cluster-refactor/scrapy/core/downloader/manager.py rename : scrapy/trunk/scrapy/core/downloader/middleware.py => scrapy/branches/cluster-refactor/scrapy/core/downloader/middleware.py rename : scrapy/trunk/scrapy/core/engine.py => scrapy/branches/cluster-refactor/scrapy/core/engine.py rename : scrapy/trunk/scrapy/core/exceptions.py => scrapy/branches/cluster-refactor/scrapy/core/exceptions.py rename : scrapy/trunk/scrapy/core/manager.py => scrapy/branches/cluster-refactor/scrapy/core/manager.py rename : scrapy/trunk/scrapy/core/prioritizers.py => scrapy/branches/cluster-refactor/scrapy/core/prioritizers.py rename : scrapy/trunk/scrapy/core/scheduler/__init__.py => scrapy/branches/cluster-refactor/scrapy/core/scheduler/__init__.py rename : scrapy/trunk/scrapy/core/scheduler/filter.py => scrapy/branches/cluster-refactor/scrapy/core/scheduler/filter.py rename : scrapy/trunk/scrapy/core/scheduler/schedulers.py => scrapy/branches/cluster-refactor/scrapy/core/scheduler/schedulers.py rename : scrapy/trunk/scrapy/core/scheduler/store.py => scrapy/branches/cluster-refactor/scrapy/core/scheduler/store.py rename : scrapy/trunk/scrapy/core/signals.py => scrapy/branches/cluster-refactor/scrapy/core/signals.py rename : scrapy/trunk/scrapy/extension/__init__.py => scrapy/branches/cluster-refactor/scrapy/extension/__init__.py rename : scrapy/trunk/scrapy/fetcher/__init__.py => scrapy/branches/cluster-refactor/scrapy/fetcher/__init__.py rename : scrapy/trunk/scrapy/http/__init__.py => scrapy/branches/cluster-refactor/scrapy/http/__init__.py rename : scrapy/trunk/scrapy/http/headers.py => scrapy/branches/cluster-refactor/scrapy/http/headers.py rename : scrapy/trunk/scrapy/http/request.py => scrapy/branches/cluster-refactor/scrapy/http/request.py rename : scrapy/trunk/scrapy/http/response.py => scrapy/branches/cluster-refactor/scrapy/http/response.py rename : scrapy/trunk/scrapy/http/url.py => scrapy/branches/cluster-refactor/scrapy/http/url.py rename : scrapy/trunk/scrapy/item/__init__.py => scrapy/branches/cluster-refactor/scrapy/item/__init__.py rename : scrapy/trunk/scrapy/item/adaptors.py => scrapy/branches/cluster-refactor/scrapy/item/adaptors.py rename : scrapy/trunk/scrapy/item/models.py => scrapy/branches/cluster-refactor/scrapy/item/models.py rename : scrapy/trunk/scrapy/item/pipeline.py => scrapy/branches/cluster-refactor/scrapy/item/pipeline.py rename : scrapy/trunk/scrapy/link/__init__.py => scrapy/branches/cluster-refactor/scrapy/link/__init__.py rename : scrapy/trunk/scrapy/link/extractors.py => scrapy/branches/cluster-refactor/scrapy/link/extractors.py rename : scrapy/trunk/scrapy/log/__init__.py => scrapy/branches/cluster-refactor/scrapy/log/__init__.py rename : scrapy/trunk/scrapy/mail/__init__.py => scrapy/branches/cluster-refactor/scrapy/mail/__init__.py rename : scrapy/trunk/scrapy/management/__init__.py => scrapy/branches/cluster-refactor/scrapy/management/__init__.py rename : scrapy/trunk/scrapy/management/telnet.py => scrapy/branches/cluster-refactor/scrapy/management/telnet.py rename : scrapy/trunk/scrapy/management/web.py => scrapy/branches/cluster-refactor/scrapy/management/web.py rename : scrapy/trunk/scrapy/patches/__init__.py => scrapy/branches/cluster-refactor/scrapy/patches/__init__.py rename : scrapy/trunk/scrapy/patches/monkeypatches.py => scrapy/branches/cluster-refactor/scrapy/patches/monkeypatches.py rename : scrapy/trunk/scrapy/replay/__init__.py => scrapy/branches/cluster-refactor/scrapy/replay/__init__.py rename : scrapy/trunk/scrapy/spider/__init__.py => scrapy/branches/cluster-refactor/scrapy/spider/__init__.py rename : scrapy/trunk/scrapy/spider/manager.py => scrapy/branches/cluster-refactor/scrapy/spider/manager.py rename : scrapy/trunk/scrapy/spider/middleware.py => scrapy/branches/cluster-refactor/scrapy/spider/middleware.py rename : scrapy/trunk/scrapy/spider/models.py => scrapy/branches/cluster-refactor/scrapy/spider/models.py rename : scrapy/trunk/scrapy/stats/__init__.py => scrapy/branches/cluster-refactor/scrapy/stats/__init__.py rename : scrapy/trunk/scrapy/stats/corestats.py => scrapy/branches/cluster-refactor/scrapy/stats/corestats.py rename : scrapy/trunk/scrapy/stats/statscollector.py => scrapy/branches/cluster-refactor/scrapy/stats/statscollector.py rename : scrapy/trunk/scrapy/store/__init__.py => scrapy/branches/cluster-refactor/scrapy/store/__init__.py rename : scrapy/trunk/scrapy/store/db.py => scrapy/branches/cluster-refactor/scrapy/store/db.py rename : scrapy/trunk/scrapy/tests/__init__.py => scrapy/branches/cluster-refactor/scrapy/tests/__init__.py rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.tar => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.tar rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz rename : scrapy/trunk/scrapy/tests/sample_data/compressed/feed-sample1.zip => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/compressed/feed-sample1.zip rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/feeds/feed-sample1.xml rename : scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/feeds/feed-sample2.xml rename : scrapy/trunk/scrapy/tests/sample_data/test_site/index.html => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/index.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/item1.html rename : scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html => scrapy/branches/cluster-refactor/scrapy/tests/sample_data/test_site/item2.html rename : scrapy/trunk/scrapy/tests/test_adaptors.py => scrapy/branches/cluster-refactor/scrapy/tests/test_adaptors.py rename : scrapy/trunk/scrapy/tests/test_c14nurls.py => scrapy/branches/cluster-refactor/scrapy/tests/test_c14nurls.py rename : scrapy/trunk/scrapy/tests/test_decompress.py => scrapy/branches/cluster-refactor/scrapy/tests/test_decompress.py rename : scrapy/trunk/scrapy/tests/test_defaultencoding.py => scrapy/branches/cluster-refactor/scrapy/tests/test_defaultencoding.py rename : scrapy/trunk/scrapy/tests/test_dependencies.py => scrapy/branches/cluster-refactor/scrapy/tests/test_dependencies.py rename : scrapy/trunk/scrapy/tests/test_engine.py => scrapy/branches/cluster-refactor/scrapy/tests/test_engine.py rename : scrapy/trunk/scrapy/tests/test_http_request.py => scrapy/branches/cluster-refactor/scrapy/tests/test_http_request.py rename : scrapy/trunk/scrapy/tests/test_http_response.py => scrapy/branches/cluster-refactor/scrapy/tests/test_http_response.py rename : scrapy/trunk/scrapy/tests/test_http_url.py => scrapy/branches/cluster-refactor/scrapy/tests/test_http_url.py rename : scrapy/trunk/scrapy/tests/test_libxml2.py => scrapy/branches/cluster-refactor/scrapy/tests/test_libxml2.py rename : scrapy/trunk/scrapy/tests/test_link.py => scrapy/branches/cluster-refactor/scrapy/tests/test_link.py rename : scrapy/trunk/scrapy/tests/test_pipeline_images.py => scrapy/branches/cluster-refactor/scrapy/tests/test_pipeline_images.py rename : scrapy/trunk/scrapy/tests/test_serialization.py => scrapy/branches/cluster-refactor/scrapy/tests/test_serialization.py rename : scrapy/trunk/scrapy/tests/test_spidermonkey.py => scrapy/branches/cluster-refactor/scrapy/tests/test_spidermonkey.py rename : scrapy/trunk/scrapy/tests/test_spiders/__init__.py => scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/__init__.py rename : scrapy/trunk/scrapy/tests/test_spiders/testplugin.py => scrapy/branches/cluster-refactor/scrapy/tests/test_spiders/testplugin.py rename : scrapy/trunk/scrapy/tests/test_stats.py => scrapy/branches/cluster-refactor/scrapy/tests/test_stats.py rename : scrapy/trunk/scrapy/tests/test_storedb.py => scrapy/branches/cluster-refactor/scrapy/tests/test_storedb.py rename : scrapy/trunk/scrapy/tests/test_utils_datatypes.py => scrapy/branches/cluster-refactor/scrapy/tests/test_utils_datatypes.py rename : scrapy/trunk/scrapy/tests/test_utils_markup.py => scrapy/branches/cluster-refactor/scrapy/tests/test_utils_markup.py rename : scrapy/trunk/scrapy/tests/test_utils_url.py => scrapy/branches/cluster-refactor/scrapy/tests/test_utils_url.py rename : scrapy/trunk/scrapy/tests/test_utils_xml.py => scrapy/branches/cluster-refactor/scrapy/tests/test_utils_xml.py rename : scrapy/trunk/scrapy/tests/test_xpath.py => scrapy/branches/cluster-refactor/scrapy/tests/test_xpath.py rename : scrapy/trunk/scrapy/utils/__init__.py => scrapy/branches/cluster-refactor/scrapy/utils/__init__.py rename : scrapy/trunk/scrapy/utils/c14n.py => scrapy/branches/cluster-refactor/scrapy/utils/c14n.py rename : scrapy/trunk/scrapy/utils/datatypes.py => scrapy/branches/cluster-refactor/scrapy/utils/datatypes.py rename : scrapy/trunk/scrapy/utils/db.py => scrapy/branches/cluster-refactor/scrapy/utils/db.py rename : scrapy/trunk/scrapy/utils/decompressor.py => scrapy/branches/cluster-refactor/scrapy/utils/decompressor.py rename : scrapy/trunk/scrapy/utils/defer.py => scrapy/branches/cluster-refactor/scrapy/utils/defer.py rename : scrapy/trunk/scrapy/utils/display.py => scrapy/branches/cluster-refactor/scrapy/utils/display.py rename : scrapy/trunk/scrapy/utils/iterators.py => scrapy/branches/cluster-refactor/scrapy/utils/iterators.py rename : scrapy/trunk/scrapy/utils/markup.py => scrapy/branches/cluster-refactor/scrapy/utils/markup.py rename : scrapy/trunk/scrapy/utils/misc.py => scrapy/branches/cluster-refactor/scrapy/utils/misc.py rename : scrapy/trunk/scrapy/utils/python.py => scrapy/branches/cluster-refactor/scrapy/utils/python.py rename : scrapy/trunk/scrapy/utils/response.py => scrapy/branches/cluster-refactor/scrapy/utils/response.py rename : scrapy/trunk/scrapy/utils/serialization.py => scrapy/branches/cluster-refactor/scrapy/utils/serialization.py rename : scrapy/trunk/scrapy/utils/url.py => scrapy/branches/cluster-refactor/scrapy/utils/url.py rename : scrapy/trunk/scrapy/utils/xml.py => scrapy/branches/cluster-refactor/scrapy/utils/xml.py rename : scrapy/trunk/scrapy/xlib/BeautifulSoup.py => scrapy/branches/cluster-refactor/scrapy/xlib/BeautifulSoup.py rename : scrapy/trunk/scrapy/xlib/__init__.py => scrapy/branches/cluster-refactor/scrapy/xlib/__init__.py rename : scrapy/trunk/scrapy/xlib/lrucache.py => scrapy/branches/cluster-refactor/scrapy/xlib/lrucache.py rename : scrapy/trunk/scrapy/xlib/lsprofcalltree.py => scrapy/branches/cluster-refactor/scrapy/xlib/lsprofcalltree.py rename : scrapy/trunk/scrapy/xlib/pydispatch/__init__.py => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/__init__.py rename : scrapy/trunk/scrapy/xlib/pydispatch/dispatcher.py => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/dispatcher.py rename : scrapy/trunk/scrapy/xlib/pydispatch/errors.py => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/errors.py rename : scrapy/trunk/scrapy/xlib/pydispatch/license.txt => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/license.txt rename : scrapy/trunk/scrapy/xlib/pydispatch/robust.py => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/robust.py rename : scrapy/trunk/scrapy/xlib/pydispatch/robustapply.py => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/robustapply.py rename : scrapy/trunk/scrapy/xlib/pydispatch/saferef.py => scrapy/branches/cluster-refactor/scrapy/xlib/pydispatch/saferef.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/INSTALL.scrapy => scrapy/branches/cluster-refactor/scrapy/xlib/spidermonkey/INSTALL.scrapy rename : scrapy/trunk/scrapy/xlib/spidermonkey/__init__.py => scrapy/branches/cluster-refactor/scrapy/xlib/spidermonkey/__init__.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/sm_settings.py => scrapy/branches/cluster-refactor/scrapy/xlib/spidermonkey/sm_settings.py rename : scrapy/trunk/scrapy/xlib/spidermonkey/spidermonkey.py => scrapy/branches/cluster-refactor/scrapy/xlib/spidermonkey/spidermonkey.py rename : scrapy/trunk/scrapy/xpath/__init__.py => scrapy/branches/cluster-refactor/scrapy/xpath/__init__.py rename : scrapy/trunk/scrapy/xpath/constructors.py => scrapy/branches/cluster-refactor/scrapy/xpath/constructors.py rename : scrapy/trunk/scrapy/xpath/document.py => scrapy/branches/cluster-refactor/scrapy/xpath/document.py rename : scrapy/trunk/scrapy/xpath/extension.py => scrapy/branches/cluster-refactor/scrapy/xpath/extension.py rename : scrapy/trunk/scrapy/xpath/selector.py => scrapy/branches/cluster-refactor/scrapy/xpath/selector.py rename : scrapy/trunk/scrapy/xpath/types.py => scrapy/branches/cluster-refactor/scrapy/xpath/types.py rename : scrapy/trunk/scripts/rpm-install.sh => scrapy/branches/cluster-refactor/scripts/rpm-install.sh rename : scrapy/trunk/setup.cfg => scrapy/branches/cluster-refactor/setup.cfg rename : scrapy/trunk/setup.py => scrapy/branches/cluster-refactor/setup.py extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40323
This commit is contained in:
Pablo Hoffman
parent
00f28e3edf
commit
5349030b31
|
|
@ -0,0 +1,32 @@
|
|||
You need:
|
||||
* Python 2.5
|
||||
* Twisted
|
||||
* libxml2 python bindings. version 2.6.28 or above is highly recommended.
|
||||
* pyopenssl for HTTPS support
|
||||
* for win32: http://webcleaner.sourceforge.net/pyOpenSSL-0.6.win32-py2.5.exe
|
||||
* spidermonkey (optional) - required for JSParser
|
||||
* simplejson (optional) - required for url2guid web service
|
||||
|
||||
The python interpreter should have sys.getdefaultencoding() == 'utf-8'
|
||||
If this is not the case, you can add the following lines to a file
|
||||
[PYTHON_INSTALL]/site-packages/sitecustomize.py:
|
||||
import sys
|
||||
sys.setdefaultencoding('utf-8')
|
||||
|
||||
|
||||
INSTALLING LIBRARIES
|
||||
--------------------
|
||||
|
||||
The procedure for installing the required third party libraries (twisted,
|
||||
libxml2 and pyopenssl) depends on the platform and OS you use.
|
||||
|
||||
Visit the project homepages for more information about it:
|
||||
|
||||
* Twisted: http://twistedmatrix.com/
|
||||
* libxml2: http://xmlsoft.org/
|
||||
* releases available here: ftp://xmlsoft.org/libxml2/
|
||||
* pyopenssl: http://pyopenssl.sourceforge.net/
|
||||
|
||||
|
||||
In Debian/Ubuntu Linux it's as easy as:
|
||||
apt-get install python-twisted python-libxml2 python-pyopenssl
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
This is Scrapy, an opensource screen scraping framework written in Python.
|
||||
|
||||
For more visit the project home page at http://scrapy.org
|
||||
Binary file not shown.
|
|
@ -0,0 +1,72 @@
|
|||
DROP TABLE IF EXISTS `url_history`;
|
||||
DROP TABLE IF EXISTS `version`;
|
||||
DROP TABLE IF EXISTS `url_status`;
|
||||
DROP TABLE IF EXISTS `ticket`;
|
||||
DROP TABLE IF EXISTS `domain_stats`;
|
||||
DROP TABLE IF EXISTS `domain_stats_history`;
|
||||
DROP TABLE IF EXISTS `domain_data_history`;
|
||||
|
||||
CREATE TABLE `ticket` (
|
||||
`guid` char(40) NOT NULL,
|
||||
`domain` varchar(255) default NULL,
|
||||
`url` varchar(2048) default NULL,
|
||||
`url_hash` char(40) default NULL, -- so we can join to url_status
|
||||
PRIMARY KEY (`guid`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE `version` (
|
||||
`id` bigint(20) NOT NULL auto_increment,
|
||||
`guid` char(40) NOT NULL,
|
||||
`version` char(40) NOT NULL,
|
||||
`seen` datetime NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
FOREIGN KEY (`guid`) REFERENCES ticket(guid) ON UPDATE CASCADE ON DELETE CASCADE,
|
||||
UNIQUE KEY (`version`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE `url_status` (
|
||||
-- see http://support.microsoft.com/kb/q208427/ for explanation of 2048
|
||||
`url_hash` char(40) NOT NULL, -- for faster searches
|
||||
`url` varchar(2048) NOT NULL,
|
||||
`parent_hash` char(40) default NULL, -- the url that was followed to this one - for reporting
|
||||
`last_version` char(40) default NULL, -- can be null if it generated an error the last time is was checked
|
||||
`last_checked` datetime NOT NULL,
|
||||
PRIMARY KEY (`url_hash`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE `url_history` (
|
||||
`url_hash` char(40) NOT NULL,
|
||||
`version` char(40) NOT NULL,
|
||||
`postdata_hash` char(40) default NULL,
|
||||
`created` datetime NOT NULL,
|
||||
PRIMARY KEY (`version`),
|
||||
FOREIGN KEY (`url_hash`) REFERENCES url_status(url_hash) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE `domain_stats` (
|
||||
`key1` varchar(128) NOT NULL,
|
||||
`key2` varchar(128) NOT NULL,
|
||||
`value` text,
|
||||
PRIMARY KEY `key1_key2` (`key1`, `key2`),
|
||||
KEY `key1` (`key1`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE `domain_stats_history` (
|
||||
`id` bigint(20) NOT NULL auto_increment,
|
||||
`key1` varchar(128) NOT NULL,
|
||||
`key2` varchar(128) NOT NULL,
|
||||
`value` varchar(2048) NOT NULL,
|
||||
`stored` datetime NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `key1_key2` (`key1`, `key2`),
|
||||
KEY `key1` (`key1`),
|
||||
KEY `stored` (`stored`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
CREATE TABLE `domain_data_history` (
|
||||
`domain` varchar(255) NOT NULL,
|
||||
`stored` datetime NOT NULL,
|
||||
`data` text,
|
||||
KEY `domain_stored` (`domain`, `stored`),
|
||||
KEY `domain` (`domain`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
"""
|
||||
Scrapy - a screen scraping framework written in Python
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
import sys, os
|
||||
|
||||
if sys.version_info < (2,5):
|
||||
print "Scrapy %s requires Python 2.5 or above" % __version__
|
||||
sys.exit(1)
|
||||
|
||||
# add external python libraries bundled into scrapy
|
||||
sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "xlib"))
|
||||
|
||||
# monkey patches to fix external library issues
|
||||
from scrapy.patches import monkeypatches
|
||||
monkeypatches.apply_patches()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env python
|
||||
"""Scrapy admin script is used to create new scrapy projects and similar
|
||||
tasks"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from optparse import OptionParser
|
||||
|
||||
import scrapy
|
||||
|
||||
usage = """
|
||||
scrapy-admin.py [options] [command]
|
||||
|
||||
Available commands:
|
||||
|
||||
startproject <project_name>
|
||||
Starts a new project with name 'project_name'
|
||||
"""
|
||||
|
||||
def main():
|
||||
parser = OptionParser(usage=usage)
|
||||
opts, args = parser.parse_args()
|
||||
|
||||
if not args:
|
||||
parser.print_help()
|
||||
|
||||
cmd = args[0]
|
||||
if cmd == "startproject":
|
||||
if len(args) >= 2:
|
||||
project_name = args[1]
|
||||
project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template")
|
||||
shutil.copytree(project_tplpath, project_name)
|
||||
else:
|
||||
print "scrapy-admin.py: missing project name"
|
||||
else:
|
||||
print "scrapy-admin.py: unknown command: %s" % cmd
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1 @@
|
|||
from scrapy.command.models import ScrapyCommand
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
import scrapy
|
||||
from scrapy import log
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.conf import settings
|
||||
|
||||
def find_commands(dir):
|
||||
try:
|
||||
return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and f.endswith('.py')]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def builtin_commands_dict():
|
||||
d = {}
|
||||
scrapy_dir = scrapy.__path__[0]
|
||||
commands_dir = os.path.join(scrapy_dir, 'command', 'commands')
|
||||
for cmdname in find_commands(commands_dir):
|
||||
modname = 'scrapy.command.commands.%s' % cmdname
|
||||
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
|
||||
if callable(command):
|
||||
d[cmdname] = command()
|
||||
else:
|
||||
print 'WARNING: Builtin command module %s exists but Command class not found' % modname
|
||||
return d
|
||||
|
||||
def custom_commands_dict():
|
||||
d = {}
|
||||
cmdsmod = settings['COMMANDS_MODULE']
|
||||
if cmdsmod:
|
||||
mod = __import__(cmdsmod, {}, {}, [''])
|
||||
for cmdname in find_commands(mod.__path__[0]):
|
||||
modname = '%s.%s' % (cmdsmod, cmdname)
|
||||
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
|
||||
if callable(command):
|
||||
d[cmdname] = command()
|
||||
else:
|
||||
print 'WARNING: Custom command module %s exists but Command class not found' % modname
|
||||
return d
|
||||
|
||||
def getcmdname(argv):
|
||||
for arg in argv[1:]:
|
||||
if not arg.startswith('-'):
|
||||
return arg
|
||||
|
||||
def usage(argv):
|
||||
s = "usage: %s <subcommand> [options] [args]\n" % argv[0]
|
||||
s += " %s <subcommand> -h\n\n" % argv[0]
|
||||
s += "Built-in subcommands:\n"
|
||||
|
||||
builtin_cmds = builtin_commands_dict()
|
||||
custom_cmds = custom_commands_dict()
|
||||
|
||||
filtered_builtin_cmds = [(name, cls) for name, cls in builtin_cmds.iteritems() if name not in custom_cmds]
|
||||
|
||||
for cmdname, cmdclass in filtered_builtin_cmds:
|
||||
s += " %s %s\n" % (cmdname, cmdclass.syntax())
|
||||
s += " %s\n" % cmdclass.short_desc()
|
||||
|
||||
if custom_cmds:
|
||||
s += "\n"
|
||||
s += "Custom (or overloaded) subcommands:\n"
|
||||
for cmdname, cmdclass in custom_cmds.iteritems():
|
||||
s += " %s %s\n" % (cmdname, cmdclass.syntax())
|
||||
s += " %s\n" % cmdclass.short_desc()
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def update_defaults(defaults, module):
|
||||
settingsdict = vars(module)
|
||||
for k, v in settingsdict.iteritems():
|
||||
if not k.startswith("_"):
|
||||
defaults[k] = v
|
||||
|
||||
def command_settings(cmdname):
|
||||
try:
|
||||
module = __import__('%s.%s' % ('scrapy.conf.commands', cmdname), {}, {}, [''])
|
||||
update_defaults(settings.defaults, module)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
basepath = settings['COMMANDS_SETTINGS_MODULE']
|
||||
if basepath:
|
||||
try:
|
||||
module = __import__('%s.%s' % (basepath, cmdname), {}, {}, [''])
|
||||
update_defaults(settings.defaults, module)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# This dict holds information about the executed command for later use
|
||||
command_executed = {}
|
||||
|
||||
def execute():
|
||||
execute_with_args(sys.argv)
|
||||
|
||||
def execute_with_args(argv):
|
||||
spiders.load()
|
||||
cmds = builtin_commands_dict()
|
||||
cmds.update(custom_commands_dict())
|
||||
|
||||
cmdname = getcmdname(argv)
|
||||
command_settings(cmdname)
|
||||
|
||||
if not cmdname:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print usage(argv)
|
||||
sys.exit()
|
||||
|
||||
parser = optparse.OptionParser()
|
||||
|
||||
if cmdname in cmds:
|
||||
cmd = cmds[cmdname]
|
||||
cmd.add_options(parser)
|
||||
parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
|
||||
parser.description = cmd.long_desc()
|
||||
else:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print "Unknown command: %s\n" % cmdname
|
||||
print 'Type "%s -h" for help' % argv[0]
|
||||
sys.exit()
|
||||
|
||||
(opts, args) = parser.parse_args(args=argv[1:])
|
||||
del args[0] # args[0] is cmdname
|
||||
|
||||
# storing command executed info for later reference
|
||||
command_executed['name'] = cmdname
|
||||
command_executed['class'] = cmd
|
||||
command_executed['args'] = args[:]
|
||||
command_executed['opts'] = opts.__dict__.copy()
|
||||
|
||||
cmd.process_options(args, opts)
|
||||
log.start() # start logging
|
||||
if opts.profile:
|
||||
log.msg("Profiling enabled. Analyze later with: python -m pstats %s" % opts.profile)
|
||||
import cProfile
|
||||
loc = locals()
|
||||
p = cProfile.Profile()
|
||||
p.runctx('ret = cmd.run(args, opts)', globals(), loc)
|
||||
p.dump_stats(opts.profile)
|
||||
try:
|
||||
import lsprofcalltree
|
||||
fn = opts.profile + ".cachegrind"
|
||||
k = lsprofcalltree.KCacheGrind(p)
|
||||
with open(fn, 'w') as f:
|
||||
k.output(f)
|
||||
except ImportError:
|
||||
pass
|
||||
ret = loc['ret']
|
||||
else:
|
||||
ret = cmd.run(args, opts)
|
||||
if ret is False:
|
||||
parser.print_help()
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.replay import Replay
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] [domain|url] ..."
|
||||
|
||||
def short_desc(self):
|
||||
return "Run the web scraping engine from the command line"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--nocache", dest="nocache", action="store_true", help="disable HTTP cache")
|
||||
parser.add_option("--nopipeline", dest="nopipeline", action="store_true", help="disable scraped item pipeline")
|
||||
parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls")
|
||||
parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE")
|
||||
parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
if opts.nopipeline:
|
||||
settings.overrides['ITEM_PIPELINES'] = []
|
||||
|
||||
if opts.nocache:
|
||||
settings.overrides['CACHE2_DIR'] = None
|
||||
|
||||
if opts.restrict:
|
||||
settings.overrides['RESTRICT_TO_URLS'] = args
|
||||
|
||||
if opts.record or opts.recorddir:
|
||||
# self.replay is used for preventing Replay signals handler from
|
||||
# disconnecting since pydispatcher uses weak references
|
||||
self.replay = Replay(opts.record or opts.recorddir, mode='record', usedir=bool(opts.recorddir))
|
||||
self.replay.record(args=args, opts=opts.__dict__)
|
||||
|
||||
def run(self, args, opts):
|
||||
scrapymanager.runonce(*args, **opts.__dict__)
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import sys, pprint
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.fetcher import fetch
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <url>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Download a URL using the Scrapy downloader"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-s", "--source", dest="source", action="store_true", help="output HTTP body only")
|
||||
parser.add_option("--headers", dest="headers", action="store_true", help="output HTTP headers only")
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A URL is required"
|
||||
return
|
||||
|
||||
responses = fetch(args)
|
||||
if responses:
|
||||
if opts.headers:
|
||||
pprint.pprint(responses[0].headers)
|
||||
else:
|
||||
sys.stdout.write(str(responses[0].body))
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import os
|
||||
import string
|
||||
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
""" Default template file name """
|
||||
template_name = 'spider.tmpl'
|
||||
|
||||
""" Childs can define custom tvars """
|
||||
custom_tvars = {}
|
||||
|
||||
def syntax(self):
|
||||
return "<spider_name> <spider_domain_name>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Generate new spider based on predefined template"
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 2:
|
||||
return False
|
||||
|
||||
name = args[0]
|
||||
site = args[1]
|
||||
spiders_dict = spiders.asdict()
|
||||
if not name in spiders_dict.keys():
|
||||
self._genspider(name, site)
|
||||
else:
|
||||
print "Spider '%s' exist" % name
|
||||
|
||||
def _genspider(self, name, site):
|
||||
""" Generate spider """
|
||||
tvars = {
|
||||
'name': name,
|
||||
'site': site,
|
||||
'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('-')])
|
||||
}
|
||||
tvars.update(self.custom_tvars)
|
||||
|
||||
spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, [''])
|
||||
spidersdir = os.path.abspath(os.path.dirname(spiders_module.__file__))
|
||||
if name[0] not in string.letters: # must start with a letter, for valid python modules
|
||||
name = "a" + name
|
||||
name = name.replace('-', '_') # - are replaced by _, for valid python modules
|
||||
self._genfiles(self.template_name, '%s/%s.py' % (spidersdir, name), tvars)
|
||||
|
||||
def _genfiles(self, template_name, source_name, tvars):
|
||||
""" Generate source from template, substitute variables """
|
||||
template_file = os.path.join(settings['TEMPLATES_DIR'], template_name)
|
||||
tmpl = open(template_file)
|
||||
clines = []
|
||||
for l in tmpl.readlines():
|
||||
for key, val in tvars.items():
|
||||
l = l.replace('@%s@' % key, val)
|
||||
clines.append(l)
|
||||
tmpl.close()
|
||||
source = ''.join(clines)
|
||||
if not os.path.exists(source_name):
|
||||
sfile = open(source_name, "w")
|
||||
sfile.write(source)
|
||||
sfile.close()
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.fetcher import fetch
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
def get_item_attr(pagedata, attr="guid"):
|
||||
spider = spiders.fromurl(pagedata.url)
|
||||
items = spider.parse(pagedata)
|
||||
attrs = [getattr(i, attr) for i in items if isinstance(i, ScrapedItem)]
|
||||
return attrs
|
||||
|
||||
def get_attr(url, attr="guid"):
|
||||
pagedatas = fetch([url])
|
||||
if pagedatas:
|
||||
return get_item_attr(pagedatas[0], attr)
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<url> <attribute>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Print an attribute from the item scraped in the given URL"
|
||||
|
||||
def run(self, args, opts):
|
||||
print get_attr(args[0], args[1])
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
from scrapy.command import ScrapyCommand, cmdline
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<command>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Provides extended help for the given command"
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
return False
|
||||
|
||||
commands = cmdline.builtin_commands_dict()
|
||||
commands.update(cmdline.custom_commands_dict())
|
||||
|
||||
cmdname = args[0]
|
||||
if cmdname in commands:
|
||||
cmd = commands[cmdname]
|
||||
help = getattr(cmd, 'help', None) or getattr(cmd, 'long_desc', None)
|
||||
print "%s: %s" % (cmdname, cmd.short_desc())
|
||||
print "usage: %s %s" % (cmdname, cmd.syntax())
|
||||
print
|
||||
print help()
|
||||
else:
|
||||
print "Unknown command: %s" % cmdname
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.spider import spiders
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return ""
|
||||
|
||||
def short_desc(self):
|
||||
return "List available spiders (both enabled and disabled)"
|
||||
|
||||
def run(self, args, opts):
|
||||
spiders_dict = spiders.asdict()
|
||||
for n, p in spiders_dict.items():
|
||||
disabled = "disabled" if getattr(p, 'disabled', False) else "enabled"
|
||||
print "%-30s %-30s %s" % (n, p.__class__.__name__, disabled)
|
||||
print "Total spiders: %d" % len(spiders_dict)
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import re
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
|
||||
log_crawled_re = re.compile(r'^(.*?) \[\w+/(.*?)\].*Crawled <(.*?)> from <(.*?)>$')
|
||||
log_scraped_re = re.compile('^(.*?) \[\w+/(.*?)\].*Scraped (.*) in <(.*?)>$')
|
||||
log_opendomain_re = re.compile('^(.*?) \[\w+.*? Started scraping (.*)$')
|
||||
log_closedomain_re = re.compile('^(.*?) \[\w+.*? Finished scraping (.*)$')
|
||||
log_debug_re = re.compile(r'\[\w+/(.*?)\] DEBUG')
|
||||
log_info_re = re.compile(r'\[\w+/(.*?)\] INFO')
|
||||
log_warning_re = re.compile(r'\[\w+/(.*?)\] WARNING')
|
||||
log_error_re = re.compile(r'\[\w+/(.*?)\] ERROR')
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <logfile>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Several tools to perform scrapy log analysis"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--sitemap", dest="sitemap", action="store_true", help="show sitemap based on given log file")
|
||||
|
||||
def sitemap(self, logfile):
|
||||
from scrapy.utils.datatypes import Sitemap
|
||||
|
||||
sm = Sitemap()
|
||||
for l in open(logfile).readlines():
|
||||
m = log_crawled_re.search(l.strip())
|
||||
if m:
|
||||
sm.add_node(m.group(3), m.group(4))
|
||||
m = log_scraped_re.search(l.strip())
|
||||
if m:
|
||||
sm.add_item(m.group(4), m.group(3))
|
||||
print sm.to_string()
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A log file is required"
|
||||
return
|
||||
|
||||
if opts.sitemap:
|
||||
self.sitemap(args[0])
|
||||
else:
|
||||
print "No analysis method specified"
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.fetcher import fetch
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.utils import display
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <url>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Parse the URL and print their results"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items")
|
||||
parser.add_option("--identify", dest="identify", action="store_true", help="try to use identify instead of parse")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output")
|
||||
|
||||
def pipeline_process(self, item, opts):
|
||||
return item
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A URL is required"
|
||||
return
|
||||
|
||||
responses = fetch([args[0]])
|
||||
if responses:
|
||||
response = responses[0]
|
||||
spider = spiders.fromurl(response.url)
|
||||
result = spider.parse(response) if not opts.identify else spider.identify(response)
|
||||
|
||||
items = [self.pipeline_process(i, opts) for i in result if isinstance(i, ScrapedItem)]
|
||||
links = [i for i in result if isinstance(i, Request)]
|
||||
|
||||
display.nocolour = opts.nocolour
|
||||
if not opts.noitems:
|
||||
print "# Scraped Items", "-"*60
|
||||
display.pprint(items)
|
||||
|
||||
if not opts.nolinks:
|
||||
print "# Links", "-"*68
|
||||
display.pprint(links)
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
import os
|
||||
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.replay import Replay
|
||||
from scrapy.utils import display
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command import cmdline
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options] <replay_file> [action]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Replay a session previously recorded with crawl --record"
|
||||
|
||||
def help(self):
|
||||
s = "Replay a session previously recorded with crawl --record\n"
|
||||
s += "\n"
|
||||
s += "Available actions:\n"
|
||||
s += " crawl: just replay the crawl (default if action omitted)\n"
|
||||
s += " diff: replay the crawl and show differences in items scraped/passed\n"
|
||||
s += " update: replay the crawl and update both scraped and passed items\n"
|
||||
s += " showitems: show items stored\n"
|
||||
s += " showpages: show all responses downloaded (not only HTML pages)\n"
|
||||
return s
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show verbose output (full items/responses)")
|
||||
parser.add_option("-t", "--item-type", dest="itype", help="item type (scraped, passed). default: scraped", metavar="TYPE")
|
||||
parser.add_option("--output", dest="outfile", help="write output to FILE. if omitted uses stdout", metavar="FILE")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true", help="disable colorized output (for console only)")
|
||||
parser.add_option("-i", "--ignore", dest="ignores", action="append", help="item attribute to ignore. can be passed multiple times", metavar="ATTR")
|
||||
parser.add_option("--target", dest="targets", action="append", help="crawl TARGET instead of recorded urls/domains. can be passed multiple times")
|
||||
# adding option to update
|
||||
parser.add_option("--pages", dest="pages", action="store_true", help="update all the pages in the replay file, recording it again.")
|
||||
parser.add_option("-q", "--quiet", dest="quiet", action="store_true", help="no verbose mode in option diff.")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
if args:
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
self.opts = opts
|
||||
self.action = args[1] if len(args) > 1 else 'crawl'
|
||||
mode = 'update' if self.action == 'update' else 'play'
|
||||
usedir = args and os.path.isdir(args[0])
|
||||
self.replay = Replay(args[0], mode=mode, usedir=usedir)
|
||||
if self.action not in ['crawl', 'diff', 'update']:
|
||||
settings.overrides['LOG_ENABLED'] = False
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A <replay_dir> is required"
|
||||
return
|
||||
|
||||
display.nocolour = opts.nocolour
|
||||
|
||||
if opts.itype == 'passed':
|
||||
self.before_db = self.replay.passed_old
|
||||
self.now_db = self.replay.passed_new
|
||||
else: # default is 'scraped'
|
||||
opts.itype = 'scraped'
|
||||
self.before_db = self.replay.scraped_old
|
||||
self.now_db = self.replay.scraped_new
|
||||
|
||||
actionfunc = getattr(self, 'action_%s' % self.action, None)
|
||||
if actionfunc:
|
||||
rep = actionfunc(opts)
|
||||
if rep:
|
||||
if opts.outfile:
|
||||
f = open(opts.outfile, "w")
|
||||
f.write(rep)
|
||||
f.close()
|
||||
else:
|
||||
print rep,
|
||||
self.replay.cleanup()
|
||||
else:
|
||||
print "Unknown replay action: %s" % self.action
|
||||
|
||||
def action_crawl(self, opts):
|
||||
self.replay.play(args=opts.targets)
|
||||
|
||||
def action_update(self, opts):
|
||||
self.replay.update(args=opts.targets, opts=opts.__dict__)
|
||||
if (opts.pages):
|
||||
args = ['scrapy-crawl', 'crawl']
|
||||
args.extend(self.replay.options['args'])
|
||||
for k in self.replay.options['opts']:
|
||||
if self.replay.options['opts'][k]:
|
||||
args.append("--%s" % k)
|
||||
if self.replay.options['opts'][k] != True:
|
||||
args.append(self.replay.options['opts'][k])
|
||||
cmdline.execute_with_args(args)
|
||||
|
||||
def action_showitems(self, opts):
|
||||
s = ""
|
||||
s += self._format_items(self.before_db.values())
|
||||
s += ">>> Total: %d items %s\n" % (len(self.before_db), opts.itype)
|
||||
return s
|
||||
|
||||
def action_showpages(self, opts):
|
||||
s = ""
|
||||
if self.opts.verbose:
|
||||
for r in self.replay.responses_old.values():
|
||||
s += ">>> %s\n" % str(r)
|
||||
s += display.pformat(r)
|
||||
else:
|
||||
s += "\n".join([str(r) for r in self.replay.responses_old.values()]) + "\n"
|
||||
s += ">>> Total: %d responses received\n" % len(self.replay.responses_old)
|
||||
return s
|
||||
|
||||
def action_diff(self, opts):
|
||||
self.action_crawl(opts)
|
||||
|
||||
guids_before = set(self.before_db.keys())
|
||||
guids_now = set(map(str, self.now_db.keys()))
|
||||
|
||||
guids_new = guids_now - guids_before
|
||||
guids_missing = guids_before - guids_now
|
||||
guids_both = guids_now & guids_before
|
||||
|
||||
changed_items, chreport = self._report_differences(self.before_db, self.now_db, guids_both)
|
||||
|
||||
ok_items = len(guids_both) - changed_items
|
||||
new_items = len(guids_new)
|
||||
missing_items = len(guids_missing)
|
||||
|
||||
if (new_items - missing_items - changed_items) == 0 and opts.quiet:
|
||||
s = ""
|
||||
else:
|
||||
s = "CRAWLING DIFFERENCES REPORT\n\n"
|
||||
|
||||
s += "Total items : %d\n" % (len(guids_both) + new_items + missing_items)
|
||||
s += " Items OK : %d\n" % ok_items
|
||||
s += " New items : %d\n" % new_items
|
||||
s += " Missing items : %d\n" % missing_items
|
||||
s += " Changed items : %d\n" % changed_items
|
||||
|
||||
s += "\n"
|
||||
s += "- NEW ITEMS (%d) -----------------------------------------\n" % new_items
|
||||
s += self._format_items([self.now_db[g] for g in guids_new])
|
||||
|
||||
s += "\n"
|
||||
s += "- MISSING ITEMS (%d) -------------------------------------\n" % missing_items
|
||||
s += self._format_items([self.before_db[g] for g in guids_missing])
|
||||
|
||||
s += "\n"
|
||||
s += "- CHANGED ITEMS (%d) -------------------------------------\n" % changed_items
|
||||
|
||||
s += chreport
|
||||
|
||||
return s
|
||||
|
||||
def _report_differences(self, old_items, new_items, guids):
|
||||
items_old = [old_items[g] for g in guids]
|
||||
items_new = [new_items[g] for g in guids]
|
||||
|
||||
c = 0
|
||||
s = ""
|
||||
for old, new in zip(items_old, items_new):
|
||||
d = self._item_diff(old, new)
|
||||
if d:
|
||||
c += 1
|
||||
s += d
|
||||
return c, s
|
||||
|
||||
def _item_diff(self, old, new):
|
||||
delta = new - old
|
||||
|
||||
s = ""
|
||||
if delta.diff:
|
||||
s += ">>> Item guid=%s name=%s\n" % (old.guid, old.name)
|
||||
s += display.pformat(delta.diff) + "\n"
|
||||
return s
|
||||
|
||||
def _format_items(self, items):
|
||||
if self.opts.verbose:
|
||||
s = display.pformat(items)
|
||||
else:
|
||||
s = ""
|
||||
for i in items:
|
||||
s += "%s\n" % str(i)
|
||||
s += " <%s>\n" % i.url
|
||||
return s
|
||||
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
from twisted.internet import reactor
|
||||
|
||||
import scrapy
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector
|
||||
from scrapy.utils.misc import load_class
|
||||
from scrapy.extension import extensions
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.core.downloader.handlers import download_any
|
||||
from scrapy.fetcher import get_or_create_spider
|
||||
from scrapy.utils.decompressor import Decompressor
|
||||
|
||||
#This code comes from twisted 8. We define here while
|
||||
#using old twisted version.
|
||||
def blockingCallFromThread(reactor, f, *a, **kw):
|
||||
"""
|
||||
Run a function in the reactor from a thread, and wait for the result
|
||||
synchronously, i.e. until the callback chain returned by the function
|
||||
get a result.
|
||||
|
||||
@param reactor: The L{IReactorThreads} provider which will be used to
|
||||
schedule the function call.
|
||||
@param f: the callable to run in the reactor thread
|
||||
@type f: any callable.
|
||||
@param a: the arguments to pass to C{f}.
|
||||
@param kw: the keyword arguments to pass to C{f}.
|
||||
|
||||
@return: the result of the callback chain.
|
||||
@raise: any error raised during the callback chain.
|
||||
"""
|
||||
import Queue
|
||||
from twisted.python import failure
|
||||
from twisted.internet import defer
|
||||
queue = Queue.Queue()
|
||||
def _callFromThread():
|
||||
result = defer.maybeDeferred(f, *a, **kw)
|
||||
result.addBoth(queue.put)
|
||||
reactor.callFromThread(_callFromThread)
|
||||
result = queue.get()
|
||||
if isinstance(result, failure.Failure):
|
||||
result.raiseException()
|
||||
return result
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[url]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Interactive scraping console"
|
||||
|
||||
def long_desc(self):
|
||||
return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html"
|
||||
|
||||
def update_vars(self):
|
||||
""" You can use this function to update the local variables that will be available in the scrape console """
|
||||
pass
|
||||
|
||||
def get_url(self, url, decompress=False):
|
||||
|
||||
print "Downloading URL... ",
|
||||
r = Request(url)
|
||||
spider = get_or_create_spider(url)
|
||||
try:
|
||||
result = blockingCallFromThread(reactor, download_any, r, spider)
|
||||
if isinstance(result, Response):
|
||||
print "Done."
|
||||
if decompress:
|
||||
print "Decompressing response...",
|
||||
d = Decompressor()
|
||||
result = d.extract(result)
|
||||
print "Done."
|
||||
result.request = r
|
||||
self.generate_vars(url, result)
|
||||
return True
|
||||
except Exception, e:
|
||||
print "Error: %s" % e
|
||||
|
||||
def generate_vars(self, url, response):
|
||||
itemcls = load_class(settings['DEFAULT_ITEM_CLASS'])
|
||||
item = itemcls()
|
||||
self.vars['item'] = item
|
||||
if url:
|
||||
self.vars['xxs'] = XmlXPathSelector(response)
|
||||
self.vars['hxs'] = HtmlXPathSelector(response)
|
||||
self.vars['url'] = url
|
||||
self.vars['response'] = response
|
||||
self.vars['spider'] = spiders.fromurl(url)
|
||||
self.update_vars()
|
||||
self.user_ns.update(self.vars)
|
||||
self.print_vars()
|
||||
|
||||
def print_vars(self):
|
||||
print '-' * 78
|
||||
print "Available local variables:"
|
||||
for key, val in self.vars.iteritems():
|
||||
if isinstance(val, basestring):
|
||||
print " %s: %s" % (key, val)
|
||||
else:
|
||||
print " %s: %s" % (key, val.__class__)
|
||||
print "Available commands:"
|
||||
print " get <url>: Fetches an url and updates all variables."
|
||||
print " getd <url>: Similar to get, but filter with decompress."
|
||||
print " scrapehelp: Prints this help."
|
||||
print '-' * 78
|
||||
|
||||
def run(self, args, opts):
|
||||
self.vars = {}
|
||||
self.user_ns = {}
|
||||
url = None
|
||||
if args:
|
||||
url = args[0]
|
||||
|
||||
print "Scrapy %s - Interactive scraping console\n" % scrapy.__version__
|
||||
|
||||
print "Enabling Scrapy extensions...",
|
||||
extensions.load()
|
||||
print "done"
|
||||
|
||||
def _console_thread():
|
||||
|
||||
def _get_magic(shell, arg):
|
||||
self.get_url(arg.strip())
|
||||
def _help_magic(shell, _):
|
||||
self.print_vars()
|
||||
def _getd_magic(shell, arg):
|
||||
self.get_url(arg.strip(), decompress=True)
|
||||
|
||||
if url:
|
||||
result = self.get_url(url)
|
||||
if not result:
|
||||
self.generate_vars(None, None)
|
||||
else:
|
||||
self.generate_vars(None, None)
|
||||
try: # use IPython if available
|
||||
import IPython
|
||||
shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns)
|
||||
ip = shell.IP.getapi()
|
||||
ip.expose_magic("get", _get_magic)
|
||||
ip.expose_magic("getd", _getd_magic)
|
||||
ip.expose_magic("scrapehelp", _help_magic)
|
||||
shell.mainloop()
|
||||
reactor.callFromThread(scrapymanager.stop)
|
||||
except ImportError:
|
||||
import code
|
||||
try: # readline module is only available on unix systems
|
||||
import readline
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
import rlcompleter
|
||||
readline.parse_and_bind("tab:complete")
|
||||
code.interact(local=self.vars)
|
||||
|
||||
reactor.callInThread(_console_thread)
|
||||
scrapymanager.start()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.core.manager import scrapymanager
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "[options]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Start the Scrapy server"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
|
||||
def run(self, args, opts):
|
||||
scrapymanager.start(*args, **opts.__dict__)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import pprint
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<domain> [domain ...]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Show all stats history stored for the given domain(s)"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-p", "--path", dest="path", help="restrict stats to PATH", metavar="PATH")
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A domain is required"
|
||||
return
|
||||
if not settings['SCRAPING_DB']:
|
||||
print "SCRAPING_DB setting is required for this command"
|
||||
return
|
||||
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history')
|
||||
|
||||
for domain in args:
|
||||
print "# %s" % domain
|
||||
pprint.pprint(list(ddh.getall(domain, opts.path)))
|
||||
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
"""
|
||||
Base class for Scrapy commands
|
||||
"""
|
||||
import os
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ScrapyCommand(object):
|
||||
def syntax(self):
|
||||
"""
|
||||
Command syntax (preferably one-line). Do not include command name.
|
||||
"""
|
||||
return ""
|
||||
|
||||
def short_desc(self):
|
||||
"""
|
||||
A short description of the command
|
||||
"""
|
||||
return ""
|
||||
|
||||
def long_desc(self):
|
||||
"""
|
||||
A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines, since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self):
|
||||
"""
|
||||
An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since not post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
||||
def add_options(self, parser):
|
||||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
parser.add_option("-f", "--logfile", dest="logfile", help="logfile to use. if omitted stderr will be used", metavar="FILE")
|
||||
parser.add_option("-o", "--loglevel", dest="loglevel", default=None, help="log level")
|
||||
parser.add_option("--default-spider", dest="default_spider", default=None, help="default spider (domain) to use if no spider is found")
|
||||
parser.add_option("--spider", dest="spider", default=None, help="Force using the given spider when the arguments are urls")
|
||||
parser.add_option("--nolog", dest="nolog", action="store_true", help="disable all log messages")
|
||||
parser.add_option("--profile", dest="profile", default=None, help="write profiling stats in FILE, to analyze later with: python -m pstats FILE", metavar="FILE")
|
||||
parser.add_option("--pidfile", dest="pidfile", help="Write process pid to file FILE", metavar="FILE")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
if opts.logfile:
|
||||
settings.overrides['LOG_ENABLED'] = True
|
||||
settings.overrides['LOGFILE'] = opts.logfile
|
||||
|
||||
if opts.loglevel:
|
||||
settings.overrides['LOG_ENABLED'] = True
|
||||
settings.overrides['LOGLEVEL'] = opts.loglevel
|
||||
|
||||
if opts.nolog:
|
||||
settings.overrides['LOG_ENABLED'] = False
|
||||
|
||||
if opts.default_spider:
|
||||
from scrapy.spider import spiders
|
||||
spiders.default_domain = opts.default_spider
|
||||
|
||||
if opts.spider:
|
||||
from scrapy.spider import spiders
|
||||
spiders.force_domain = opts.spider
|
||||
|
||||
if opts.pidfile:
|
||||
pid = os.getpid()
|
||||
open(opts.pidfile, "w").write(str(pid))
|
||||
|
||||
def run(self, args, opts):
|
||||
"""
|
||||
Entry point for running commands
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
import os
|
||||
import cPickle as pickle
|
||||
|
||||
SETTINGS_MODULE = os.environ.get('SCRAPYSETTINGS_MODULE', 'scrapy_settings')
|
||||
|
||||
class Settings(object):
|
||||
"""Class to obtain configuration values from settings module
|
||||
which can be overriden by environment variables prepended by SCRAPY_"""
|
||||
|
||||
# settings in precedence order
|
||||
overrides = None
|
||||
settings = None
|
||||
defaults = None
|
||||
core = None
|
||||
|
||||
def __init__(self):
|
||||
pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
|
||||
self.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
|
||||
self.settings = self._import(SETTINGS_MODULE)
|
||||
self.defaults = {}
|
||||
self.core = self._import('scrapy.conf.core_settings')
|
||||
|
||||
def _import(self, modulepath):
|
||||
return __import__(modulepath, {}, {}, [''])
|
||||
|
||||
def __getitem__(self, opt_name):
|
||||
if opt_name in self.overrides:
|
||||
return self.overrides[opt_name]
|
||||
|
||||
if 'SCRAPY_' + opt_name in os.environ:
|
||||
return os.environ['SCRAPY_' + opt_name]
|
||||
|
||||
if hasattr(self.settings, opt_name):
|
||||
return getattr(self.settings, opt_name)
|
||||
|
||||
if opt_name in self.defaults:
|
||||
return self.defaults[opt_name]
|
||||
|
||||
if hasattr(self.core, opt_name):
|
||||
return getattr(self.core, opt_name)
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self[name] if self[name] is not None else default
|
||||
|
||||
def getbool(self, name, default=False):
|
||||
"""
|
||||
True is: 1, '1', True
|
||||
False is: 0, '0', False, None
|
||||
"""
|
||||
return bool(int(self.get(name, default)))
|
||||
|
||||
def getint(self, name, default=0):
|
||||
return int(self.get(name, default))
|
||||
|
||||
def getfloat(self, name, default=0.0):
|
||||
return float(self.get(name, default))
|
||||
|
||||
def getlist(self, name, default=None):
|
||||
value = self.get(name)
|
||||
if value is None:
|
||||
return []
|
||||
elif hasattr(value, '__iter__'):
|
||||
return value
|
||||
else:
|
||||
return str(value).split(',')
|
||||
|
||||
settings = Settings()
|
||||
|
|
@ -0,0 +1 @@
|
|||
LOG_STDOUT = True
|
||||
|
|
@ -0,0 +1 @@
|
|||
LOG_ENABLED = False
|
||||
|
|
@ -0,0 +1 @@
|
|||
LOG_ENABLED = False
|
||||
|
|
@ -0,0 +1 @@
|
|||
LOG_ENABLED = False
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
LOG_ENABLED = False
|
||||
|
||||
|
|
@ -0,0 +1 @@
|
|||
LOG_ENABLED = False
|
||||
|
|
@ -0,0 +1 @@
|
|||
LOG_ENABLED = False
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import scrapy
|
||||
# Scrapy core settings
|
||||
|
||||
BOT_NAME = 'scrapy'
|
||||
BOT_VERSION = scrapy.__version__
|
||||
|
||||
ENGINE_DEBUG = False
|
||||
|
||||
# Download configuration options
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid (a negative value means "never expires")
|
||||
|
||||
LOG_ENABLED = True #
|
||||
LOGLEVEL = 'DEBUG' # default loglevel
|
||||
LOGFILE = None # None means sys.stderr by default
|
||||
LOG_STDOUT = False #
|
||||
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
|
||||
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
|
||||
MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore'
|
||||
PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
|
||||
|
||||
EXTENSIONS = []
|
||||
|
||||
# contrib.middleware.retry.RetryMiddleware default settings
|
||||
RETRY_TIMES = 3
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200']
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# Define here the models for your scraped items
|
||||
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
class MyItem(ScrapedItem):
|
||||
pass
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
execute()
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
import $project_name
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# - Scrapy settings for $project_name -
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BOT_NAME = 'scrapybot'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['$project_name.spiders']
|
||||
NEWSPIDER_MODULE = '$project_name.spiders'
|
||||
TEMPLATES_DIR = '%s/templates' % $project_name.__path__[0]
|
||||
ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % $project_name.__path__[0]
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
DOWNLOAD_TIMEOUT = 600
|
||||
|
||||
# uncomment if you want to add your own custom scrapy commands
|
||||
#COMMANDS_MODULE = '$project_name.commands'
|
||||
#COMMANDS_SETTINGS_MODULE = '$project_name.conf.commands'
|
||||
|
||||
#Global timeout between sucessive downloads (can be overrided by spider
|
||||
#attribute download_timeout
|
||||
#DOWNLOAD_TIMEOUT = 0
|
||||
|
||||
MYSQL_CONNECTION_SETTINGS = {"charset": "utf8" }
|
||||
MYSQL_CONNECTION_PING_PERIOD = 600
|
||||
|
||||
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
|
||||
SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO
|
||||
|
||||
#CACHE2_DIR = '/tmp/cache2' # if set, enables HTTP cache
|
||||
#CACHE2_IGNORE_MISSING = 0 # ignore requests not in cache
|
||||
#CACHE2_SECTORIZE = 1 # sectorize domains to distribute storage among servers
|
||||
|
||||
#STATS_ENABLED = 1 # enable stats
|
||||
#STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory)
|
||||
#STATS_DEBUG = 0 # log stats on domain closed
|
||||
|
||||
EXTENSIONS = (
|
||||
'scrapy.management.web.WebConsole',
|
||||
'scrapy.management.telnet.TelnetConsole',
|
||||
)
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = (
|
||||
# Engine side
|
||||
'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
|
||||
'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
|
||||
# Downloader side
|
||||
)
|
||||
|
||||
SPIDER_MIDDLEWARES = (
|
||||
# Engine side
|
||||
'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware',
|
||||
# Spider side
|
||||
)
|
||||
|
||||
# Item pipelines are usually configured by commands (see conf/commands)
|
||||
#ITEM_PIPELINES = (
|
||||
#)
|
||||
|
||||
#DEPTH_LIMIT = 10 # limit the maximum link depth to follow
|
||||
#DEPTH_STATS = 1 # enable depth stats
|
||||
|
||||
# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html
|
||||
URLLENGTH_LIMIT = 2083
|
||||
|
||||
#WEBCONSOLE_ENABLED = 1
|
||||
#WEBCONSOLE_PORT = 8060 # if not set uses a dynamic port
|
||||
|
||||
#TELNETCONSOLE_ENABLED = 1
|
||||
#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port
|
||||
|
||||
# global mail sending settings
|
||||
#MAIL_HOST = 'localhost'
|
||||
#MAIL_FROM = 'scrapybot@localhost'
|
||||
|
||||
# scrapy webservice
|
||||
WS_ENABLED = 0
|
||||
|
||||
SPIDERPROFILER_ENABLED = 0
|
||||
|
|
@ -0,0 +1 @@
|
|||
# Place here all your scrapy spiders
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
from scrapy.item.adaptors import AdaptorPipe as BaseAdaptorPipe
|
||||
from scrapy.utils.serialization import serialize
|
||||
|
||||
class DuplicatedAdaptorName(Exception): pass
|
||||
|
||||
class _Adaptor(object):
|
||||
"""
|
||||
Adaptors instances should be instantiated and used only
|
||||
inside the AdaptorPipe.
|
||||
"""
|
||||
def __init__(self, function, match_function):
|
||||
self.basefunction = function
|
||||
self.match_function = match_function
|
||||
def __repr__(self):
|
||||
return self.basefunction.func_name
|
||||
def __call__(self, *args):
|
||||
return self.basefunction(*args)
|
||||
|
||||
class AdaptorPipe(BaseAdaptorPipe):
|
||||
|
||||
def __init__(self, attribute_names, adaptors=None):
|
||||
"""
|
||||
If "adaptors" is given, constructs pipeline from this.
|
||||
"adaptors" is an ordered tuple of 2-elements tuples, each of which
|
||||
has the same parameters you give to the insertadaptor method, except
|
||||
'after' and 'before', because you define the adaptors order in the tuple.
|
||||
Example:
|
||||
(
|
||||
(my_function, lambda x: x in my_list)
|
||||
...
|
||||
)
|
||||
"""
|
||||
self.__attribute_names = [ n for n in attribute_names ]
|
||||
self.__adaptorspipe = []
|
||||
self.pipes = {}
|
||||
if adaptors:
|
||||
for entry in adaptors:
|
||||
self.insertadaptor(compile_pipe=False, *entry)
|
||||
self._compile_pipe()
|
||||
|
||||
@property
|
||||
def adaptors_names(self):
|
||||
_adaptors = []
|
||||
for a in self.__adaptorspipe:
|
||||
_adaptors.append(a.basefunction.func_name)
|
||||
return _adaptors
|
||||
|
||||
def insertadaptor(self, function, match_function=lambda x: True, compile_pipe=True, after=None, before=None):
|
||||
"""
|
||||
Inserts a "function" as an adaptor that will apply when match_function returns True (by
|
||||
default always apply)
|
||||
If "after" is given, inserts the adaptor after the already inserted adaptor
|
||||
of the name given in this parameter, If "before" is given, inserts it before
|
||||
the adaptor of the given name. "name" is the name of the adaptor.
|
||||
"""
|
||||
if function.func_name in self.adaptors_names:
|
||||
raise DuplicatedAdaptorName(function.func_name)
|
||||
else:
|
||||
adaptor = _Adaptor(function, match_function)
|
||||
#by default append adaptor at end of pipe
|
||||
pos = len(self.adaptors_names)
|
||||
if after:
|
||||
pos = self.adaptors_names.index(after) + 1
|
||||
elif before:
|
||||
pos = self.adaptors_names.index(before)
|
||||
self.__adaptorspipe.insert(pos, adaptor)
|
||||
if compile_pipe:
|
||||
self._compile_pipe()
|
||||
return pos
|
||||
|
||||
def removeadaptor(self, adaptorname):
|
||||
pos = self.adaptors_names.index(adaptorname)
|
||||
self.__adaptorspipe.pop(pos)
|
||||
self._compile_pipe()
|
||||
|
||||
def _compile_pipe(self):
|
||||
for attrname in self.__attribute_names:
|
||||
adaptors_pipe = []
|
||||
for adaptor in self.__adaptorspipe:
|
||||
if adaptor.match_function(attrname):
|
||||
adaptors_pipe.append(adaptor)
|
||||
self.pipes[attrname] = adaptors_pipe
|
||||
|
||||
def __repr__(self):
|
||||
return serialize(self.pipes, "pprint")
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
"""
|
||||
CloseDomain is an extension that forces spiders to be closed after a given
|
||||
time has expired.
|
||||
|
||||
"""
|
||||
import datetime
|
||||
import pprint
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.mail import MailSender
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CloseDomain(object):
|
||||
def __init__(self):
|
||||
self.timeout = settings.getint('CLOSEDOMAIN_TIMEOUT')
|
||||
if not self.timeout:
|
||||
raise NotConfigured
|
||||
|
||||
self.tasks = {}
|
||||
self.mail = MailSender()
|
||||
self.notify = settings.getlist('CLOSEDOMAIN_NOTIFY')
|
||||
|
||||
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
|
||||
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
|
||||
|
||||
def domain_opened(self, domain):
|
||||
self.tasks[domain] = scrapyengine.addtask(self.close_domain, self.timeout, args=[domain])
|
||||
|
||||
def close_domain(self, domain):
|
||||
log.msg("Domain was opened for more than %d seconds, closing it..." % self.timeout, domain=domain)
|
||||
scrapyengine.close_domain(domain)
|
||||
if self.notify:
|
||||
body = "Closed domain %s because it remained opened for more than %s\n\n" % (domain, datetime.timedelta(seconds=self.timeout))
|
||||
body += "DOMAIN STATS ------------------------------------------------------\n\n"
|
||||
body += pprint.pformat(stats.get(domain, None))
|
||||
subj = "Closed domain by timeout: %s" % domain
|
||||
self.mail.send(self.notify, subj, body)
|
||||
|
||||
def domain_closed(self, domain):
|
||||
scrapyengine.removetask(self.tasks[domain])
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
"""
|
||||
Extensions for debugging Scrapy
|
||||
"""
|
||||
import signal
|
||||
import traceback
|
||||
|
||||
class StackTraceDebug(object):
|
||||
def __init__(self):
|
||||
try:
|
||||
signal.signal(signal.SIGUSR1, self.dump_stacktrace)
|
||||
except AttributeError:
|
||||
# win32 platforms don't support SIGUSR signals
|
||||
pass
|
||||
|
||||
def dump_stacktrace(self, signum, frame):
|
||||
print "Got signal. Dumping stack trace..."
|
||||
traceback.print_stack(frame)
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import os
|
||||
import sys
|
||||
import hashlib
|
||||
import datetime
|
||||
import urlparse
|
||||
import cPickle as pickle
|
||||
from pydispatch import dispatcher
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.http import Response, Headers
|
||||
from scrapy.http.headers import headers_dict_to_raw
|
||||
from scrapy.core.exceptions import UsageError, NotConfigured, HttpException, IgnoreRequest
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class CacheMiddleware(object):
|
||||
def __init__(self):
|
||||
if not settings['CACHE2_DIR']:
|
||||
raise NotConfigured
|
||||
self.cache = Cache(settings['CACHE2_DIR'], sectorize=settings.getbool('CACHE2_SECTORIZE'))
|
||||
self.ignore_missing = settings.getbool('CACHE2_IGNORE_MISSING')
|
||||
dispatcher.connect(self.open_domain, signal=signals.domain_open)
|
||||
|
||||
def open_domain(self, domain):
|
||||
self.cache.open_domain(domain)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if not is_cacheable(request):
|
||||
return
|
||||
|
||||
key = request.fingerprint()
|
||||
domain = spider.domain_name
|
||||
try:
|
||||
response = self.cache.retrieve_response(domain, key)
|
||||
except:
|
||||
log.msg("Corrupt cache for %s" % request.url, log.WARNING)
|
||||
response = False
|
||||
if response:
|
||||
response.cached = True
|
||||
if not 200 <= int(response.status) < 300:
|
||||
raise HttpException(response.status, None, response)
|
||||
return response
|
||||
elif self.ignore_missing:
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if not is_cacheable(request):
|
||||
return response
|
||||
|
||||
if isinstance(response, Response) and not response.cached:
|
||||
key = request.fingerprint()
|
||||
domain = spider.domain_name
|
||||
self.cache.store(domain, key, request, response)
|
||||
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if not is_cacheable(request):
|
||||
return
|
||||
|
||||
if isinstance(exception, HttpException) and isinstance(exception.response, Response):
|
||||
key = request.fingerprint()
|
||||
domain = spider.domain_name
|
||||
self.cache.store(domain,key, request, exception.response)
|
||||
|
||||
def is_cacheable(request):
|
||||
scheme, _, _, _, _ = urlparse.urlsplit(request.url)
|
||||
return scheme in ['http', 'https']
|
||||
|
||||
|
||||
class Cache(object):
|
||||
DOMAIN_SECTORDIR = 'data'
|
||||
DOMAIN_LINKDIR = 'domains'
|
||||
|
||||
def __init__(self, cachedir, sectorize=False):
|
||||
self.cachedir = cachedir
|
||||
self.sectorize = sectorize
|
||||
|
||||
self.baselinkpath = os.path.join(self.cachedir, self.DOMAIN_LINKDIR)
|
||||
if not os.path.exists(self.baselinkpath):
|
||||
os.makedirs(self.baselinkpath)
|
||||
|
||||
self.basesectorpath = os.path.join(self.cachedir, self.DOMAIN_SECTORDIR)
|
||||
if not os.path.exists(self.basesectorpath):
|
||||
os.makedirs(self.basesectorpath)
|
||||
|
||||
def domainsectorpath(self, domain):
|
||||
sector = hashlib.sha1(domain).hexdigest()[0]
|
||||
return os.path.join(self.basesectorpath, sector, domain)
|
||||
|
||||
def domainlinkpath(self, domain):
|
||||
return os.path.join(self.baselinkpath, domain)
|
||||
|
||||
def requestpath(self, domain, key):
|
||||
linkpath = self.domainlinkpath(domain)
|
||||
return os.path.join(linkpath, key[0:2], key)
|
||||
|
||||
def open_domain(self, domain):
|
||||
if domain:
|
||||
linkpath = self.domainlinkpath(domain)
|
||||
if self.sectorize:
|
||||
sectorpath = self.domainsectorpath(domain)
|
||||
if not os.path.exists(sectorpath):
|
||||
os.makedirs(sectorpath)
|
||||
if not os.path.exists(linkpath):
|
||||
try:
|
||||
os.symlink(sectorpath, linkpath)
|
||||
except:
|
||||
os.makedirs(linkpath) # windows filesystem
|
||||
else:
|
||||
if not os.path.exists(linkpath):
|
||||
os.makedirs(linkpath)
|
||||
|
||||
def is_cached(self, domain, key):
|
||||
requestpath = self.requestpath(domain, key)
|
||||
if os.path.exists(requestpath):
|
||||
with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f:
|
||||
metadata = pickle.load(f)
|
||||
expiration_secs = settings.getint('CACHE2_EXPIRATION_SECS')
|
||||
if expiration_secs >= 0:
|
||||
if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs):
|
||||
return True
|
||||
else:
|
||||
log.msg('dropping old cached response from %s' % metadata['timestamp'])
|
||||
return False
|
||||
else:
|
||||
# disabled cache expiration
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def retrieve_response(self, domain, key):
|
||||
"""
|
||||
Return response dictionary if request has correspondent cache record;
|
||||
return None if not.
|
||||
"""
|
||||
if not self.is_cached(domain, key):
|
||||
return None # not cached
|
||||
|
||||
requestpath = self.requestpath(domain, key)
|
||||
metadata = responsebody = responseheaders = None
|
||||
with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f:
|
||||
metadata = pickle.load(f)
|
||||
with open(os.path.join(requestpath, 'response_body')) as f:
|
||||
responsebody = f.read()
|
||||
with open(os.path.join(requestpath, 'response_headers')) as f:
|
||||
responseheaders = f.read()
|
||||
|
||||
url = metadata['url']
|
||||
original_url = metadata.get('original_url', url)
|
||||
headers = Headers(responseheaders)
|
||||
status = metadata['status']
|
||||
|
||||
response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody)
|
||||
response.cached = True
|
||||
return response
|
||||
|
||||
def store(self, domain, key, request, response):
|
||||
requestpath = self.requestpath(domain, key)
|
||||
if not os.path.exists(requestpath):
|
||||
os.makedirs(requestpath)
|
||||
|
||||
metadata = {
|
||||
'url':request.url,
|
||||
'method': request.method,
|
||||
'status': response.status,
|
||||
'domain': response.domain,
|
||||
'original_url': response.original_url,
|
||||
'timestamp': datetime.datetime.utcnow(),
|
||||
}
|
||||
|
||||
# metadata
|
||||
with open(os.path.join(requestpath, 'meta_data'), 'w') as f:
|
||||
f.write(repr(metadata))
|
||||
# pickled metadata (to recover without using eval)
|
||||
with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f:
|
||||
pickle.dump(metadata, f)
|
||||
# response
|
||||
with open(os.path.join(requestpath, 'response_headers'), 'w') as f:
|
||||
f.write(headers_dict_to_raw(response.headers))
|
||||
with open(os.path.join(requestpath, 'response_body'), 'w') as f:
|
||||
f.write(response.body.get_content())
|
||||
# request
|
||||
with open(os.path.join(requestpath, 'request_headers'), 'w') as f:
|
||||
f.write(headers_dict_to_raw(request.headers))
|
||||
if request.body:
|
||||
with open(os.path.join(requestpath, 'request_body'), 'w') as f:
|
||||
f.write(request.body)
|
||||
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
class CommonMiddleware(object):
|
||||
"""This middleware provides common/basic functionality, and should always
|
||||
be enabled"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
request.headers.setdefault('Accept-Language', 'en')
|
||||
request.headers.setdefault('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
|
||||
if request.method == 'POST':
|
||||
request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
|
||||
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import zlib
|
||||
from gzip import GzipFile
|
||||
from cStringIO import StringIO
|
||||
|
||||
from scrapy.http import Response, ResponseBody
|
||||
|
||||
|
||||
class CompressionMiddleware(object):
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if isinstance(response, Response):
|
||||
content_encoding = response.headers.get('Content-Encoding')
|
||||
if content_encoding:
|
||||
encoding = content_encoding[0].lower()
|
||||
raw_body = response.body.get_content()
|
||||
declared_encoding = response.body.declared_encoding
|
||||
decoded_body = self._decode(raw_body, encoding)
|
||||
response.body = ResponseBody(decoded_body, declared_encoding)
|
||||
response.headers['Content-Encoding'] = content_encoding[1:]
|
||||
return response
|
||||
|
||||
def _decode(self, body, encoding):
|
||||
if encoding == 'gzip':
|
||||
body = GzipFile(fileobj=StringIO(body)).read()
|
||||
|
||||
if encoding == 'deflate':
|
||||
try:
|
||||
body = zlib.decompress(body)
|
||||
except zlib.error:
|
||||
# ugly hack to work with raw deflate content that may
|
||||
# be sent by microsof servers. For more information, see:
|
||||
# http://carsten.codimi.de/gzip.yaws/
|
||||
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
|
||||
# http://www.gzip.org/zlib/zlib_faq.html#faq38
|
||||
body = zlib.decompress(body, -15)
|
||||
return body
|
||||
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.utils.misc import dict_updatedefault
|
||||
|
||||
class CookiesMiddleware(object):
|
||||
"""This middleware enables working with sites that need cookies"""
|
||||
|
||||
def __init__(self):
|
||||
self.cookies = {}
|
||||
dispatcher.connect(self.domain_open, signals.domain_open)
|
||||
dispatcher.connect(self.domain_closed, signals.domain_closed)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
dict_updatedefault(request.cookies, self.cookies[spider.domain_name])
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
cookies = self.cookies[spider.domain_name]
|
||||
cookies.update(request.cookies)
|
||||
return response
|
||||
|
||||
def domain_open(self, domain):
|
||||
self.cookies[domain] = {}
|
||||
|
||||
def domain_closed(self, domain):
|
||||
del self.cookies[domain]
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CrawlDebug(object):
|
||||
def __init__(self):
|
||||
self.enabled = settings.getbool('CRAWL_DEBUG')
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if self.enabled:
|
||||
log.msg("Crawling %s" % repr(request), domain=spider.domain_name, level=log.DEBUG)
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if self.enabled:
|
||||
log.msg("Crawl exception %s in %s" % (exception, repr(request)), domain=spider.domain_name, level=log.DEBUG)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if self.enabled:
|
||||
log.msg("Fetched %s from %s" % (response.info(), repr(request)), domain=spider.domain_name, level=log.DEBUG)
|
||||
return response
|
||||
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from scrapy.core.exceptions import HttpException
|
||||
|
||||
class ErrorPagesMiddleware(object):
|
||||
"""This middleware allows the spiders to receive error (non 200) responses,
|
||||
the same way the receive normal responses"""
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, HttpException):
|
||||
statuses = getattr(spider, 'handle_httpstatus_list', None)
|
||||
httpstatus = exception.response.status
|
||||
if statuses and httpstatus in statuses:
|
||||
return exception.response
|
||||
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
class HttpAuthMiddleware(object):
|
||||
"""This middleware allows spiders to use HTTP auth in a cleaner way
|
||||
(http_user and http_pass spider class attributes)"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if getattr(spider, 'http_user', None) or getattr(spider, 'http_pass', None):
|
||||
request.httpauth(spider.http_user, spider.http_pass)
|
||||
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import re
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.utils.url import urljoin_rfc as urljoin
|
||||
|
||||
class RedirectLoop(Exception):
|
||||
pass
|
||||
|
||||
META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);url=([^"\']+)', re.IGNORECASE)
|
||||
# some sites use meta-refresh for redirecting to a session expired page, so we
|
||||
# restrict automatic redirection to a maximum delay (in number of seconds)
|
||||
META_REFRESH_MAXSEC = 100
|
||||
MAX_REDIRECT_LOOP = 10
|
||||
|
||||
class RedirectMiddleware(object):
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, HttpException):
|
||||
status = exception.status
|
||||
response = exception.response
|
||||
|
||||
if status in ['302', '303']:
|
||||
redirected_url = urljoin(request.url, response.headers['location'][0])
|
||||
if not getattr(spider, "no_redirect", False):
|
||||
redirected = request.copy()
|
||||
redirected.url = redirected_url
|
||||
redirected.method = 'GET'
|
||||
redirected.body = None
|
||||
# This is needed to avoid redirection loops with requests that contain dont_filter = True
|
||||
# Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks
|
||||
if isinstance(redirected.dont_filter, int):
|
||||
if not hasattr(redirected, "original_dont_filter"):
|
||||
redirected.original_dont_filter = redirected.dont_filter
|
||||
if redirected.dont_filter <= -MAX_REDIRECT_LOOP:
|
||||
raise RedirectLoop("Exited redirect loop with %s consecutive visits to the same url." % (redirected.original_dont_filter + MAX_REDIRECT_LOOP) )
|
||||
redirected.dont_filter -= 1
|
||||
else:
|
||||
redirected.dont_filter = False
|
||||
log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
|
||||
return redirected
|
||||
log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
|
||||
return response
|
||||
|
||||
if status in ['301', '307']:
|
||||
redirected_url = urljoin(request.url, response.headers['location'][0])
|
||||
if not getattr(spider, "no_redirect", False):
|
||||
redirected = request.copy()
|
||||
redirected.url = redirected_url
|
||||
# This is needed to avoid redirection loops with requests that contain dont_filter = True
|
||||
# Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks
|
||||
redirected.dont_filter = False
|
||||
log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
|
||||
return redirected
|
||||
log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
|
||||
return response
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if isinstance(response, Response):
|
||||
m = META_REFRESH_RE.search(response.body.to_string()[0:4096])
|
||||
if m and int(m.group(1)) < META_REFRESH_MAXSEC:
|
||||
redirected = request.copy()
|
||||
redirected.url = urljoin(request.url, m.group(2))
|
||||
log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name)
|
||||
return redirected
|
||||
return response
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
An extension to retry failed requests that are potentially caused by temporary
|
||||
problems such as a connection timeout or HTTP 500 error.
|
||||
|
||||
You can change the behaviour of this moddileware by modifing the scraping settings:
|
||||
RETRY_TIMES - how many times to retry a failed page
|
||||
RETRY_HTTP_CODES - which HTTP response codes to retry
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non failed) pages. Once
|
||||
there is no more failed pages to retry this middleware sends a signal
|
||||
(retry_complete), so other extensions could connect to that signal.
|
||||
|
||||
Default values are located in scrapy.conf.core_settings
|
||||
|
||||
About HTTP errors to consider:
|
||||
|
||||
- You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP
|
||||
protocol. It's included by default because it's a common code used to
|
||||
indicate server overload, which would be something we want to retry
|
||||
- 200 is included by default (and shoudln't be removed) to check for partial
|
||||
downloads errors, which means the TCP connection has broken in the middle of
|
||||
a HTTP download
|
||||
"""
|
||||
|
||||
from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError
|
||||
from twisted.internet.defer import TimeoutError as UserTimeoutError
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.conf import settings
|
||||
|
||||
class RetryMiddleware(object):
|
||||
|
||||
EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError)
|
||||
|
||||
def __init__(self):
|
||||
self.failed_count = {}
|
||||
self.max_retries = settings.getint('RETRY_TIMES')
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
retry = False
|
||||
|
||||
if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
|
||||
retry = True
|
||||
elif isinstance(exception, HttpException):
|
||||
if exception.status in settings.getlist('RETRY_HTTP_CODES'):
|
||||
retry = True
|
||||
|
||||
if retry:
|
||||
fp = request.fingerprint()
|
||||
count = self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
|
||||
|
||||
if self.failed_count[fp] < self.max_retries:
|
||||
log.msg("Retrying %s (failed %d times): %s" % (request, count, exception), level=log.DEBUG, domain=spider.domain_name)
|
||||
retryreq = request.copy()
|
||||
retryreq.dont_filter = True
|
||||
return retryreq
|
||||
else:
|
||||
log.msg("Discarding %s (failed %d times): %s" % (request, count, exception), domain=spider.domain_name, level=log.DEBUG)
|
||||
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
"""
|
||||
jhis is an expertimental middleware to respect robots.txt policies. The biggest
|
||||
problem it has is that it uses urllib directly (in RobotFileParser.read()
|
||||
method) and that conflicts with twisted networking, so it should be ported to
|
||||
use twisted networking API, but that is not as trivial as it may seem.
|
||||
|
||||
This code is left here for future reference, when we resume the work on this
|
||||
subject.
|
||||
|
||||
"""
|
||||
|
||||
import re
|
||||
import urlparse
|
||||
import robotparser
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import IgnoreRequest
|
||||
from scrapy.conf import settings
|
||||
|
||||
BASEURL_RE = re.compile("http://.*?/")
|
||||
|
||||
class RobotsMiddleware(object):
|
||||
|
||||
def __init__(self):
|
||||
self._parsers = {}
|
||||
self._spiderdomains = {}
|
||||
self._pending = {}
|
||||
dispatcher.connect(self.domain_open, signals.domain_open)
|
||||
dispatcher.connect(self.domain_closed, signals.domain_closed)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
agent = getattr(spider, 'user_agent', None) or settings['USER_AGENT']
|
||||
rp = self.robot_parser(request.url, spider.domain_name)
|
||||
if rp and not rp.can_fetch(agent, request.url):
|
||||
raise IgnoreRequest("URL forbidden by robots.txt: %s" % request.url)
|
||||
|
||||
def robot_parser(self, url, spiderdomain):
|
||||
urldomain = urlparse.urlparse(url).hostname
|
||||
if urldomain in self._parsers:
|
||||
rp = self._parsers[urldomain]
|
||||
else:
|
||||
rp = robotparser.RobotFileParser()
|
||||
m = BASEURL_RE.search(url)
|
||||
if m:
|
||||
rp.set_url("%srobots.txt" % m.group())
|
||||
rp.read()
|
||||
self._parsers[urldomain] = rp
|
||||
self._spiderdomains[spiderdomain].add(urldomain)
|
||||
return rp
|
||||
|
||||
def domain_open(self, domain):
|
||||
self._spiderdomains[domain] = set()
|
||||
|
||||
def domain_closed(self, domain):
|
||||
for urldomain in self._spiderdomains[domain]:
|
||||
del self._parsers[urldomain]
|
||||
del self._spiderdomains[domain]
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
class UserAgentMiddleware(object):
|
||||
"""This middleware allows spiders to override the user_agent"""
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if getattr(spider, 'user_agent', None):
|
||||
request.headers.setdefault('User-Agent', spider.user_agent)
|
||||
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
"""
|
||||
Extensions to override scrapy settings with per-group settings according to the
|
||||
group the spider belongs to. It only overrides the settings when running the
|
||||
crawl command with *only one domain as argument*.
|
||||
"""
|
||||
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.command.cmdline import command_executed
|
||||
|
||||
class GroupSettings(object):
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool("GROUPSETTINGS_ENABLED"):
|
||||
raise NotConfigured
|
||||
|
||||
if command_executed and command_executed['name'] == 'crawl':
|
||||
mod = __import__(settings['GROUPSETTINGS_MODULE'], {}, {}, [''])
|
||||
args = command_executed['args']
|
||||
if len(args) == 1 and not args[0].startswith('http://'):
|
||||
domain = args[0]
|
||||
settings.overrides.update(mod.default_settings)
|
||||
for group, domains in mod.group_spiders.iteritems():
|
||||
if domain in domains:
|
||||
settings.overrides.update(mod.group_settings.get(group, {}))
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
from scrapy.contrib.history.history import ItemHistory
|
||||
from scrapy.contrib.history.scheduler import RulesScheduler
|
||||
from scrapy.contrib.history.store import SQLHistoryStore
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
"""
|
||||
History management
|
||||
"""
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import MySQLdb
|
||||
|
||||
class History(object):
|
||||
"""Base class for tracking different kinds of histories"""
|
||||
|
||||
def __init__(self, db_uri):
|
||||
self.db_uri = db_uri
|
||||
self._mysql_conn = None
|
||||
|
||||
def connect(self):
|
||||
"""
|
||||
Connect to PDB and open mysql connect to PRODUCT_DB
|
||||
"""
|
||||
m = re.search(r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/]+)/(?P<db>.*)$", self.db_uri)
|
||||
if m:
|
||||
d = m.groupdict()
|
||||
if d['passwd'] is None:
|
||||
del(d['passwd'])
|
||||
|
||||
d['charset'] = "utf8"
|
||||
self._mysql_conn = MySQLdb.connect(**d)
|
||||
|
||||
def get_mysql_conn(self):
|
||||
if self._mysql_conn is None:
|
||||
self.connect()
|
||||
return self._mysql_conn
|
||||
mysql_conn = property(get_mysql_conn)
|
||||
|
||||
|
||||
class ItemHistory(History):
|
||||
"""
|
||||
Class instance registers item guids, and versions with check_item method.
|
||||
It also gives access to stored ItemTicket and ItemVersion objects.
|
||||
"""
|
||||
NEW = 0
|
||||
UPDATE = 1
|
||||
DUPLICATE = 2
|
||||
|
||||
def check_item(self, domain, item):
|
||||
"""
|
||||
Store item's guid and version to the database along
|
||||
with date and time of last occurence.
|
||||
Return:
|
||||
* ItemHistory.NEW - if guid hasn't been met
|
||||
* ItemHistory.UPDATE - if history already contains guid, but versions doesn't match
|
||||
* ItemHistory.DUPLICATE - both guid and version are not new
|
||||
"""
|
||||
version = item.version
|
||||
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
|
||||
def add_version(version):
|
||||
insert = "INSERT INTO version (guid, version, seen) VALUES (%s,%s,%s)"
|
||||
c.execute(insert, (item.guid, version, datetime.now()))
|
||||
|
||||
select = "SELECT * FROM ticket WHERE guid=%s"
|
||||
c.execute(select, item.guid)
|
||||
r = c.fetchone()
|
||||
if r:
|
||||
select = "SELECT * FROM version WHERE version=%s"
|
||||
if c.execute(select, version):
|
||||
update = "UPDATE version SET seen=%s WHERE version=%s"
|
||||
c.execute(update, (datetime.now(), version))
|
||||
self.mysql_conn.commit()
|
||||
return ItemHistory.DUPLICATE
|
||||
else:
|
||||
add_version(version)
|
||||
self.mysql_conn.commit()
|
||||
return ItemHistory.UPDATE
|
||||
else:
|
||||
insert = "INSERT INTO ticket (guid, domain, url, url_hash) VALUES (%s,%s,%s,%s)"
|
||||
c.execute(insert, (item.guid, domain, item.url, hash(item.url)))
|
||||
add_version(version)
|
||||
self.mysql_conn.commit()
|
||||
return ItemHistory.NEW
|
||||
|
||||
def get_ticket(self, guid):
|
||||
"""
|
||||
Return ItemTicket object for guid.
|
||||
ItemVersion objects can be accessed via 'versions' list.
|
||||
"""
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM ticket WHERE guid=%s"
|
||||
c.execute(select, guid)
|
||||
ticket = c.fetchone()
|
||||
if not ticket:
|
||||
raise Exception("Item ticket with guid = '%s' not found" % guid)
|
||||
ticket['versions'] = []
|
||||
select = "SELECT * FROM version WHERE guid=%s"
|
||||
c.execute(select, guid)
|
||||
for version in c.fetchall():
|
||||
ticket['versions'].append(version)
|
||||
return ticket
|
||||
|
||||
def delete_ticket(self, guid):
|
||||
"""Delete item ticket and associated versions from DB"""
|
||||
c = self.mysql_conn.cursor()
|
||||
delete = "DELETE FROM ticket WHERE guid=%s"
|
||||
c.execute(delete, guid)
|
||||
self.mysql_conn.commit()
|
||||
|
||||
|
||||
class URLHistory(History):
|
||||
"""
|
||||
Access URL status and history for the scraping engine
|
||||
|
||||
This is degsigned to have an instance per domain where typically
|
||||
a call will be made to get_url_status, followed by either
|
||||
update_checked or record_version.
|
||||
"""
|
||||
|
||||
def get_url_status(self, urlkey):
|
||||
"""
|
||||
Get the url status (url, last_version, last_checked),
|
||||
or None if the url data has not been seen before
|
||||
"""
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM url_status WHERE url_hash=%s"
|
||||
c.execute(select, urlkey)
|
||||
r = c.fetchone()
|
||||
return (r['url'], r['last_version'], r['last_checked']) if r else None
|
||||
|
||||
def record_version(self, urlkey, url, parent_key, version, postdata_hash=None):
|
||||
"""
|
||||
Record a version of a page and update the last checked time.
|
||||
|
||||
If the same version (or None) is passed, the last checked time is still updated.
|
||||
"""
|
||||
now = datetime.now()
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM url_status WHERE url_hash=%s"
|
||||
c.execute(select, urlkey)
|
||||
r = c.fetchone()
|
||||
|
||||
if not r:
|
||||
insert = "INSERT INTO url_status (url_hash, url, parent_hash, last_version, last_checked) VALUES (%s,%s,%s,%s,%s)"
|
||||
c.execute(insert, (urlkey, url, parent_key, version, now))
|
||||
else:
|
||||
update = "UPDATE url_status SET last_version=%s, last_checked=%s WHERE url_hash=%s"
|
||||
c.execute(update, (version, now, urlkey))
|
||||
self.mysql_conn.commit()
|
||||
|
||||
last_version = r['last_version'] if r else None
|
||||
if version and version != last_version:
|
||||
if not c.execute("SELECT url_hash FROM url_history WHERE version=%s", version):
|
||||
insert = "INSERT INTO url_history (url_hash, version, postdata_hash, created) VALUES (%s,%s,%s,%s)"
|
||||
c.execute(insert, (urlkey, version, postdata_hash, now))
|
||||
self.mysql_conn.commit()
|
||||
|
||||
def get_version_info(self, version):
|
||||
"""Simple accessor method"""
|
||||
c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
select = "SELECT * FROM url_history WHERE version=%s"
|
||||
c.execute(select, version)
|
||||
r = c.fetchone()
|
||||
return (r['url_hash'], r['created']) if r else None
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import hashlib
|
||||
from datetime import datetime
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.utils.misc import load_class
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.conf import settings
|
||||
|
||||
class HistoryMiddleware(object):
|
||||
# How often we should re-check links we know about
|
||||
MIN_CHECK_DAYS = 4
|
||||
# How often we should process pages that have not changed (need to include depth)
|
||||
MIN_PROCESS_UNCHANGED_DAYS = 12
|
||||
|
||||
def __init__(self):
|
||||
historycls = load_class(settings['MEMORYSTORE'])
|
||||
if not historycls:
|
||||
raise NotConfigured
|
||||
self.historydata = historycls()
|
||||
dispatcher.connect(self.open_domain, signal=signals.domain_open)
|
||||
dispatcher.connect(self.close_domain, signal=signals.domain_closed)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
key = urlkey(request.url)
|
||||
status = self.historydata.status(domain, key)
|
||||
if status:
|
||||
_url, version, last_checked = status
|
||||
d = datetime.now() - last_checked
|
||||
if d.days < self.MIN_CHECK_DAYS:
|
||||
raise IgnoreRequest("Not scraping %s (scraped %s ago)" % (request.url, d))
|
||||
request.context['history_response_version'] = version
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
version = request.context.get('history_response_version')
|
||||
if version == response.version():
|
||||
del request.content['history_response_version']
|
||||
hist = self.historydata.version_info(domain, version)
|
||||
if hist:
|
||||
versionkey, created = hist
|
||||
# if versionkey != urlkey(url) this means
|
||||
# the same content is available on a different url
|
||||
delta = datetime.now() - created
|
||||
if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS:
|
||||
message = "skipping %s: unchanged for %s" % (response.url, delta)
|
||||
raise IgnoreRequest(message)
|
||||
self.record_visit(domain, request, response)
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
self.record_visit(spider.domain_name, request, None)
|
||||
|
||||
def open_domain(self, domain):
|
||||
self.historydata.open(domain)
|
||||
|
||||
def close_domain(self, domain):
|
||||
self.historydata.close_site(domain)
|
||||
|
||||
def record_visit(self, domain, request, response):
|
||||
"""record the fact that the url has been visited"""
|
||||
url = request.url
|
||||
post_version = hash(request.body)
|
||||
key = urlkey(url)
|
||||
if response:
|
||||
redirect_url = response.url
|
||||
parentkey = urlkey(response.request.headers.get('referer')) if response.request else None
|
||||
version = response.version()
|
||||
else:
|
||||
redirect_url, parentkey, version = url, None, None
|
||||
self.historydata.store(domain, key, url, parentkey, version, post_version)
|
||||
|
||||
|
||||
def urlkey(url):
|
||||
"""Generate a 'key' for a given url
|
||||
|
||||
>>> urlkey("http://www.example.com/")
|
||||
'89e6a0649e06d83370cdf2cbfb05f363934a8d0c'
|
||||
>>> urlkey("http://www.example.com/") == urlkey("http://www.example.com/?")
|
||||
True
|
||||
"""
|
||||
from scrapy.utils.c14n import canonicalize
|
||||
return hash(canonicalize(url))
|
||||
|
||||
|
||||
def hash(value):
|
||||
return hashlib.sha1(value).hexdigest() if value else None
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
from datetime import datetime
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.core.scheduler import Scheduler
|
||||
from scrapy.core.exceptions import IgnoreRequest
|
||||
|
||||
class RulesScheduler(Scheduler):
|
||||
"""Scheduler that uses rules to determine if we should follow links
|
||||
|
||||
TODO:
|
||||
* take into account where in chain of links we are (less depth should
|
||||
be crawled more often)
|
||||
* Be more strict about scraping product pages that rarely lead to new
|
||||
versions of products. The same applies to pages with links. Particularly
|
||||
useful for filtering out when there are many urls for the same product.
|
||||
(but be careful to also filter out pages that almost always lead to new
|
||||
output).
|
||||
"""
|
||||
|
||||
# if these parameters change, then update bin/unavailable.py
|
||||
|
||||
# How often we should re-check links we know about
|
||||
MIN_CHECK_DAYS = 4
|
||||
|
||||
# How often we should process pages that have not changed (need to include depth)
|
||||
MIN_PROCESS_UNCHANGED_DAYS = 12
|
||||
|
||||
def enqueue_request(self, domain, request, priority=1):
|
||||
"""Add a page to be scraped for a domain that is currently being scraped.
|
||||
|
||||
The url will only be added if we have not checked it already within
|
||||
a specified time period.
|
||||
"""
|
||||
requestid = request.fingerprint()
|
||||
added = self.groupfilter.add(domain, requestid)
|
||||
|
||||
if request.dont_filter or added:
|
||||
key = urlkey(request.url) # we can not use fingerprint unless lost crawled history
|
||||
status = self.historydata.status(domain, key)
|
||||
now = datetime.now()
|
||||
version = None
|
||||
if status:
|
||||
_url, version, last_checked = status
|
||||
d = now - last_checked
|
||||
if d.days < self.MIN_CHECK_DAYS:
|
||||
log.msg("Not scraping %s (scraped %s ago)" % (request.url, d), level=log.DEBUG)
|
||||
return
|
||||
# put the version in the pending pages to avoid querying DB again
|
||||
record = (request, version, now)
|
||||
self.pending_requests[domain].put(record, priority)
|
||||
|
||||
def next_request(self, domain):
|
||||
"""Get the next page from the superclass. This will add a callback
|
||||
to prevent processing the page unless its content has been
|
||||
changed.
|
||||
|
||||
In the event that it a page is not processed, the record_visit method
|
||||
is called to update the last_checked time.
|
||||
"""
|
||||
pending_list = self.pending_requests.get(domain)
|
||||
if not pending_list :
|
||||
return None
|
||||
request, version, timestamp = pending_list.get_nowait()[1]
|
||||
post_version = hash(request.body)
|
||||
|
||||
def callback(pagedata):
|
||||
"""process other callback if we pass the checks"""
|
||||
if version == pagedata.version():
|
||||
hist = self.historydata.version_info(domain, version)
|
||||
if hist:
|
||||
versionkey, created = hist
|
||||
# if versionkey != urlkey(url) this means
|
||||
# the same content is available on a different url
|
||||
delta = timestamp - created
|
||||
if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS:
|
||||
message = "skipping %s: unchanged for %s" % (pagedata.url, delta)
|
||||
raise IgnoreRequest(message)
|
||||
self.record_visit(domain, request.url, pagedata.url,
|
||||
pagedata.parent, pagedata.version(),
|
||||
post_version)
|
||||
return pagedata
|
||||
|
||||
def errback(error) :
|
||||
self.record_visit(domain, request.url, request.url, None, None,
|
||||
post_version)
|
||||
return error
|
||||
|
||||
d = defer.Deferred()
|
||||
d.addCallbacks(callback, errback)
|
||||
request.prepend_callback(d)
|
||||
|
||||
return request
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
"""
|
||||
SQLHistoryStore
|
||||
|
||||
Persistent history storage using relational database
|
||||
"""
|
||||
from scrapy.contrib.history.history import URLHistory
|
||||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
|
||||
class SQLHistoryStore(object) :
|
||||
"""Implementation of a data store that stores information in a relation
|
||||
database
|
||||
|
||||
This maintains a URLHistory object per site. That means each domain
|
||||
has it's own session and is isolated from the others.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._store = {}
|
||||
self._dbinfo = settings['SCRAPING_DB']
|
||||
self._debug = settings['DEBUG_SQL_HISTORY_STORE']
|
||||
|
||||
def open(self, site):
|
||||
self._store[site] = URLHistory(self._dbinfo)
|
||||
|
||||
def close_site(self, site):
|
||||
self._store[site].close()
|
||||
del self._store[site]
|
||||
|
||||
def store(self, site, key, url, parent=None, version=None, post_version=None):
|
||||
history = self._store[site]
|
||||
if self._debug:
|
||||
log.msg("record_version(key=%s, url=%s, parent=%s, version=%s, post_version=%s)" %
|
||||
(key, url, parent, version, post_version), domain=site, level=log.DEBUG)
|
||||
history.record_version(key, url, parent, version, post_version)
|
||||
|
||||
def has_site(self, site):
|
||||
return site in self._store
|
||||
|
||||
def status(self, site, key):
|
||||
if site in self._store:
|
||||
history = self._store[site]
|
||||
return history.get_url_status(key)
|
||||
|
||||
def version_info(self, site, version):
|
||||
history = self._store[site]
|
||||
return history.get_version_info(version)
|
||||
|
|
@ -0,0 +1 @@
|
|||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
"""
|
||||
This module contains some extra base models for scraped items which could be
|
||||
useful in some Scrapy implementations
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from pprint import PrettyPrinter
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.core.exceptions import UsageError, DropItem
|
||||
|
||||
class ValidationError(DropItem):
|
||||
"""Indicates a data validation error"""
|
||||
def __init__(self,problem,value=None):
|
||||
self.problem = problem
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
if self.value is not None:
|
||||
return '%s "%s"' % (self.problem, self.value)
|
||||
else:
|
||||
return '%s' % (self.problem)
|
||||
|
||||
class ValidationPipeline(object):
|
||||
def process_item(self, domain, response, item):
|
||||
item.validate()
|
||||
return item
|
||||
|
||||
class RobustScrapedItem(ScrapedItem):
|
||||
"""
|
||||
A more robust scraped item class with a built-in validation mechanism and
|
||||
minimal versioning support
|
||||
"""
|
||||
|
||||
ATTRIBUTES = {
|
||||
'guid': basestring, # a global unique identifier
|
||||
'url': basestring, # the main URL where this item was scraped from
|
||||
}
|
||||
|
||||
def __init__(self, data=None):
|
||||
"""
|
||||
A scraped item can be initialised with a dictionary that will be
|
||||
squirted directly into the object.
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
for attr, value in data.iteritems():
|
||||
setattr(self, attr, value)
|
||||
elif data is not None:
|
||||
raise UsageError("Initialize with dict, not %s" % data.__class__.__name__)
|
||||
|
||||
self.__dict__['_version'] = None
|
||||
|
||||
def __getattr__(self, attr):
|
||||
# Return None for valid attributes not set, raise AttributeError for invalid attributes
|
||||
# Note that this method is called only when the attribute is not found in
|
||||
# self.__dict__ or the class/instance methods.
|
||||
if attr in self.ATTRIBUTES:
|
||||
return None
|
||||
else:
|
||||
raise AttributeError(attr)
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
"""
|
||||
Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES
|
||||
"""
|
||||
if not attr.startswith('_') and attr not in self.ATTRIBUTES:
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__))
|
||||
|
||||
if value is None:
|
||||
self.__dict__.pop(attr, None)
|
||||
return
|
||||
|
||||
type1 = self.ATTRIBUTES[attr]
|
||||
if hasattr(type1, '__iter__'):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise TypeError('Attribute "%s" must be a sequence' % attr)
|
||||
type2 = type1[0]
|
||||
for i in value:
|
||||
if not isinstance(i, type2):
|
||||
raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__))
|
||||
else:
|
||||
if not isinstance(value, type1):
|
||||
raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__))
|
||||
|
||||
self.__dict__[attr] = value
|
||||
self.__dict__['_version'] = None
|
||||
|
||||
def __delattr__(self, attr):
|
||||
"""
|
||||
Delete an attribute from the ScrapedItem instance if it exists.
|
||||
If not, raise an AttributeError.
|
||||
"""
|
||||
if attr in self.__dict__:
|
||||
del self.__dict__[attr]
|
||||
self.__dict__['_version'] = None
|
||||
else:
|
||||
raise AttributeError("Attribute '%s' doesn't exist" % attr)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, type(self)):
|
||||
return self.version == other.version
|
||||
|
||||
def __ne__(self, other):
|
||||
return self.version != other.version
|
||||
|
||||
def __sub__(self, other):
|
||||
return RobustItemDelta(other, self)
|
||||
|
||||
def __repr__(self):
|
||||
# Generate this format so that it can be deserialized easily:
|
||||
# ClassName({...})
|
||||
reprdict = {}
|
||||
for k, v in self.__dict__.iteritems():
|
||||
if not k.startswith('_'):
|
||||
reprdict[k] = v
|
||||
return "%s(%s)" % (self.__class__.__name__, repr(reprdict))
|
||||
|
||||
def __str__(self) :
|
||||
return "%s: GUID=%s, url=%s" % ( self.__class__.__name__ , self.guid, self.url )
|
||||
|
||||
def validate(self):
|
||||
"""Method used to validate item attributes data"""
|
||||
if not self.guid:
|
||||
raise ValidationError('A guid is required')
|
||||
|
||||
def copy(self):
|
||||
"""Create a new ScrapedItem object based on the current one"""
|
||||
import copy
|
||||
return copy.deepcopy(self)
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
"""
|
||||
Return a (cached) 40 char hash of all the item attributes.
|
||||
|
||||
WARNING: This cached version won't work if mutable products are
|
||||
modified directly like:
|
||||
|
||||
item.features.append('feature')
|
||||
"""
|
||||
if self._version:
|
||||
return self._version
|
||||
hash_ = hashlib.sha1()
|
||||
hash_.update("".join(["".join([n, str(v)]) for n,v in sorted(self.__dict__.iteritems())]))
|
||||
return hash_.hexdigest()
|
||||
|
||||
|
||||
class RobustItemDelta(object):
|
||||
"""
|
||||
This class represents the difference between
|
||||
a pair of RobustScrapedItems.
|
||||
"""
|
||||
|
||||
def __init__(self, old_item, new_item):
|
||||
if not isinstance(old_item, RobustScrapedItem) or \
|
||||
not isinstance(new_item, RobustScrapedItem):
|
||||
raise TypeError("Both arguments must be RobustScrapedItem instances")
|
||||
|
||||
if old_item.guid != new_item.guid:
|
||||
raise AttributeError("Item GUIDs must be equal in order to create a RobustItemDelta object")
|
||||
|
||||
self.old_item = old_item
|
||||
self.new_item = new_item
|
||||
self.diff = self.do_diff()
|
||||
|
||||
def do_diff(self):
|
||||
"""
|
||||
This method should retreive a dictionary
|
||||
containing the changes between both items
|
||||
as in this example:
|
||||
|
||||
>>> delta.do_diff()
|
||||
>>> {'attrib': {'new': 'New value', 'old': 'Old value'}, # Common attributes
|
||||
'attrib2': {'new': 'New value 2', 'old': 'Old value 2'},
|
||||
'attrib3': [{'new': 'New list value', 'old': 'Old list value'}, # List attributes
|
||||
{'new': 'New list value 2', 'old': 'Old list value 2'}]}
|
||||
"""
|
||||
|
||||
if self.old_item == self.new_item:
|
||||
return {}
|
||||
|
||||
diff = {}
|
||||
for key, value in self.old_item.__dict__.items():
|
||||
if key in self.old_item.ATTRIBUTES.keys():
|
||||
new_value = getattr(self.new_item, key)
|
||||
if value != new_value:
|
||||
diff[key] = {'new': new_value, 'old': value}
|
||||
for key, value in self.new_item.__dict__.items():
|
||||
if value and key in self.new_item.ATTRIBUTES.keys():
|
||||
if not getattr(self.old_item, key):
|
||||
diff[key] = {'new': value, 'old': None}
|
||||
return diff
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, RobustItemDelta):
|
||||
if other.old_item == self.old_item and \
|
||||
other.new_item == self.new_item and \
|
||||
other.diff == self.diff:
|
||||
return True
|
||||
return False
|
||||
|
||||
def __repr__(self):
|
||||
if self.diff:
|
||||
pp = PrettyPrinter(indent=3)
|
||||
return pp.pformat(self.diff)
|
||||
else:
|
||||
return 'No differences found between the provided items.'
|
||||
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
import pprint
|
||||
import gc
|
||||
import socket
|
||||
|
||||
import libxml2
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.mail import MailSender
|
||||
from scrapy.stats import stats
|
||||
from scrapy.extension import extensions
|
||||
from scrapy.conf import settings
|
||||
|
||||
class MemoryDebugger(object):
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('MEMDEBUG_ENABLED'):
|
||||
raise NotConfigured
|
||||
|
||||
self.mail = MailSender()
|
||||
self.rcpts = settings.getlist('MEMDEBUG_NOTIFY')
|
||||
|
||||
self.domains_scraped = []
|
||||
dispatcher.connect(self.domain_opened, signals.domain_opened)
|
||||
dispatcher.connect(self.engine_started, signals.engine_started)
|
||||
dispatcher.connect(self.engine_stopped, signals.engine_stopped)
|
||||
|
||||
def engine_started(self):
|
||||
libxml2.debugMemory(1)
|
||||
|
||||
def engine_stopped(self):
|
||||
figures = self.collect_figures()
|
||||
report = self.create_report(figures)
|
||||
self.print_or_send_report(report)
|
||||
|
||||
def collect_figures(self):
|
||||
libxml2.cleanupParser()
|
||||
gc.collect()
|
||||
|
||||
figures = []
|
||||
if 'MemoryUsage' in extensions.enabled:
|
||||
memusage = extensions.enabled['MemoryUsage']
|
||||
memusage.update()
|
||||
figures.append(("Memory usage at startup", int(memusage.data['startup']/1024/1024), "Mb"))
|
||||
figures.append(("Maximum memory usage", int(memusage.data['max']/1024/1024), "Mb"))
|
||||
figures.append(("Memory usage at shutdown", int(memusage.virtual/1024/1024), "Mb"))
|
||||
figures.append(("Objects in gc.garbage", len(gc.garbage), ""))
|
||||
figures.append(("libxml2 memory leak", libxml2.debugMemory(1), "bytes"))
|
||||
return figures
|
||||
|
||||
def create_report(self, figures):
|
||||
s = ""
|
||||
s += "SCRAPY MEMORY DEBUGGER RESULTS\n\n"
|
||||
for f in figures:
|
||||
s += "%-30s : %s %s\n" % f
|
||||
s += "\n"
|
||||
if stats:
|
||||
s += "SCRAPING STATS --------------------------------------------------\n\n"
|
||||
s += pprint.pformat(stats)
|
||||
return s
|
||||
|
||||
def print_or_send_report(self, report):
|
||||
if self.rcpts:
|
||||
self.mail.send(self.rcpts, "Scrapy Memory Debugger results at %s" % socket.gethostname(), report)
|
||||
print report
|
||||
|
||||
def domain_opened(self, domain):
|
||||
self.domains_scraped.append(domain)
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
import sys
|
||||
import os
|
||||
import pprint
|
||||
import socket
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.mail import MailSender
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
class MemoryUsage(object):
|
||||
|
||||
_proc_status = '/proc/%d/status' % os.getpid()
|
||||
_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
|
||||
'KB': 1024.0, 'MB': 1024.0*1024.0}
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('MEMUSAGE_ENABLED'):
|
||||
raise NotConfigured
|
||||
if sys.platform != 'linux2':
|
||||
raise NotConfigured("MemoryUsage extension is only available on Linux")
|
||||
|
||||
self.warned = False
|
||||
|
||||
self.data = {}
|
||||
self.data['startup'] = 0
|
||||
self.data['max'] = 0
|
||||
|
||||
scrapyengine.addtask(self.update, 60.0, now=True)
|
||||
|
||||
self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY')
|
||||
self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
|
||||
self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
|
||||
self.report = settings.getbool('MEMUSAGE_REPORT')
|
||||
|
||||
if self.limit:
|
||||
scrapyengine.addtask(self._check_limit, 60.0, now=True)
|
||||
if self.warning:
|
||||
scrapyengine.addtask(self._check_warning, 60.0, now=True)
|
||||
|
||||
self.mail = MailSender()
|
||||
|
||||
dispatcher.connect(self.engine_started, signal=signals.engine_started)
|
||||
|
||||
|
||||
@property
|
||||
def virtual(self):
|
||||
return self._vmvalue('VmSize:')
|
||||
|
||||
@property
|
||||
def resident(self):
|
||||
return self._vmvalue('VmRSS:')
|
||||
|
||||
@property
|
||||
def stacksize(self):
|
||||
return self._vmvalue('VmStk:')
|
||||
|
||||
def engine_started(self):
|
||||
self.data['startup'] = self.virtual
|
||||
|
||||
def update(self):
|
||||
if self.virtual > self.data['max']:
|
||||
self.data['max'] = self.virtual
|
||||
|
||||
def _vmvalue(self, VmKey):
|
||||
# get pseudo file /proc/<pid>/status
|
||||
try:
|
||||
t = open(self._proc_status)
|
||||
v = t.read()
|
||||
t.close()
|
||||
except:
|
||||
return 0.0 # non-Linux?
|
||||
# get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
|
||||
i = v.index(VmKey)
|
||||
v = v[i:].split(None, 3) # whitespace
|
||||
if len(v) < 3:
|
||||
return 0.0 # invalid format?
|
||||
# convert Vm value to bytes
|
||||
return float(v[1]) * self._scale[v[2]]
|
||||
|
||||
def _check_limit(self):
|
||||
if self.virtual > self.limit:
|
||||
mem = self.limit/1024/1024
|
||||
log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR)
|
||||
if self.notify_mails:
|
||||
subj = "%s terminated: memory usage exceeded %dM at %s" % (settings['BOT_NAME'], mem, socket.gethostname())
|
||||
self._send_report(self.notify_mails, subj)
|
||||
scrapymanager.stop()
|
||||
|
||||
def _check_warning(self):
|
||||
if self.warned: # warn only once
|
||||
return
|
||||
if self.virtual > self.warning:
|
||||
mem = self.warning/1024/1024
|
||||
log.msg("Memory usage reached %dM" % mem, level=log.WARNING)
|
||||
if self.notify_mails:
|
||||
subj = "%s warning: memory usage reached %dM at %s" % (settings['BOT_NAME'], mem, socket.gethostname())
|
||||
self._send_report(self.notify_mails, subj)
|
||||
self.warned = True
|
||||
|
||||
def _send_report(self, rcpts, subject):
|
||||
"""send notification mail with some additional useful info"""
|
||||
s = "Memory usage at engine startup : %dM\r\n" % (self.data['startup']/1024/1024)
|
||||
s += "Maximum memory usage : %dM\r\n" % (self.data['max']/1024/1024)
|
||||
s += "Current memory usage : %dM\r\n" % (self.virtual/1024/1024)
|
||||
|
||||
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
|
||||
s += "\r\n"
|
||||
s += scrapyengine.getstatus()
|
||||
s += "\r\n"
|
||||
|
||||
if stats:
|
||||
s += "SCRAPING STATS ------------------------------------------------------ \r\n"
|
||||
s += "\r\n"
|
||||
s += pprint.pformat(stats)
|
||||
self.mail.send(rcpts, subject, s)
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
from scrapy.contrib.pbcluster.worker.manager import ClusterWorker
|
||||
from scrapy.contrib.pbcluster.master.web import ClusterMasterWeb
|
||||
from scrapy.contrib.pbcluster.crawler.manager import ClusterCrawler
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import os
|
||||
|
||||
from twisted.spread import pb
|
||||
from twisted.internet import reactor
|
||||
|
||||
from scrapy.conf import settings
|
||||
from scrapy import log
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
|
||||
class Broker(pb.Referenceable):
|
||||
def __init__(self, crawler, remote):
|
||||
self.__remote = remote
|
||||
self.__crawler = crawler
|
||||
try:
|
||||
deferred = self.__remote.callRemote("register_crawler", os.getpid(), self)
|
||||
except pb.DeadReferenceError:
|
||||
self._set_status(None)
|
||||
log.msg("Lost connection to node %s." % (self.name), log.ERROR)
|
||||
else:
|
||||
deferred.addCallbacks(callback=lambda x: None, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
def remote_stop(self):
|
||||
scrapymanager.stop()
|
||||
|
||||
class ClusterCrawler:
|
||||
def __init__(self):
|
||||
if not settings.getbool('CLUSTER_CRAWLER_ENABLED'):
|
||||
raise NotConfigured
|
||||
|
||||
self.worker = None
|
||||
|
||||
factory = pb.PBClientFactory()
|
||||
reactor.connectTCP("localhost", settings.getint('CLUSTER_WORKER_PORT'), factory)
|
||||
d = factory.getRootObject()
|
||||
def _set_worker(obj):
|
||||
self.worker = Broker(self, obj)
|
||||
d.addCallbacks(callback=_set_worker, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
|
||||
|
|
@ -0,0 +1,330 @@
|
|||
import sys, datetime
|
||||
import pickle
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from twisted.spread import pb
|
||||
from twisted.internet import reactor
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
|
||||
DEFAULT_PRIORITY = settings.getint("DEFAULT_PRIORITY", 20)
|
||||
|
||||
def my_import(name):
|
||||
mod = __import__(name)
|
||||
components = name.split('.')
|
||||
for comp in components[1:]:
|
||||
mod = getattr(mod, comp)
|
||||
return mod
|
||||
|
||||
class Broker(pb.Referenceable):
|
||||
def __init__(self, remote, name, master):
|
||||
self.__remote = remote
|
||||
self.alive = False
|
||||
self.name = name
|
||||
self.master = master
|
||||
self.available = True
|
||||
try:
|
||||
deferred = self.__remote.callRemote("set_master", self)
|
||||
except pb.DeadReferenceError:
|
||||
self._set_status(None)
|
||||
log.msg("Lost connection to node %s." % (self.name), log.ERROR)
|
||||
else:
|
||||
deferred.addCallbacks(callback=self._set_status, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
|
||||
def status_as_dict(self, verbosity=1):
|
||||
if verbosity == 0:
|
||||
return
|
||||
status = {"alive": self.alive}
|
||||
if self.alive:
|
||||
if verbosity == 1:
|
||||
#dont show spider settings
|
||||
status["running"] = []
|
||||
for proc in self.running:
|
||||
proccopy = proc.copy()
|
||||
del proccopy["settings"]
|
||||
status["running"].append(proccopy)
|
||||
elif verbosity == 2:
|
||||
status["running"] = self.running
|
||||
status["maxproc"] = self.maxproc
|
||||
status["freeslots"] = self.maxproc - len(self.running)
|
||||
status["available"] = self.available
|
||||
status["starttime"] = self.starttime
|
||||
status["timestamp"] = self.timestamp
|
||||
status["loadavg"] = self.loadavg
|
||||
return status
|
||||
|
||||
def _set_status(self, status):
|
||||
if not status:
|
||||
self.alive = False
|
||||
else:
|
||||
self.alive = True
|
||||
self.running = status['running']
|
||||
self.maxproc = status['maxproc']
|
||||
self.starttime = status['starttime']
|
||||
self.timestamp = status['timestamp']
|
||||
self.loadavg = status['loadavg']
|
||||
self.logdir = status['logdir']
|
||||
free_slots = self.maxproc - len(self.running)
|
||||
|
||||
#load domains by one, so to mix up better the domain loading between nodes. The next one in the same node will be loaded
|
||||
#when there is no loading domain or in the next status update. This way also we load the nodes softly
|
||||
if self.available and free_slots > 0 and self.master.pending:
|
||||
pending = self.master.pending.pop(0)
|
||||
#if domain already running in some node, reschedule with same priority (so will be moved to run later)
|
||||
if pending['domain'] in self.master.running or pending['domain'] in self.master.loading:
|
||||
self.master.schedule([pending['domain']], pending['settings'], pending['priority'])
|
||||
else:
|
||||
self.run(pending)
|
||||
self.master.loading.append(pending['domain'])
|
||||
|
||||
def update_status(self):
|
||||
try:
|
||||
deferred = self.__remote.callRemote("status")
|
||||
except pb.DeadReferenceError:
|
||||
self._set_status(None)
|
||||
log.msg("Lost connection to node %s." % (self.name), log.ERROR)
|
||||
else:
|
||||
deferred.addCallbacks(callback=self._set_status, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
|
||||
def stop(self, domain):
|
||||
try:
|
||||
deferred = self.__remote.callRemote("stop", domain)
|
||||
except pb.DeadReferenceError:
|
||||
self._set_status(None)
|
||||
log.msg("Lost connection to node %s." % (self.name), log.ERROR)
|
||||
else:
|
||||
deferred.addCallbacks(callback=self._set_status, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
|
||||
def run(self, pending):
|
||||
|
||||
def _run_errback(reason):
|
||||
log.msg(reason, log.ERROR)
|
||||
self.master.loading.remove(pending['domain'])
|
||||
self.master.schedule([pending['domain']], pending['settings'], pending['priority'] - 1)
|
||||
log.msg("Domain %s rescheduled: lost connection to node." % pending['domain'], log.WARNING)
|
||||
|
||||
def _run_callback(status):
|
||||
if status['callresponse'][0] == 1:
|
||||
#slots are complete. Reschedule in master with priority reduced by one.
|
||||
#self.master.loading check should avoid this to happen
|
||||
self.master.loading.remove(pending['domain'])
|
||||
self.master.schedule([pending['domain']], pending['settings'], pending['priority'] - 1)
|
||||
log.msg("Domain %s rescheduled: no proc space in node." % pending['domain'], log.WARNING)
|
||||
elif status['callresponse'][0] == 2:
|
||||
#domain already running in node. Reschedule with same priority.
|
||||
#self.master.loading check should avoid this to happen
|
||||
self.master.loading.remove(pending['domain'])
|
||||
self.master.schedule([pending['domain']], pending['settings'], pending['priority'])
|
||||
log.msg("Domain %s rescheduled: already running in node." % pending['domain'], log.WARNING)
|
||||
|
||||
try:
|
||||
deferred = self.__remote.callRemote("run", pending["domain"], pending["settings"])
|
||||
except pb.DeadReferenceError:
|
||||
self._set_status(None)
|
||||
log.msg("Lost connection to node %s." % (self.name), log.ERROR)
|
||||
else:
|
||||
deferred.addCallbacks(callback=_run_callback, errback=_run_errback)
|
||||
|
||||
def remote_update(self, status, domain, domain_status):
|
||||
self._set_status(status)
|
||||
if domain in self.master.loading and domain_status == "running":
|
||||
self.master.loading.remove(domain)
|
||||
self.master.statistics["domains"]["running"].add(domain)
|
||||
elif domain_status == "scraped":
|
||||
self.master.statistics["domains"]["running"].remove(domain)
|
||||
self.master.statistics["domains"]["scraped"][domain] = self.master.statistics["domains"]["scraped"].get(domain, 0) + 1
|
||||
self.master.statistics["scraped_count"] = self.master.statistics.get("scraped_count", 0) + 1
|
||||
if domain in self.master.statistics["domains"]["lost"]:
|
||||
self.master.statistics["domains"]["lost"].remove(domain)
|
||||
|
||||
class ScrapyPBClientFactory(pb.PBClientFactory):
|
||||
def __init__(self, master, nodename):
|
||||
pb.PBClientFactory.__init__(self)
|
||||
self.master = master
|
||||
self.nodename = nodename
|
||||
|
||||
def clientConnectionLost(self, *args, **kargs):
|
||||
pb.PBClientFactory.clientConnectionLost(self, *args, **kargs)
|
||||
del self.master.nodes[self.nodename]
|
||||
log.msg("Removed node %s." % self.nodename )
|
||||
|
||||
class ClusterMaster:
|
||||
|
||||
def __init__(self):
|
||||
|
||||
if not (settings.getbool('CLUSTER_MASTER_ENABLED')):
|
||||
raise NotConfigured
|
||||
|
||||
#import groups settings
|
||||
if settings.getbool('GROUPSETTINGS_ENABLED'):
|
||||
self.get_spider_groupsettings = my_import(settings["GROUPSETTINGS_MODULE"]).get_spider_groupsettings
|
||||
else:
|
||||
self.get_spider_groupsettings = lambda x: {}
|
||||
#load pending domains
|
||||
try:
|
||||
self.pending = pickle.load( open(settings["CLUSTER_MASTER_CACHEFILE"], "r") )
|
||||
except IOError:
|
||||
self.pending = []
|
||||
self.loading = []
|
||||
self.nodes = {}
|
||||
self.start_time = datetime.datetime.utcnow()
|
||||
#on how statistics works, see self.update_nodes() and Broker.remote_update()
|
||||
self.statistics = {"domains": {"running": set(), "scraped": {}, "lost_count": {}, "lost": set()}, "scraped_count": 0 }
|
||||
self.global_settings = {}
|
||||
#load cluster global settings
|
||||
for sname in settings.getlist('GLOBAL_CLUSTER_SETTINGS'):
|
||||
self.global_settings[sname] = settings[sname]
|
||||
|
||||
dispatcher.connect(self._engine_started, signal=signals.engine_started)
|
||||
dispatcher.connect(self._engine_stopped, signal=signals.engine_stopped)
|
||||
|
||||
def load_nodes(self):
|
||||
"""Loads nodes listed in CLUSTER_MASTER_NODES setting"""
|
||||
for name, url in settings.get('CLUSTER_MASTER_NODES', {}).iteritems():
|
||||
self.load_node(name, url)
|
||||
|
||||
def load_node(self, name, url):
|
||||
"""Creates the remote reference for each worker node"""
|
||||
def _make_callback(_factory, _name, _url):
|
||||
|
||||
def _errback(_reason):
|
||||
log.msg("Could not get remote node %s in %s: %s." % (_name, _url, _reason), log.ERROR)
|
||||
|
||||
d = _factory.getRootObject()
|
||||
d.addCallbacks(callback=lambda obj: self.add_node(obj, _name), errback=_errback)
|
||||
|
||||
server, port = url.split(":")
|
||||
port = int(port)
|
||||
log.msg("Connecting to cluster worker %s..." % name)
|
||||
log.msg("Server: %s, Port: %s" % (server, port))
|
||||
factory = ScrapyPBClientFactory(self, name)
|
||||
try:
|
||||
reactor.connectTCP(server, port, factory)
|
||||
except Exception, err:
|
||||
log.msg("Could not connect to node %s in %s: %s." % (name, url, reason), log.ERROR)
|
||||
else:
|
||||
_make_callback(factory, name, url)
|
||||
|
||||
def update_nodes(self):
|
||||
for name, url in settings.get('CLUSTER_MASTER_NODES', {}).iteritems():
|
||||
if name in self.nodes and self.nodes[name].alive:
|
||||
log.msg("Updating node. name: %s, url: %s" % (name, url) )
|
||||
self.nodes[name].update_status()
|
||||
else:
|
||||
log.msg("Reloading node. name: %s, url: %s" % (name, url) )
|
||||
self.load_node(name, url)
|
||||
|
||||
real_running = set(self.running.keys())
|
||||
lost = self.statistics["domains"]["running"].difference(real_running)
|
||||
for domain in lost:
|
||||
self.statistics["domains"]["lost_count"][domain] = self.statistics["domains"]["lost_count"].get(domain, 0) + 1
|
||||
self.statistics["domains"]["lost"] = self.statistics["domains"]["lost"].union(lost)
|
||||
|
||||
def add_node(self, cworker, name):
|
||||
"""Add node given its node"""
|
||||
node = Broker(cworker, name, self)
|
||||
self.nodes[name] = node
|
||||
log.msg("Added cluster worker %s" % name)
|
||||
|
||||
def disable_node(self, name):
|
||||
self.nodes[name].available = False
|
||||
|
||||
def enable_node(self, name):
|
||||
self.nodes[name].available = True
|
||||
|
||||
def remove_node(self, nodename):
|
||||
raise NotImplemented
|
||||
|
||||
def schedule(self, domains, spider_settings=None, priority=DEFAULT_PRIORITY):
|
||||
i = 0
|
||||
for p in self.pending:
|
||||
if p['priority'] <= priority:
|
||||
i += 1
|
||||
else:
|
||||
break
|
||||
for domain in domains:
|
||||
pd = self.find_inpending(domain)
|
||||
if pd: #domain already pending, so just change priority if new is higher
|
||||
if priority < pd['priority']:
|
||||
self.pending.remove(pd)
|
||||
pd['priority'] = priority
|
||||
self.pending.insert(i, pd)
|
||||
else:
|
||||
final_spider_settings = self.get_spider_groupsettings(domain)
|
||||
final_spider_settings.update(self.global_settings)
|
||||
final_spider_settings.update(spider_settings or {})
|
||||
self.pending.insert(i, {'domain': domain, 'settings': final_spider_settings, 'priority': priority})
|
||||
|
||||
def stop(self, domains):
|
||||
to_stop = {}
|
||||
for domain in domains:
|
||||
node = self.running.get(domain, None)
|
||||
if node:
|
||||
if node.name not in to_stop:
|
||||
to_stop[node.name] = []
|
||||
to_stop[node.name].append(domain)
|
||||
|
||||
for nodename, domains in to_stop.iteritems():
|
||||
for domain in domains:
|
||||
self.nodes[nodename].stop(domain)
|
||||
|
||||
def remove(self, domains):
|
||||
"""Remove all scheduled instances of the given domains (if it hasn't
|
||||
started yet). Otherwise use stop()"""
|
||||
|
||||
for domain in domains:
|
||||
to_remove = []
|
||||
for p in self.pending:
|
||||
if p['domain'] == domain:
|
||||
to_remove.append(p)
|
||||
|
||||
for p in to_remove:
|
||||
self.pending.remove(p)
|
||||
|
||||
def discard(self, domains):
|
||||
"""Stop and remove all running and pending instances of the given
|
||||
domains"""
|
||||
self.remove(domains)
|
||||
self.stop(domains)
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
"""Return dict of running domains as domain -> node"""
|
||||
d = {}
|
||||
for node in self.nodes.itervalues():
|
||||
for proc in node.running:
|
||||
d[proc['domain']] = node
|
||||
return d
|
||||
|
||||
@property
|
||||
def available_nodes(self):
|
||||
return (node for node in self.nodes.itervalues() if node.available)
|
||||
|
||||
def find_inpending(self, domain):
|
||||
for p in self.pending:
|
||||
if domain == p['domain']:
|
||||
return p
|
||||
|
||||
def print_pending(self, verbosity=1):
|
||||
if verbosity == 1:
|
||||
pending = []
|
||||
for p in self.pending:
|
||||
pp = p.copy()
|
||||
del pp["settings"]
|
||||
pending.append(pp)
|
||||
return pending
|
||||
elif verbosity == 2:
|
||||
return self.pending
|
||||
return
|
||||
|
||||
def _engine_started(self):
|
||||
self.load_nodes()
|
||||
scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL'))
|
||||
def _engine_stopped(self):
|
||||
pickle.dump( self.pending, open(settings["CLUSTER_MASTER_CACHEFILE"], "w") )
|
||||
log.msg("Pending saved in %s" % settings["CLUSTER_MASTER_CACHEFILE"])
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
import datetime
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.management.web import banner, webconsole_discover_module
|
||||
from scrapy.contrib.pbcluster.master.manager import ClusterMaster, DEFAULT_PRIORITY
|
||||
from scrapy.utils.serialization import serialize
|
||||
|
||||
class ClusterMasterWeb(ClusterMaster):
|
||||
webconsole_id = 'cluster_master'
|
||||
webconsole_name = 'Cluster master'
|
||||
|
||||
def __init__(self):
|
||||
ClusterMaster.__init__(self)
|
||||
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
changes = ""
|
||||
if wc_request.path == '/cluster_master/nodes/':
|
||||
return self.render_nodes(wc_request)
|
||||
elif wc_request.path == '/cluster_master/domains/':
|
||||
return self.render_domains(wc_request)
|
||||
elif wc_request.path == '/cluster_master/ws/':
|
||||
return self.webconsole_control(wc_request, ws=True)
|
||||
elif wc_request.args:
|
||||
changes = self.webconsole_control(wc_request)
|
||||
|
||||
s = self.render_header()
|
||||
|
||||
s += "<h2>Home</h2>\n"
|
||||
|
||||
s += "<table border='1'>\n"
|
||||
s += "<tr><th> </th><th>Name</th><th>Available</th><th>Running</th><th>Load.avg</th></tr>\n"
|
||||
for node in self.nodes.itervalues():
|
||||
#chkbox = "<input type='checkbox' name='shutdown' value='%s' />" % domain if node.status in ["up", "idle"] else " "
|
||||
nodelink = "<a href='nodes/#%s'>%s</a>" % (node.name, node.name)
|
||||
chkbox = " "
|
||||
loadavg = "%.2f %.2f %.2f" % node.loadavg
|
||||
s += "<tr><td>%s</td><td>%s</td><td>%s</td><td>%d/%d</td><td>%s</td></tr>\n" % \
|
||||
(chkbox, nodelink, node.available, len(node.running), node.maxproc, loadavg)
|
||||
s += "</table>\n"
|
||||
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return str(s)
|
||||
|
||||
def webconsole_control(self, wc_request, ws=False):
|
||||
args = wc_request.args
|
||||
if "updatenodes" in args:
|
||||
self.update_nodes()
|
||||
if ws:
|
||||
return self.ws_status(wc_request)
|
||||
|
||||
if "schedule" in args:
|
||||
if ws:
|
||||
sep = ","
|
||||
domains = args["schedule"][0].split(sep)
|
||||
else:
|
||||
sep = "\r"
|
||||
domains = args["schedule"]
|
||||
priority = int(args.get("priority", [DEFAULT_PRIORITY])[0])
|
||||
|
||||
#spider settings
|
||||
slist = args.get("settings", [""])[0].split(sep)
|
||||
spider_settings = {}
|
||||
for s in slist:
|
||||
try:
|
||||
k, v = s.strip().split("=")
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
spider_settings[k] = v
|
||||
|
||||
self.schedule(domains, spider_settings, priority)
|
||||
if ws:
|
||||
return self.ws_status(wc_request, verbosity=0)
|
||||
|
||||
if "stop" in args:
|
||||
if ws:
|
||||
domains = args["stop"][0].split(",")
|
||||
else:
|
||||
domains=args["stop"]
|
||||
self.stop(domains)
|
||||
if ws:
|
||||
return self.ws_status(wc_request)
|
||||
|
||||
if "remove" in args:
|
||||
if ws:
|
||||
domains = args["remove"][0].split(",")
|
||||
else:
|
||||
domains=args["remove"]
|
||||
self.remove(domains)
|
||||
if ws:
|
||||
return self.ws_status(wc_request)
|
||||
if "disable_node" in args:
|
||||
self.disable_node(args["disable_node"][0])
|
||||
if ws:
|
||||
return self.ws_status(wc_request)
|
||||
if "enable_node" in args:
|
||||
self.enable_node(args["enable_node"][0])
|
||||
if ws:
|
||||
return self.ws_status(wc_request)
|
||||
if "statistics" in args:
|
||||
if ws:
|
||||
return self.ws_statistics(wc_request)
|
||||
|
||||
if ws:
|
||||
return self.ws_status(wc_request)
|
||||
else:
|
||||
return ""
|
||||
|
||||
def render_nodes(self, wc_request):
|
||||
if wc_request.args:
|
||||
self.webconsole_control(wc_request)
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
|
||||
s = self.render_header()
|
||||
for node in self.nodes.itervalues():
|
||||
if node.available:
|
||||
s += "<h2><a name='%s'>%s</h2>\n" % (node.name, node.name)
|
||||
|
||||
s += "<h3>Running domains</h3>\n"
|
||||
if node.running:
|
||||
s += "<form method='post' action='.'>\n"
|
||||
s += "<table border='1'>\n"
|
||||
s += "<tr><th> </th><th>PID</th><th>Domain</th><th>Status</th><th>Running time</th><th>Log file</th></tr>\n"
|
||||
for proc in node.running:
|
||||
chkbox = "<input type='checkbox' name='stop' value='%s' />" % proc['domain'] if proc['status'] == "running" else " "
|
||||
start_time = proc.get('starttime', None)
|
||||
elapsed = now - start_time if start_time else None
|
||||
s += "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n" % \
|
||||
(chkbox, proc['pid'], proc['domain'], proc['status'], elapsed, proc['logfile'])
|
||||
s += "</table>\n"
|
||||
s += "<input type='hidden' name='node' value='%s'>\n" % node.name
|
||||
s += "<p><input type='submit' value='Stop selected domains on %s'></p>\n" % node.name
|
||||
s += "</form>\n"
|
||||
else:
|
||||
s += "<p>No running domains on %s</p>\n" % node.name
|
||||
|
||||
return str(s)
|
||||
|
||||
def render_domains(self, wc_request):
|
||||
if wc_request.args:
|
||||
self.webconsole_control(wc_request)
|
||||
|
||||
enabled_domains = set(spiders.asdict(include_disabled=False).keys())
|
||||
print "Enabled domains: %s" % len(enabled_domains)
|
||||
inactive_domains = enabled_domains - set(self.running.keys() + [p['domain'] for p in self.pending])
|
||||
|
||||
s = self.render_header()
|
||||
|
||||
s += "<h2>Schedule domains</h2>\n"
|
||||
|
||||
s += "Inactive domains (not running or pending)<br />"
|
||||
s += "<form method='post' action='.'>\n"
|
||||
s += "<select name='schedule' multiple='multiple' size='10'>\n"
|
||||
for domain in sorted(inactive_domains):
|
||||
s += "<option>%s</option>\n" % domain
|
||||
s += "</select>\n"
|
||||
s += "<br />\n"
|
||||
|
||||
s += "Priority:<br />\n"
|
||||
s += "<input type='text' name='priority'>%s</input>" % DEFAULT_PRIORITY
|
||||
s += "<br />\n"
|
||||
|
||||
#spider settings
|
||||
s += "Overrided spider settings:<br />\n"
|
||||
s += "<textarea name='settings' rows='4'>\n"
|
||||
s += "UNAVAILABLES_NOTIFY=2\n"
|
||||
s += "</textarea>\n"
|
||||
s += "<br />\n"
|
||||
|
||||
s += "<p><input type='submit' value='Schedule selected domains'></p>\n"
|
||||
s += "</form>\n"
|
||||
|
||||
s += "<h2>Domains</h2>\n"
|
||||
|
||||
s += "<table border='1'>\n"
|
||||
s += "<tr><th>Domain</th><th>Status</th><th>Node</th></tr>\n"
|
||||
s += self._domains_table(self.running, '<b>running</b>')
|
||||
s += "</table>\n"
|
||||
|
||||
# pending domains
|
||||
s += "<h3>Pending domains</h3>\n"
|
||||
if self.pending:
|
||||
s += "<form method='post' action='.'>\n"
|
||||
s += "<select name='remove' multiple='multiple' size='10'>\n"
|
||||
for p in self.pending:
|
||||
s += "<option value='%s'>%s (P:%s)</option>\n" % (p['domain'], p['domain'],p['priority'])
|
||||
s += "</select>\n"
|
||||
s += "<p><input type='submit' value='Remove selected pending domains'></p>\n"
|
||||
s += "</form>\n"
|
||||
else:
|
||||
s += "<p>No pending domains</p>\n"
|
||||
|
||||
return str(s)
|
||||
|
||||
def render_header(self):
|
||||
s = banner(self)
|
||||
s += "<p>Nav: "
|
||||
s += "<a href='/cluster_master/'>Home</a> | "
|
||||
s += "<a href='/cluster_master/domains/'>Domains</a> | "
|
||||
s += "<a href='/cluster_master/nodes/'>Nodes</a> (<a href='/cluster_master/nodes/?updatenodes=1'>update</a>)"
|
||||
s += "</p>"
|
||||
return s
|
||||
|
||||
def _domains_table(self, dict_, status):
|
||||
s = ""
|
||||
for domain, node in dict_.iteritems():
|
||||
s += "<tr><td>%s</td><td>%s</td><td>%s</td></tr>\n" % (domain, status, node.name)
|
||||
return s
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
return self
|
||||
|
||||
def ws_status(self, wc_request, verbosity=1):
|
||||
format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json'
|
||||
verbosity = int(wc_request.args['verbosity'][0]) if 'verbosity' in wc_request.args else verbosity
|
||||
wc_request.setHeader('content-type', 'text/plain')
|
||||
status = {}
|
||||
nodes_status = {}
|
||||
if verbosity > 0:
|
||||
for d, n in self.nodes.iteritems():
|
||||
nodes_status[d] = n.status_as_dict(verbosity)
|
||||
status["nodes"] = nodes_status
|
||||
status["pending"] = self.print_pending(verbosity)
|
||||
status["loading"] = self.loading
|
||||
content = serialize(status, format)
|
||||
return content
|
||||
return ""
|
||||
|
||||
def ws_statistics(self, wc_request):
|
||||
format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json'
|
||||
content = serialize(self.statistics, format)
|
||||
return content
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
Cluster Webservice API
|
||||
======================
|
||||
|
||||
The webservice API is available at
|
||||
|
||||
http://server:port/cluster_master/ws/
|
||||
|
||||
With no parameters, webservice returns the cluster status.
|
||||
|
||||
Query parameters
|
||||
================
|
||||
|
||||
- `format`: the answer format. By default, format=json. Other formats: pprint, pickle.
|
||||
|
||||
- `schedule`: schedules a comma separated list of domains. Schedule function takes optional parameters:
|
||||
|
||||
"priority": sets the queue priority for the specified domains (an integer). The default is setted by "DEFAULT_PRIORITY"
|
||||
setting (20 if not given). A lower priority number implies more priority.
|
||||
|
||||
"settings": run settings for the specified domains. This is a comma separated list of <setting_name>=<value> pairs. By default it is empty.
|
||||
|
||||
- `remove`: removes from pending list a comma separated list of domains.
|
||||
|
||||
- `stop`: stops comma separated list of domains (they have to be running in some node)
|
||||
|
||||
- `disable_node`: disables a node so no more domains will be loaded in it until enabled again (but it will finish to run the running domains)
|
||||
|
||||
- `enable_node`: revert the state setted by 'disable_node'
|
||||
|
||||
- `verbosity`: sets the output verbosity level (1 is the default minimal, 2 includes domain settings, 0 disables output)
|
||||
|
||||
- `statistics`: shows the pending/running/scraped/lost statistics
|
||||
|
||||
Examples:
|
||||
---------
|
||||
|
||||
1) Schedule argos.co.uk, diy.com, littlewoodsdirect.com spiders, with priority=0, and settings UNAVAILABLES_NOTIFY=2 and UNAVAILABLES_DAYS_BACK=3. Answer with pprint format
|
||||
|
||||
http://localhost:8080/cluster_master/ws/?format=pprint&schedule=argos.co.uk,diy.com,littlewoodsdirect.com&priority=0&settings=UNAVAILABLES_NOTIFY=2,UNAVAILABLES_DAYS_BACK=3
|
||||
|
||||
2) Get status with pprint format:
|
||||
|
||||
http://localhost:8080/cluster_master/ws/?format=pprint
|
||||
|
||||
3) Remove from pending lists domains argos.co.uk and diy.com. Answer with pprint format:
|
||||
|
||||
http://localhost:8080/cluster_master/ws/?remove=argos.co.uk,diy.com
|
||||
|
||||
4) Stop running domain littlewoodsdirect.com:
|
||||
|
||||
http://localhost:8080/cluster_master/ws/?stop=littlewoodsdirect.com
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
#!/usr/bin/env python
|
||||
"""
|
||||
Cluster control script
|
||||
"""
|
||||
|
||||
from optparse import OptionParser
|
||||
import urllib
|
||||
|
||||
def main():
|
||||
parser = OptionParser(usage="Usage: scrapy-cluster-ctl.py [domain [domain [...]]] [options]" )
|
||||
parser.add_option("--disablenode", dest="disable_node", help="Disable given node (by name) so it will no accept more run requests.")
|
||||
parser.add_option("--enablenode", dest="enable_node", help="Enable given node (by name) so it will accept again run requests")
|
||||
parser.add_option("--format", dest="format", help="Output format. Default: pprint.", default="pprint")
|
||||
parser.add_option("--list", metavar="FILE", dest="list", help="Specify a file from where to read domains, one per line.")
|
||||
parser.add_option("--now", action="store_true", dest="now", help="Schedule domains to run with priority now.")
|
||||
parser.add_option("--output", metavar="FILE", dest="output", help="Output file. If not given, output to stdout.")
|
||||
parser.add_option("--port", dest="port", type="int", help="Cluster master port. Default: 8060.", default=8060)
|
||||
parser.add_option("--remove", dest="remove", action="store_true", help="Remove from schedule domains given as args.")
|
||||
parser.add_option("--schedule", dest="schedule", action="store_true", help="Schedule domains given as args.")
|
||||
parser.add_option("--server", dest="server", help="Cluster master server name. Default: localhost.", default="localhost")
|
||||
parser.add_option("--status", dest="status", action="store_true", help="Print cluster master status and quit.")
|
||||
parser.add_option("--statistics", dest="statistics", action="store_true", help="Print cluster statistics")
|
||||
parser.add_option("--stop", dest="stop", action="store_true", help="Stops a running domain.")
|
||||
parser.add_option("--verbosity", dest="verbosity", type="int", help="Sets the report status verbosity.")
|
||||
(opts, args) = parser.parse_args()
|
||||
|
||||
output = ""
|
||||
domains = []
|
||||
urlstring = "http://%s:%s/cluster_master/ws/" % (opts.server, opts.port)
|
||||
post = {"format":opts.format}
|
||||
if isinstance(opts.verbosity, int):
|
||||
post["verbosity"] = opts.verbosity
|
||||
|
||||
if args:
|
||||
domains = ",".join(args)
|
||||
elif opts.list:
|
||||
try:
|
||||
domainlist = []
|
||||
for d in open(opts.list, "r").readlines():
|
||||
domainlist.append(d.strip())
|
||||
domains = ",".join(domainlist)
|
||||
except IOError:
|
||||
print "Can't open file %s" % opts.list
|
||||
|
||||
if opts.status:
|
||||
pass
|
||||
elif opts.statistics:
|
||||
post["statistics"] = True
|
||||
elif opts.schedule and domains:
|
||||
post["schedule"] = domains
|
||||
if opts.now:
|
||||
post["priority"] = "0"
|
||||
post["settings"] = "UNAVAILABLES_NOTIFY=2"
|
||||
elif opts.remove and domains:
|
||||
post["remove"] = domains
|
||||
elif opts.stop and domains:
|
||||
post["stop"] = domains
|
||||
elif opts.disable_node:
|
||||
post["disable_node"] = opts.disable_node
|
||||
elif opts.enable_node:
|
||||
post["enable_node"] = opts.enable_node
|
||||
else:
|
||||
parser.print_help()
|
||||
return
|
||||
|
||||
f = urllib.urlopen(urlstring, urllib.urlencode(post))
|
||||
output=f.read()
|
||||
if not output:
|
||||
return
|
||||
if not opts.output:
|
||||
print output
|
||||
else:
|
||||
try:
|
||||
open(opts.output, "w").write(output)
|
||||
except IOError:
|
||||
open("/tmp/scrapy-cluster-schedule.tmp", "w").write(output)
|
||||
print "Could not open file %s for writing. Output dumped to /tmp/scrapy-cluster-schedule.tmp instead." % opts.output
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
import sys, os, time, datetime, pickle
|
||||
|
||||
from twisted.internet import protocol, reactor
|
||||
from twisted.spread import pb
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.engine import scrapyengine
|
||||
|
||||
class ScrapyProcessProtocol(protocol.ProcessProtocol):
|
||||
def __init__(self, procman, domain, logfile=None, spider_settings=None):
|
||||
self.procman = procman
|
||||
self.domain = domain
|
||||
self.logfile = logfile
|
||||
self.start_time = datetime.datetime.utcnow()
|
||||
self.status = "starting"
|
||||
self.pid = -1
|
||||
self.env = {}
|
||||
#We conserve original setting format for info purposes (avoid lots of unnecesary "SCRAPY_")
|
||||
self.scrapy_settings = spider_settings or {}
|
||||
self.scrapy_settings.update({'LOGFILE': self.logfile, 'CLUSTER_WORKER_ENABLED': 0, 'CLUSTER_CRAWLER_ENABLED': 1, 'WEBCONSOLE_ENABLED': 0})
|
||||
pickled_settings = pickle.dumps(self.scrapy_settings)
|
||||
self.env["SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE"] = pickled_settings
|
||||
self.env["PYTHONPATH"] = ":".join(sys.path)#this is need so this crawl process knows where to locate local_scrapy_settings.
|
||||
|
||||
def __str__(self):
|
||||
return "<ScrapyProcess domain=%s, pid=%s, status=%s>" % (self.domain, self.pid, self.status)
|
||||
|
||||
def as_dict(self):
|
||||
return {"domain": self.domain, "pid": self.pid, "status": self.status, "settings": self.scrapy_settings, "logfile": self.logfile, "starttime": self.start_time}
|
||||
|
||||
def connectionMade(self):
|
||||
self.pid = self.transport.pid
|
||||
log.msg("ClusterWorker: started domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile))
|
||||
self.transport.closeStdin()
|
||||
self.status = "running"
|
||||
self.procman.update_master(self.domain, "running")
|
||||
|
||||
def processEnded(self, reason):
|
||||
log.msg("ClusterWorker: finished domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile))
|
||||
log.msg("Reason type: %s. value: %s" % (reason.type, reason.value) )
|
||||
del self.procman.running[self.domain]
|
||||
del self.procman.crawlers[self.pid]
|
||||
self.procman.update_master(self.domain, "scraped")
|
||||
|
||||
class ClusterWorker(pb.Root):
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('CLUSTER_WORKER_ENABLED'):
|
||||
raise NotConfigured
|
||||
|
||||
self.maxproc = settings.getint('CLUSTER_WORKER_MAXPROC')
|
||||
self.logdir = settings['CLUSTER_LOGDIR']
|
||||
self.running = {}#a dict domain->ScrapyProcessControl
|
||||
self.crawlers = {}#a dict pid->scrapy process remote pb connection
|
||||
self.starttime = datetime.datetime.utcnow()
|
||||
port = settings.getint('CLUSTER_WORKER_PORT')
|
||||
scrapyengine.listenTCP(port, pb.PBServerFactory(self))
|
||||
log.msg("PYTHONPATH: %s" % repr(sys.path))
|
||||
|
||||
def status(self, rcode=0, rstring=None):
|
||||
status = {}
|
||||
status["running"] = [ self.running[k].as_dict() for k in self.running.keys() ]
|
||||
status["starttime"] = self.starttime
|
||||
status["timestamp"] = datetime.datetime.utcnow()
|
||||
status["maxproc"] = self.maxproc
|
||||
status["loadavg"] = os.getloadavg()
|
||||
status["logdir"] = self.logdir
|
||||
status["callresponse"] = (rcode, rstring) if rstring else (0, "Status Response.")
|
||||
return status
|
||||
|
||||
def update_master(self, domain, domain_status):
|
||||
try:
|
||||
deferred = self.__master.callRemote("update", self.status(), domain, domain_status)
|
||||
except pb.DeadReferenceError:
|
||||
self.__master = None
|
||||
log.msg("Lost connection to node %s." % (self.name), log.ERROR)
|
||||
else:
|
||||
deferred.addCallbacks(callback=lambda x: x, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
|
||||
def remote_set_master(self, master):
|
||||
self.__master = master
|
||||
return self.status()
|
||||
|
||||
def remote_stop(self, domain):
|
||||
"""Stop running domain."""
|
||||
if domain in self.running:
|
||||
proc = self.running[domain]
|
||||
log.msg("ClusterWorker: Sending shutdown signal to domain=%s, pid=%d" % (domain, proc.pid))
|
||||
d = self.crawlers[proc.pid].callRemote("stop")
|
||||
def _close():
|
||||
proc.status = "closing"
|
||||
d.addCallbacks(callback=_close, errback=lambda reason: log.msg(reason, log.ERROR))
|
||||
return self.status(0, "Stopped process %s" % proc)
|
||||
else:
|
||||
return self.status(1, "%s: domain not running." % domain)
|
||||
|
||||
def remote_status(self):
|
||||
return self.status()
|
||||
|
||||
def remote_run(self, domain, spider_settings=None):
|
||||
"""Spawn process to run the given domain."""
|
||||
if len(self.running) < self.maxproc:
|
||||
if not domain in self.running:
|
||||
logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log"))
|
||||
if not os.path.exists(os.path.dirname(logfile)):
|
||||
os.makedirs(os.path.dirname(logfile))
|
||||
scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings)
|
||||
args = [sys.executable, sys.argv[0], 'crawl', domain]
|
||||
self.running[domain] = scrapy_proc
|
||||
try:
|
||||
import pysvn
|
||||
c = pysvn.Client()
|
||||
r = c.update(settings.get("CLUSTER_WORKER_SVNWORKDIR", "."))
|
||||
log.msg("Updated to revision %s." %r[0].number, level=log.DEBUG)
|
||||
except pysvn.ClientError, e:
|
||||
log.msg("Unable to svn update: %s" % e, level=log.WARNING)
|
||||
except ImportError:
|
||||
log.msg("pysvn module not available.", level=log.WARNING)
|
||||
proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env)
|
||||
return self.status(0, "Started process %s." % scrapy_proc)
|
||||
return self.status(2, "Domain %s already running." % domain )
|
||||
return self.status(1, "No free slot to run another process.")
|
||||
|
||||
def remote_register_crawler(self, pid, crawler):
|
||||
self.crawlers[pid] = crawler
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/python2.5
|
||||
|
||||
from twisted.spread import pb
|
||||
from twisted.internet import reactor
|
||||
from twisted.python import util
|
||||
import sys
|
||||
|
||||
factory = pb.PBClientFactory()
|
||||
reactor.connectTCP("localhost", 8789, factory)
|
||||
d = factory.getRootObject()
|
||||
|
||||
sys.argv.pop(0)
|
||||
|
||||
if not sys.argv:
|
||||
d.addCallback(lambda object: object.callRemote("status"))
|
||||
elif sys.argv[0] == "-s":
|
||||
d.addCallback(lambda object: object.callRemote("stop", sys.argv[1]))
|
||||
elif sys.argv[0] == "-r":
|
||||
d.addCallback(lambda object: object.callRemote("run", sys.argv[1]))
|
||||
elif sys.argv[0] == "-t":
|
||||
d.addCallback(lambda object: object.callRemote("statistics"))
|
||||
|
||||
d.addCallbacks(callback = util.println, errback = lambda reason: 'error: '+str(reason.value))
|
||||
d.addCallback(lambda _: reactor.stop())
|
||||
reactor.run()
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
import re
|
||||
import os
|
||||
import time
|
||||
import hashlib
|
||||
import urllib
|
||||
import urlparse
|
||||
from cStringIO import StringIO
|
||||
|
||||
import Image
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.stats import stats
|
||||
from scrapy.core.exceptions import DropItem, NotConfigured
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.conf import settings
|
||||
|
||||
from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
|
||||
# the age at which we download images again
|
||||
IMAGE_EXPIRES = settings.getint('IMAGES_EXPIRES', 90)
|
||||
|
||||
class NoimagesDrop(DropItem):
|
||||
pass
|
||||
|
||||
class ImageException(Exception):
|
||||
"""General image error exception"""
|
||||
|
||||
|
||||
class ImagesPipeline(MediaPipeline):
|
||||
MEDIA_TYPE = 'image'
|
||||
THUMBS = None
|
||||
# THUMBS = (
|
||||
# ("50", (50, 50)),
|
||||
# ("110", (110, 110)),
|
||||
# ("270", (270, 270))
|
||||
# )
|
||||
MIN_WIDTH = 0
|
||||
MIN_HEIGHT = 0
|
||||
|
||||
def __init__(self):
|
||||
if not settings['IMAGES_DIR']:
|
||||
raise NotConfigured
|
||||
|
||||
self.BASEDIRNAME = settings['IMAGES_DIR']
|
||||
self.mkdir(self.BASEDIRNAME)
|
||||
|
||||
self.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0)
|
||||
self.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0)
|
||||
MediaPipeline.__init__(self)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
relative, absolute = self._get_paths(request)
|
||||
if not should_download(absolute):
|
||||
self.inc_stats(info.domain, 'uptodate')
|
||||
referer = request.headers.get('Referer')
|
||||
log.msg('Image (uptodate): Downloaded %s from %s referred in <%s>' % \
|
||||
(self.MEDIA_TYPE, request, referer), level=log.DEBUG, domain=info.domain)
|
||||
return relative
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
mtype = self.MEDIA_TYPE
|
||||
referer = request.headers.get('Referer')
|
||||
|
||||
if not response or not response.body.to_string():
|
||||
msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: Empty image (no-content)' % (mtype, request, referer)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
result = self.save_image(response, request, info) # save and thumbs response
|
||||
|
||||
status = 'cached' if getattr(response, 'cached', False) else 'downloaded'
|
||||
msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer)
|
||||
log.msg(msg, level=log.DEBUG, domain=info.domain)
|
||||
self.inc_stats(info.domain, status)
|
||||
return result
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
referer = request.headers.get('Referer')
|
||||
errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure)
|
||||
msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' % (self.MEDIA_TYPE, request, referer, errmsg)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
def save_image(self, response, request, info):
|
||||
mtype = self.MEDIA_TYPE
|
||||
relpath, abspath = self._get_paths(request)
|
||||
dirname = os.path.dirname(abspath)
|
||||
self.mkdir(dirname, info)
|
||||
|
||||
try:
|
||||
save_image_with_thumbnails(response, abspath, self.THUMBS, self.MIN_WIDTH, self.MIN_HEIGHT)
|
||||
except ImageException, ex:
|
||||
log.msg(str(ex), level=log.WARNING, domain=info.domain)
|
||||
raise ex
|
||||
except Exception, ex:
|
||||
referer = request.headers.get('Referer')
|
||||
msg = 'Image (processing-error): Error thumbnailing %s from %s referred in <%s>: %s' % (mtype, request, referer, ex)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
return relpath # success value sent as input result for item_media_downloaded
|
||||
|
||||
def _get_paths(self, request):
|
||||
relative = image_path(request.url)
|
||||
absolute = os.path.join(self.BASEDIRNAME, relative)
|
||||
return relative, absolute
|
||||
|
||||
def mkdir(self, dirname, info=None):
|
||||
already_created = info.extra.setdefault('created_directories', set()) if info else set()
|
||||
if dirname not in already_created:
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
already_created.add(dirname)
|
||||
|
||||
def inc_stats(self, domain, status):
|
||||
stats.incpath('%s/image_count' % domain)
|
||||
stats.incpath('%s/image_status_count/%s' % (domain, status))
|
||||
|
||||
|
||||
|
||||
def should_download(path):
|
||||
"""Should the image downloader download the image to the location specified
|
||||
"""
|
||||
try:
|
||||
mtime = os.path.getmtime(path)
|
||||
age_seconds = time.time() - mtime
|
||||
age_days = age_seconds / 60 / 60 / 24
|
||||
return age_days > IMAGE_EXPIRES
|
||||
except:
|
||||
return True
|
||||
|
||||
_MULTIPLE_SLASHES_REGEXP = re.compile(r"\/{2,}")
|
||||
_FINAL_SLASH_REGEXP = re.compile(r"\/$")
|
||||
def image_path(url):
|
||||
"""Return the relative path on the target filesystem for an image to be
|
||||
downloaded to.
|
||||
"""
|
||||
_, netloc, urlpath, query, _ = urlparse.urlsplit(url)
|
||||
urlpath = _MULTIPLE_SLASHES_REGEXP.sub('/', urlpath)
|
||||
urlpath = _FINAL_SLASH_REGEXP.sub('.jpg', urlpath)
|
||||
if os.sep != '/':
|
||||
urlpath.replace('/', os.sep)
|
||||
if query:
|
||||
img_path = os.path.join(netloc, hashlib.sha1(url).hexdigest())
|
||||
else:
|
||||
img_path = os.path.join(netloc, urlpath[1:])
|
||||
return urllib.unquote(img_path)
|
||||
|
||||
|
||||
def thumbnail_name(image, sizestr):
|
||||
"""Get the name of a thumbnail image given the name of the original file.
|
||||
|
||||
There will can be many types of thumbnails, so we will have a "name" for
|
||||
each type.
|
||||
"""
|
||||
return os.path.splitext(image)[0] + '_' + sizestr + '.jpg'
|
||||
|
||||
def save_scaled_image(image, img_path, name, size):
|
||||
thumb = image.copy() if image.mode == 'RGB' else image.convert('RGB')
|
||||
thumb.thumbnail(size, Image.ANTIALIAS)
|
||||
filename = thumbnail_name(img_path, name)
|
||||
thumb.save(filename, 'JPEG')
|
||||
|
||||
def save_image_with_thumbnails(response, path, thumbsizes, min_width=0, min_height=0):
|
||||
memoryfile = StringIO(response.body.to_string())
|
||||
im = Image.open(memoryfile)
|
||||
if im.mode != 'RGB':
|
||||
log.msg("Found non-RGB image during scraping %s" % path, level=log.WARNING)
|
||||
for name, size in thumbsizes or []:
|
||||
save_scaled_image(im, path, name, size)
|
||||
try:
|
||||
im.save(path)
|
||||
except Exception, ex:
|
||||
log.msg("Image (processing-error): cannot process %s, so writing direct file: Error: %s" % (path, ex))
|
||||
f = open(path, 'wb')
|
||||
f.write(response.body.to_string())
|
||||
f.close()
|
||||
width, height = im.size
|
||||
if width < min_width or height < min_height:
|
||||
raise ImageException("Image too small (%dx%d < %dx%d): %s" % (width, height, min_width, min_height, response.url))
|
||||
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
from twisted.internet import defer
|
||||
|
||||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy import log
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.spider import spiders
|
||||
|
||||
|
||||
class DomainInfo(object):
|
||||
def __init__(self, domain):
|
||||
self.domain = domain
|
||||
self.spider = spiders.fromdomain(domain)
|
||||
self.downloading = {}
|
||||
self.downloaded = {}
|
||||
self.waiting = {}
|
||||
self.extra = {}
|
||||
|
||||
|
||||
class MediaPipeline(object):
|
||||
def __init__(self):
|
||||
self.domaininfo = {}
|
||||
|
||||
def open_domain(self, domain):
|
||||
self.domaininfo[domain] = DomainInfo(domain)
|
||||
|
||||
def close_domain(self, domain):
|
||||
del self.domaininfo[domain]
|
||||
|
||||
def process_item(self, domain, response, item):
|
||||
info = self.domaininfo[domain]
|
||||
requests = self.get_media_requests(item, info)
|
||||
assert requests is None or hasattr(requests, '__iter__'), \
|
||||
'get_media_requests should return None or iterable'
|
||||
|
||||
def _bugtrap(_failure, request):
|
||||
log.msg('Unhandled ERROR in MediaPipeline.item_media_{downloaded,failed} for %s: %s' % (request, _failure), log.ERROR, domain=domain)
|
||||
|
||||
lst = []
|
||||
for request in requests or ():
|
||||
dfd = self._enqueue(request, info)
|
||||
dfd.addCallbacks(
|
||||
callback=self.item_media_downloaded,
|
||||
callbackArgs=(item, request, info),
|
||||
errback=self.item_media_failed,
|
||||
errbackArgs=(item, request, info),
|
||||
)
|
||||
dfd.addErrback(_bugtrap, request)
|
||||
lst.append(dfd)
|
||||
|
||||
dlst = defer.DeferredList(lst, consumeErrors=False)
|
||||
dlst.addBoth(lambda _: self.item_completed(item, info))
|
||||
return dlst
|
||||
|
||||
def _enqueue(self, request, info):
|
||||
wad = request.deferred or defer.Deferred()
|
||||
|
||||
fp = request.fingerprint()
|
||||
if fp in info.downloaded:
|
||||
cached = info.downloaded[fp]
|
||||
defer_result(cached).chainDeferred(wad)
|
||||
else:
|
||||
info.waiting.setdefault(fp, []).append(wad)
|
||||
if fp not in info.downloading:
|
||||
self._download(request, info, fp)
|
||||
|
||||
return wad
|
||||
|
||||
def _download(self, request, info, fp):
|
||||
def _bugtrap(_failure):
|
||||
log.msg('Unhandled ERROR in MediaPipeline._downloaded: %s' % (_failure), log.ERROR, domain=info.domain)
|
||||
|
||||
result = self.media_to_download(request, info)
|
||||
if result is not None:
|
||||
dwld = defer_result(result)
|
||||
else:
|
||||
dwld = mustbe_deferred(self.download, request, info)
|
||||
dwld.addCallbacks(
|
||||
callback=self.media_downloaded,
|
||||
callbackArgs=(request, info),
|
||||
errback=self.media_failed,
|
||||
errbackArgs=(request, info),
|
||||
)
|
||||
|
||||
dwld.addBoth(self._downloaded, info, fp)
|
||||
dwld.addErrback(_bugtrap)
|
||||
info.downloading[fp] = (request, dwld)
|
||||
|
||||
def _downloaded(self, result, info, fp):
|
||||
info.downloaded[fp] = result # cache result
|
||||
waiting = info.waiting[fp] # client list
|
||||
del info.waiting[fp]
|
||||
del info.downloading[fp]
|
||||
for wad in waiting:
|
||||
defer_result(result).chainDeferred(wad)
|
||||
|
||||
|
||||
### Overradiable Interface
|
||||
def download(self, request, info):
|
||||
""" Defines how to request the download of media
|
||||
|
||||
Default gives high priority to media requests and use scheduler,
|
||||
shouldn't be necessary to override.
|
||||
|
||||
This methods is called only if result for request isn't cached,
|
||||
request fingerprint is used as cache key.
|
||||
|
||||
"""
|
||||
return scrapyengine.schedule(request, info.spider, priority=0)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
""" Ongoing request hook pre-cache
|
||||
|
||||
This method is called every time a media is requested for download, and
|
||||
only once for the same request because return value is cached as media
|
||||
result.
|
||||
|
||||
returning a non-None value implies:
|
||||
- the return value is cached and piped into `item_media_downloaded` or `item_media_failed`
|
||||
- prevents downloading, this means calling `download` method.
|
||||
- `media_downloaded` or `media_failed` isn't called.
|
||||
|
||||
"""
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
""" Return a list of Request objects to download for this item
|
||||
|
||||
Should return None or an iterable
|
||||
|
||||
Defaults return None (no media to download)
|
||||
|
||||
"""
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
""" Method called on success download of media request
|
||||
|
||||
Return value is cached and used as input for `item_media_downloaded` method.
|
||||
Default implementation returns None.
|
||||
|
||||
WARNING: returning the response object can eat your memory.
|
||||
|
||||
"""
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
""" Method called when media request failed due to any kind of download error.
|
||||
|
||||
Return value is cached and used as input for `item_media_failed` method.
|
||||
Default implementation returns same Failure object.
|
||||
"""
|
||||
return failure
|
||||
|
||||
def item_media_downloaded(self, result, item, request, info):
|
||||
""" Method to handle result of requested media for item.
|
||||
|
||||
result is the return value of `media_downloaded` hook, or the non-Failure instance
|
||||
returned by `media_failed` hook.
|
||||
|
||||
return value of this method isn't important and is recommended to return None.
|
||||
"""
|
||||
|
||||
def item_media_failed(self, failure, item, request, info):
|
||||
""" Method to handle failed result of requested media for item.
|
||||
|
||||
result is the returned Failure instance of `media_failed` hook, or Failure instance
|
||||
of an exception raised by `media_downloaded` hook.
|
||||
|
||||
return value of this method isn't important and is recommended to return None.
|
||||
"""
|
||||
|
||||
def item_completed(self, item, info):
|
||||
""" Method called when all media requests for a single item has returned a result or failure.
|
||||
|
||||
The return value of this method is used as output of pipeline stage.
|
||||
|
||||
`item_completed` can return item itself or raise DropItem exception.
|
||||
|
||||
Default returns item
|
||||
"""
|
||||
return item
|
||||
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
import time
|
||||
import hashlib
|
||||
import rfc822
|
||||
from cStringIO import StringIO
|
||||
|
||||
import Image
|
||||
import boto
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.stats import stats
|
||||
from scrapy.core.exceptions import DropItem, NotConfigured
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
from scrapy.conf import settings
|
||||
|
||||
class NoimagesDrop(DropItem):
|
||||
"""Product with no images exception"""
|
||||
|
||||
class ImageException(Exception):
|
||||
"""General image error exception"""
|
||||
|
||||
class S3ImagesPipeline(MediaPipeline):
|
||||
MEDIA_TYPE = 'image'
|
||||
THUMBS = (
|
||||
("50", (50, 50)),
|
||||
("110", (110, 110)),
|
||||
("270", (270, 270))
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
if not settings['S3_IMAGES']:
|
||||
raise NotConfigured
|
||||
|
||||
# days to wait before redownloading images
|
||||
self.image_refresh_days = settings.getint('IMAGES_REFRESH_DAYS', 90)
|
||||
|
||||
self.bucket_name = settings['S3_BUCKET']
|
||||
self.prefix = settings['S3_PREFIX']
|
||||
access_key = settings['AWS_ACCESS_KEY_ID']
|
||||
secret_key = settings['AWS_SECRET_ACCESS_KEY']
|
||||
conn = boto.connect_s3(access_key, secret_key)
|
||||
self.bucket = conn.get_bucket(self.bucket_name)
|
||||
|
||||
MediaPipeline.__init__(self)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
key = self.s3_image_key(request.url)
|
||||
if not self.s3_should_download(request.url):
|
||||
self.inc_stats(info.domain, 'uptodate')
|
||||
referer = request.headers.get('Referer')
|
||||
log.msg('Image (uptodate) type=%s at <%s> referred from <%s>' % \
|
||||
(self.MEDIA_TYPE, request.url, referer), level=log.DEBUG, domain=info.domain)
|
||||
return key
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
mtype = self.MEDIA_TYPE
|
||||
referer = request.headers.get('Referer')
|
||||
|
||||
if not response or not response.body.to_string():
|
||||
msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: Empty image (no-content)' % (mtype, request, referer)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
result = self.save_image(response, request, info) # save and thumbs response
|
||||
|
||||
status = 'cached' if getattr(response, 'cached', False) else 'downloaded'
|
||||
msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer)
|
||||
log.msg(msg, level=log.DEBUG, domain=info.domain)
|
||||
self.inc_stats(info.domain, status)
|
||||
return result
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
referer = request.headers.get('Referer')
|
||||
errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure)
|
||||
msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' % (self.MEDIA_TYPE, request, referer, errmsg)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
def save_image(self, response, request, info):
|
||||
try:
|
||||
key = self.s3_image_key(request.url)
|
||||
self.s3_store_image(response, request.url)
|
||||
except ImageException, ex:
|
||||
log.msg(str(ex), level=log.WARNING, domain=info.domain)
|
||||
raise ex
|
||||
except Exception, ex:
|
||||
log.msg(str(ex), level=log.WARNING, domain=info.domain)
|
||||
raise ex
|
||||
|
||||
return key # success value sent as input result for item_media_downloaded
|
||||
|
||||
def inc_stats(self, domain, status):
|
||||
stats.incpath('%s/image_count' % domain)
|
||||
stats.incpath('%s/image_status_count/%s' % (domain, status))
|
||||
|
||||
def s3_image_key(self, url):
|
||||
"""Return the relative path on the target filesystem for an image to be
|
||||
downloaded to.
|
||||
"""
|
||||
image_guid = hashlib.sha1(url).hexdigest()
|
||||
return '%s/full/%s.jpg' % (self.prefix, image_guid)
|
||||
|
||||
def s3_thumb_key(self, url, thumb_id):
|
||||
"""Return the relative path on the target filesystem for an image to be
|
||||
downloaded to.
|
||||
"""
|
||||
image_guid = hashlib.sha1(url).hexdigest()
|
||||
return '%s/thumbs/%s/%s.jpg' % (self.prefix, thumb_id, image_guid)
|
||||
|
||||
def s3_should_download(self, url):
|
||||
"""Return if the image should be downloaded by checking if it's already in
|
||||
the S3 storage and not too old"""
|
||||
key = self.s3_image_key(url)
|
||||
k = self.bucket.get_key(key)
|
||||
if k is None:
|
||||
return True
|
||||
modified_tuple = rfc822.parsedate_tz(k.last_modified)
|
||||
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
|
||||
age_seconds = time.time() - modified_stamp
|
||||
age_days = age_seconds / 60 / 60 / 24
|
||||
return age_days > self.image_refresh_days
|
||||
|
||||
def s3_store_image(self, response, url):
|
||||
"""Upload image to S3 storage"""
|
||||
buf = StringIO(response.body.to_string())
|
||||
image = Image.open(buf)
|
||||
key = self.s3_image_key(url)
|
||||
self._s3_put_image(image, key)
|
||||
self.s3_store_thumbnails(image, url)
|
||||
|
||||
def s3_store_thumbnails(self, image, url):
|
||||
"""Upload image thumbnails to S3 storage"""
|
||||
for thumb_id, size in self.THUMBS or []:
|
||||
thumb = image.copy() if image.mode == 'RGB' else image.convert('RGB')
|
||||
thumb.thumbnail(size, Image.ANTIALIAS)
|
||||
key = self.s3_thumb_key(url, thumb_id)
|
||||
self._s3_put_image(thumb, key)
|
||||
|
||||
def s3_public_url(self, key):
|
||||
return "http://%s.s3.amazonaws.com/%s" % (self.bucket_name, key)
|
||||
|
||||
def _s3_put_image(self, image, key):
|
||||
buf = StringIO()
|
||||
try:
|
||||
image.save(buf, 'JPEG')
|
||||
except Exception, ex:
|
||||
raise ImageException("Cannot process image. Error: %s" % ex)
|
||||
|
||||
buf.seek(0)
|
||||
k = self.bucket.new_key(key)
|
||||
k.content_type = 'image/jpeg'
|
||||
k.set_contents_from_file(buf, policy='public-read')
|
||||
log.msg("Uploaded to S3: %s" % self.s3_public_url(key), level=log.DEBUG)
|
||||
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
"""
|
||||
A pipeline to persist objects using shove.
|
||||
|
||||
New is a "new generation" shelve. For more information see:
|
||||
http://pypi.python.org/pypi/shove
|
||||
"""
|
||||
|
||||
from string import Template
|
||||
|
||||
from shove import Shove
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ShoveItemPipeline(object):
|
||||
|
||||
def __init__(self):
|
||||
self.uritpl = settings['SHOVEITEM_STORE_URI']
|
||||
if not self.uritpl:
|
||||
raise NotConfigured
|
||||
self.opts = settings['SHOVEITEM_STORE_OPT'] or {}
|
||||
self.stores = {}
|
||||
|
||||
dispatcher.connect(self.domain_open, signal=signals.domain_open)
|
||||
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
|
||||
|
||||
def process_item(self, domain, response, item):
|
||||
self.stores[domain][str(item.guid)] = item
|
||||
return item
|
||||
|
||||
def domain_open(self, domain):
|
||||
uri = Template(self.uritpl).substitute(domain=domain)
|
||||
self.stores[domain] = Shove(uri, **self.opts)
|
||||
|
||||
def domain_closed(self, domain):
|
||||
self.stores[domain].sync()
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
"""
|
||||
Pipeline to print Items
|
||||
"""
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ShowItemPipeline(object):
|
||||
def __init__(self):
|
||||
if not settings['DEBUG_SHOWITEM']:
|
||||
raise NotConfigured
|
||||
|
||||
def process_item(self, domain, response, item):
|
||||
log.msg("Scraped: \n%s" % repr(item), log.DEBUG, domain=domain)
|
||||
return item
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
import time
|
||||
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
from scrapy.conf import settings
|
||||
|
||||
class LessScrapedPrioritizer(object):
|
||||
"""
|
||||
A spider prioritizer based on these few simple rules:
|
||||
1. if spider was never scraped, it has top priority
|
||||
2. if spider was scraped before, then the less recently the spider
|
||||
has been scraped, the more priority it has
|
||||
"""
|
||||
def __init__(self, elements):
|
||||
if not settings['SCRAPING_DB']:
|
||||
raise NotConfigured("SCRAPING_DB setting is required")
|
||||
|
||||
self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history')
|
||||
domains_to_scrape = set(elements)
|
||||
|
||||
self.priorities = {}
|
||||
|
||||
for domain in domains_to_scrape:
|
||||
stat = self.ddh.getlast(domain, path="start_time")
|
||||
if stat and stat[1]:
|
||||
last_started = stat[1]
|
||||
# spider is the timestamp of last start time
|
||||
self.priorities[domain] = time.mktime(last_started.timetuple())
|
||||
else:
|
||||
# if domain was never scraped, it has top priority
|
||||
self.priorities[domain] = 1
|
||||
|
||||
def get_priority(self, element):
|
||||
return self.priorities[element]
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
"""
|
||||
The ResponseSoup extension causes the Response objects to grow a new method
|
||||
("getsoup") which returns a (cached) BeautifulSoup object of its body, and a
|
||||
"soup" attribute with the same effect. The soup argument is provided for
|
||||
convenience, but you cannot pass any BeautifulSoup constructor arguments (which
|
||||
you can do with the getsoup() method).
|
||||
|
||||
For more information about BeautifulSoup see:
|
||||
http://www.crummy.com/software/BeautifulSoup/documentation.html
|
||||
"""
|
||||
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
|
||||
from scrapy.http import Response
|
||||
|
||||
|
||||
class ResponseSoup(object):
|
||||
def __init__(self):
|
||||
setattr(Response, 'getsoup', getsoup)
|
||||
setattr(Response, 'soup', property(getsoup))
|
||||
|
||||
def getsoup(response, **kwargs):
|
||||
if not hasattr(response, '_soup'):
|
||||
body = response.body.to_string() if response.body is not None else ""
|
||||
setattr(response, '_soup', BeautifulSoup(body, **kwargs))
|
||||
return response._soup
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
"""
|
||||
SpiderProfiler is an extension that hooks itself into every Request callback
|
||||
returned from spiders to measure the processing time and memory allocation
|
||||
caused by spiders code.
|
||||
|
||||
The results are collected using the StatsCollector.
|
||||
|
||||
This extension introduces a big impact on crawling performance, so enable only
|
||||
when needed.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
|
||||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.extension import extensions
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
class SpiderProfiler(object):
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('SPIDERPROFILER_ENABLED'):
|
||||
raise NotConfigured
|
||||
dispatcher.connect(self._request_received, signals.request_received)
|
||||
dispatcher.connect(self._engine_started, signals.engine_started)
|
||||
|
||||
def _engine_started(self):
|
||||
self.memusage = extensions.enabled.get('MemoryUsage', None)
|
||||
|
||||
def _request_received(self, request, spider):
|
||||
old_cbs = request.deferred.callbacks[0]
|
||||
new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], old_cbs[0][2]), old_cbs[1])
|
||||
request.deferred.callbacks[0] = new_cbs
|
||||
|
||||
def _profiled_callback(self, function, spider):
|
||||
def new_callback(*args, **kwargs):
|
||||
tbefore = datetime.datetime.now()
|
||||
mbefore = self._memusage()
|
||||
r = function(*args, **kwargs)
|
||||
tafter = datetime.datetime.now()
|
||||
mafter = self._memusage()
|
||||
ct = tafter-tbefore
|
||||
tcc = stats.getpath('%s/profiling/total_callback_time' % spider.domain_name, datetime.timedelta(0))
|
||||
sct = stats.getpath('%s/profiling/slowest_callback_time' % spider.domain_name, datetime.timedelta(0))
|
||||
stats.setpath('%s/profiling/total_callback_time' % spider.domain_name, tcc+ct)
|
||||
if ct > sct:
|
||||
stats.setpath('%s/profiling/slowest_callback_time' % spider.domain_name, ct)
|
||||
stats.setpath('%s/profiling/slowest_callback_name' % spider.domain_name, function.__name__)
|
||||
stats.setpath('%s/profiling/slowest_callback_url' % spider.domain_name, args[0].url)
|
||||
if self.memusage:
|
||||
tma = stats.getpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, 0)
|
||||
stats.setpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, tma+mafter-mbefore)
|
||||
return r
|
||||
return new_callback
|
||||
|
||||
def _memusage(self):
|
||||
return self.memusage.virtual if self.memusage else 0.0
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
"""
|
||||
Reload spider modules once they are finished scraping
|
||||
|
||||
This is to release any resources held on to by scraping spiders.
|
||||
"""
|
||||
import sys
|
||||
from pydispatch import dispatcher
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
|
||||
class SpiderReloader(object):
|
||||
|
||||
def __init__(self):
|
||||
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
|
||||
|
||||
def domain_closed(self, domain, spider):
|
||||
module = spider.__module__
|
||||
log.msg("reloading module %s" % module, domain=domain)
|
||||
reload(sys.modules[module])
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
"""
|
||||
DepthMiddleware is a scrape middleware used for tracking the depth of each
|
||||
Request inside the site being scraped. It can be used to limit the maximum
|
||||
depth to scrape or things like that
|
||||
"""
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import Request
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
class DepthMiddleware(object):
|
||||
|
||||
def __init__(self):
|
||||
self.maxdepth = settings.getint('DEPTH_LIMIT')
|
||||
self.stats = settings.getbool('DEPTH_STATS')
|
||||
if self.stats and self.maxdepth:
|
||||
stats.setpath('_envinfo/request_depth_limit', self.maxdepth)
|
||||
|
||||
def process_result(self, response, result, spider):
|
||||
def _filter(request):
|
||||
if isinstance(request, Request):
|
||||
request.depth = response.request.depth + 1
|
||||
if self.maxdepth and request.depth > self.maxdepth:
|
||||
log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name)
|
||||
return False
|
||||
elif self.stats:
|
||||
stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, request.depth))
|
||||
if request.depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0):
|
||||
stats.setpath('%s/request_depth_max' % spider.domain_name, request.depth)
|
||||
return True
|
||||
|
||||
if self.stats and response.request.depth == 0: # otherwise we loose stats for depth=0
|
||||
stats.incpath('%s/request_depth_count/0' % spider.domain_name)
|
||||
|
||||
return (r for r in result or () if _filter(r))
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
"""
|
||||
RequestLimitMiddleware: Limits the scheduler request queue from the point of
|
||||
view of the spider. That is, if the scheduler queue contains an equal or
|
||||
greater ammount of requests than the specified limit, the new requests
|
||||
(generated by the spider) will be ignored.
|
||||
|
||||
The limit is setted from the spider attribute "requests_queue_size". If not
|
||||
found, from the scrapy setting "REQUESTS_QUEUE_SIZE". If not found, no limit
|
||||
will be applied. If given a value of 0, no limit will be applied.
|
||||
"""
|
||||
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.conf import settings
|
||||
from scrapy.http import Request
|
||||
from scrapy import log
|
||||
|
||||
class RequestLimitMiddleware(object):
|
||||
#_last_queue_size = 0
|
||||
def process_result(self, response, result, spider):
|
||||
requests = []
|
||||
other = []
|
||||
[requests.append(r) if isinstance(r, Request) else other.append(r) for r in result]
|
||||
|
||||
max_pending = spider.requests_queue_size if hasattr(spider,"requests_queue_size") else settings.getint("REQUESTS_QUEUE_SIZE")
|
||||
if not max_pending:
|
||||
accepted = requests
|
||||
else:
|
||||
free_slots = max_pending - len(scrapyengine.scheduler.pending_requests[spider.domain_name])
|
||||
accepted = requests[:free_slots]
|
||||
dropped = set(requests) - set(accepted)
|
||||
if dropped:
|
||||
for r in dropped:
|
||||
log.msg("Ignoring link (max schedule queue size reached): %s " % r.url, level=log.WARNING, domain=spider.domain_name)
|
||||
#actual_size = len(scrapyengine.scheduler.pending_requests[spider.domain_name])
|
||||
#log.msg("queue size: %d (%+d)" % (actual_size, actual_size - self._last_queue_size) )
|
||||
#self._last_queue_size = actual_size
|
||||
return accepted + other
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
"""
|
||||
OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by
|
||||
the spider.
|
||||
"""
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
|
||||
class OffsiteMiddleware(object):
|
||||
def process_result(self, response, result, spider):
|
||||
def _filter(r):
|
||||
if isinstance(r, Request) and not url_is_from_spider(r.url, spider):
|
||||
log.msg("Ignoring link (offsite): %s " % r.url, level=log.DEBUG, domain=spider.domain_name)
|
||||
return False
|
||||
return True
|
||||
return (r for r in result or () if _filter(r))
|
||||
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
"""
|
||||
RefererMiddleware: populates Request referer field, based on the Response which
|
||||
originated it.
|
||||
"""
|
||||
|
||||
from scrapy.http import Request
|
||||
|
||||
class RefererMiddleware(object):
|
||||
def process_result(self, response, result, spider):
|
||||
def _set_referer(r):
|
||||
if isinstance(r, Request):
|
||||
r.headers.setdefault('Referer', response.url)
|
||||
return r
|
||||
return (_set_referer(r) for r in result or ())
|
||||
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
"""
|
||||
RestrictMiddleware: restricts crawling to fixed set of particular URLs
|
||||
"""
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
|
||||
class RestrictMiddleware(object):
|
||||
def __init__(self):
|
||||
self.allowed_urls = set(settings.getlist('RESTRICT_TO_URLS'))
|
||||
if not self.allowed_urls:
|
||||
raise NotConfigured
|
||||
|
||||
def process_result(self, response, result, spider):
|
||||
def _filter(r):
|
||||
if isinstance(r, Request) and r.url not in self.allowed_urls:
|
||||
return False
|
||||
return True
|
||||
return (r for r in result or () if _filter(r))
|
||||
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
"""
|
||||
UrlFilterMiddleware: canonicalizes URLs to filter out duplicated ones
|
||||
"""
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.url import canonicalize_url
|
||||
|
||||
class UrlFilterMiddleware(object):
|
||||
def process_result(self, response, result, spider):
|
||||
disabled = getattr(spider, 'urlfilter_disabled', False)
|
||||
for r in result:
|
||||
if isinstance(r, Request) and not disabled:
|
||||
curl = canonicalize_url(r.url)
|
||||
# only assign if different to avoid re-calculating fingerprint
|
||||
if curl != r.url:
|
||||
r.url = curl
|
||||
yield r
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
"""
|
||||
UrlLengthMiddleware: Filters out requests with URLs longer than URLLENGTH_LIMIT
|
||||
"""
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import Request
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
|
||||
class UrlLengthMiddleware(object):
|
||||
def __init__(self):
|
||||
self.maxlength = settings.getint('URLLENGTH_LIMIT')
|
||||
if not self.maxlength:
|
||||
raise NotConfigured
|
||||
|
||||
def process_result(self, response, result, spider):
|
||||
def _filter(request):
|
||||
if isinstance(request, Request) and len(request.url) > self.maxlength:
|
||||
log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
return (r for r in result or () if _filter(r))
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
"""
|
||||
This module contains BasicSpider, a spider class which provides support for
|
||||
basic crawling.
|
||||
"""
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.core.exceptions import UsageError
|
||||
|
||||
class BasicSpider(BaseSpider):
|
||||
"""BasicSpider extends BaseSpider by providing support for simple crawling
|
||||
by following links contained in web pages.
|
||||
|
||||
With BasicSpider you can write a basic spider very easily and quickly. For
|
||||
more information refer to the Scrapy tutorial"""
|
||||
|
||||
def __init__(self):
|
||||
super(BaseSpider, self).__init__()
|
||||
|
||||
self._links_callback = []
|
||||
|
||||
for attr in dir(self):
|
||||
if attr.startswith('links_'):
|
||||
suffix = attr.split('_', 1)[1]
|
||||
value = getattr(self, attr)
|
||||
try:
|
||||
callback = getattr(self, 'parse_%s' % suffix)
|
||||
except AttributeError:
|
||||
raise UsageError("%s defines links_%s but doesn't provide a parse_%s method" % \
|
||||
(type(self).__name__, suffix, suffix))
|
||||
self._links_callback.append((value, callback))
|
||||
|
||||
def parse(self, response):
|
||||
"""This function is called by the core for all the start_urls. Do not
|
||||
override this function, override parse_start_url instead."""
|
||||
return self._parse_wrapper(response, self.parse_start_url)
|
||||
|
||||
def parse_start_url(self, response):
|
||||
"""Callback function for processing start_urls. It must return a list
|
||||
of ScrapedItems and/or Requests."""
|
||||
return []
|
||||
|
||||
def _links_to_follow(self, response):
|
||||
res = []
|
||||
links_to_follow = {}
|
||||
for lx, callback in self._links_callback:
|
||||
for url, link_text in lx.extract_urls(response).iteritems():
|
||||
links_to_follow[url] = (callback, link_text)
|
||||
|
||||
for url, cb_link in links_to_follow.iteritems():
|
||||
request = Request(url=url, link_text=link_text)
|
||||
request.append_callback(self._parse_wrapper, callback)
|
||||
res.append(request)
|
||||
return res
|
||||
|
||||
def _parse_wrapper(self, response, callback):
|
||||
res = self._links_to_follow(response)
|
||||
res += callback(response) or ()
|
||||
return res
|
||||
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
"""A django alike request-response model
|
||||
|
||||
most of this code is borrowed from django
|
||||
"""
|
||||
|
||||
from Cookie import SimpleCookie
|
||||
from scrapy.utils.datatypes import MultiValueDict, CaselessDict
|
||||
|
||||
|
||||
def build_httprequest(twistedrequest):
|
||||
"""Translate twisted request object to a django request approach"""
|
||||
request = HttpRequest()
|
||||
request.path = twistedrequest.path
|
||||
request.method = twistedrequest.method.upper()
|
||||
request.COOKIES = SimpleCookie(twistedrequest.received_cookies)
|
||||
request.HEADERS = Headers(twistedrequest.received_headers)
|
||||
request.ARGS = MultiValueDict(twistedrequest.args)
|
||||
request.FILES = {} # not yet supported
|
||||
request.content = twistedrequest.content
|
||||
request.twistedrequest = twistedrequest
|
||||
return request
|
||||
|
||||
|
||||
class HttpRequest(object):
|
||||
def __init__(self):
|
||||
self.path = ''
|
||||
self.method = None
|
||||
self.COOKIES = {}
|
||||
self.HEADERS = {}
|
||||
self.ARGS = {}
|
||||
self.FILES = {}
|
||||
|
||||
|
||||
class HttpResponse(object):
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, content='', status=None, content_type=None):
|
||||
content_type = content_type or "text/html; charset=utf-8"
|
||||
self._headers = {'content-type': content_type}
|
||||
self.content = content
|
||||
self.cookies = SimpleCookie()
|
||||
self.status_code = status
|
||||
|
||||
def __str__(self):
|
||||
"Full HTTP message, including headers"
|
||||
return '\n'.join(['%s: %s' % (key, value)
|
||||
for key, value in self._headers.items()]) \
|
||||
+ '\n\n' + self.content
|
||||
|
||||
def __setitem__(self, header, value):
|
||||
self._headers[header.lower()] = value
|
||||
|
||||
def __delitem__(self, header):
|
||||
try:
|
||||
del self._headers[header.lower()]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def __getitem__(self, header):
|
||||
return self._headers[header.lower()]
|
||||
|
||||
def has_header(self, header):
|
||||
"Case-insensitive check for a header"
|
||||
return self._headers.has_key(header.lower())
|
||||
|
||||
__contains__ = has_header
|
||||
|
||||
def items(self):
|
||||
return self._headers.items()
|
||||
|
||||
def get(self, header, alternate):
|
||||
return self._headers.get(header, alternate)
|
||||
|
||||
def set_cookie(self, key, value='', max_age=None, expires=None, path='/', domain=None, secure=None):
|
||||
self.cookies[key] = value
|
||||
for var in ('max_age', 'path', 'domain', 'secure', 'expires'):
|
||||
val = locals()[var]
|
||||
if val is not None:
|
||||
self.cookies[key][var.replace('_', '-')] = val
|
||||
|
||||
def delete_cookie(self, key, path='/', domain=None):
|
||||
self.cookies[key] = ''
|
||||
if path is not None:
|
||||
self.cookies[key]['path'] = path
|
||||
if domain is not None:
|
||||
self.cookies[key]['domain'] = domain
|
||||
self.cookies[key]['expires'] = 0
|
||||
self.cookies[key]['max-age'] = 0
|
||||
|
||||
|
||||
class Headers(CaselessDict):
|
||||
def __init__(self, source=None, encoding='utf-8'):
|
||||
self.encoding = encoding
|
||||
|
||||
if getattr(source, 'iteritems', None):
|
||||
d = source.iteritems()
|
||||
else:
|
||||
d = source # best effort
|
||||
|
||||
# can't use CaselessDict.__init__(self, d) because it doesn't call __setitem__
|
||||
for k,v in d:
|
||||
self.__setitem__(k.lower(), v)
|
||||
|
||||
def normkey(self, key):
|
||||
return key.title() # 'Content-Type' styles headers
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""Headers must not be unicode"""
|
||||
if isinstance(key, unicode):
|
||||
key = key.encode(self.encoding)
|
||||
if isinstance(value, unicode):
|
||||
value = value.encode(self.encoding)
|
||||
super(Headers, self).__setitem__(key, value)
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
from functools import wraps
|
||||
|
||||
from scrapy.utils.defer import mustbe_deferred
|
||||
from scrapy.utils.serialization import serialize as _serialize, unserialize as _unserialize
|
||||
|
||||
from .http import HttpResponse
|
||||
|
||||
JSON_CONTENT_TYPES = ('application/json',)
|
||||
|
||||
serialize = lambda x: _serialize(x, 'json')
|
||||
unserialize = lambda x: _unserialize(x, 'json')
|
||||
|
||||
class JsonException(Exception):
|
||||
pass
|
||||
|
||||
class JsonResponse(HttpResponse):
|
||||
def __init__(self, content=None, callback=None, serialize=serialize, *args, **kwargs):
|
||||
content = serialize(content)
|
||||
if callback: # JSONP support
|
||||
status, content = 200, '%s(%s)' % (callback, content)
|
||||
kwargs.setdefault('content_type', 'application/x-javascript')
|
||||
HttpResponse.__init__(self, content=content, *args, **kwargs)
|
||||
|
||||
class JsonResponseAccepted(JsonResponse):
|
||||
status_code = 202
|
||||
|
||||
class JsonResponseNoContent(JsonResponse):
|
||||
status_code = 204
|
||||
|
||||
class JsonResponseNotModified(JsonResponse):
|
||||
status_code = 304
|
||||
|
||||
class JsonResponseBadRequest(JsonResponse):
|
||||
status_code = 400
|
||||
|
||||
class JsonResponseUnauthorized(JsonResponse):
|
||||
status_code = 401
|
||||
|
||||
class JsonResponseForbidden(JsonResponse):
|
||||
status_code = 403
|
||||
|
||||
class JsonResponseNotFound(JsonResponse):
|
||||
status_code = 404
|
||||
|
||||
class JsonResponseInternalServerError(JsonResponse):
|
||||
status_code = 500
|
||||
|
||||
class JsonResponseNotImplemented(JsonResponse):
|
||||
status_code = 501
|
||||
|
||||
|
||||
def json(func):
|
||||
""" Decorator to wrap a json prepared view and return a JsonResponse
|
||||
|
||||
if content-type is application/json, sets request.JSON to unserialized request body.
|
||||
in case of unserialization failure, returns JsonResponseBadRequest()
|
||||
|
||||
if returned data from func is a dictionary, serialize it and returns JsonResponse()
|
||||
"""
|
||||
if not hasattr(func, '__call__'):
|
||||
raise TypeError('The argument should be a callable')
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(request, *args, **kwargs):
|
||||
json_callback = request.ARGS.get('callback') # JSONP support
|
||||
request.method = method = _x_http_method_override(request)
|
||||
request.content_type = ct = content_type(request)
|
||||
request.JSON = None
|
||||
|
||||
if method in ('POST', 'PUT'):
|
||||
if ct in JSON_CONTENT_TYPES:
|
||||
body = request.content.read()
|
||||
try:
|
||||
request.JSON = unserialize(body)
|
||||
except Exception, e:
|
||||
return JsonResponseBadRequest('Invalid json: %s' % e )
|
||||
|
||||
def _onsuccess(response):
|
||||
if not isinstance(response, HttpResponse):
|
||||
return JsonResponse(response, json_callback) # best effort
|
||||
return response
|
||||
|
||||
ret = mustbe_deferred(func, request, *args, **kwargs)
|
||||
ret.addCallback(_onsuccess)
|
||||
return ret
|
||||
return wrapper
|
||||
|
||||
def content_type(request):
|
||||
ct = request.HEADERS.get('content-type','')
|
||||
return ct.split(';')[0].strip()
|
||||
|
||||
def _x_http_method_override(request):
|
||||
""" support for X-Http-Method-Override hack
|
||||
|
||||
some clients does not support methods others than GET and POST, that clients
|
||||
has a chance to set an extra header to indicate intended method.
|
||||
"""
|
||||
return request.HEADERS.get('x-http-method-override', request.method).upper()
|
||||
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
import re
|
||||
import urllib
|
||||
import hashlib
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.core.engine import scrapyengine
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
from scrapy.utils.misc import memoize
|
||||
from lrucache import LRUCache
|
||||
|
||||
from .site import WebSite, WebResource
|
||||
from .http import HttpResponse
|
||||
from .json import JsonResponse
|
||||
|
||||
JSONCALLBACK_RE = '^[a-zA-Z][a-zA-Z_.-]*$'
|
||||
CACHESIZE = settings.get('WS_CACHESIZE', 20)
|
||||
|
||||
|
||||
def _urlhash(request):
|
||||
h = hashlib.sha1()
|
||||
for a in sorted(request.ARGS):
|
||||
h.update(request.ARGS[a])
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@memoize(cache=LRUCache(CACHESIZE), hash=_urlhash)
|
||||
def url_to_guid(httprequest):
|
||||
url = httprequest.ARGS.get('url')
|
||||
if not url:
|
||||
return HttpResponse('Bad Request', 400)
|
||||
url = urllib.unquote(url)
|
||||
|
||||
jsoncb = httprequest.ARGS.get('callback')
|
||||
if jsoncb and not re.match(JSONCALLBACK_RE, jsoncb):
|
||||
return HttpResponse('Bad callback argument', 400)
|
||||
|
||||
def _response(guids=(), message=None):
|
||||
content = {
|
||||
'guids': list(guids),
|
||||
'domain': getattr(spider, 'domain_name', None),
|
||||
'message': message,
|
||||
}
|
||||
return JsonResponse(content=content, callback=jsoncb)
|
||||
|
||||
spider = spiders.fromurl(url)
|
||||
if not spider:
|
||||
return _response(message='No crawler found for site')
|
||||
|
||||
if httprequest.ARGS.get('dontcrawl'):
|
||||
return _response()
|
||||
|
||||
|
||||
def _on_error(_failure):
|
||||
return _response(message='Error downloading url from site')
|
||||
|
||||
def _on_success(pagedata):
|
||||
try:
|
||||
items = spider.identify(pagedata)
|
||||
except Exception, ex:
|
||||
return _response(message='Error processing url')
|
||||
|
||||
guids = [i.guid for i in items if isinstance(i, ScrapedItem)]
|
||||
return _response(guids=guids)
|
||||
|
||||
deferred = defer.Deferred().addCallbacks(_on_success, _on_error)
|
||||
request = Request(url=url, callback=deferred, dont_filter=True)
|
||||
schd = scrapyengine.schedule(request, spider)
|
||||
schd.chainDeferred(deferred)
|
||||
return deferred
|
||||
|
||||
|
||||
urlmapping = (
|
||||
('^ws/tools/url_to_guid/$', url_to_guid),
|
||||
)
|
||||
|
||||
|
||||
class UrlToGuidService(WebSite):
|
||||
def __init__(self):
|
||||
if not settings.getbool('WS_ENABLED'):
|
||||
raise NotConfigured
|
||||
|
||||
port = settings.getint('WS_PORT') or 8088
|
||||
timeout = settings.getint('WS_TIMEOUT') or 15 # seconds
|
||||
resource = WebResource(urlmapping, timeout=timeout)
|
||||
WebSite.__init__(self, port=port, resource=resource)
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
"""Twisted website object as django
|
||||
|
||||
################################################################################
|
||||
## Simple Usage example:
|
||||
|
||||
from twisted.internet import reactor
|
||||
from scrapy.contrib.web.http import WebSite, HttpResponse
|
||||
|
||||
def helloword(request):
|
||||
return HttpResponse('Hello World!')
|
||||
|
||||
def hello(request, name):
|
||||
return HttpResponse('Hello %s' % name)
|
||||
|
||||
urls = (
|
||||
('^hello/(?P<name>\w+)/$', hello),
|
||||
('^$', helloword),
|
||||
)
|
||||
|
||||
|
||||
resource = WebResource(urls)
|
||||
site = WebSite(port=8081, resource=resource)
|
||||
reactor.run()
|
||||
|
||||
# now go to http://localhost:8081/
|
||||
|
||||
|
||||
|
||||
################################################################################
|
||||
## Complex usage example:
|
||||
|
||||
from twisted.internet import reactor, defer
|
||||
from scrapy.contrib.web.http import WebSite, HttpResponse
|
||||
|
||||
def delayed(request):
|
||||
def _callback(result):
|
||||
return HttpResponse('Heavy task completed: %s' % result)
|
||||
|
||||
def _errback(_failure):
|
||||
return HttpResponse('Internal Server Error: %s' % _failure, status=500)
|
||||
|
||||
def heavytask(_):
|
||||
import random
|
||||
assert random.randint(0,1), "Exception found processing request"
|
||||
return _
|
||||
|
||||
d = defer.Deferred().addCallback(heavytask)
|
||||
d.addCallbacks(_callback, _errback)
|
||||
reactor.callLater(1, d.callback, "Well done")
|
||||
return d
|
||||
|
||||
urls = (('^delayed/$', delayed),)
|
||||
|
||||
resource = WebResource(urls)
|
||||
site = WebSite(port=8081, resource=resource)
|
||||
reactor.run()
|
||||
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from twisted.web import server, resource
|
||||
from twisted.internet import reactor
|
||||
from scrapy.utils.defer import mustbe_deferred
|
||||
|
||||
from .http import HttpResponse, build_httprequest
|
||||
|
||||
|
||||
def urlresolver(urls, path):
|
||||
"""Simple path to view mapper"""
|
||||
path = path.lstrip('/')
|
||||
for pathre, view in urls:
|
||||
m = re.search(pathre, path)
|
||||
if m:
|
||||
kwargs = m.groupdict()
|
||||
args = () if kwargs else m.groups()
|
||||
return view, args, kwargs
|
||||
return None, (), {}
|
||||
|
||||
|
||||
class WebSite(server.Site):
|
||||
def __init__(self, port=None, *args, **kwargs):
|
||||
server.Site.__init__(self, *args, **kwargs)
|
||||
if port:
|
||||
self.bind(port)
|
||||
|
||||
def bind(self, port):
|
||||
from scrapy.core.engine import scrapyengine
|
||||
scrapyengine.listenTCP(port, self)
|
||||
|
||||
|
||||
class WebResource(resource.Resource):
|
||||
"""Translate twisted web approach to django alike way"""
|
||||
isLeaf = True
|
||||
debug = True
|
||||
|
||||
def __init__(self, urls, timeout=3, urlresolver=urlresolver):
|
||||
resource.Resource.__init__(self)
|
||||
self.urlresolver = urlresolver
|
||||
self.timeout = timeout
|
||||
self.urls = urls
|
||||
|
||||
def render(self, twistedrequest):
|
||||
httprequest = build_httprequest(twistedrequest)
|
||||
|
||||
def _send_response(response):
|
||||
assert isinstance(response, HttpResponse), 'view should return a HttpResponse object'
|
||||
twistedrequest.setResponseCode(response.status_code or 200)
|
||||
for key, val in response.items():
|
||||
twistedrequest.setHeader(key, response[key])
|
||||
twistedrequest.write(response.content)
|
||||
twistedrequest.finish()
|
||||
|
||||
def _on_error(_failure):
|
||||
content = _failure.getTraceback() if self.debug else 'Internal Error'
|
||||
response = HttpResponse(content=str(_failure), status=500)
|
||||
return _send_response(response)
|
||||
|
||||
view, args, kwargs = self.urlresolver(self.urls, httprequest.path)
|
||||
if not view:
|
||||
response = HttpResponse(content='Not Found', status=404)
|
||||
_send_response(response)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
deferred = mustbe_deferred(view, httprequest, *args, **kwargs)
|
||||
deferred.addCallback(_send_response)
|
||||
deferred.addErrback(_on_error)
|
||||
if not deferred.timeoutCall:
|
||||
deferred.setTimeout(self.timeout)
|
||||
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue