fix get function (strip and canonicalize url)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40197
This commit is contained in:
olveyra 2008-09-02 14:12:17 +00:00
parent 7913a86250
commit 2a30073ece
1 changed files with 6 additions and 4 deletions

View File

@ -8,6 +8,10 @@ from scrapy.utils.misc import load_class
from scrapy.extension import extensions
from scrapy.conf import settings
from scrapy.core.manager import scrapymanager
from scrapy.http import Request, Response
from scrapy.core.downloader.handlers import download_any
from scrapy.fetcher import get_or_create_spider
from scrapy.utils.url import canonicalize_url
#This code comes from twisted 8. We define here while
#using old twisted version.
@ -68,15 +72,13 @@ class Command(ScrapyCommand):
#print _failure
print "Downloading URL... ",
from scrapy.http import Request, Response
from scrapy.core.downloader.handlers import download_any
from scrapy.fetcher import get_or_create_spider
url = canonicalize_url(url.strip())
r = Request(url)
spider = get_or_create_spider(url)
try:
result = blockingCallFromThread(reactor, download_any, r, spider)
if isinstance(result, Response):
print "OK"
print "Done."
self.generate_vars(url, result)
return True
except Exception, e: