RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

some code refactoring for the scrapy shell command 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401034
This commit is contained in:
Pablo Hoffman 2009-04-03 03:13:22 +00:00
parent b521ca4d36
commit 7e7823654c
3 changed files with 62 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapy/trunk/docs/topics/shell.rst
View File

@ -90,9 +90,9 @@ Example of shell session
Here's an example of a typical shell session where we start by scraping the
http://scrapy.org page, and then the http://slashdot.org page. Note, however,
that the data extracted may not be the same when you try this as those pages
that the data extracted here may not be the same when you try, as those pages
are not static and could have changed by the time you test this. The purpose of
this example is only to get you familiarized with how the Scrapy shell works.
this example is just to get you familiarized with how the Scrapy shell works.
::
@ -104,12 +104,12 @@ this example is only to get you familiarized with how the Scrapy shell works.
------------------------------------------------------------
Available Scrapy variables:
xxs: <class 'scrapy.xpath.selector.XmlXPathSelector'>
xxs: <XmlXPathSelector (http://scrapy.org)>
url: http://scrapy.org
spider: <class 'scrapy.spider.models.BaseSpider'>
hxs: <class 'scrapy.xpath.selector.HtmlXPathSelector'>
hxs: <HtmlXPathSelector (http://scrapy.org)>
item: <class 'myproject.models.Item'>
response: <class 'scrapy.http.response.html.HtmlResponse'>
response: <http://scrapy.org>
Available commands:
get <url>: Fetches an url and updates all variables.
scrapehelp: Prints this help.
@ -130,13 +130,12 @@ this example is only to get you familiarized with how the Scrapy shell works.
Downloading URL... Done.
------------------------------------------------------------
Available Scrapy variables:
xxs: <class 'scrapy.xpath.selector.XmlXPathSelector'>
xxs: <XmlXPathSelector (http://slashdot.org)>
url: http://slashdot.org
spider: <class 'scrapy.spider.models.BaseSpider'>
hxs: <class 'scrapy.xpath.selector.HtmlXPathSelector'>
hxs: <HtmlXPathSelector (http://slashdot.org)>
item: <class 'myproject.models.Item'>
r: <class 'scrapy.http.response.html.HtmlResponse'>
response: <class 'scrapy.http.response.html.HtmlResponse'>
response: <http://slashdot.org>
Available commands:
get <url>: Fetches an url and updates all variables.
scrapehelp: Prints this help.

66
scrapy/trunk/scrapy/command/commands/shell.py
View File

@ -1,49 +1,19 @@
from twisted.internet import reactor
import os
import urllib
import urlparse
from twisted.internet import reactor, threads
import os, urllib, urlparse
import scrapy
from scrapy.command import ScrapyCommand
from scrapy.spider import spiders
from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector
from scrapy.utils.misc import load_object
from scrapy.extension import extensions
from scrapy.conf import settings
from scrapy.core.manager import scrapymanager
from scrapy.core.engine import scrapyengine
from scrapy.http import Request
from scrapy.fetcher import get_or_create_spider
#This code comes from twisted 8. We define here while
#using old twisted version.
def blockingCallFromThread(reactor, f, *a, **kw):
"""
Run a function in the reactor from a thread, and wait for the result
synchronously, i.e. until the callback chain returned by the function
get a result.
@param reactor: The L{IReactorThreads} provider which will be used to
schedule the function call.
@param f: the callable to run in the reactor thread
@type f: any callable.
@param a: the arguments to pass to C{f}.
@param kw: the keyword arguments to pass to C{f}.
@return: the result of the callback chain.
@raise: any error raised during the callback chain.
"""
import Queue
from twisted.python import failure
from twisted.internet import defer
queue = Queue.Queue()
def _callFromThread():
result = defer.maybeDeferred(f, *a, **kw)
result.addBoth(queue.put)
reactor.callFromThread(_callFromThread)
result = queue.get()
if isinstance(result, failure.Failure):
result.raiseException()
return result
class Command(ScrapyCommand):
def syntax(self):
return "[url]"
@ -55,7 +25,7 @@ class Command(ScrapyCommand):
return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html"
def update_vars(self):
""" You can use this function to update the local variables that will be available in the scrape console """
""" You can use this function to update the Scrapy objects that will be available in the shell"""
pass
def get_url(self, url):
@ -74,12 +44,11 @@ class Command(ScrapyCommand):
self.result = response
return []
print "Downloading URL... ",
print "Downloading URL..."
r = Request(url, callback=_get_response)
spider = get_or_create_spider(url)
blockingCallFromThread(reactor, scrapyengine.crawl, r, spider)
threads.blockingCallFromThread(reactor, scrapyengine.crawl, r, spider)
if self.result:
print "Done."
self.result.request = r
self.generate_vars(url, self.result)
return True
@ -99,17 +68,14 @@ class Command(ScrapyCommand):
self.print_vars()
def print_vars(self):
print '-' * 78
print "Available local variables:"
print '-' * 60
print "Available Scrapy objects:"
for key, val in self.vars.iteritems():
if isinstance(val, basestring):
print " %s: %s" % (key, val)
else:
print " %s: %s" % (key, val.__class__)
print " %s: %s" % (key, val)
print "Available commands:"
print " get <url>: Fetches an url and updates all variables."
print " scrapehelp: Prints this help."
print '-' * 78
print " get <url>: Fetches a new page and updates all Scrapy objects."
print " shelp: Prints this help."
print '-' * 60
def run(self, args, opts):
self.vars = {}
@ -118,7 +84,7 @@ class Command(ScrapyCommand):
if args:
url = args[0]
print "Scrapy %s - Interactive scraping console\n" % scrapy.__version__
print "Welcome to Scrapy shell!"
def _console_thread():
@ -138,7 +104,7 @@ class Command(ScrapyCommand):
shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns)
ip = shell.IP.getapi()
ip.expose_magic("get", _get_magic)
ip.expose_magic("scrapehelp", _help_magic)
ip.expose_magic("shelp", _help_magic)
shell.mainloop()
reactor.callFromThread(scrapymanager.stop)
except ImportError:

46
scrapy/trunk/scrapy/patches/monkeypatches.py
View File

@ -1,18 +1,13 @@
"""
Monkey patches
Monkey patches for supporting Twisted 2.5.0
"""
These are generally a bad idea.
"""
import twisted
# Extend limit for BeautifulSoup parsing loops
#import sys
#sys.setrecursionlimit(7400)
def apply_patches():
if twisted.__version__ < '8.0.0':
patch_HTTPPageGetter_handleResponse()
add_missing_blockingCallFromThread()
# bugfix not present in twisted 2.5 for handling empty response of HEAD requests
@ -43,3 +38,38 @@ def patch_HTTPPageGetter_handleResponse():
# stupid...
self.transport.loseConnection()
setattr(HTTPPageGetter, 'handleResponse', _handleResponse)
# This function comes bundled with Twisted 8.x and above
def add_missing_blockingCallFromThread():
import Queue
from twisted.internet import defer
from twisted.python import failure
def blockingCallFromThread(reactor, f, *a, **kw):
"""
Run a function in the reactor from a thread, and wait for the result
synchronously, i.e. until the callback chain returned by the function
get a result.
@param reactor: The L{IReactorThreads} provider which will be used to
schedule the function call.
@param f: the callable to run in the reactor thread
@type f: any callable.
@param a: the arguments to pass to C{f}.
@param kw: the keyword arguments to pass to C{f}.
@return: the result of the callback chain.
@raise: any error raised during the callback chain.
"""
queue = Queue.Queue()
def _callFromThread():
result = defer.maybeDeferred(f, *a, **kw)
result.addBoth(queue.put)
reactor.callFromThread(_callFromThread)
result = queue.get()
if isinstance(result, failure.Failure):
result.raiseException()
return result
from twisted.internet import threads
threads.blockingCallFromThread = blockingCallFromThread