RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1710 from redapple/1550-shell_file-cont 

[MRG+1] shell command's ability to open local files + tests
This commit is contained in:
Mikhail Korobov 2016-01-29 21:03:40 +05:00
parent 505e87258c 78f00401cd
commit a35aec71e9
6 changed files with 155 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/topics/commands.rst
View File

@ -373,7 +373,9 @@ shell
* Requires project: *no*
Starts the Scrapy shell for the given URL (if given) or empty if no URL is
given. See :ref:`topics-shell` for more info.
given. Also supports UNIX-style local file paths, either relative with
``./`` or ``../`` prefixes or absolute file paths.
See :ref:`topics-shell` for more info.
Usage example::

32
docs/topics/shell.rst
View File

@ -53,6 +53,38 @@ this::
Where the ``<url>`` is the URL you want to scrape.
:command:`shell` also works for local files. This can be handy if you want
to play around with a local copy of a web page. :command:`shell` understands
the following syntaxes for local files::
# UNIX-style
scrapy shell ./path/to/file.html
scrapy shell ../other/path/to/file.html
scrapy shell /absolute/path/to/file.html
# File URI
scrapy shell file:///absolute/path/to/file.html
.. note:: When using relative file paths, be explicit and prepend them
with ``./`` (or ``../`` when relevant).
``scrapy shell index.html`` will not work as one might expect (and
this is by design, not a bug).
Because :command:`shell` favors HTTP URLs over File URIs,
and ``index.html`` being syntactically similar to ``example.com``,
:command:`shell` will treat ``index.html`` as a domain name and trigger
a DNS lookup error::
$ scrapy shell index.html
[ ... scrapy shell starts ... ]
[ ... traceback ... ]
twisted.internet.error.DNSLookupError: DNS lookup failed:
address 'index.html' not found: [Errno -5] No address associated with hostname.
:command:`shell` will not test beforehand if a file called ``index.html``
exists in the current directory. Again, be explicit.
Using the shell
===============

7
scrapy/commands/shell.py
View File

@ -3,14 +3,13 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
from threading import Thread
from scrapy.commands import ScrapyCommand
from scrapy.shell import Shell
from scrapy.http import Request
from scrapy.utils.url import add_http_if_no_scheme
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.url import guess_scheme
class Command(ScrapyCommand):
@ -47,7 +46,9 @@ class Command(ScrapyCommand):
def run(self, args, opts):
url = args[0] if args else None
if url:
url = add_http_if_no_scheme(url)
# first argument may be a local file
url = guess_scheme(url)
spider_loader = self.crawler_process.spider_loader
spidercls = DefaultSpider

34
scrapy/utils/url.py
View File

@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved
to the w3lib.url module. Always import those from there instead.
"""
import posixpath
import re
from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
urlparse, parse_qsl, urlencode,
unquote)
@ -114,10 +115,31 @@ def escape_ajax(url):
def add_http_if_no_scheme(url):
"""Add http as the default scheme if it is missing from the url."""
if url.startswith('//'):
url = 'http:' + url
return url
parser = parse_url(url)
if not parser.scheme or not parser.netloc:
url = 'http://' + url
match = re.match(r"^\w+://", url, flags=re.I)
if not match:
parts = urlparse(url)
scheme = "http:" if parts.netloc else "http://"
url = scheme + url
return url
def guess_scheme(url):
"""Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise."""
parts = urlparse(url)
if parts.scheme:
return url
# Note: this does not match Windows filepath
if re.match(r'''^ # start with...
(
\. # ...a single dot,
(
\. | [^/\.]+ # optionally followed by
)? # either a second dot or some characters
)? # optional match of ".", ".." or ".blabla"
/ # at least one "/" for a file path,
. # and something after the "/"
''', parts.path, flags=re.VERBOSE):
return any_to_uri(url)
else:
return add_http_if_no_scheme(url)

26
tests/test_command_shell.py
View File

@ -1,9 +1,13 @@
from os.path import join
from twisted.trial import unittest
from twisted.internet import defer
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
from tests import tests_datadir
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@ -51,3 +55,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
code = "fetch('{0}') or fetch(response.request.replace(method='POST'))"
errcode, out, _ = yield self.execute(['-c', code.format(url)])
self.assertEqual(errcode, 0, out)
@defer.inlineCallbacks
def test_local_file(self):
filepath = join(tests_datadir, 'test_site/index.html')
_, out, _ = yield self.execute([filepath, '-c', 'item'])
assert b'{}' in out
@defer.inlineCallbacks
def test_local_nofile(self):
filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
errcode, out, err = yield self.execute([filepath, '-c', 'item'],
check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'No such file or directory', err)
@defer.inlineCallbacks
def test_dns_failures(self):
url = 'www.somedomainthatdoesntexi.st'
errcode, out, err = yield self.execute([url, '-c', 'item'],
check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'DNS lookup failed', err)

65
tests/test_utils_url.py
View File

@ -4,7 +4,8 @@ import unittest
import six
from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
canonicalize_url, add_http_if_no_scheme)
canonicalize_url, add_http_if_no_scheme,
guess_scheme)
__doctests__ = ['scrapy.utils.url']
@ -188,7 +189,7 @@ class CanonicalizeUrlTest(unittest.TestCase):
class AddHttpIfNoScheme(unittest.TestCase):
def test_add_scheme(self):
self.assertEqual(add_http_if_no_scheme('www.example.com'),
'http://www.example.com')
@ -216,7 +217,7 @@ class AddHttpIfNoScheme(unittest.TestCase):
def test_username_password(self):
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
'http://username:password@www.example.com')
def test_complete_url(self):
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
@ -294,5 +295,63 @@ class AddHttpIfNoScheme(unittest.TestCase):
'ftp://www.example.com')
class GuessSchemeTest(unittest.TestCase):
pass
def create_guess_scheme_t(args):
def do_expected(self):
url = guess_scheme(args[0])
assert url.startswith(args[1]), \
'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
args[0], url, args[1])
return do_expected
def create_skipped_scheme_t(args):
def do_expected(self):
raise unittest.SkipTest(args[2])
url = guess_scheme(args[0])
assert url.startswith(args[1])
return do_expected
for k, args in enumerate ([
('/index', 'file://'),
('/index.html', 'file://'),
('./index.html', 'file://'),
('../index.html', 'file://'),
('../../index.html', 'file://'),
('./data/index.html', 'file://'),
('.hidden/data/index.html', 'file://'),
('/home/user/www/index.html', 'file://'),
('//home/user/www/index.html', 'file://'),
('file:///home/user/www/index.html', 'file://'),
('index.html', 'http://'),
('example.com', 'http://'),
('www.example.com', 'http://'),
('www.example.com/index.html', 'http://'),
('http://example.com', 'http://'),
('http://example.com/index.html', 'http://'),
('localhost', 'http://'),
('localhost/index.html', 'http://'),
# some corner cases (default to http://)
('/', 'http://'),
('.../test', 'http://'),
], start=1):
t_method = create_guess_scheme_t(args)
t_method.__name__ = 'test_uri_%03d' % k
setattr (GuessSchemeTest, t_method.__name__, t_method)
# TODO: the following tests do not pass with current implementation
for k, args in enumerate ([
('C:\absolute\path\to\a\file.html', 'file://',
'Windows filepath are not supported for scrapy shell'),
], start=1):
t_method = create_skipped_scheme_t(args)
t_method.__name__ = 'test_uri_skipped_%03d' % k
setattr (GuessSchemeTest, t_method.__name__, t_method)
if __name__ == "__main__":
unittest.main()