From da1f6d31440dfacd7ef36c6213286a1b2a181b0d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 23 Aug 2013 13:03:28 -0300 Subject: [PATCH] remove unused imports and some assorted pylint-ing --- scrapy/__init__.py | 2 +- scrapy/commands/crawl.py | 2 +- scrapy/commands/shell.py | 1 - scrapy/contrib/exporter/__init__.py | 2 +- scrapy/contrib_exp/downloadermiddleware/decompression.py | 1 - scrapy/core/downloader/handlers/http11.py | 3 +-- scrapy/core/downloader/webclient.py | 2 -- scrapy/resolver.py | 2 -- scrapy/spider.py | 1 - scrapy/tests/test_http_request.py | 1 - scrapy/tests/test_utils_defer.py | 4 ++-- scrapy/tests/test_utils_misc/__init__.py | 4 ++-- 12 files changed, 8 insertions(+), 17 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 17bf64202..e32618ebb 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,7 +7,7 @@ version_info = tuple(__version__.split('.')[:3]) import sys, os, warnings -if sys.version_info < (2,6): +if sys.version_info < (2, 6): print "Scrapy %s requires Python 2.6 or above" % __version__ sys.exit(1) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index e1167cbc0..75e8e7720 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -34,7 +34,7 @@ class Command(ScrapyCommand): self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats)) + raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index fe45e7dba..7209ae16a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,7 +8,6 @@ from threading import Thread from scrapy.command import ScrapyCommand from scrapy.shell import Shell -from scrapy import log class Command(ScrapyCommand): diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 584786a87..a20c92b54 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -198,7 +198,7 @@ class PickleItemExporter(BaseItemExporter): def __init__(self, file, protocol=2, **kwargs): self._configure(kwargs) - self.file =file + self.file = file self.protocol = protocol def export_item(self, item): diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index d18d51eb5..d67794d67 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -10,7 +10,6 @@ from cStringIO import StringIO from tempfile import mktemp from scrapy import log -from scrapy.http import Response from scrapy.responsetypes import responsetypes diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c74581991..6690bbe96 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,10 +8,9 @@ from zope.interface import implements from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer -from twisted.web.http import PotentialDataLoss from twisted.internet.error import TimeoutError from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, TCP4ClientEndpoint, ResponseFailed + HTTPConnectionPool, TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ea69dc5ca..3b239ddec 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,11 +1,9 @@ from time import time from urlparse import urlparse, urlunparse, urldefrag -from twisted.internet.ssl import ClientContextFactory from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient from twisted.internet import defer -from OpenSSL import SSL from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6ba89ac94..7d9811727 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,5 +1,3 @@ -import socket - from twisted.internet import defer from twisted.internet.base import ThreadedResolver diff --git a/scrapy/spider.py b/scrapy/spider.py index 36e91d636..25a59036d 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -6,7 +6,6 @@ See documentation in docs/topics/spiders.rst from scrapy import log from scrapy.http import Request -from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index d3378148d..29358399e 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,7 +1,6 @@ import cgi import unittest import xmlrpclib -from cStringIO import StringIO from urlparse import urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse diff --git a/scrapy/tests/test_utils_defer.py b/scrapy/tests/test_utils_defer.py index 4473002dc..670116b34 100644 --- a/scrapy/tests/test_utils_defer.py +++ b/scrapy/tests/test_utils_defer.py @@ -14,7 +14,7 @@ class MustbeDeferredTest(unittest.TestCase): return steps dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred steps.append(2) # add another value, that should be catched by assertEqual return dfd @@ -27,7 +27,7 @@ class MustbeDeferredTest(unittest.TestCase): return dfd dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred steps.append(2) # add another value, that should be catched by assertEqual return dfd diff --git a/scrapy/tests/test_utils_misc/__init__.py b/scrapy/tests/test_utils_misc/__init__.py index 916b5e25b..143c9b644 100644 --- a/scrapy/tests/test_utils_misc/__init__.py +++ b/scrapy/tests/test_utils_misc/__init__.py @@ -64,14 +64,14 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(None), '__iter__') assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') - assert hasattr(arg_to_iter([1,2,3]), '__iter__') + assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) - self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) + self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])