diff --git a/docs/faq.rst b/docs/faq.rst index abbef29ea..16ce4f1e2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -29,16 +29,13 @@ comparing `jinja2`_ to `Django`_. What Python versions does Scrapy support? ----------------------------------------- -Scrapy runs in Python 2.5, 2.6 and 2.7. But it's recommended you use Python 2.6 -or above, since the Python 2.5 standard library has a few bugs in their URL -handling libraries. Some of these Python 2.5 bugs not only affect Scrapy but -any user code, such as spiders. +Scrapy runs in Python 2.6 and 2.7. Does Scrapy work with Python 3.0? --------------------------------- No, and there are no plans to port Scrapy to Python 3.0 yet. At the moment, -Scrapy works with Python 2.5, 2.6 and 2.7. +Scrapy works with Python 2.6 and 2.7. .. seealso:: :ref:`faq-python-versions`. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c16d82bd3..fda0d7058 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -11,7 +11,7 @@ This document describes how to install Scrapy on Linux, Windows and Mac OS X. Requirements ============ -* `Python`_ 2.5, 2.6, 2.7 (3.x is not yet supported) +* `Python`_ 2.6, 2.7 (3.x is not yet supported) * `Twisted`_ 2.5.0, 8.0 or above (Windows users: you'll need to install `Zope.Interface`_ and maybe `pywin32`_ because of `this Twisted bug`_) @@ -32,7 +32,7 @@ Install Python First, you need to install Python, if you haven't done so already. -Scrapy works with Python 2.5, 2.6 or 2.7, which you can get at +Scrapy works with Python 2.6 or 2.7, which you can get at http://www.python.org/download/ .. seealso:: :ref:`faq-python-versions` diff --git a/extras/scrapy-ws.py b/extras/scrapy-ws.py index 675094708..c5de44c52 100755 --- a/extras/scrapy-ws.py +++ b/extras/scrapy-ws.py @@ -11,11 +11,10 @@ command). """ -import sys, optparse, urllib +import sys, optparse, urllib, json from urlparse import urljoin from scrapy.utils.jsonrpc import jsonrpc_client_call, JsonRpcError -from scrapy.utils.py26 import json def get_commands(): return { diff --git a/scrapy/__init__.py b/scrapy/__init__.py index b426285d5..99c366eb7 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,8 +7,8 @@ __version__ = "0.15.1" import sys, os, warnings -if sys.version_info < (2,5): - print "Scrapy %s requires Python 2.5 or above" % __version__ +if sys.version_info < (2,6): + print "Scrapy %s requires Python 2.6 or above" % __version__ sys.exit(1) # ignore noisy twisted deprecation warnings diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 79a8727aa..cd33381fc 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import sys import os import optparse diff --git a/scrapy/command.py b/scrapy/command.py index c115fc210..be92d9780 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -2,8 +2,6 @@ Base class for Scrapy commands """ -from __future__ import with_statement - import os from optparse import OptionGroup diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py index 7f2fe62a7..ce9cb3f42 100644 --- a/scrapy/commands/deploy.py +++ b/scrapy/commands/deploy.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import sys import os import glob @@ -8,6 +6,7 @@ import shutil import time import urllib2 import netrc +import json from urlparse import urlparse, urljoin from subprocess import Popen, PIPE, check_call @@ -15,7 +14,6 @@ from w3lib.form import encode_multipart from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.py26 import json from scrapy.utils.http import basic_auth_header from scrapy.utils.conf import get_config, closest_scrapy_cfg diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index a4f615f6e..a38a4981b 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -3,11 +3,11 @@ import string import re import shutil from os.path import join, exists +from shutil import copytree, ignore_patterns import scrapy from scrapy.command import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase -from scrapy.utils.py26 import ignore_patterns, copytree from scrapy.exceptions import UsageError TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project') diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index a3bae921c..9eba51fdf 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import os from os.path import join, exists from time import time diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 30b164dbb..1419d76f6 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -5,11 +5,10 @@ Item Exporters are used to export/serialize items into different formats. import csv import pprint import marshal +import json import cPickle as pickle from xml.sax.saxutils import XMLGenerator -from scrapy.utils.py26 import json - __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \ 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \ diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 8eed8b943..3d05e0900 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import os from time import time import cPickle as pickle diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 62aca6de4..15a6f037f 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -4,7 +4,6 @@ Images Pipeline See documentation in topics/images.rst """ -from __future__ import with_statement import os import time import hashlib diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py index 89f0dc38d..0347ccbbb 100644 --- a/scrapy/contrib/spiderstate.py +++ b/scrapy/contrib/spiderstate.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import os, cPickle as pickle from scrapy import signals diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 8beb4e923..bd97c4f01 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,13 +1,11 @@ -from __future__ import with_statement - import os +import json from os.path import join, exists from scrapy.utils.pqueue import PriorityQueue from scrapy.utils.reqser import request_to_dict, request_from_dict from scrapy.utils.misc import load_object from scrapy.utils.job import job_dir -from scrapy.utils.py26 import json from scrapy.stats import stats from scrapy import log diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 0d5b44bb7..6cb4fc04b 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -4,14 +4,13 @@ based on different criterias. """ -from os.path import abspath, dirname, join from mimetypes import MimeTypes +from pkgutil import get_data from cStringIO import StringIO from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import isbinarytext -from scrapy.utils.py26 import get_data from scrapy.conf import settings class ResponseTypes(object): diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 374872e54..449dcfe1c 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import sys import os import subprocess diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 5039fcf17..4d6b10524 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,9 +1,8 @@ -import unittest, cPickle as pickle +import unittest, json, cPickle as pickle from cStringIO import StringIO from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode -from scrapy.utils.py26 import json from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \ JsonItemExporter diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d11625559..1a64c5347 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - from unittest import TestCase from scrapy.http import Response, Request diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 4385fbae1..b77226df6 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - from unittest import TestCase from os.path import join, abspath, dirname from cStringIO import StringIO diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 11476ac65..9ab5dee33 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import gzip, warnings from cStringIO import StringIO diff --git a/scrapy/tests/test_utils_gz.py b/scrapy/tests/test_utils_gz.py index 3dfd1a2ce..eb2acdd13 100644 --- a/scrapy/tests/test_utils_gz.py +++ b/scrapy/tests/test_utils_gz.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import unittest from os.path import join diff --git a/scrapy/tests/test_utils_jsonrpc.py b/scrapy/tests/test_utils_jsonrpc.py index fe62fb319..d186d2234 100644 --- a/scrapy/tests/test_utils_jsonrpc.py +++ b/scrapy/tests/test_utils_jsonrpc.py @@ -1,11 +1,10 @@ -import unittest +import unittest, json from cStringIO import StringIO from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \ JsonRpcError, jsonrpc_errors from scrapy.utils.serialize import ScrapyJSONDecoder from scrapy.tests.test_utils_serialize import CrawlerMock -from scrapy.utils.py26 import json class urllib_mock(object): def __init__(self, result=None, error=None): diff --git a/scrapy/tests/test_utils_serialize.py b/scrapy/tests/test_utils_serialize.py index 108bad6f2..5bfb87c77 100644 --- a/scrapy/tests/test_utils_serialize.py +++ b/scrapy/tests/test_utils_serialize.py @@ -1,11 +1,11 @@ import unittest import datetime +import json from decimal import Decimal from twisted.internet import defer from scrapy.utils.serialize import SpiderReferencer, ScrapyJSONEncoder, ScrapyJSONDecoder -from scrapy.utils.py26 import json from scrapy.spider import BaseSpider from scrapy.http import Request, Response diff --git a/scrapy/utils/jsonrpc.py b/scrapy/utils/jsonrpc.py index 686a1d235..c1458b29c 100644 --- a/scrapy/utils/jsonrpc.py +++ b/scrapy/utils/jsonrpc.py @@ -4,9 +4,9 @@ http://groups.google.com/group/json-rpc/web/json-rpc-2-0 """ import urllib +import json import traceback -from scrapy.utils.py26 import json from scrapy.utils.serialize import ScrapyJSONDecoder # JSON-RPC 2.0 errors, as defined in: diff --git a/scrapy/utils/memory.py b/scrapy/utils/memory.py index 3340a0a6e..aad0c2935 100644 --- a/scrapy/utils/memory.py +++ b/scrapy/utils/memory.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import os import sys import struct diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py deleted file mode 100644 index 9c590481d..000000000 --- a/scrapy/utils/py26.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -This module provides functions added in Python 2.6, which weren't yet available -in Python 2.5. The Python 2.6 function is used when available. -""" - -import sys -import os -import fnmatch -import pkgutil -from shutil import copy2, copystat - -__all__ = ['cpu_count', 'copytree', 'ignore_patterns'] - -try: - import multiprocessing - cpu_count = multiprocessing.cpu_count -except ImportError: - def cpu_count(): - ''' - Returns the number of CPUs in the system - ''' - if sys.platform == 'win32': - try: - num = int(os.environ['NUMBER_OF_PROCESSORS']) - except (ValueError, KeyError): - num = 0 - elif 'bsd' in sys.platform or sys.platform == 'darwin': - try: - num = int(os.popen('sysctl -n hw.ncpu').read()) - except ValueError: - num = 0 - else: - try: - num = os.sysconf('SC_NPROCESSORS_ONLN') - except (ValueError, OSError, AttributeError): - num = 0 - - if num >= 1: - return num - else: - raise NotImplementedError('cannot determine number of cpus') - -if sys.version_info >= (2, 6): - from shutil import copytree, ignore_patterns -else: - try: - WindowsError - except NameError: - WindowsError = None - - class Error(EnvironmentError): - pass - - def ignore_patterns(*patterns): - def _ignore_patterns(path, names): - ignored_names = [] - for pattern in patterns: - ignored_names.extend(fnmatch.filter(names, pattern)) - return set(ignored_names) - return _ignore_patterns - - def copytree(src, dst, symlinks=False, ignore=None): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except (IOError, os.error), why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error, err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except OSError, why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.extend((src, dst, str(why))) - if errors: - raise Error, errors - -try: - import json -except ImportError: - import simplejson as json - - -def _get_data(package, resource): - loader = pkgutil.get_loader(package) - if loader is None or not hasattr(loader, 'get_data'): - return None - mod = sys.modules.get(package) or loader.load_module(package) - if mod is None or not hasattr(mod, '__file__'): - return None - - # Modify the resource name to be compatible with the loader.get_data - # signature - an os.path format "filename" starting with the dirname of - # the package's __file__ - parts = resource.split('/') - parts.insert(0, os.path.dirname(mod.__file__)) - resource_name = os.path.join(*parts) - return loader.get_data(resource_name) - -# pkgutil.get_data() not available in python 2.5 -# see http://docs.python.org/release/2.5/lib/module-pkgutil.html -try: - get_data = pkgutil.get_data -except AttributeError: - get_data = _get_data diff --git a/scrapy/utils/py27.py b/scrapy/utils/py27.py index ca523d851..02411a5fa 100644 --- a/scrapy/utils/py27.py +++ b/scrapy/utils/py27.py @@ -1,5 +1,6 @@ """ -Similar to scrapy.utils.py26, but for Python 2.7 +This module provides functions added in Python 2.7, which weren't yet available +in Python 2.6. The Python 2.7 function is used when available. """ __all__ = ['OrderedDict'] diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5eb4cbeaf..82c953abc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -2,7 +2,7 @@ This module contains essential stuff that should've come with Python itself ;) It also contains functions (or functionality) which is in Python versions -higher than 2.5 which is the lowest version supported by Scrapy. +higher than 2.5 which used to be the lowest version supported by Scrapy. """ import os diff --git a/scrapy/utils/queue.py b/scrapy/utils/queue.py index 1192b38ca..fe6e9c75f 100644 --- a/scrapy/utils/queue.py +++ b/scrapy/utils/queue.py @@ -1,12 +1,9 @@ -from __future__ import with_statement - import os import struct import glob +import json from collections import deque -from scrapy.utils.py26 import json - class FifoMemoryQueue(object): """Memory FIFO queue.""" diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 530c925c4..1c0e603c7 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,12 +1,12 @@ import re import datetime import decimal +import json from twisted.internet import defer from scrapy.spider import BaseSpider from scrapy.http import Request, Response -from scrapy.utils.py26 import json class SpiderReferencer(object): diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 19ad64809..1cfcd82ee 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,5 +1,4 @@ """Helper functions for working with templates""" -from __future__ import with_statement import os import re diff --git a/scrapy/utils/txweb.py b/scrapy/utils/txweb.py index b2124d8e1..6b76114c3 100644 --- a/scrapy/utils/txweb.py +++ b/scrapy/utils/txweb.py @@ -1,6 +1,6 @@ -from twisted.web import resource +import json -from scrapy.utils.py26 import json +from twisted.web import resource class JsonResource(resource.Resource): diff --git a/scrapyd/config.py b/scrapyd/config.py index d2be8722c..c13ed28f8 100644 --- a/scrapyd/config.py +++ b/scrapyd/config.py @@ -1,11 +1,9 @@ import glob from cStringIO import StringIO +from pkgutil import get_data from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError from scrapy.utils.conf import closest_scrapy_cfg -from scrapy.utils.py26 import get_data - -__package__ = 'scrapyd' # required for compatibility with python 2.5 class Config(object): """A ConfigParser wrapper to support defaults when calling instance diff --git a/scrapyd/eggstorage.py b/scrapyd/eggstorage.py index fbeb9e9c4..552b19172 100644 --- a/scrapyd/eggstorage.py +++ b/scrapyd/eggstorage.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - from glob import glob from os import path, makedirs, remove from shutil import copyfileobj, rmtree diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index f1b219bea..cac5ccf7e 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -1,11 +1,11 @@ import sys from datetime import datetime +from multiprocessing import cpu_count from twisted.internet import reactor, defer, protocol, error from twisted.application.service import Service from twisted.python import log -from scrapy.utils.py26 import cpu_count from scrapy.utils.python import stringify_dict from scrapyd.utils import get_crawl_args from .interfaces import IPoller, IEnvironment diff --git a/scrapyd/runner.py b/scrapyd/runner.py index c0acd0fea..1b089ce48 100644 --- a/scrapyd/runner.py +++ b/scrapyd/runner.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import sys import os import shutil diff --git a/scrapyd/sqlite.py b/scrapyd/sqlite.py index 0cbc7217b..69f44a66e 100644 --- a/scrapyd/sqlite.py +++ b/scrapyd/sqlite.py @@ -1,9 +1,8 @@ import sqlite3 import cPickle +import json from UserDict import DictMixin -from scrapy.utils.py26 import json - class SqliteDict(DictMixin): """SQLite-backed dictionary""" diff --git a/scrapyd/tests/test_utils.py b/scrapyd/tests/test_utils.py index e01311eeb..4c3be2fcf 100644 --- a/scrapyd/tests/test_utils.py +++ b/scrapyd/tests/test_utils.py @@ -1,17 +1,13 @@ -from __future__ import with_statement - import os +from pkgutil import get_data from cStringIO import StringIO from twisted.trial import unittest -from scrapy.utils.py26 import get_data from scrapyd.interfaces import IEggStorage from scrapyd.utils import get_crawl_args, get_spider_list from scrapyd import get_application -__package__ = 'scrapyd.tests' # required for compatibility with python 2.5 - class UtilsTest(unittest.TestCase): def test_get_crawl_args(self): diff --git a/setup.py b/setup.py index 98b109e6a..069c8cdf0 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,6 @@ # It doesn't depend on setuptools, but if setuptools is available it'll use # some of its features, like package dependencies. -from __future__ import with_statement - from distutils.command.install_data import install_data from distutils.command.install import INSTALL_SCHEMES from subprocess import Popen, PIPE @@ -104,7 +102,6 @@ setup_args = { 'scripts': scripts, 'classifiers': [ 'Programming Language :: Python', - 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'License :: OSI Approved :: BSD License',