Dropped support for Python 2.5. See: http://blog.scrapy.org/scrapy-dropping-support-for-python-25

docs/faq.rst

@@ -29,16 +29,13 @@ comparing `jinja2`_ to `Django`_.
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy runs in Python 2.5, 2.6 and 2.7. But it's recommended you use Python 2.6
-or above, since the Python 2.5 standard library has a few bugs in their URL
-handling libraries. Some of these Python 2.5 bugs not only affect Scrapy but
-any user code, such as spiders.
+Scrapy runs in Python 2.6 and 2.7.
 
 Does Scrapy work with Python 3.0?
 ---------------------------------
 
 No, and there are no plans to port Scrapy to Python 3.0 yet. At the moment,
-Scrapy works with Python 2.5, 2.6 and 2.7.
+Scrapy works with Python 2.6 and 2.7.
 
 .. seealso:: :ref:`faq-python-versions`.
 

docs/intro/install.rst

@@ -11,7 +11,7 @@ This document describes how to install Scrapy on Linux, Windows and Mac OS X.
 Requirements
 ============
 
-* `Python`_ 2.5, 2.6, 2.7 (3.x is not yet supported)
+* `Python`_ 2.6, 2.7 (3.x is not yet supported)
 
 * `Twisted`_ 2.5.0, 8.0 or above (Windows users: you'll need to install
   `Zope.Interface`_ and maybe `pywin32`_ because of `this Twisted bug`_)
@@ -32,7 +32,7 @@ Install Python
 
 First, you need to install Python, if you haven't done so already.
 
-Scrapy works with Python 2.5, 2.6 or 2.7, which you can get at
+Scrapy works with Python 2.6 or 2.7, which you can get at
 http://www.python.org/download/
 
 .. seealso:: :ref:`faq-python-versions`

extras/scrapy-ws.py

@@ -11,11 +11,10 @@ command).
 
 """
 
-import sys, optparse, urllib
+import sys, optparse, urllib, json
 from urlparse import urljoin
 
 from scrapy.utils.jsonrpc import jsonrpc_client_call, JsonRpcError
-from scrapy.utils.py26 import json
 
 def get_commands():
     return {

scrapy/__init__.py

@@ -7,8 +7,8 @@ __version__ = "0.15.1"
 
 import sys, os, warnings
 
-if sys.version_info < (2,5):
-    print "Scrapy %s requires Python 2.5 or above" % __version__
+if sys.version_info < (2,6):
+    print "Scrapy %s requires Python 2.6 or above" % __version__
     sys.exit(1)
 
 # ignore noisy twisted deprecation warnings

scrapy/cmdline.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import sys
 import os
 import optparse

scrapy/command.py

@@ -2,8 +2,6 @@
 Base class for Scrapy commands
 """
 
-from __future__ import with_statement
-
 import os
 from optparse import OptionGroup
 

scrapy/commands/deploy.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import sys
 import os
 import glob
@@ -8,6 +6,7 @@ import shutil
 import time
 import urllib2
 import netrc
+import json
 from urlparse import urlparse, urljoin
 from subprocess import Popen, PIPE, check_call
 
@@ -15,7 +14,6 @@ from w3lib.form import encode_multipart
 
 from scrapy.command import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.py26 import json
 from scrapy.utils.http import basic_auth_header
 from scrapy.utils.conf import get_config, closest_scrapy_cfg
 

scrapy/commands/startproject.py

@@ -3,11 +3,11 @@ import string
 import re
 import shutil
 from os.path import join, exists
+from shutil import copytree, ignore_patterns
 
 import scrapy
 from scrapy.command import ScrapyCommand
 from scrapy.utils.template import render_templatefile, string_camelcase
-from scrapy.utils.py26 import ignore_patterns, copytree
 from scrapy.exceptions import UsageError
 
 TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project')

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import os
 from os.path import join, exists
 from time import time

scrapy/contrib/exporter/__init__.py

@@ -5,11 +5,10 @@ Item Exporters are used to export/serialize items into different formats.
 import csv
 import pprint
 import marshal
+import json
 import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
-from scrapy.utils.py26 import json
-
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \
     'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \

scrapy/contrib/httpcache.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import os
 from time import time
 import cPickle as pickle

scrapy/contrib/pipeline/images.py

@@ -4,7 +4,6 @@ Images Pipeline
 See documentation in topics/images.rst
 """
 
-from __future__ import with_statement
 import os
 import time
 import hashlib

scrapy/contrib/spiderstate.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import os, cPickle as pickle
 
 from scrapy import signals

scrapy/core/scheduler.py

@@ -1,13 +1,11 @@
-from __future__ import with_statement
-
 import os
+import json
 from os.path import join, exists
 
 from scrapy.utils.pqueue import PriorityQueue
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 from scrapy.utils.misc import load_object
 from scrapy.utils.job import job_dir
-from scrapy.utils.py26 import json
 from scrapy.stats import stats
 from scrapy import log
 

scrapy/responsetypes.py

@@ -4,14 +4,13 @@ based on different criterias.
 
 """
 
-from os.path import abspath, dirname, join
 from mimetypes import MimeTypes
+from pkgutil import get_data
 from cStringIO import StringIO
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import isbinarytext
-from scrapy.utils.py26 import get_data
 from scrapy.conf import settings
 
 class ResponseTypes(object):

scrapy/tests/test_commands.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import sys
 import os
 import subprocess

scrapy/tests/test_contrib_exporter.py

@@ -1,9 +1,8 @@
-import unittest, cPickle as pickle
+import unittest, json, cPickle as pickle
 from cStringIO import StringIO
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import str_to_unicode
-from scrapy.utils.py26 import json
 from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \
     PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \
     JsonItemExporter

scrapy/tests/test_downloadermiddleware_cookies.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 from unittest import TestCase
 
 from scrapy.http import Response, Request

scrapy/tests/test_downloadermiddleware_httpcompression.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 from unittest import TestCase
 from os.path import join, abspath, dirname
 from cStringIO import StringIO

scrapy/tests/test_spider.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import gzip, warnings
 from cStringIO import StringIO
 

scrapy/tests/test_utils_gz.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import unittest
 from os.path import join
 

scrapy/tests/test_utils_jsonrpc.py

@@ -1,11 +1,10 @@
-import unittest
+import unittest, json
 from cStringIO import StringIO
 
 from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \
     JsonRpcError, jsonrpc_errors
 from scrapy.utils.serialize import ScrapyJSONDecoder
 from scrapy.tests.test_utils_serialize import CrawlerMock
-from scrapy.utils.py26 import json
 
 class urllib_mock(object):
     def __init__(self, result=None, error=None):

scrapy/tests/test_utils_serialize.py

@@ -1,11 +1,11 @@
 import unittest
 import datetime
+import json
 from decimal import Decimal
 
 from twisted.internet import defer
 
 from scrapy.utils.serialize import SpiderReferencer, ScrapyJSONEncoder, ScrapyJSONDecoder
-from scrapy.utils.py26 import json
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
 

scrapy/utils/jsonrpc.py

@@ -4,9 +4,9 @@ http://groups.google.com/group/json-rpc/web/json-rpc-2-0
 """
 
 import urllib
+import json
 import traceback
 
-from scrapy.utils.py26 import json
 from scrapy.utils.serialize import ScrapyJSONDecoder
 
 # JSON-RPC 2.0 errors, as defined in:

scrapy/utils/memory.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import os
 import sys
 import struct

scrapy/utils/py26.py

@@ -1,129 +0,0 @@
-"""
-This module provides functions added in Python 2.6, which weren't yet available
-in Python 2.5. The Python 2.6 function is used when available.
-"""
-
-import sys
-import os
-import fnmatch
-import pkgutil
-from shutil import copy2, copystat
-
-__all__ = ['cpu_count', 'copytree', 'ignore_patterns']
-
-try:
-    import multiprocessing
-    cpu_count = multiprocessing.cpu_count
-except ImportError:
-    def cpu_count():
-        '''
-        Returns the number of CPUs in the system
-        '''
-        if sys.platform == 'win32':
-            try:
-                num = int(os.environ['NUMBER_OF_PROCESSORS'])
-            except (ValueError, KeyError):
-                num = 0
-        elif 'bsd' in sys.platform or sys.platform == 'darwin':
-            try:
-                num = int(os.popen('sysctl -n hw.ncpu').read())
-            except ValueError:
-                num = 0
-        else:
-            try:
-                num = os.sysconf('SC_NPROCESSORS_ONLN')
-            except (ValueError, OSError, AttributeError):
-                num = 0
-
-        if num >= 1:
-            return num
-        else:
-            raise NotImplementedError('cannot determine number of cpus')
-
-if sys.version_info >= (2, 6):
-    from shutil import copytree, ignore_patterns
-else:
-    try:
-        WindowsError
-    except NameError:
-        WindowsError = None
-
-    class Error(EnvironmentError):
-        pass
-
-    def ignore_patterns(*patterns):
-        def _ignore_patterns(path, names):
-            ignored_names = []
-            for pattern in patterns:
-                ignored_names.extend(fnmatch.filter(names, pattern))
-            return set(ignored_names)
-        return _ignore_patterns
-
-    def copytree(src, dst, symlinks=False, ignore=None):
-        names = os.listdir(src)
-        if ignore is not None:
-            ignored_names = ignore(src, names)
-        else:
-            ignored_names = set()
-
-        os.makedirs(dst)
-        errors = []
-        for name in names:
-            if name in ignored_names:
-                continue
-            srcname = os.path.join(src, name)
-            dstname = os.path.join(dst, name)
-            try:
-                if symlinks and os.path.islink(srcname):
-                    linkto = os.readlink(srcname)
-                    os.symlink(linkto, dstname)
-                elif os.path.isdir(srcname):
-                    copytree(srcname, dstname, symlinks, ignore)
-                else:
-                    copy2(srcname, dstname)
-                # XXX What about devices, sockets etc.?
-            except (IOError, os.error), why:
-                errors.append((srcname, dstname, str(why)))
-            # catch the Error from the recursive copytree so that we can
-            # continue with other files
-            except Error, err:
-                errors.extend(err.args[0])
-        try:
-            copystat(src, dst)
-        except OSError, why:
-            if WindowsError is not None and isinstance(why, WindowsError):
-                # Copying file access times may fail on Windows
-                pass
-            else:
-                errors.extend((src, dst, str(why)))
-        if errors:
-            raise Error, errors
-
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-
-def _get_data(package, resource):
-    loader = pkgutil.get_loader(package)
-    if loader is None or not hasattr(loader, 'get_data'):
-        return None
-    mod = sys.modules.get(package) or loader.load_module(package)
-    if mod is None or not hasattr(mod, '__file__'):
-        return None
-
-    # Modify the resource name to be compatible with the loader.get_data
-    # signature - an os.path format "filename" starting with the dirname of
-    # the package's __file__
-    parts = resource.split('/')
-    parts.insert(0, os.path.dirname(mod.__file__))
-    resource_name = os.path.join(*parts)
-    return loader.get_data(resource_name)
-
-# pkgutil.get_data() not available in python 2.5
-# see http://docs.python.org/release/2.5/lib/module-pkgutil.html
-try:
-    get_data = pkgutil.get_data
-except AttributeError:
-    get_data = _get_data

scrapy/utils/py27.py

@@ -1,5 +1,6 @@
 """
-Similar to scrapy.utils.py26, but for Python 2.7
+This module provides functions added in Python 2.7, which weren't yet available
+in Python 2.6. The Python 2.7 function is used when available.
 """
 
 __all__ = ['OrderedDict']

scrapy/utils/python.py

@@ -2,7 +2,7 @@
 This module contains essential stuff that should've come with Python itself ;)
 
 It also contains functions (or functionality) which is in Python versions
-higher than 2.5 which is the lowest version supported by Scrapy.
+higher than 2.5 which used to be the lowest version supported by Scrapy.
 
 """
 import os

scrapy/utils/queue.py

@@ -1,12 +1,9 @@
-from __future__ import with_statement
-
 import os
 import struct
 import glob
+import json
 from collections import deque
 
-from scrapy.utils.py26 import json
-
 
 class FifoMemoryQueue(object):
     """Memory FIFO queue."""

scrapy/utils/serialize.py

@@ -1,12 +1,12 @@
 import re
 import datetime
 import decimal
+import json
 
 from twisted.internet import defer
 
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
-from scrapy.utils.py26 import json
 
 
 class SpiderReferencer(object):

scrapy/utils/template.py

@@ -1,5 +1,4 @@
 """Helper functions for working with templates"""
-from __future__ import with_statement
 
 import os
 import re

scrapy/utils/txweb.py

@@ -1,6 +1,6 @@
-from twisted.web import resource
+import json
 
-from scrapy.utils.py26 import json
+from twisted.web import resource
 
 class JsonResource(resource.Resource):
 

scrapyd/config.py

@@ -1,11 +1,9 @@
 import glob
 from cStringIO import StringIO
+from pkgutil import get_data
 from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError
 
 from scrapy.utils.conf import closest_scrapy_cfg
-from scrapy.utils.py26 import get_data
-
-__package__ = 'scrapyd' # required for compatibility with python 2.5
 
 class Config(object):
     """A ConfigParser wrapper to support defaults when calling instance

scrapyd/eggstorage.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 from glob import glob
 from os import path, makedirs, remove
 from shutil import copyfileobj, rmtree

scrapyd/launcher.py

@@ -1,11 +1,11 @@
 import sys
 from datetime import datetime
+from multiprocessing import cpu_count
 
 from twisted.internet import reactor, defer, protocol, error
 from twisted.application.service import Service
 from twisted.python import log
 
-from scrapy.utils.py26 import cpu_count
 from scrapy.utils.python import stringify_dict
 from scrapyd.utils import get_crawl_args
 from .interfaces import IPoller, IEnvironment

scrapyd/runner.py

@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 import sys
 import os
 import shutil

scrapyd/sqlite.py

@@ -1,9 +1,8 @@
 import sqlite3
 import cPickle
+import json
 from UserDict import DictMixin
 
-from scrapy.utils.py26 import json
-
 
 class SqliteDict(DictMixin):
     """SQLite-backed dictionary"""

scrapyd/tests/test_utils.py

@@ -1,17 +1,13 @@
-from __future__ import with_statement
-
 import os
+from pkgutil import get_data
 from cStringIO import StringIO
 
 from twisted.trial import unittest
 
-from scrapy.utils.py26 import get_data
 from scrapyd.interfaces import IEggStorage
 from scrapyd.utils import get_crawl_args, get_spider_list
 from scrapyd import get_application
 
-__package__ = 'scrapyd.tests' # required for compatibility with python 2.5
-
 class UtilsTest(unittest.TestCase):
 
     def test_get_crawl_args(self):

setup.py

@@ -3,8 +3,6 @@
 # It doesn't depend on setuptools, but if setuptools is available it'll use
 # some of its features, like package dependencies.
 
-from __future__ import with_statement
-
 from distutils.command.install_data import install_data
 from distutils.command.install import INSTALL_SCHEMES
 from subprocess import Popen, PIPE
@@ -104,7 +102,6 @@ setup_args = {
     'scripts': scripts,
     'classifiers': [
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.5',
         'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'License :: OSI Approved :: BSD License',