Modernize setup.py

Makefile.buildbot

@@ -1,8 +1,5 @@
 TRIAL := $(shell which trial)
 BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
-ifeq ($(BRANCH),master)
-export SCRAPY_VERSION_FROM_GIT=1
-endif
 export PYTHONPATH=$(PWD)
 
 test:
@@ -11,6 +8,7 @@ test:
 	-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
 
 build:
+	test $(BRANCH) != master || git describe >scrapy/VERSION
 	python extras/makedeb.py build
 
 clean:

scrapy/core/downloader/handlers/ftp.py

@@ -83,7 +83,7 @@ class FTPDownloadHandler(object):
                         callbackArgs=(request, protocol),
                         errback=self._failed,
                         errbackArgs=(request,))
-    
+
     def _build_response(self, result, request, protocol):
         self.result = result
         respcls = responsetypes.from_args(url=request.url)

setup.py

@@ -1,128 +1,40 @@
-# Scrapy setup.py script
+from os.path import dirname, join
-#
+from setuptools import setup, find_packages
-# It doesn't depend on setuptools, but if setuptools is available it'll use
-# some of its features, like package dependencies.
-
-from distutils.command.install_data import install_data
-from distutils.command.install import INSTALL_SCHEMES
-from subprocess import Popen, PIPE
-import os
-import sys
-
-class osx_install_data(install_data):
-    # On MacOS, the platform-specific lib dir is /System/Library/Framework/Python/.../
-    # which is wrong. Python 2.5 supplied with MacOS 10.5 has an Apple-specific fix
-    # for this in distutils.command.install_data#306. It fixes install_lib but not
-    # install_data, which is why we roll our own install_data class.
-
-    def finalize_options(self):
-        # By the time finalize_options is called, install.install_lib is set to the
-        # fixed directory, so we set the installdir to install_lib. The
-        # install_data class uses ('install_data', 'install_dir') instead.
-        self.set_undefined_options('install', ('install_lib', 'install_dir'))
-        install_data.finalize_options(self)
-
-if sys.platform == "darwin":
-    cmdclasses = {'install_data': osx_install_data}
-else:
-    cmdclasses = {'install_data': install_data}
-
-def fullsplit(path, result=None):
-    """
-    Split a pathname into components (the opposite of os.path.join) in a
-    platform-neutral way.
-    """
-    if result is None:
-        result = []
-    head, tail = os.path.split(path)
-    if head == '':
-        return [tail] + result
-    if head == path:
-        return result
-    return fullsplit(head, [tail] + result)
-
-# Tell distutils to put the data_files in platform-specific installation
-# locations. See here for an explanation:
-# http://groups.google.com/group/comp.lang.python/browse_thread/thread/35ec7b2fed36eaec/2105ee4d9e8042cb
-for scheme in INSTALL_SCHEMES.values():
-    scheme['data'] = scheme['purelib']
-
-# Compile the list of packages available, because distutils doesn't have
-# an easy way to do this.
-packages, data_files = [], []
-root_dir = os.path.dirname(__file__)
-if root_dir != '':
-    os.chdir(root_dir)
-
-def is_not_module(filename):
-    return os.path.splitext(filename)[1] not in ['.py', '.pyc', '.pyo']
-
-for scrapy_dir in ['scrapy']:
-    for dirpath, dirnames, filenames in os.walk(scrapy_dir):
-        # Ignore dirnames that start with '.'
-        for i, dirname in enumerate(dirnames):
-            if dirname.startswith('.'): del dirnames[i]
-        if '__init__.py' in filenames:
-            packages.append('.'.join(fullsplit(dirpath)))
-            data = [f for f in filenames if is_not_module(f)]
-            if data:
-                data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]])
-        elif filenames:
-            data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]])
-
-# Small hack for working with bdist_wininst.
-# See http://mail.python.org/pipermail/distutils-sig/2004-August/004134.html
-if len(sys.argv) > 1 and sys.argv[1] == 'bdist_wininst':
-    for file_info in data_files:
-        file_info[0] = '\\PURELIB\\%s' % file_info[0]
-
-scripts = ['bin/scrapy']
-if os.name == 'nt':
-    scripts.append('extras/scrapy.bat')
-
-if os.environ.get('SCRAPY_VERSION_FROM_GIT'):
-    v = Popen("git describe", shell=True, stdout=PIPE).communicate()[0]
-    with open('scrapy/VERSION', 'w+') as f:
-        f.write(v.strip())
-with open(os.path.join(os.path.dirname(__file__), 'scrapy/VERSION')) as f:
-    version = f.read().strip()
 
 
-setup_args = {
+with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
-    'name': 'Scrapy',
+    version = f.read().decode('ascii').strip()
-    'version': version,
+
-    'url': 'http://scrapy.org',
+
-    'description': 'A high-level Python Screen Scraping framework',
+setup(
-    'long_description': open('README.rst').read(),
+    name='Scrapy',
-    'author': 'Scrapy developers',
+    version=version,
-    'maintainer': 'Pablo Hoffman',
+    url='http://scrapy.org',
-    'maintainer_email': 'pablo@pablohoffman.com',
+    description='A high-level Python Screen Scraping framework',
-    'license': 'BSD',
+    long_description=open('README.rst').read(),
-    'packages': packages,
+    author='Scrapy developers',
-    'cmdclass': cmdclasses,
+    maintainer='Pablo Hoffman',
-    'data_files': data_files,
+    maintainer_email='pablo@pablohoffman.com',
-    'scripts': scripts,
+    license='BSD',
-    'include_package_data': True,
+    packages=find_packages(exclude=['tests']),
-    'classifiers': [
+    include_package_data=True,
-        'Programming Language :: Python',
+    entry_points={
-        'Programming Language :: Python :: 2.7',
+        'console_scripts': ['scrapy = scrapy.cmdline:execute']
+    },
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
-        'Development Status :: 5 - Production/Stable',
+        'Programming Language :: Python',
-        'Intended Audience :: Developers',
+        'Programming Language :: Python :: 2',
-        'Environment :: Console',
+        'Programming Language :: Python :: 2.7',
+        'Topic :: Internet :: WWW/HTTP',
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
-        'Topic :: Internet :: WWW/HTTP',
+    ],
-    ]
+    install_requires=[
-}
-
-try:
-    from setuptools import setup
-except ImportError:
-    from distutils.core import setup
-else:
-    setup_args['install_requires'] = [
         'Twisted>=10.0.0',
         'w3lib>=1.2',
         'queuelib',
@@ -130,6 +42,5 @@ else:
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-    ]
+    ],
-
+)
-setup(**setup_args)