diff --git a/Makefile.buildbot b/Makefile.buildbot index 5af1f6b20..7917fa0fd 100644 --- a/Makefile.buildbot +++ b/Makefile.buildbot @@ -1,8 +1,5 @@ TRIAL := $(shell which trial) BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -ifeq ($(BRANCH),master) -export SCRAPY_VERSION_FROM_GIT=1 -endif export PYTHONPATH=$(PWD) test: @@ -11,6 +8,7 @@ test: -s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/ build: + test $(BRANCH) != master || git describe >scrapy/VERSION python extras/makedeb.py build clean: diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 6ac02cc2b..d96e37fef 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -83,7 +83,7 @@ class FTPDownloadHandler(object): callbackArgs=(request, protocol), errback=self._failed, errbackArgs=(request,)) - + def _build_response(self, result, request, protocol): self.result = result respcls = responsetypes.from_args(url=request.url) diff --git a/setup.py b/setup.py index 6efe64074..48ee0adce 100644 --- a/setup.py +++ b/setup.py @@ -1,128 +1,40 @@ -# Scrapy setup.py script -# -# It doesn't depend on setuptools, but if setuptools is available it'll use -# some of its features, like package dependencies. - -from distutils.command.install_data import install_data -from distutils.command.install import INSTALL_SCHEMES -from subprocess import Popen, PIPE -import os -import sys - -class osx_install_data(install_data): - # On MacOS, the platform-specific lib dir is /System/Library/Framework/Python/.../ - # which is wrong. Python 2.5 supplied with MacOS 10.5 has an Apple-specific fix - # for this in distutils.command.install_data#306. It fixes install_lib but not - # install_data, which is why we roll our own install_data class. - - def finalize_options(self): - # By the time finalize_options is called, install.install_lib is set to the - # fixed directory, so we set the installdir to install_lib. The - # install_data class uses ('install_data', 'install_dir') instead. - self.set_undefined_options('install', ('install_lib', 'install_dir')) - install_data.finalize_options(self) - -if sys.platform == "darwin": - cmdclasses = {'install_data': osx_install_data} -else: - cmdclasses = {'install_data': install_data} - -def fullsplit(path, result=None): - """ - Split a pathname into components (the opposite of os.path.join) in a - platform-neutral way. - """ - if result is None: - result = [] - head, tail = os.path.split(path) - if head == '': - return [tail] + result - if head == path: - return result - return fullsplit(head, [tail] + result) - -# Tell distutils to put the data_files in platform-specific installation -# locations. See here for an explanation: -# http://groups.google.com/group/comp.lang.python/browse_thread/thread/35ec7b2fed36eaec/2105ee4d9e8042cb -for scheme in INSTALL_SCHEMES.values(): - scheme['data'] = scheme['purelib'] - -# Compile the list of packages available, because distutils doesn't have -# an easy way to do this. -packages, data_files = [], [] -root_dir = os.path.dirname(__file__) -if root_dir != '': - os.chdir(root_dir) - -def is_not_module(filename): - return os.path.splitext(filename)[1] not in ['.py', '.pyc', '.pyo'] - -for scrapy_dir in ['scrapy']: - for dirpath, dirnames, filenames in os.walk(scrapy_dir): - # Ignore dirnames that start with '.' - for i, dirname in enumerate(dirnames): - if dirname.startswith('.'): del dirnames[i] - if '__init__.py' in filenames: - packages.append('.'.join(fullsplit(dirpath))) - data = [f for f in filenames if is_not_module(f)] - if data: - data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]]) - elif filenames: - data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]]) - -# Small hack for working with bdist_wininst. -# See http://mail.python.org/pipermail/distutils-sig/2004-August/004134.html -if len(sys.argv) > 1 and sys.argv[1] == 'bdist_wininst': - for file_info in data_files: - file_info[0] = '\\PURELIB\\%s' % file_info[0] - -scripts = ['bin/scrapy'] -if os.name == 'nt': - scripts.append('extras/scrapy.bat') - -if os.environ.get('SCRAPY_VERSION_FROM_GIT'): - v = Popen("git describe", shell=True, stdout=PIPE).communicate()[0] - with open('scrapy/VERSION', 'w+') as f: - f.write(v.strip()) -with open(os.path.join(os.path.dirname(__file__), 'scrapy/VERSION')) as f: - version = f.read().strip() +from os.path import dirname, join +from setuptools import setup, find_packages -setup_args = { - 'name': 'Scrapy', - 'version': version, - 'url': 'http://scrapy.org', - 'description': 'A high-level Python Screen Scraping framework', - 'long_description': open('README.rst').read(), - 'author': 'Scrapy developers', - 'maintainer': 'Pablo Hoffman', - 'maintainer_email': 'pablo@pablohoffman.com', - 'license': 'BSD', - 'packages': packages, - 'cmdclass': cmdclasses, - 'data_files': data_files, - 'scripts': scripts, - 'include_package_data': True, - 'classifiers': [ - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', +with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: + version = f.read().decode('ascii').strip() + + +setup( + name='Scrapy', + version=version, + url='http://scrapy.org', + description='A high-level Python Screen Scraping framework', + long_description=open('README.rst').read(), + author='Scrapy developers', + maintainer='Pablo Hoffman', + maintainer_email='pablo@pablohoffman.com', + license='BSD', + packages=find_packages(exclude=['tests']), + include_package_data=True, + entry_points={ + 'console_scripts': ['scrapy = scrapy.cmdline:execute'] + }, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Environment :: Console', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Internet :: WWW/HTTP', - ] -} - -try: - from setuptools import setup -except ImportError: - from distutils.core import setup -else: - setup_args['install_requires'] = [ + ], + install_requires=[ 'Twisted>=10.0.0', 'w3lib>=1.2', 'queuelib', @@ -130,6 +42,5 @@ else: 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - ] - -setup(**setup_args) + ], +)