diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py index 44a416a55..64e211e61 100644 --- a/scrapy/utils/py26.py +++ b/scrapy/utils/py26.py @@ -3,7 +3,12 @@ This module provides functions added in Python 2.6, which weren't yet available in Python 2.5. The Python 2.6 function is used when available. """ -import sys, os +import sys +import os +import fnmatch +from shutil import copytree, ignore_patterns, copy2, copystat + +__all__ = ['cpu_count', 'copytree', 'ignore_patterns'] try: import multiprocessing @@ -34,4 +39,59 @@ except ImportError: else: raise NotImplementedError('cannot determine number of cpus') +if sys.version_info < (2, 6): + try: + WindowsError + except NameError: + WindowsError = None + class Error(EnvironmentError): + pass + + def ignore_patterns(*patterns): + def _ignore_patterns(path, names): + ignored_names = [] + for pattern in patterns: + ignored_names.extend(fnmatch.filter(names, pattern)) + return set(ignored_names) + return _ignore_patterns + + def copytree(src, dst, symlinks=False, ignore=None): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + os.makedirs(dst) + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + copytree(srcname, dstname, symlinks, ignore) + else: + copy2(srcname, dstname) + # XXX What about devices, sockets etc.? + except (IOError, os.error), why: + errors.append((srcname, dstname, str(why))) + # catch the Error from the recursive copytree so that we can + # continue with other files + except Error, err: + errors.extend(err.args[0]) + try: + copystat(src, dst) + except OSError, why: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) + if errors: + raise Error, errors diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 99aefcd74..abf3625e3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,12 +6,9 @@ higher than 2.5 which is the lowest version supported by Scrapy. """ import re -import os -import fnmatch import inspect import weakref from functools import wraps -from shutil import copy2, copystat from sgmllib import SGMLParser class FixedSGMLParser(SGMLParser): @@ -146,68 +143,6 @@ def isbinarytext(text): assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ return any(c in _BINARYCHARS for c in text) - -# ----- shutil.copytree function from Python 2.6 adds ignore argument ---- # - -try: - WindowsError -except NameError: - WindowsError = None - -class Error(EnvironmentError): - pass - -def ignore_patterns(*patterns): - def _ignore_patterns(path, names): - ignored_names = [] - for pattern in patterns: - ignored_names.extend(fnmatch.filter(names, pattern)) - return set(ignored_names) - return _ignore_patterns - -def copytree(src, dst, symlinks=False, ignore=None): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except (IOError, os.error), why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error, err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except OSError, why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.extend((src, dst, str(why))) - if errors: - raise Error, errors - -# ----- end of shutil.copytree function from Python 2.6 ---- # - - def get_func_args(func): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -221,7 +156,6 @@ def get_func_args(func): raise TypeError('%s is not callable' % type(func)) return func_args - def equal_attributes(obj1, obj2, attributes): """Compare two objects attributes""" # not attributes given return False by default