diff --git a/pytest.ini b/pytest.ini index 0e289866c..0797e8ac8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -27,18 +27,16 @@ flake8-ignore = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused + scrapy/__init__.py E402 scrapy/core/downloader/handlers/http.py F401 scrapy/http/__init__.py F401 scrapy/linkextractors/__init__.py E402 F401 scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/__init__.py E402 scrapy/selector/__init__.py F403 - scrapy/spiders/__init__.py E402 scrapy/utils/http.py F403 scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 - tests/test_webclient.py E402 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index fb8357f3c..e791deaa6 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,33 +2,11 @@ Scrapy - a web crawling and web scraping framework written for Python """ -__all__ = ['__version__', 'version_info', 'twisted_version', - 'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field'] - -# Scrapy version import pkgutil -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() -version_info = tuple(int(v) if v.isdigit() else v - for v in __version__.split('.')) -del pkgutil - -# Check minimum required Python version import sys -if sys.version_info < (3, 5): - print("Scrapy %s requires Python 3.5" % __version__) - sys.exit(1) - -# Ignore noisy twisted deprecation warnings import warnings -warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') -del warnings - -# Apply monkey patches to fix issues in external libraries -from scrapy import _monkeypatches -del _monkeypatches from twisted import version as _txv -twisted_version = (_txv.major, _txv.minor, _txv.micro) # Declare top-level shortcuts from scrapy.spiders import Spider @@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest from scrapy.selector import Selector from scrapy.item import Item, Field + +__all__ = [ + '__version__', 'version_info', 'twisted_version', 'Spider', + 'Request', 'FormRequest', 'Selector', 'Item', 'Field', +] + + +# Scrapy and Twisted versions +__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) +twisted_version = (_txv.major, _txv.minor, _txv.micro) + + +# Check minimum required Python version +if sys.version_info < (3, 5): + print("Scrapy %s requires Python 3.5" % __version__) + sys.exit(1) + + +# Ignore noisy twisted deprecation warnings +warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') + + +del pkgutil del sys +del warnings diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py deleted file mode 100644 index f74f89bda..000000000 --- a/scrapy/_monkeypatches.py +++ /dev/null @@ -1,11 +0,0 @@ -import copyreg - - -# Undo what Twisted's perspective broker adds to pickle register -# to prevent bugs like Twisted#7989 while serializing requests -import twisted.persisted.styles # NOQA -# Remove only entries with twisted serializers for non-twisted types. -for k, v in frozenset(copyreg.dispatch_table.items()): - if not str(getattr(k, '__module__', '')).startswith('twisted') \ - and str(getattr(v, '__module__', '')).startswith('twisted'): - copyreg.dispatch_table.pop(k) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ae019c70f..984a5c4e1 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -133,4 +133,4 @@ class FilteringLinkExtractor: # Top-level imports -from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index ba1c866f8..02f87f8f5 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -110,6 +110,6 @@ class Spider(object_ref): # Top-level imports -from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401 -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401 -from scrapy.spiders.sitemap import SitemapSpider # noqa: F401 +from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.sitemap import SitemapSpider