diff --git a/.travis.yml b/.travis.yml index cb54c8008..bcbf75a43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ matrix: python: 3.8 - env: TOXENV=flake8 python: 3.8 + - env: TOXENV=pylint + python: 3.8 - env: TOXENV=docs python: 3.7 # Keep in sync with .readthedocs.yml diff --git a/docs/conf.py b/docs/conf.py index 813417bae..8ab38a090 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Scrapy documentation build configuration file, created by # sphinx-quickstart on Mon Nov 24 12:02:52 2008. # diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 6290adbe2..9acfc3b23 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -14,50 +14,57 @@ Author: dufferzafar import re -# Used for remembering the file (and its contents) -# so we don't have to open the same file again. -_filename = None -_contents = None -# A regex that matches standard linkcheck output lines -line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +def main(): -# Read lines from the linkcheck output file -try: - with open("build/linkcheck/output.txt") as out: - output_lines = out.readlines() -except IOError: - print("linkcheck output not found; please run linkcheck first.") - exit(1) + # Used for remembering the file (and its contents) + # so we don't have to open the same file again. + _filename = None + _contents = None -# For every line, fix the respective file -for line in output_lines: - match = re.match(line_re, line) + # A regex that matches standard linkcheck output lines + line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') - if match: - newfilename = match.group(1) - errortype = match.group(2) + # Read lines from the linkcheck output file + try: + with open("build/linkcheck/output.txt") as out: + output_lines = out.readlines() + except IOError: + print("linkcheck output not found; please run linkcheck first.") + exit(1) - # Broken links can't be fixed and - # I am not sure what do with the local ones. - if errortype.lower() in ["broken", "local"]: - print("Not Fixed: " + line) + # For every line, fix the respective file + for line in output_lines: + match = re.match(line_re, line) + + if match: + newfilename = match.group(1) + errortype = match.group(2) + + # Broken links can't be fixed and + # I am not sure what do with the local ones. + if errortype.lower() in ["broken", "local"]: + print("Not Fixed: " + line) + else: + # If this is a new file + if newfilename != _filename: + + # Update the previous file + if _filename: + with open(_filename, "w") as _file: + _file.write(_contents) + + _filename = newfilename + + # Read the new file to memory + with open(_filename) as _file: + _contents = _file.read() + + _contents = _contents.replace(match.group(3), match.group(4)) else: - # If this is a new file - if newfilename != _filename: + # We don't understand what the current line means! + print("Not Understood: " + line) - # Update the previous file - if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) - _filename = newfilename - - # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() - - _contents = _contents.replace(match.group(3), match.group(4)) - else: - # We don't understand what the current line means! - print("Not Understood: " + line) +if __name__ == '__main__': + main() diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..129c7bf7d --- /dev/null +++ b/pylintrc @@ -0,0 +1,113 @@ +[MASTER] +persistent=no +jobs=1 # >1 hides results + +[MESSAGES CONTROL] +disable=abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-classmethod-argument, + bad-continuation, + bad-indentation, + bad-mcs-classmethod-argument, + bad-super-call, + bad-whitespace, + bare-except, + blacklisted-name, + broad-except, + c-extension-no-member, + catching-non-exception, + cell-var-from-loop, + comparison-with-callable, + consider-iterating-dictionary, + consider-using-in, + consider-using-set-comprehension, + consider-using-sys-exit, + cyclic-import, + dangerous-default-value, + deprecated-method, + deprecated-module, + duplicate-code, # https://github.com/PyCQA/pylint/issues/214 + eval-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + import-error, + import-outside-toplevel, + import-self, + inconsistent-return-statements, + inherit-non-class, + invalid-name, + invalid-overridden-method, + isinstance-second-argument-not-valid-type, + keyword-arg-before-vararg, + line-too-long, + logging-format-interpolation, + logging-not-lazy, + lost-exception, + method-hidden, + misplaced-comparison-constant, + missing-docstring, + missing-final-newline, + multiple-imports, + multiple-statements, + no-else-continue, + no-else-raise, + no-else-return, + no-init, + no-member, + no-method-argument, + no-name-in-module, + no-self-argument, + no-self-use, + no-value-for-parameter, + not-an-iterable, + not-callable, + pointless-statement, + pointless-string-statement, + protected-access, + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + reimported, + signature-differs, + singleton-comparison, + super-init-not-called, + superfluous-parens, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-format-args, + too-many-function-args, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-public-methods, + too-many-return-statements, + trailing-newlines, + trailing-whitespace, + unbalanced-tuple-unpacking, + undefined-variable, + undefined-loop-variable, + unexpected-special-method-signature, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-comprehension, + unnecessary-lambda, + unnecessary-pass, + unreachable, + unsubscriptable-object, + unused-argument, + unused-import, + unused-variable, + unused-wildcard-import, + used-before-assignment, + useless-object-inheritance, # Required for Python 2 support + useless-return, + useless-super-delegation, + wildcard-import, + wrong-import-order, + wrong-import-position diff --git a/pytest.ini b/pytest.ini index 44b0e070c..f9769467d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -96,15 +96,15 @@ flake8-ignore = scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/__init__.py E501 - scrapy/pipelines/files.py E116 E501 + scrapy/pipelines/files.py E501 scrapy/pipelines/images.py E501 scrapy/pipelines/media.py E501 # scrapy/selector scrapy/selector/__init__.py F403 - scrapy/selector/unified.py E501 E111 + scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 + scrapy/settings/default_settings.py E501 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -205,7 +205,7 @@ flake8-ignore = tests/test_item.py E501 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 - tests/test_loader.py E501 E741 E117 + tests/test_loader.py E501 E741 tests/test_logformatter.py E501 tests/test_mail.py E501 tests/test_middleware.py E501 @@ -221,10 +221,10 @@ flake8-ignore = tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E501 E121 - tests/test_spidermiddleware_offsite.py E501 E111 + tests/test_spidermiddleware_httperror.py E501 + tests/test_spidermiddleware_offsite.py E501 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E501 E121 + tests/test_spidermiddleware_referer.py E501 F841 E501 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index ad7a81e6b..4e12a5044 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re import logging diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ceb37c5f1..1615d44d7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,8 @@ """ Link extractor based on lxml.html """ +import operator +from functools import partial from urllib.parse import urljoin import lxml.etree as etree @@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link +from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url -from scrapy.linkextractors import FilteringLinkExtractor # from lxml/src/lxml/html/__init__.py @@ -27,19 +29,24 @@ def _nons(tag): return tag +def _identity(x): + return x + + +def _canonicalize_link_url(link): + return canonicalize_url(link.url, keep_fragments=True) + + class LxmlParserLinkExtractor: - def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True, canonicalized=False): - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v + def __init__( + self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False + ): + self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) + self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) + self.process_attr = process if callable(process) else _identity self.unique = unique self.strip = strip - if canonicalized: - self.link_key = lambda link: link.url - else: - self.link_key = lambda link: canonicalize_url(link.url, - keep_fragments=True) + self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url def _iter_links(self, document): for el in document.iter(etree.Element): @@ -93,25 +100,44 @@ class LxmlParserLinkExtractor: class LxmlLinkExtractor(FilteringLinkExtractor): - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=False, - unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True, restrict_text=None): + def __init__( + self, + allow=(), + deny=(), + allow_domains=(), + deny_domains=(), + restrict_xpaths=(), + tags=('a', 'area'), + attrs=('href',), + canonicalize=False, + unique=True, + process_value=None, + deny_extensions=None, + restrict_css=(), + strip=True, + restrict_text=None, + ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) lx = LxmlParserLinkExtractor( - tag=lambda x: x in tags, - attr=lambda x: x in attrs, + tag=partial(operator.contains, tags), + attr=partial(operator.contains, attrs), unique=unique, process=process_value, strip=strip, canonicalized=canonicalize ) - - super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions, - restrict_text=restrict_text) + super(LxmlLinkExtractor, self).__init__( + link_extractor=lx, + allow=allow, + deny=deny, + allow_domains=allow_domains, + deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, + restrict_css=restrict_css, + canonicalize=canonicalize, + deny_extensions=deny_extensions, + restrict_text=restrict_text, + ) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the @@ -124,9 +150,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor): """ base_url = get_base_url(response) if self.restrict_xpaths: - docs = [subdoc - for x in self.restrict_xpaths - for subdoc in response.xpath(x)] + docs = [ + subdoc + for x in self.restrict_xpaths + for subdoc in response.xpath(x) + ] else: docs = [response.selector] all_links = [] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a9066986b..cd3e29057 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -83,8 +83,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in - # FilesPipeline.from_settings. + POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a08955dc9..85a9bb526 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref): selectorlist_cls = SelectorList def __init__(self, response=None, text=None, type=None, root=None, **kwargs): - if not(response is None or text is None): - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + if response is not None and text is not None: + raise ValueError('%s.__init__() received both response and text' + % self.__class__.__name__) st = _st(response, type or self._default_type) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 63da55718..db4193430 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import traceback import warnings from collections import defaultdict diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index a12d08414..88a18331c 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your scraped items # # See documentation in: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index b3e58ff94..6490f52a7 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your spider middleware # # See documentation in: diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4876526a9..ce0edd335 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index cb220eafc..a414b5fde 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Scrapy settings for $project_name project # # For simplicity, this file contains only settings considered important or diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 1cfe9cc9d..e9112bc95 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 878425125..356496487 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index c2e4bacfe..cbcbe9e2c 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 863c9772f..5aa2aa8b0 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 83c359bd4..5207690f4 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import sys import warnings diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 6e81b33ff..c3c5e329b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import OpenSSL import OpenSSL._util as pyOpenSSLutil diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 52c4d71a6..22f23d7b5 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import unittest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index a1645ed96..b9452a0e7 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 04f58d305..357f4d727 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from w3lib.encoding import resolve_encoding diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 7e1b62b7f..2a2650480 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,3 +1,4 @@ +import pickle import re import unittest from warnings import catch_warnings @@ -462,6 +463,10 @@ class Base: Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), ]) + def test_pickle_extractor(self): + lx = self.extractor_cls() + self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 188c5c3cf..6d15aaf31 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os import shutil diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 8cdf7a176..9e63ac924 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.responsetypes import responsetypes diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 714279ae0..e032b247c 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider): def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) self.start_urls = [ - self.mockserver.url("/status?n=200"), - self.mockserver.url("/status?n=404"), - self.mockserver.url("/status?n=402"), - self.mockserver.url("/status?n=500"), + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), ] self.failed = set() self.skipped = set() diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b17e17f2f..adef66c1d 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import inspect import unittest from unittest import mock diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3a58cc36b..824170d71 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os from twisted.trial import unittest diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 21100aeb8..25cd904bc 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import sys import logging import unittest diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f8388957..16e7449c9 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.spiders import Spider diff --git a/tox.ini b/tox.ini index 2102fc602..69b1bdfdd 100644 --- a/tox.ini +++ b/tox.ini @@ -37,6 +37,19 @@ deps = pytest-flake8 commands = py.test --flake8 {posargs:docs scrapy tests} + +[testenv:pylint] +basepython = python3 +deps = + {[testenv]deps} + # Optional dependencies + boto + reppy + robotexclusionrulesparser + # Test dependencies + pylint +commands = + pylint conftest.py docs extras scrapy setup.py tests [testenv:pypy3] basepython = pypy3