From 3b6f7ac9f2f5b48b9f2f3ce106d1205599d2164f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 19:43:02 +0200 Subject: [PATCH 01/11] Use pylint --- .travis.yml | 2 + docs/utils/linkfix.py | 85 ++++++++++++++++++++++------------------- pylintrc | 88 +++++++++++++++++++++++++++++++++++++++++++ tox.ini | 14 +++++++ 4 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 pylintrc diff --git a/.travis.yml b/.travis.yml index 0190a7f4d..28a19f4f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: + - env: TOXENV=pylint + python: 3.7 - env: TOXENV=py27 python: 2.7 - env: TOXENV=py27-pinned diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 6290adbe2..9acfc3b23 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -14,50 +14,57 @@ Author: dufferzafar import re -# Used for remembering the file (and its contents) -# so we don't have to open the same file again. -_filename = None -_contents = None -# A regex that matches standard linkcheck output lines -line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +def main(): -# Read lines from the linkcheck output file -try: - with open("build/linkcheck/output.txt") as out: - output_lines = out.readlines() -except IOError: - print("linkcheck output not found; please run linkcheck first.") - exit(1) + # Used for remembering the file (and its contents) + # so we don't have to open the same file again. + _filename = None + _contents = None -# For every line, fix the respective file -for line in output_lines: - match = re.match(line_re, line) + # A regex that matches standard linkcheck output lines + line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') - if match: - newfilename = match.group(1) - errortype = match.group(2) + # Read lines from the linkcheck output file + try: + with open("build/linkcheck/output.txt") as out: + output_lines = out.readlines() + except IOError: + print("linkcheck output not found; please run linkcheck first.") + exit(1) - # Broken links can't be fixed and - # I am not sure what do with the local ones. - if errortype.lower() in ["broken", "local"]: - print("Not Fixed: " + line) + # For every line, fix the respective file + for line in output_lines: + match = re.match(line_re, line) + + if match: + newfilename = match.group(1) + errortype = match.group(2) + + # Broken links can't be fixed and + # I am not sure what do with the local ones. + if errortype.lower() in ["broken", "local"]: + print("Not Fixed: " + line) + else: + # If this is a new file + if newfilename != _filename: + + # Update the previous file + if _filename: + with open(_filename, "w") as _file: + _file.write(_contents) + + _filename = newfilename + + # Read the new file to memory + with open(_filename) as _file: + _contents = _file.read() + + _contents = _contents.replace(match.group(3), match.group(4)) else: - # If this is a new file - if newfilename != _filename: + # We don't understand what the current line means! + print("Not Understood: " + line) - # Update the previous file - if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) - _filename = newfilename - - # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() - - _contents = _contents.replace(match.group(3), match.group(4)) - else: - # We don't understand what the current line means! - print("Not Understood: " + line) +if __name__ == '__main__': + main() diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..b83bc9f82 --- /dev/null +++ b/pylintrc @@ -0,0 +1,88 @@ +[MASTER] +persistent=no +jobs=1 # >1 hides results + +[MESSAGES CONTROL] +disable=abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-classmethod-argument, + bad-continuation, + bad-indentation, + bad-mcs-classmethod-argument, + bad-whitespace, + broad-except, + c-extension-no-member, + catching-non-exception, + cell-var-from-loop, + comparison-with-callable, + consider-using-in, + cyclic-import, + dangerous-default-value, + deprecated-method, + deprecated-module, + duplicate-code, # https://github.com/PyCQA/pylint/issues/214 + eval-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + import-error, + import-outside-toplevel, + inconsistent-return-statements, + inherit-non-class, + invalid-name, + keyword-arg-before-vararg, + line-too-long, + logging-format-interpolation, + logging-not-lazy, + lost-exception, + method-hidden, + missing-docstring, + missing-final-newline, + multiple-imports, + multiple-statements, + no-else-continue, + no-else-raise, + no-else-return, + no-init, + no-member, + no-method-argument, + no-name-in-module, + no-self-argument, + no-self-use, + pointless-string-statement, + protected-access, + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + reimported, + signature-differs, + super-init-not-called, + superfluous-parens, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-function-args, + too-many-instance-attributes, + too-many-locals, + too-many-return-statements, + trailing-newlines, + trailing-whitespace, + unexpected-special-method-signature, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-comprehension, + unnecessary-pass, + unsubscriptable-object, + unused-argument, + unused-import, + unused-variable, + unused-wildcard-import, + used-before-assignment, + useless-object-inheritance, # Required for Python 2 support + wildcard-import, + wrong-import-order, + wrong-import-position diff --git a/tox.ini b/tox.ini index ffe7360d3..e7d366fe9 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,20 @@ deps = {[testenv:py35]deps} commands = py.test {posargs:scrapy tests} +[testenv:pylint] +basepython = python3.7 +deps = + {[testenv:py35]deps} + # Optional dependencies + boto + reppy + robotexclusionrulesparser + # Test dependencies + pylint + +commands = + pylint scrapy + [docs] changedir = docs deps = From 02577f55a0586bc3e6c13a4a3ea572c7eefc82b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 24 Oct 2019 13:25:11 +0200 Subject: [PATCH 02/11] Have PyLint cover all Python files in the repository --- pylintrc | 19 +++++++++++++++++++ tox.ini | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index b83bc9f82..ca3ea1c57 100644 --- a/pylintrc +++ b/pylintrc @@ -11,13 +11,18 @@ disable=abstract-method, bad-continuation, bad-indentation, bad-mcs-classmethod-argument, + bad-super-call, bad-whitespace, + blacklisted-name, broad-except, c-extension-no-member, catching-non-exception, cell-var-from-loop, comparison-with-callable, + consider-iterating-dictionary, consider-using-in, + consider-using-set-comprehension, + consider-using-sys-exit, cyclic-import, dangerous-default-value, deprecated-method, @@ -30,6 +35,7 @@ disable=abstract-method, global-statement, import-error, import-outside-toplevel, + import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -39,6 +45,7 @@ disable=abstract-method, logging-not-lazy, lost-exception, method-hidden, + misplaced-comparison-constant, missing-docstring, missing-final-newline, multiple-imports, @@ -52,6 +59,9 @@ disable=abstract-method, no-name-in-module, no-self-argument, no-self-use, + no-value-for-parameter, + not-callable, + pointless-statement, pointless-string-statement, protected-access, redefined-argument-from-local, @@ -59,6 +69,7 @@ disable=abstract-method, redefined-outer-name, reimported, signature-differs, + singleton-comparison, super-init-not-called, superfluous-parens, too-few-public-methods, @@ -67,15 +78,21 @@ disable=abstract-method, too-many-branches, too-many-function-args, too-many-instance-attributes, + too-many-lines, too-many-locals, + too-many-public-methods, too-many-return-statements, trailing-newlines, trailing-whitespace, + unbalanced-tuple-unpacking, + undefined-variable, unexpected-special-method-signature, ungrouped-imports, unidiomatic-typecheck, unnecessary-comprehension, + unnecessary-lambda, unnecessary-pass, + unreachable, unsubscriptable-object, unused-argument, unused-import, @@ -83,6 +100,8 @@ disable=abstract-method, unused-wildcard-import, used-before-assignment, useless-object-inheritance, # Required for Python 2 support + useless-return, + useless-super-delegation, wildcard-import, wrong-import-order, wrong-import-position diff --git a/tox.ini b/tox.ini index e7d366fe9..428571ef2 100644 --- a/tox.ini +++ b/tox.ini @@ -110,7 +110,7 @@ deps = pylint commands = - pylint scrapy + pylint conftest.py docs extras scrapy setup.py tests [docs] changedir = docs From c7f9b955bdf2405fce58907b0395abce2400a66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Dec 2019 12:44:52 +0100 Subject: [PATCH 03/11] Pylint: ignore not-an-iterable --- pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/pylintrc b/pylintrc index ca3ea1c57..c52a4c2d0 100644 --- a/pylintrc +++ b/pylintrc @@ -60,6 +60,7 @@ disable=abstract-method, no-self-argument, no-self-use, no-value-for-parameter, + not-an-iterable, not-callable, pointless-statement, pointless-string-statement, From b59dfb75fa72346f8268b83dedd2c1f9af460c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 May 2020 14:14:59 +0200 Subject: [PATCH 04/11] Update disabled Pylint checks --- pylintrc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pylintrc b/pylintrc index c52a4c2d0..129c7bf7d 100644 --- a/pylintrc +++ b/pylintrc @@ -13,6 +13,7 @@ disable=abstract-method, bad-mcs-classmethod-argument, bad-super-call, bad-whitespace, + bare-except, blacklisted-name, broad-except, c-extension-no-member, @@ -39,6 +40,8 @@ disable=abstract-method, inconsistent-return-statements, inherit-non-class, invalid-name, + invalid-overridden-method, + isinstance-second-argument-not-valid-type, keyword-arg-before-vararg, line-too-long, logging-format-interpolation, @@ -77,6 +80,7 @@ disable=abstract-method, too-many-ancestors, too-many-arguments, too-many-branches, + too-many-format-args, too-many-function-args, too-many-instance-attributes, too-many-lines, @@ -87,6 +91,7 @@ disable=abstract-method, trailing-whitespace, unbalanced-tuple-unpacking, undefined-variable, + undefined-loop-variable, unexpected-special-method-signature, ungrouped-imports, unidiomatic-typecheck, From cf09af787eafa6770bc5ab00bb0ee9759c75df23 Mon Sep 17 00:00:00 2001 From: Antonio Gordillo Toledo Date: Fri, 8 May 2020 06:45:19 -0700 Subject: [PATCH 05/11] Remove Python 2 encoding header from files (#4553) --- docs/conf.py | 2 -- scrapy/downloadermiddlewares/ajaxcrawl.py | 1 - scrapy/spiderloader.py | 1 - scrapy/templates/project/module/items.py.tmpl | 2 -- scrapy/templates/project/module/middlewares.py.tmpl | 2 -- scrapy/templates/project/module/pipelines.py.tmpl | 2 -- scrapy/templates/project/module/settings.py.tmpl | 2 -- scrapy/templates/spiders/basic.tmpl | 1 - scrapy/templates/spiders/crawl.tmpl | 1 - scrapy/templates/spiders/csvfeed.tmpl | 1 - scrapy/templates/spiders/xmlfeed.tmpl | 1 - scrapy/utils/log.py | 2 -- scrapy/utils/ssl.py | 2 -- tests/test_downloadermiddleware_redirect.py | 2 -- tests/test_downloadermiddleware_robotstxt.py | 1 - tests/test_http_response.py | 1 - tests/test_pipeline_crawl.py | 1 - tests/test_responsetypes.py | 1 - tests/test_utils_deprecate.py | 1 - tests/test_utils_iterators.py | 1 - tests/test_utils_log.py | 1 - tests/test_utils_url.py | 1 - 22 files changed, 30 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 813417bae..8ab38a090 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Scrapy documentation build configuration file, created by # sphinx-quickstart on Mon Nov 24 12:02:52 2008. # diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index ad7a81e6b..4e12a5044 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re import logging diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 63da55718..db4193430 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import traceback import warnings from collections import defaultdict diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index a12d08414..88a18331c 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your scraped items # # See documentation in: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index b3e58ff94..6490f52a7 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your spider middleware # # See documentation in: diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4876526a9..ce0edd335 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index cb220eafc..a414b5fde 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Scrapy settings for $project_name project # # For simplicity, this file contains only settings considered important or diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 1cfe9cc9d..e9112bc95 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 878425125..356496487 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index c2e4bacfe..cbcbe9e2c 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 863c9772f..5aa2aa8b0 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 5998dc33b..203842fc8 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import sys import warnings diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 6e81b33ff..c3c5e329b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import OpenSSL import OpenSSL._util as pyOpenSSLutil diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 551e124ab..61c9eddbc 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import unittest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index a1645ed96..b9452a0e7 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 522ec4875..43d6d936a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from w3lib.encoding import resolve_encoding diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 24c516473..e2578a9c9 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os import shutil diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 8cdf7a176..9e63ac924 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.responsetypes import responsetypes diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b17e17f2f..adef66c1d 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import inspect import unittest from unittest import mock diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 46aaaecbc..69339256e 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os from twisted.trial import unittest diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 21100aeb8..25cd904bc 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import sys import logging import unittest diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f8388957..16e7449c9 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.spiders import Spider From b852fff6f82e24c535c0dd9b4ef7fa78e7946497 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 15:19:22 -0300 Subject: [PATCH 06/11] Style changes in link extractor --- scrapy/linkextractors/lxmlhtml.py | 52 ++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ceb37c5f1..9ebf8e7c7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -28,8 +28,9 @@ def _nons(tag): class LxmlParserLinkExtractor: - def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True, canonicalized=False): + def __init__( + self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False + ): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v @@ -93,10 +94,23 @@ class LxmlParserLinkExtractor: class LxmlLinkExtractor(FilteringLinkExtractor): - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=False, - unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True, restrict_text=None): + def __init__( + self, + allow=(), + deny=(), + allow_domains=(), + deny_domains=(), + restrict_xpaths=(), + tags=('a', 'area'), + attrs=('href',), + canonicalize=False, + unique=True, + process_value=None, + deny_extensions=None, + restrict_css=(), + strip=True, + restrict_text=None, + ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) lx = LxmlParserLinkExtractor( tag=lambda x: x in tags, @@ -106,12 +120,18 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - - super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions, - restrict_text=restrict_text) + super(LxmlLinkExtractor, self).__init__( + link_extractor=lx, + allow=allow, + deny=deny, + allow_domains=allow_domains, + deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, + restrict_css=restrict_css, + canonicalize=canonicalize, + deny_extensions=deny_extensions, + restrict_text=restrict_text, + ) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the @@ -124,9 +144,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor): """ base_url = get_base_url(response) if self.restrict_xpaths: - docs = [subdoc - for x in self.restrict_xpaths - for subdoc in response.xpath(x)] + docs = [ + subdoc + for x in self.restrict_xpaths + for subdoc in response.xpath(x) + ] else: docs = [response.selector] all_links = [] From 3ebf2a0d82b0ae5b9e701972ba2a9e1420c6d2c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 15:17:33 -0300 Subject: [PATCH 07/11] Remove lambdas in link extractor --- scrapy/linkextractors/lxmlhtml.py | 28 +++++++++++++++++----------- tests/test_linkextractors.py | 5 +++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 9ebf8e7c7..1615d44d7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,8 @@ """ Link extractor based on lxml.html """ +import operator +from functools import partial from urllib.parse import urljoin import lxml.etree as etree @@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link +from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url -from scrapy.linkextractors import FilteringLinkExtractor # from lxml/src/lxml/html/__init__.py @@ -27,20 +29,24 @@ def _nons(tag): return tag +def _identity(x): + return x + + +def _canonicalize_link_url(link): + return canonicalize_url(link.url, keep_fragments=True) + + class LxmlParserLinkExtractor: def __init__( self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False ): - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v + self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) + self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) + self.process_attr = process if callable(process) else _identity self.unique = unique self.strip = strip - if canonicalized: - self.link_key = lambda link: link.url - else: - self.link_key = lambda link: canonicalize_url(link.url, - keep_fragments=True) + self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url def _iter_links(self, document): for el in document.iter(etree.Element): @@ -113,8 +119,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor): ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) lx = LxmlParserLinkExtractor( - tag=lambda x: x in tags, - attr=lambda x: x in attrs, + tag=partial(operator.contains, tags), + attr=partial(operator.contains, attrs), unique=unique, process=process_value, strip=strip, diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 68e8514ba..46d8c13af 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,3 +1,4 @@ +import pickle import re import unittest from warnings import catch_warnings @@ -462,6 +463,10 @@ class Base: Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), ]) + def test_pickle_extractor(self): + lx = self.extractor_cls() + self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor From 81d0b2f61ac119efab4d5970bd235dbc288496ef Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:23:53 -0300 Subject: [PATCH 08/11] Flake8: remove E111 --- pytest.ini | 4 ++-- scrapy/selector/unified.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index 604bbfe1d..20639baa3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -102,7 +102,7 @@ flake8-ignore = scrapy/pipelines/media.py E501 # scrapy/selector scrapy/selector/__init__.py F403 - scrapy/selector/unified.py E501 E111 + scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 scrapy/settings/default_settings.py E501 E114 E116 @@ -224,7 +224,7 @@ flake8-ignore = tests/test_spider.py E501 tests/test_spidermiddleware.py E501 tests/test_spidermiddleware_httperror.py E128 E501 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 + tests/test_spidermiddleware_offsite.py E501 E128 tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a08955dc9..85a9bb526 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref): selectorlist_cls = SelectorList def __init__(self, response=None, text=None, type=None, root=None, **kwargs): - if not(response is None or text is None): - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + if response is not None and text is not None: + raise ValueError('%s.__init__() received both response and text' + % self.__class__.__name__) st = _st(response, type or self._default_type) From 1a157f2e26274455405b7e272bab5aede0fa59fa Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:27:21 -0300 Subject: [PATCH 09/11] Flake8: remove E116 --- pytest.ini | 4 ++-- scrapy/pipelines/files.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 20639baa3..295609435 100644 --- a/pytest.ini +++ b/pytest.ini @@ -97,7 +97,7 @@ flake8-ignore = scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/__init__.py E501 - scrapy/pipelines/files.py E116 E501 + scrapy/pipelines/files.py E501 scrapy/pipelines/images.py E501 scrapy/pipelines/media.py E501 # scrapy/selector @@ -105,7 +105,7 @@ flake8-ignore = scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 + scrapy/settings/default_settings.py E501 E114 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a9066986b..cd3e29057 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -83,8 +83,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in - # FilesPipeline.from_settings. + POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } From 83ce82f400d9686f7fbaa526f21f635fca8491de Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:28:26 -0300 Subject: [PATCH 10/11] Flake8: remove E114 and E117 (unused) --- pytest.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 295609435..59cce9ac4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -105,7 +105,7 @@ flake8-ignore = scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 + scrapy/settings/default_settings.py E501 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -207,7 +207,7 @@ flake8-ignore = tests/test_item.py E128 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 E117 + tests/test_loader.py E501 E741 E128 tests/test_logformatter.py E128 E501 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 From c2c3054ac13838b89ad5062d6583cc52dd0ba317 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:32:02 -0300 Subject: [PATCH 11/11] Flake8: remove E121 --- pytest.ini | 4 ++-- tests/test_spidermiddleware_httperror.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index 59cce9ac4..61139f7fe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -223,10 +223,10 @@ flake8-ignore = tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E121 + tests/test_spidermiddleware_httperror.py E128 E501 tests/test_spidermiddleware_offsite.py E501 E128 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E501 E121 + tests/test_spidermiddleware_referer.py E501 F841 E501 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 6b61df56f..29584f21b 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider): def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) self.start_urls = [ - self.mockserver.url("/status?n=200"), - self.mockserver.url("/status?n=404"), - self.mockserver.url("/status?n=402"), - self.mockserver.url("/status?n=500"), + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), ] self.failed = set() self.skipped = set()