mirror of https://github.com/scrapy/scrapy.git
Mikhail Korobov
GitHub
commit
794ab19660
|
|
@ -11,6 +11,8 @@ matrix:
|
|||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pylint
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
|
||||
|
|
|
|||
|
|
@ -14,50 +14,57 @@ Author: dufferzafar
|
|||
|
||||
import re
|
||||
|
||||
# Used for remembering the file (and its contents)
|
||||
# so we don't have to open the same file again.
|
||||
_filename = None
|
||||
_contents = None
|
||||
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
def main():
|
||||
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
with open("build/linkcheck/output.txt") as out:
|
||||
output_lines = out.readlines()
|
||||
except IOError:
|
||||
print("linkcheck output not found; please run linkcheck first.")
|
||||
exit(1)
|
||||
# Used for remembering the file (and its contents)
|
||||
# so we don't have to open the same file again.
|
||||
_filename = None
|
||||
_contents = None
|
||||
|
||||
# For every line, fix the respective file
|
||||
for line in output_lines:
|
||||
match = re.match(line_re, line)
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
|
||||
if match:
|
||||
newfilename = match.group(1)
|
||||
errortype = match.group(2)
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
with open("build/linkcheck/output.txt") as out:
|
||||
output_lines = out.readlines()
|
||||
except IOError:
|
||||
print("linkcheck output not found; please run linkcheck first.")
|
||||
exit(1)
|
||||
|
||||
# Broken links can't be fixed and
|
||||
# I am not sure what do with the local ones.
|
||||
if errortype.lower() in ["broken", "local"]:
|
||||
print("Not Fixed: " + line)
|
||||
# For every line, fix the respective file
|
||||
for line in output_lines:
|
||||
match = re.match(line_re, line)
|
||||
|
||||
if match:
|
||||
newfilename = match.group(1)
|
||||
errortype = match.group(2)
|
||||
|
||||
# Broken links can't be fixed and
|
||||
# I am not sure what do with the local ones.
|
||||
if errortype.lower() in ["broken", "local"]:
|
||||
print("Not Fixed: " + line)
|
||||
else:
|
||||
# If this is a new file
|
||||
if newfilename != _filename:
|
||||
|
||||
# Update the previous file
|
||||
if _filename:
|
||||
with open(_filename, "w") as _file:
|
||||
_file.write(_contents)
|
||||
|
||||
_filename = newfilename
|
||||
|
||||
# Read the new file to memory
|
||||
with open(_filename) as _file:
|
||||
_contents = _file.read()
|
||||
|
||||
_contents = _contents.replace(match.group(3), match.group(4))
|
||||
else:
|
||||
# If this is a new file
|
||||
if newfilename != _filename:
|
||||
# We don't understand what the current line means!
|
||||
print("Not Understood: " + line)
|
||||
|
||||
# Update the previous file
|
||||
if _filename:
|
||||
with open(_filename, "w") as _file:
|
||||
_file.write(_contents)
|
||||
|
||||
_filename = newfilename
|
||||
|
||||
# Read the new file to memory
|
||||
with open(_filename) as _file:
|
||||
_contents = _file.read()
|
||||
|
||||
_contents = _contents.replace(match.group(3), match.group(4))
|
||||
else:
|
||||
# We don't understand what the current line means!
|
||||
print("Not Understood: " + line)
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
[MASTER]
|
||||
persistent=no
|
||||
jobs=1 # >1 hides results
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
disable=abstract-method,
|
||||
anomalous-backslash-in-string,
|
||||
arguments-differ,
|
||||
attribute-defined-outside-init,
|
||||
bad-classmethod-argument,
|
||||
bad-continuation,
|
||||
bad-indentation,
|
||||
bad-mcs-classmethod-argument,
|
||||
bad-super-call,
|
||||
bad-whitespace,
|
||||
bare-except,
|
||||
blacklisted-name,
|
||||
broad-except,
|
||||
c-extension-no-member,
|
||||
catching-non-exception,
|
||||
cell-var-from-loop,
|
||||
comparison-with-callable,
|
||||
consider-iterating-dictionary,
|
||||
consider-using-in,
|
||||
consider-using-set-comprehension,
|
||||
consider-using-sys-exit,
|
||||
cyclic-import,
|
||||
dangerous-default-value,
|
||||
deprecated-method,
|
||||
deprecated-module,
|
||||
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
|
||||
eval-used,
|
||||
expression-not-assigned,
|
||||
fixme,
|
||||
function-redefined,
|
||||
global-statement,
|
||||
import-error,
|
||||
import-outside-toplevel,
|
||||
import-self,
|
||||
inconsistent-return-statements,
|
||||
inherit-non-class,
|
||||
invalid-name,
|
||||
invalid-overridden-method,
|
||||
isinstance-second-argument-not-valid-type,
|
||||
keyword-arg-before-vararg,
|
||||
line-too-long,
|
||||
logging-format-interpolation,
|
||||
logging-not-lazy,
|
||||
lost-exception,
|
||||
method-hidden,
|
||||
misplaced-comparison-constant,
|
||||
missing-docstring,
|
||||
missing-final-newline,
|
||||
multiple-imports,
|
||||
multiple-statements,
|
||||
no-else-continue,
|
||||
no-else-raise,
|
||||
no-else-return,
|
||||
no-init,
|
||||
no-member,
|
||||
no-method-argument,
|
||||
no-name-in-module,
|
||||
no-self-argument,
|
||||
no-self-use,
|
||||
no-value-for-parameter,
|
||||
not-an-iterable,
|
||||
not-callable,
|
||||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
protected-access,
|
||||
redefined-argument-from-local,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
reimported,
|
||||
signature-differs,
|
||||
singleton-comparison,
|
||||
super-init-not-called,
|
||||
superfluous-parens,
|
||||
too-few-public-methods,
|
||||
too-many-ancestors,
|
||||
too-many-arguments,
|
||||
too-many-branches,
|
||||
too-many-format-args,
|
||||
too-many-function-args,
|
||||
too-many-instance-attributes,
|
||||
too-many-lines,
|
||||
too-many-locals,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
trailing-newlines,
|
||||
trailing-whitespace,
|
||||
unbalanced-tuple-unpacking,
|
||||
undefined-variable,
|
||||
undefined-loop-variable,
|
||||
unexpected-special-method-signature,
|
||||
ungrouped-imports,
|
||||
unidiomatic-typecheck,
|
||||
unnecessary-comprehension,
|
||||
unnecessary-lambda,
|
||||
unnecessary-pass,
|
||||
unreachable,
|
||||
unsubscriptable-object,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
useless-super-delegation,
|
||||
wildcard-import,
|
||||
wrong-import-order,
|
||||
wrong-import-position
|
||||
13
tox.ini
13
tox.ini
|
|
@ -37,6 +37,19 @@ deps =
|
|||
pytest-flake8
|
||||
commands =
|
||||
py.test --flake8 {posargs:docs scrapy tests}
|
||||
|
||||
[testenv:pylint]
|
||||
basepython = python3
|
||||
deps =
|
||||
{[testenv]deps}
|
||||
# Optional dependencies
|
||||
boto
|
||||
reppy
|
||||
robotexclusionrulesparser
|
||||
# Test dependencies
|
||||
pylint
|
||||
commands =
|
||||
pylint conftest.py docs extras scrapy setup.py tests
|
||||
|
||||
[testenv:pypy3]
|
||||
basepython = pypy3
|
||||
|
|
|
|||
Loading…
Reference in New Issue