RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into bytes_received_signal

This commit is contained in:
Eugenio Lacuesta 2020-04-06 09:17:35 -03:00
parent aa6075e08c 8845773d44
commit e3342669ae
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
105 changed files with 244 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/news.rst
View File

@ -3,6 +3,22 @@
Release notes
=============
.. _release-2.0.1:
Scrapy 2.0.1 (2020-03-18)
-------------------------
* :meth:`Response.follow_all <scrapy.http.Response.follow_all>` now supports
an empty URL iterable as input (:issue:`4408`, :issue:`4420`)
* Removed top-level :mod:`~twisted.internet.reactor` imports to prevent
errors about the wrong Twisted reactor being installed when setting a
different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`,
:issue:`4406`)
* Fixed tests (:issue:`4422`)
.. _release-2.0.0:
Scrapy 2.0.0 (2020-03-03)

2
docs/topics/exporters.rst
View File

@ -42,7 +42,7 @@ value of one of their fields::
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline(object):
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
def open_spider(self, spider):

2
docs/topics/extensions.rst
View File

@ -107,7 +107,7 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging(object):
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count

10
docs/topics/item-pipeline.rst
View File

@ -81,7 +81,7 @@ contain a price::
from scrapy.exceptions import DropItem
class PricePipeline(object):
class PricePipeline:
vat_factor = 1.15
@ -103,7 +103,7 @@ format::
import json
class JsonWriterPipeline(object):
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'w')
@ -132,7 +132,7 @@ method and how to clean up the resources properly.::
import pymongo
class MongoPipeline(object):
class MongoPipeline:
collection_name = 'scrapy_items'
@ -180,7 +180,7 @@ it saves the screenshot to a file and adds filename to the item.
from urllib.parse import quote
class ScreenshotPipeline(object):
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
every Scrapy item."""
@ -219,7 +219,7 @@ returns multiples items with the same id::
from scrapy.exceptions import DropItem
class DuplicatesPipeline(object):
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()

66
docs/topics/leaks.rst
View File

@ -17,8 +17,8 @@ what is known as a "memory leak".
To help debugging memory leaks, Scrapy provides a built-in mechanism for
tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
and you can also use a third-party library called :ref:`Guppy
<topics-leaks-guppy>` for more advanced memory debugging (see below for more
and you can also use a third-party library called :ref:`muppy
<topics-leaks-muppy>` for more advanced memory debugging (see below for more
info). Both mechanisms must be used from the :ref:`Telnet Console
<topics-telnetconsole>`.
@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
.. class:: object_ref
Inherit from this class (instead of object) if you want to track live
Inherit from this class if you want to track live
instances with the ``trackref`` module.
.. function:: print_live_refs(class_name, ignore=NoneType)
@ -193,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
``None`` if none is found. Use :func:`print_live_refs` first to get a list
of all tracked live objects per class name.
.. _topics-leaks-guppy:
.. _topics-leaks-muppy:
Debugging memory leaks with Guppy
Debugging memory leaks with muppy
=================================
``trackref`` provides a very convenient mechanism for tracking down memory
@ -203,63 +203,9 @@ leaks, but it only keeps track of the objects that are more likely to cause
memory leaks (Requests, Responses, Items, and Selectors). However, there are
other cases where the memory leaks could come from other (more or less obscure)
objects. If this is your case, and you can't find your leaks using ``trackref``,
you still have another resource: the `Guppy library`_.
If you're using Python3, see :ref:`topics-leaks-muppy`.
you still have another resource: the muppy library.
.. _Guppy library: https://pypi.org/project/guppy/
If you use ``pip``, you can install Guppy with the following command::
pip install guppy
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
Guppy heap objects. Here's an example to view all Python objects available in
the heap using Guppy:
>>> x = hpy.heap()
>>> x.bytype
Partition of a set of 297033 objects. Total size = 52587824 bytes.
Index Count % Size % Cumulative % Type
0 22307 8 16423880 31 16423880 31 dict
1 122285 41 12441544 24 28865424 55 str
2 68346 23 5966696 11 34832120 66 tuple
3 227 0 5836528 11 40668648 77 unicode
4 2461 1 2222272 4 42890920 82 type
5 16870 6 2024400 4 44915320 85 function
6 13949 5 1673880 3 46589200 89 types.CodeType
7 13422 5 1653104 3 48242304 92 list
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
<1676 more rows. Type e.g. '_.more' to view.>
You can see that most space is used by dicts. Then, if you want to see from
which attribute those dicts are referenced, you could do:
>>> x.bytype[0].byvia
Partition of a set of 22307 objects. Total size = 16423880 bytes.
Index Count % Size % Cumulative % Referred Via:
0 10982 49 9416336 57 9416336 57 '.__dict__'
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
2 3097 14 1122904 7 13220744 80
3 990 4 277200 2 13497944 82 "['cookies']"
4 987 4 276360 2 13774304 84 "['cache']"
5 985 4 275800 2 14050104 86 "['meta']"
6 897 4 251160 2 14301264 87 '[2]'
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
9 27 0 155016 1 14841328 90 '[1]'
<333 more rows. Type e.g. '_.more' to view.>
As you can see, the Guppy module is very powerful but also requires some deep
knowledge about Python internals. For more info about Guppy, refer to the
`Guppy documentation`_.
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
.. _topics-leaks-muppy:
Debugging memory leaks with muppy
=================================
You can use muppy from `Pympler`_.
.. _Pympler: https://pypi.org/project/Pympler/

2
docs/topics/settings.rst
View File

@ -124,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
extensions, middlewares and item pipelines::
class MyExtension(object):
class MyExtension:
def __init__(self, log_is_enabled=False):
if log_is_enabled:
print("log is enabled!")

2
docs/topics/stats.rst
View File

@ -32,7 +32,7 @@ Common Stats Collector uses
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
class ExtensionThatAccessStats(object):
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats

76
extras/scrapy_zsh_completion
View File

@ -14,40 +14,40 @@ _scrapy() {
;;
args)
case $words[1] in
bench)
(bench)
_scrapy_glb_opts
;;
fetch)
(fetch)
local options=(
'--headers[print response HTTP headers instead of body]'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
genspider)
(genspider)
local options=(
{-l,--list}'[List available templates]'
{-e,--edit}'[Edit spider after creating it]'
{'(--list)-l','(-l)--list'}'[List available templates]'
{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
'--force[If the spider already exists, overwrite it with the template]'
{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
'1:name:(NAME)'
'2:domain:_httpie_urls'
)
_scrapy_glb_opts $options
;;
runspider)
(runspider)
local options=(
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py'
)
_scrapy_glb_opts $options
;;
settings)
(settings)
local options=(
'--get=[print raw setting value]:option:(SETTING)'
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
@ -57,77 +57,77 @@ _scrapy() {
)
_scrapy_glb_opts $options
;;
shell)
(shell)
local options=(
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'::file:_files -g \*.html'
'::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
startproject)
(startproject)
local options=(
'1:name:(NAME)'
'2:dir:_dir_list'
)
_scrapy_glb_opts $options
;;
version)
(version)
local options=(
{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
)
_scrapy_glb_opts $options
;;
view)
(view)
local options=(
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
check)
(check)
local options=(
'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
{-v,--verbose}'[print contract tests for all spiders]'
{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
crawl)
(crawl)
local options=(
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
edit)
(edit)
local options=(
'1:spider:_scrapy_spiders'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
list)
(list)
_scrapy_glb_opts
;;
parse)
(parse)
local options=(
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
'--pipelines[process items through pipelines]'
"--nolinks[don't show links to follow (extracted requests)]"
"--noitems[don't show scraped items]"
'--nocolour[avoid using pygments to colorize the output]'
{-r,--rules}'[use CrawlSpider rules to discover the callback]'
{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{-v,--verbose}'[print each depth level one by one]'
{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
@ -162,7 +162,7 @@ _scrapy_cmds() {
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
commands=(${commands[@]} ${project_commands[@]})
fi
_describe -t common-commands 'common commands' commands
_describe -t common-commands 'common commands' commands && ret=0
}
_scrapy_glb_opts() {
@ -172,13 +172,13 @@ _scrapy_glb_opts() {
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
'--pidfile=[write process ID to FILE]:file:_files'
'--profile=[write python cProfile stats to FILE]:file:_files'
'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
'(-L --loglevel --logfile)--nolog[disable logging completely]'
'--pdb[enable pdb on failure]'
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
)
options=(${options[@]} "$@")
_arguments $options
_arguments -A "-*" $options && ret=0
}
_httpie_urls() {

2
scrapy/cmdline.py
View File

@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
def _iter_command_classes(module_name):
@ -118,7 +117,6 @@ def execute(argv=None, settings=None):
pass
else:
settings['EDITOR'] = editor
check_deprecated_settings(settings)
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)

2
scrapy/commands/__init__.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict
from scrapy.exceptions import UsageError
class ScrapyCommand(object):
class ScrapyCommand:
requires_project = False
crawler_process = None

2
scrapy/commands/bench.py
View File

@ -25,7 +25,7 @@ class Command(ScrapyCommand):
self.crawler_process.start()
class _BenchServer(object):
class _BenchServer:
def __enter__(self):
from scrapy.utils.test import get_testenv

4
scrapy/contracts/__init__.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
class ContractsManager(object):
class ContractsManager:
contracts = {}
def __init__(self, contracts):
@ -107,7 +107,7 @@ class ContractsManager(object):
request.errback = eb_wrapper
class Contract(object):
class Contract:
""" Abstract class for contracts """
request_cls = None

4
scrapy/core/downloader/__init__.py
View File

@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
class Slot:
"""Downloader slot"""
def __init__(self, concurrency, delay, randomize_delay):
@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings):
return concurrency, delay
class Downloader(object):
class Downloader:
DOWNLOAD_SLOT = 'download_slot'

4
scrapy/core/engine.py
View File

@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
logger = logging.getLogger(__name__)
class Slot(object):
class Slot:
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
@ -53,7 +53,7 @@ class Slot(object):
self.closing.callback(None)
class ExecutionEngine(object):
class ExecutionEngine:
def __init__(self, crawler, spider_closed_callback):
self.crawler = crawler

2
scrapy/core/scheduler.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
logger = logging.getLogger(__name__)
class Scheduler(object):
class Scheduler:
"""
Scrapy Scheduler. It allows to enqueue requests and then get
a next request to download. Scheduler is also handling duplication

4
scrapy/core/scraper.py
View File

@ -21,7 +21,7 @@ from scrapy.core.spidermw import SpiderMiddlewareManager
logger = logging.getLogger(__name__)
class Slot(object):
class Slot:
"""Scraper slot (one per running spider)"""
MIN_RESPONSE_SIZE = 1024
@ -62,7 +62,7 @@ class Slot(object):
return self.active_size > self.max_active_size
class Scraper(object):
class Scraper:
def __init__(self, crawler):
self.slot = None

56
scrapy/crawler.py
View File

@ -4,7 +4,15 @@ import signal
import warnings
from twisted.internet import defer
from zope.interface.verify import DoesNotImplement, verifyClass
from zope.interface.exceptions import DoesNotImplement
try:
# zope >= 5.0 only supports MultipleInvalid
from zope.interface.exceptions import MultipleInvalid
except ImportError:
MultipleInvalid = None
from zope.interface.verify import verifyClass
from scrapy import signals, Spider
from scrapy.core.engine import ExecutionEngine
@ -68,17 +76,6 @@ class Crawler:
self.spider = None
self.engine = None
@property
def spiders(self):
if not hasattr(self, '_spiders'):
warnings.warn("Crawler.spiders is deprecated, use "
"CrawlerRunner.spider_loader or instantiate "
"scrapy.spiderloader.SpiderLoader with your "
"settings.",
category=ScrapyDeprecationWarning, stacklevel=2)
self._spiders = _get_spider_loader(self.settings.frozencopy())
return self._spiders
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
assert not self.crawling, "Crawling already taking place"
@ -130,11 +127,28 @@ class CrawlerRunner:
":meth:`crawl` and managed by this class."
)
@staticmethod
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
try:
verifyClass(ISpiderLoader, loader_cls)
except excs:
warnings.warn(
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
'Please add all missing methods to avoid unexpected runtime errors.',
category=ScrapyDeprecationWarning, stacklevel=2
)
return loader_cls.from_settings(settings.frozencopy())
def __init__(self, settings=None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.settings = settings
self.spider_loader = _get_spider_loader(settings)
self.spider_loader = self._get_spider_loader(settings)
self._crawlers = set()
self._active = set()
self.bootstrap_failed = False
@ -327,19 +341,3 @@ class CrawlerProcess(CrawlerRunner):
if self.settings.get("TWISTED_REACTOR"):
install_reactor(self.settings["TWISTED_REACTOR"])
super()._handle_twisted_reactor()
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
try:
verifyClass(ISpiderLoader, loader_cls)
except DoesNotImplement:
warnings.warn(
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
'Please add all missing methods to avoid unexpected runtime errors.',
category=ScrapyDeprecationWarning, stacklevel=2
)
return loader_cls.from_settings(settings.frozencopy())

2
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse
logger = logging.getLogger(__name__)
class AjaxCrawlMiddleware(object):
class AjaxCrawlMiddleware:
"""
Handle 'AJAX crawlable' pages marked as crawlable via meta tag.
For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.

21
scrapy/downloadermiddlewares/chunked.py
View File

@ -1,21 +0,0 @@
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.http import decode_chunked_transfer
warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
"chunked transfers are supported by default.",
ScrapyDeprecationWarning, stacklevel=2)
class ChunkedTransferMiddleware(object):
"""This middleware adds support for chunked transfer encoding, as
documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
"""
def process_response(self, request, response, spider):
if response.headers.get('Transfer-Encoding') == 'chunked':
body = decode_chunked_transfer(response.body)
return response.replace(body=body)
return response

2
scrapy/downloadermiddlewares/cookies.py
View File

@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
class CookiesMiddleware(object):
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
def __init__(self, debug=False):

2
scrapy/downloadermiddlewares/decompression.py
View File

@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes
logger = logging.getLogger(__name__)
class DecompressionMiddleware(object):
class DecompressionMiddleware:
""" This middleware tries to recognise and extract the possibly compressed
responses that may arrive. """

2
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
from scrapy.utils.python import without_none_values
class DefaultHeadersMiddleware(object):
class DefaultHeadersMiddleware:
def __init__(self, headers):
self._headers = headers

2
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
from scrapy import signals
class DownloadTimeoutMiddleware(object):
class DownloadTimeoutMiddleware:
def __init__(self, timeout=180):
self._timeout = timeout

2
scrapy/downloadermiddlewares/httpauth.py
View File

@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header
from scrapy import signals
class HttpAuthMiddleware(object):
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header
(http_user and http_pass spider class attributes)"""

2
scrapy/downloadermiddlewares/httpcache.py
View File

@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.utils.misc import load_object
class HttpCacheMiddleware(object):
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
ConnectionRefusedError, ConnectionDone, ConnectError,

2
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -15,7 +15,7 @@ except ImportError:
pass
class HttpCompressionMiddleware(object):
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
@classmethod

2
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -7,7 +7,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
class HttpProxyMiddleware(object):
class HttpProxyMiddleware:
def __init__(self, auth_encoding='latin-1'):
self.auth_encoding = auth_encoding

2
scrapy/downloadermiddlewares/redirect.py
View File

@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
class BaseRedirectMiddleware(object):
class BaseRedirectMiddleware:
enabled_setting = 'REDIRECT_ENABLED'

2
scrapy/downloadermiddlewares/retry.py
View File

@ -25,7 +25,7 @@ from scrapy.utils.python import global_object_name
logger = logging.getLogger(__name__)
class RetryMiddleware(object):
class RetryMiddleware:
# IOError is raised by the HttpCompression middleware when trying to
# decompress an empty response

2
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -16,7 +16,7 @@ from scrapy.utils.misc import load_object
logger = logging.getLogger(__name__)
class RobotsTxtMiddleware(object):
class RobotsTxtMiddleware:
DOWNLOAD_PRIORITY = 1000
def __init__(self, crawler):

2
scrapy/downloadermiddlewares/stats.py
View File

@ -4,7 +4,7 @@ from scrapy.utils.response import response_httprepr
from scrapy.utils.python import global_object_name
class DownloaderStats(object):
class DownloaderStats:
def __init__(self, stats):
self.stats = stats

2
scrapy/downloadermiddlewares/useragent.py
View File

@ -3,7 +3,7 @@
from scrapy import signals
class UserAgentMiddleware(object):
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""
def __init__(self, user_agent='Scrapy'):

2
scrapy/dupefilters.py
View File

@ -5,7 +5,7 @@ from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, request_fingerprint
class BaseDupeFilter(object):
class BaseDupeFilter:
@classmethod
def from_settings(cls, settings):

2
scrapy/exporters.py
View File

@ -21,7 +21,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
'JsonItemExporter', 'MarshalItemExporter']
class BaseItemExporter(object):
class BaseItemExporter:
def __init__(self, *, dont_fail=False, **kwargs):
self._kwargs = kwargs

2
scrapy/extensions/closespider.py
View File

@ -10,7 +10,7 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured
class CloseSpider(object):
class CloseSpider:
def __init__(self, crawler):
self.crawler = crawler

2
scrapy/extensions/corestats.py
View File

@ -6,7 +6,7 @@ from datetime import datetime
from scrapy import signals
class CoreStats(object):
class CoreStats:
def __init__(self, stats):
self.stats = stats

4
scrapy/extensions/debug.py
View File

@ -17,7 +17,7 @@ from scrapy.utils.trackref import format_live_refs
logger = logging.getLogger(__name__)
class StackTraceDump(object):
class StackTraceDump:
def __init__(self, crawler=None):
self.crawler = crawler
@ -52,7 +52,7 @@ class StackTraceDump(object):
return dumps
class Debugger(object):
class Debugger:
def __init__(self):
try:
signal.signal(signal.SIGUSR2, self._enter_debugger)

10
scrapy/extensions/feedexport.py
View File

@ -44,7 +44,7 @@ class IFeedStorage(Interface):
@implementer(IFeedStorage)
class BlockingFeedStorage(object):
class BlockingFeedStorage:
def open(self, spider):
path = spider.crawler.settings['FEED_TEMPDIR']
@ -61,7 +61,7 @@ class BlockingFeedStorage(object):
@implementer(IFeedStorage)
class StdoutFeedStorage(object):
class StdoutFeedStorage:
def __init__(self, uri, _stdout=None):
if not _stdout:
@ -76,7 +76,7 @@ class StdoutFeedStorage(object):
@implementer(IFeedStorage)
class FileFeedStorage(object):
class FileFeedStorage:
def __init__(self, uri):
self.path = file_uri_to_path(uri)
@ -179,7 +179,7 @@ class FTPFeedStorage(BlockingFeedStorage):
)
class _FeedSlot(object):
class _FeedSlot:
def __init__(self, file, exporter, storage, uri, format, store_empty):
self.file = file
self.exporter = exporter
@ -203,7 +203,7 @@ class _FeedSlot(object):
self._exporting = False
class FeedExporter(object):
class FeedExporter:
@classmethod
def from_crawler(cls, crawler):

8
scrapy/extensions/httpcache.py
View File

@ -20,7 +20,7 @@ from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
class DummyPolicy(object):
class DummyPolicy:
def __init__(self, settings):
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
@ -39,7 +39,7 @@ class DummyPolicy(object):
return True
class RFC2616Policy(object):
class RFC2616Policy:
MAXAGE = 3600 * 24 * 365 # one year
@ -213,7 +213,7 @@ class RFC2616Policy(object):
return currentage
class DbmCacheStorage(object):
class DbmCacheStorage:
def __init__(self, settings):
self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
@ -270,7 +270,7 @@ class DbmCacheStorage(object):
return request_fingerprint(request)
class FilesystemCacheStorage(object):
class FilesystemCacheStorage:
def __init__(self, settings):
self.cachedir = data_path(settings['HTTPCACHE_DIR'])

2
scrapy/extensions/logstats.py
View File

@ -8,7 +8,7 @@ from scrapy import signals
logger = logging.getLogger(__name__)
class LogStats(object):
class LogStats:
"""Log basic scraping stats periodically"""
def __init__(self, stats, interval=60.0):

2
scrapy/extensions/memdebug.py
View File

@ -11,7 +11,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.utils.trackref import live_refs
class MemoryDebugger(object):
class MemoryDebugger:
def __init__(self, stats):
self.stats = stats

2
scrapy/extensions/memusage.py
View File

@ -19,7 +19,7 @@ from scrapy.utils.engine import get_engine_status
logger = logging.getLogger(__name__)
class MemoryUsage(object):
class MemoryUsage:
def __init__(self, crawler):
if not crawler.settings.getbool('MEMUSAGE_ENABLED'):

2
scrapy/extensions/spiderstate.py
View File

@ -6,7 +6,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
class SpiderState(object):
class SpiderState:
"""Store and load spider state during a scraping job"""
def __init__(self, jobdir=None):

2
scrapy/extensions/statsmailer.py
View File

@ -8,7 +8,7 @@ from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
class StatsMailer(object):
class StatsMailer:
def __init__(self, stats, recipients, mail):
self.stats = stats

6
scrapy/extensions/telnet.py
View File

@ -26,11 +26,6 @@ from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
from scrapy.utils.decorators import defers
try:
import guppy
hpy = guppy.hpy()
except ImportError:
hpy = None
logger = logging.getLogger(__name__)
@ -110,7 +105,6 @@ class TelnetConsole(protocol.ServerFactory):
'est': lambda: print_engine_status(self.crawler.engine),
'p': pprint.pprint,
'prefs': print_live_refs,
'hpy': hpy,
'help': "This is Scrapy telnet console. For more info see: "
"https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
}

2
scrapy/extensions/throttle.py
View File

@ -6,7 +6,7 @@ from scrapy import signals
logger = logging.getLogger(__name__)
class AutoThrottle(object):
class AutoThrottle:
def __init__(self, crawler):
self.crawler = crawler

8
scrapy/http/cookies.py
View File

@ -5,7 +5,7 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
class CookieJar(object):
class CookieJar:
def __init__(self, policy=None, check_expired_frequency=10000):
self.policy = policy or DefaultCookiePolicy()
self.jar = _CookieJar(self.policy)
@ -100,7 +100,7 @@ def potential_domain_matches(domain):
return matches + ['.' + d for d in matches]
class _DummyLock(object):
class _DummyLock:
def acquire(self):
pass
@ -108,7 +108,7 @@ class _DummyLock(object):
pass
class WrappedRequest(object):
class WrappedRequest:
"""Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
see http://docs.python.org/library/urllib2.html#urllib2.Request
@ -178,7 +178,7 @@ class WrappedRequest(object):
self.request.headers.appendlist(name, value)
class WrappedResponse(object):
class WrappedResponse:
def __init__(self, response):
self.response = response

2
scrapy/link.py
View File

@ -6,7 +6,7 @@ its documentation in: docs/topics/link-extractors.rst
"""
class Link(object):
class Link:
"""Link objects represent an extracted link by the LinkExtractor."""
__slots__ = ['url', 'text', 'fragment', 'nofollow']

2
scrapy/linkextractors/__init__.py
View File

@ -49,7 +49,7 @@ _matches = lambda url, regexs: any(r.search(url) for r in regexs)
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor(object):
class FilteringLinkExtractor:
_csstranslator = HTMLTranslator()

2
scrapy/linkextractors/lxmlhtml.py
View File

@ -27,7 +27,7 @@ def _nons(tag):
return tag
class LxmlParserLinkExtractor(object):
class LxmlParserLinkExtractor:
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True, canonicalized=False):
self.scan_tag = tag if callable(tag) else lambda t: t == tag

2
scrapy/loader/__init__.py
View File

@ -25,7 +25,7 @@ def unbound_method(method):
return method
class ItemLoader(object):
class ItemLoader:
default_item_class = Item
default_input_processor = Identity()

12
scrapy/loader/processors.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter
from scrapy.loader.common import wrap_loader_context
class MapCompose(object):
class MapCompose:
def __init__(self, *functions, **default_loader_context):
self.functions = functions
@ -36,7 +36,7 @@ class MapCompose(object):
return values
class Compose(object):
class Compose:
def __init__(self, *functions, **default_loader_context):
self.functions = functions
@ -61,7 +61,7 @@ class Compose(object):
return value
class TakeFirst(object):
class TakeFirst:
def __call__(self, values):
for value in values:
@ -69,13 +69,13 @@ class TakeFirst(object):
return value
class Identity(object):
class Identity:
def __call__(self, values):
return values
class SelectJmes(object):
class SelectJmes:
"""
Query the input string for the jmespath (given at instantiation),
and return the answer
@ -95,7 +95,7 @@ class SelectJmes(object):
return self.compiled_path.search(value)
class Join(object):
class Join:
def __init__(self, separator=u' '):
self.separator = separator

2
scrapy/logformatter.py
View File

@ -14,7 +14,7 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
class LogFormatter(object):
class LogFormatter:
"""Class for generating log messages for different actions.
All methods must return a dictionary listing the parameters ``level``, ``msg``

2
scrapy/mail.py
View File

@ -27,7 +27,7 @@ def _to_bytes_or_none(text):
return to_bytes(text)
class MailSender(object):
class MailSender:
def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
self.smtphost = smtphost

2
scrapy/middleware.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.defer import process_parallel, process_chain, process_chain_bo
logger = logging.getLogger(__name__)
class MiddlewareManager(object):
class MiddlewareManager:
"""Base class for implementing middleware managers"""
component_name = 'foo middleware'

8
scrapy/pipelines/files.py
View File

@ -37,7 +37,7 @@ class FileException(Exception):
"""General media error exception"""
class FSFilesStore(object):
class FSFilesStore:
def __init__(self, basedir):
if '://' in basedir:
basedir = basedir.split('://', 1)[1]
@ -75,7 +75,7 @@ class FSFilesStore(object):
seen.add(dirname)
class S3FilesStore(object):
class S3FilesStore:
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
AWS_ENDPOINT_URL = None
@ -213,7 +213,7 @@ class S3FilesStore(object):
return extra
class GCSFilesStore(object):
class GCSFilesStore:
GCS_PROJECT_ID = None
@ -259,7 +259,7 @@ class GCSFilesStore(object):
)
class FTPFilesStore(object):
class FTPFilesStore:
FTP_USERNAME = None
FTP_PASSWORD = None

4
scrapy/pipelines/media.py
View File

@ -14,11 +14,11 @@ from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
class MediaPipeline(object):
class MediaPipeline:
LOG_FAILED_RESULTS = True
class SpiderInfo(object):
class SpiderInfo:
def __init__(self, spider):
self.spider = spider
self.downloading = set()

6
scrapy/pqueues.py
View File

@ -110,7 +110,7 @@ class ScrapyPriorityQueue:
return sum(len(x) for x in self.queues.values()) if self.queues else 0
class DownloaderInterface(object):
class DownloaderInterface:
def __init__(self, crawler):
self.downloader = crawler.engine.downloader
@ -129,8 +129,8 @@ class DownloaderInterface(object):
return len(self.downloader.slots[slot].active)
class DownloaderAwarePriorityQueue(object):
""" PriorityQueue which takes Downloader activity in account:
class DownloaderAwarePriorityQueue:
""" PriorityQueue which takes Downloader activity into account:
domains (slots) with the least amount of active downloads are dequeued
first.
"""

2
scrapy/responsetypes.py
View File

@ -11,7 +11,7 @@ from scrapy.utils.misc import load_object
from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
class ResponseTypes(object):
class ResponseTypes:
CLASSES = {
'text/html': 'scrapy.http.HtmlResponse',

2
scrapy/settings/__init__.py
View File

@ -28,7 +28,7 @@ def get_settings_priority(priority):
return priority
class SettingsAttribute(object):
class SettingsAttribute:
"""Class for storing data related to settings attributes.

23
scrapy/settings/deprecated.py
View File

@ -1,23 +0,0 @@
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
DEPRECATED_SETTINGS = [
('TRACK_REFS', 'no longer needed (trackref is always enabled)'),
('RESPONSE_CLASSES', 'no longer supported'),
('DEFAULT_RESPONSE_ENCODING', 'no longer supported'),
('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'),
('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'),
('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'),
('SQLITE_DB', 'no longer supported'),
('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
]
def check_deprecated_settings(settings):
deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None]
if deprecated:
msg = "You are using the following settings which are deprecated or obsolete"
msg += " (ask scrapy-users@googlegroups.com for alternatives):"
msg = msg + "\n " + "\n ".join("%s: %s" % x for x in deprecated)
warnings.warn(msg, ScrapyDeprecationWarning)

2
scrapy/shell.py
View File

@ -24,7 +24,7 @@ from scrapy.utils.conf import get_config
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
class Shell(object):
class Shell:
relevant_classes = (Crawler, Spider, Request, Response, BaseItem,
Settings)

2
scrapy/signalmanager.py
View File

@ -2,7 +2,7 @@ from pydispatch import dispatcher
from scrapy.utils import signal as _signal
class SignalManager(object):
class SignalManager:
def __init__(self, sender=dispatcher.Anonymous):
self.sender = sender

2
scrapy/spiderloader.py
View File

@ -11,7 +11,7 @@ from scrapy.utils.spider import iter_spider_classes
@implementer(ISpiderLoader)
class SpiderLoader(object):
class SpiderLoader:
"""
SpiderLoader is a class which locates and loads spiders
in a Scrapy project.

2
scrapy/spidermiddlewares/depth.py
View File

@ -11,7 +11,7 @@ from scrapy.http import Request
logger = logging.getLogger(__name__)
class DepthMiddleware(object):
class DepthMiddleware:
def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
self.maxdepth = maxdepth

2
scrapy/spidermiddlewares/httperror.py
View File

@ -18,7 +18,7 @@ class HttpError(IgnoreRequest):
super(HttpError, self).__init__(*args, **kwargs)
class HttpErrorMiddleware(object):
class HttpErrorMiddleware:
@classmethod
def from_crawler(cls, crawler):

2
scrapy/spidermiddlewares/offsite.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.httpobj import urlparse_cached
logger = logging.getLogger(__name__)
class OffsiteMiddleware(object):
class OffsiteMiddleware:
def __init__(self, stats):
self.stats = stats

4
scrapy/spidermiddlewares/referer.py
View File

@ -28,7 +28,7 @@ POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy(object):
class ReferrerPolicy:
NOREFERRER_SCHEMES = LOCAL_SCHEMES
@ -284,7 +284,7 @@ def _load_policy_class(policy, warning_only=False):
return None
class RefererMiddleware(object):
class RefererMiddleware:
def __init__(self, settings=None):
self.default_policy = DefaultReferrerPolicy

2
scrapy/spidermiddlewares/urllength.py
View File

@ -12,7 +12,7 @@ from scrapy.exceptions import NotConfigured
logger = logging.getLogger(__name__)
class UrlLengthMiddleware(object):
class UrlLengthMiddleware:
def __init__(self, maxlength):
self.maxlength = maxlength

2
scrapy/spiders/crawl.py
View File

@ -34,7 +34,7 @@ def _get_method(method, spider):
_default_link_extractor = LinkExtractor()
class Rule(object):
class Rule:
def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
process_links=None, process_request=None, errback=None):

2
scrapy/statscollectors.py
View File

@ -7,7 +7,7 @@ import logging
logger = logging.getLogger(__name__)
class StatsCollector(object):
class StatsCollector:
def __init__(self, crawler):
self._dump = crawler.settings.getbool('STATS_DUMP')

4
scrapy/templates/project/module/middlewares.py.tmpl
View File

@ -8,7 +8,7 @@
from scrapy import signals
class ${ProjectName}SpiderMiddleware(object):
class ${ProjectName}SpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@ -56,7 +56,7 @@ class ${ProjectName}SpiderMiddleware(object):
spider.logger.info('Spider opened: %s' % spider.name)
class ${ProjectName}DownloaderMiddleware(object):
class ${ProjectName}DownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.

2
scrapy/templates/project/module/pipelines.py.tmpl
View File

@ -6,6 +6,6 @@
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
class ${ProjectName}Pipeline(object):
class ${ProjectName}Pipeline:
def process_item(self, item, spider):
return item

2
scrapy/utils/datatypes.py
View File

@ -109,7 +109,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
return None # key is not weak-referenceable, it's not cached
class SequenceExclude(object):
class SequenceExclude:
"""Object to test if an item is NOT within some sequence."""
def __init__(self, seq):

2
scrapy/utils/deprecate.py
View File

@ -144,7 +144,7 @@ def method_is_overridden(subclass, base_class, method_name):
Return True if a method named ``method_name`` of a ``base_class``
is overridden in a ``subclass``.
>>> class Base(object):
>>> class Base:
... def foo(self):
... pass
>>> class Sub1(Base):

2
scrapy/utils/iterators.py
View File

@ -52,7 +52,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
yield xs.xpath(selxpath)[0]
class _StreamReader(object):
class _StreamReader:
def __init__(self, obj):
self._ptr = 0

2
scrapy/utils/log.py
View File

@ -152,7 +152,7 @@ def log_scrapy_info(settings):
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
class StreamLogger(object):
class StreamLogger:
"""Fake file-like stream object that redirects writes to a logger instance
Taken from:

4
scrapy/utils/python.py
View File

@ -223,7 +223,7 @@ def get_spec(func):
>>> get_spec(re.match)
(['pattern', 'string'], {'flags': 0})
>>> class Test(object):
>>> class Test:
... def __call__(self, val):
... pass
... def method(self, val, flags=0):
@ -272,7 +272,7 @@ def equal_attributes(obj1, obj2, attributes):
return True
class WeakKeyCache(object):
class WeakKeyCache:
def __init__(self, default_factory):
self.default_factory = default_factory

2
scrapy/utils/reactor.py
View File

@ -24,7 +24,7 @@ def listen_tcp(portrange, host, factory):
raise
class CallLaterOnce(object):
class CallLaterOnce:
"""Schedule a function to be called in the next reactor loop, but only if
it hasn't been already scheduled since the last time it ran.
"""

2
scrapy/utils/sitemap.py
View File

@ -10,7 +10,7 @@ from urllib.parse import urljoin
import lxml.etree
class Sitemap(object):
class Sitemap:
"""Class to parse Sitemap (type=urlset) and Sitemap Index
(type=sitemapindex) files"""

2
scrapy/utils/testproc.py
View File

@ -4,7 +4,7 @@ import os
from twisted.internet import defer, protocol
class ProcessTest(object):
class ProcessTest:
command = None
prefix = [sys.executable, '-m', 'scrapy.cmdline']

2
scrapy/utils/testsite.py
View File

@ -3,7 +3,7 @@ from urllib.parse import urljoin
from twisted.web import server, resource, static, util
class SiteTest(object):
class SiteTest:
def setUp(self):
from twisted.internet import reactor

5
scrapy/utils/trackref.py
View File

@ -19,9 +19,8 @@ NoneType = type(None)
live_refs = defaultdict(weakref.WeakKeyDictionary)
class object_ref(object):
"""Inherit from this class (instead of object) to a keep a record of live
instances"""
class object_ref:
"""Inherit from this class to a keep a record of live instances"""
__slots__ = ()

5
setup.py
View File

@ -30,6 +30,11 @@ setup(
name='Scrapy',
version=version,
url='https://scrapy.org',
project_urls = {
'Documentation': 'https://docs.scrapy.org/',
'Source': 'https://github.com/scrapy/scrapy',
'Tracker': 'https://github.com/scrapy/scrapy/issues',
},
description='A high-level Web Crawling and Web Scraping framework',
long_description=open('README.rst').read(),
author='Scrapy developers',

4
tests/pipelines.py
View File

@ -3,7 +3,7 @@ Some pipelines used for testing
"""
class ZeroDivisionErrorPipeline(object):
class ZeroDivisionErrorPipeline:
def open_spider(self, spider):
a = 1 / 0
@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline(object):
return item
class ProcessWithZeroDivisionErrorPipiline(object):
class ProcessWithZeroDivisionErrorPipiline:
def process_item(self, item, spider):
1 / 0

4
tests/test_cmdline/extensions.py
View File

@ -1,7 +1,7 @@
"""A test extension used to check the settings loading order"""
class TestExtension(object):
class TestExtension:
def __init__(self, settings):
settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started'))
@ -11,5 +11,5 @@ class TestExtension(object):
return cls(crawler.settings)
class DummyExtension(object):
class DummyExtension:
pass

4
tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
View File

@ -1,4 +1,4 @@
class TestSpiderPipeline(object):
class TestSpiderPipeline:
def open_spider(self, spider):
pass
@ -7,7 +7,7 @@ class TestSpiderPipeline(object):
return item
class TestSpiderExceptionPipeline(object):
class TestSpiderExceptionPipeline:
def open_spider(self, spider):
raise Exception('exception')

2
tests/test_command_parse.py
View File

@ -89,7 +89,7 @@ class MyBadCrawlSpider(CrawlSpider):
f.write("""
import logging
class MyPipeline(object):
class MyPipeline:
component_name = 'my_pipeline'
def process_item(self, item, spider):

2
tests/test_contracts.py
View File

@ -25,7 +25,7 @@ class TestItem(Item):
url = Field()
class ResponseMock(object):
class ResponseMock:
url = 'http://scrapy.org'

17
tests/test_crawler.py
View File

@ -33,21 +33,6 @@ class CrawlerTestCase(BaseCrawlerTest):
def setUp(self):
self.crawler = Crawler(DefaultSpider, Settings())
def test_deprecated_attribute_spiders(self):
with warnings.catch_warnings(record=True) as w:
spiders = self.crawler.spiders
self.assertEqual(len(w), 1)
self.assertIn("Crawler.spiders", str(w[0].message))
sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
self.assertIsInstance(spiders, sl_cls)
self.crawler.spiders
is_one_warning = len(w) == 1
if not is_one_warning:
for warning in w:
print(warning)
self.assertTrue(is_one_warning, "Warn deprecated access only once")
def test_populate_spidercls_settings(self):
spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@ -141,7 +126,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
class SpiderLoaderWithWrongInterface(object):
class SpiderLoaderWithWrongInterface:
def unneeded_method(self):
pass

2
tests/test_downloadermiddleware.py
View File

@ -106,7 +106,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
def test_download_func_not_called(self):
resp = Response('http://example.com/index.html')
class ResponseMiddleware(object):
class ResponseMiddleware:
def process_request(self, request, spider):
return resp

2
tests/test_dupefilters.py
View File

@ -35,7 +35,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter):
return df
class DirectDupeFilter(object):
class DirectDupeFilter:
method = 'n/a'

2
tests/test_engine.py
View File

@ -100,7 +100,7 @@ def start_test_site(debug=False):
return port
class CrawlerRun(object):
class CrawlerRun:
"""A class to run the crawler and keep track of events occurred"""
def __init__(self, spider_class):

2
tests/test_feedexport.py
View File

@ -373,7 +373,7 @@ class StdoutFeedStorageTest(unittest.TestCase):
self.assertEqual(out.getvalue(), b"content")
class FromCrawlerMixin(object):
class FromCrawlerMixin:
init_with_crawler = False
@classmethod

2
tests/test_item.py
View File

@ -213,7 +213,7 @@ class ItemTest(unittest.TestCase):
class B(A):
pass
class C(object):
class C:
fields = {'load': Field(default='C')}
not_allowed = Field(default='not_allowed')
save = Field(default='C')

2
tests/test_loader.py
View File

@ -456,7 +456,7 @@ class BasicItemLoaderTest(unittest.TestCase):
[u'marta', u'other'], Compose(float))
class InitializationTestMixin(object):
class InitializationTestMixin:
item_class = None

2
tests/test_logformatter.py
View File

@ -170,7 +170,7 @@ class SkipMessagesLogFormatter(LogFormatter):
return None
class DropSomeItemsPipeline(object):
class DropSomeItemsPipeline:
drop = True
def process_item(self, item, spider):

8
tests/test_middleware.py
View File

@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.middleware import MiddlewareManager
class M1(object):
class M1:
def open_spider(self, spider):
pass
@ -17,7 +17,7 @@ class M1(object):
pass
class M2(object):
class M2:
def open_spider(self, spider):
pass
@ -28,13 +28,13 @@ class M2(object):
pass
class M3(object):
class M3:
def process(self, response, request, spider):
pass
class MOff(object):
class MOff:
def open_spider(self, spider):
pass

4
tests/test_request_cb_kwargs.py
View File

@ -8,7 +8,7 @@ from tests.spiders import MockServerSpider
from tests.mockserver import MockServer
class InjectArgumentsDownloaderMiddleware(object):
class InjectArgumentsDownloaderMiddleware:
"""
Make sure downloader middlewares are able to update the keyword arguments
"""
@ -23,7 +23,7 @@ class InjectArgumentsDownloaderMiddleware(object):
return response
class InjectArgumentsSpiderMiddleware(object):
class InjectArgumentsSpiderMiddleware:
"""
Make sure spider middlewares are able to update the keyword arguments
"""

6
tests/test_scheduler.py
View File

@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader'])
MockSlot = collections.namedtuple('MockSlot', ['active'])
class MockDownloader(object):
class MockDownloader:
def __init__(self):
self.slots = dict()
@ -57,7 +57,7 @@ class MockCrawler(Crawler):
self.engine = MockEngine(downloader=MockDownloader())
class SchedulerHandler(object):
class SchedulerHandler:
priority_queue_cls = None
jobdir = None
@ -245,7 +245,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots):
return True
class DownloaderAwareSchedulerTestMixin(object):
class DownloaderAwareSchedulerTestMixin:
priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
reopen = False

Some files were not shown because too many files have changed in this diff Show More