mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into bytes_received_signal
This commit is contained in:
Eugenio Lacuesta
commit
7fd2085725
14
.travis.yml
14
.travis.yml
|
|
@ -15,19 +15,25 @@ matrix:
|
|||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
|
||||
- env: TOXENV=pypy3
|
||||
- python: 3.5
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5.2
|
||||
- python: 3.6
|
||||
- python: 3.7
|
||||
- env: PYPI_RELEASE_JOB=true
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
|
|
|
|||
|
|
@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
|
|||
.. note::
|
||||
|
||||
HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
|
||||
unless you use Twisted<11.1, or if you really want to use HTTP/1.0
|
||||
and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
|
||||
accordingly, i.e. to
|
||||
``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
|
||||
unless you really want to use HTTP/1.0 and override
|
||||
:setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
|
||||
i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
|
||||
|
||||
|
|
@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
|
|||
Scrapy also has another context factory class that you can set,
|
||||
``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
|
||||
which uses the platform's certificates to validate remote endpoints.
|
||||
**This is only available if you use Twisted>=14.0.**
|
||||
|
||||
If you do use a custom ContextFactory, make sure its ``__init__`` method
|
||||
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
|
||||
|
|
@ -494,10 +492,6 @@ This setting must be one of these string values:
|
|||
- ``'TLSv1.2'``: forces TLS version 1.2
|
||||
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
|
||||
or above (Twisted>=14.0 if you can).
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
||||
|
|
@ -660,8 +654,6 @@ If you want to disable it set to 0.
|
|||
spider attribute and per-request using :reqmeta:`download_maxsize`
|
||||
Request.meta key.
|
||||
|
||||
This feature needs Twisted >= 11.1.
|
||||
|
||||
.. setting:: DOWNLOAD_WARNSIZE
|
||||
|
||||
DOWNLOAD_WARNSIZE
|
||||
|
|
@ -679,8 +671,6 @@ If you want to disable it set to 0.
|
|||
spider attribute and per-request using :reqmeta:`download_warnsize`
|
||||
Request.meta key.
|
||||
|
||||
This feature needs Twisted >= 11.1.
|
||||
|
||||
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
||||
DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
|
|
|||
64
pytest.ini
64
pytest.ini
|
|
@ -35,21 +35,21 @@ flake8-ignore =
|
|||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128
|
||||
scrapy/commands/fetch.py E501 E128
|
||||
scrapy/commands/genspider.py E128 E501
|
||||
scrapy/commands/parse.py E128 E501
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501
|
||||
scrapy/commands/startproject.py E127 E501 E128
|
||||
scrapy/commands/startproject.py E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/__init__.py E501
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E501 E128 E127
|
||||
scrapy/core/engine.py E501 E128
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E128 W504
|
||||
scrapy/core/scraper.py E501 E128
|
||||
scrapy/core/spidermw.py E501 E126
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
|
|
@ -57,7 +57,7 @@ flake8-ignore =
|
|||
scrapy/core/downloader/tls.py E501
|
||||
scrapy/core/downloader/webclient.py E501 E128 E126
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
|
|
@ -68,29 +68,29 @@ flake8-ignore =
|
|||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E501 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/redirect.py E501
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E128 E123
|
||||
scrapy/extensions/closespider.py E501 E128
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/telnet.py E501
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/form.py E501
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128
|
||||
scrapy/http/response/text.py E501 E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E501 E402 W504
|
||||
scrapy/linkextractors/__init__.py E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E128
|
||||
|
|
@ -99,7 +99,7 @@ flake8-ignore =
|
|||
scrapy/pipelines/__init__.py E501
|
||||
scrapy/pipelines/files.py E116 E501
|
||||
scrapy/pipelines/images.py E501
|
||||
scrapy/pipelines/media.py E125 E501
|
||||
scrapy/pipelines/media.py E501
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
|
|
@ -110,7 +110,7 @@ flake8-ignore =
|
|||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py E501 E129 W504
|
||||
scrapy/spidermiddlewares/referer.py E501 E129
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
|
|
@ -124,8 +124,8 @@ flake8-ignore =
|
|||
scrapy/utils/datatypes.py E501
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127
|
||||
scrapy/utils/gz.py E501 W504
|
||||
scrapy/utils/deprecate.py E501
|
||||
scrapy/utils/gz.py E501
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501
|
||||
|
|
@ -137,7 +137,7 @@ flake8-ignore =
|
|||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E501
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/request.py E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
|
|
@ -169,10 +169,10 @@ flake8-ignore =
|
|||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/mockserver.py E501 E126
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/spiders.py E501
|
||||
tests/test_closespider.py E501
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128
|
||||
tests/test_command_shell.py E501 E128
|
||||
|
|
@ -181,28 +181,28 @@ flake8-ignore =
|
|||
tests/test_crawl.py E501 E741
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
|
||||
tests/test_downloader_handlers.py E124 E128 E501 E126
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126
|
||||
tests/test_downloadermiddleware_decompression.py E501
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128 E127
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E501 E741 E128 E124
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_engine.py E501 E128
|
||||
tests/test_exporters.py E501 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
|
||||
tests/test_http_request.py E402 E501 E128 E128 E126
|
||||
tests/test_http_response.py E501 E128
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_link.py E501
|
||||
|
|
@ -219,14 +219,14 @@ flake8-ignore =
|
|||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501
|
||||
tests/test_robotstxt_interface.py E501 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_scheduler.py E501 E126
|
||||
tests/test_selector.py E501
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E128
|
||||
|
|
@ -234,7 +234,7 @@ flake8-ignore =
|
|||
tests/test_utils_datatypes.py E402 E501
|
||||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_http.py E501 E128
|
||||
tests/test_utils_iterators.py E501 E128 E129
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501
|
||||
|
|
@ -243,8 +243,8 @@ flake8-ignore =
|
|||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E123 E126
|
||||
tests/test_utils_url.py E501 E501 E126
|
||||
tests/test_webclient.py E501 E128 E122 E402 E126
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
|
|
|
|||
|
|
@ -17,10 +17,10 @@ class ContractsManager:
|
|||
self.contracts[contract.name] = contract
|
||||
|
||||
def tested_methods_from_spidercls(self, spidercls):
|
||||
is_method = re.compile(r"^\s*@", re.MULTILINE).search
|
||||
methods = []
|
||||
for key, value in getmembers(spidercls):
|
||||
if (callable(value) and value.__doc__ and
|
||||
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
|
||||
if callable(value) and value.__doc__ and is_method(value.__doc__):
|
||||
methods.append(key)
|
||||
|
||||
return methods
|
||||
|
|
|
|||
|
|
@ -94,11 +94,12 @@ class FTPDownloadHandler:
|
|||
def gotClient(self, client, request, filepath):
|
||||
self.client = client
|
||||
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
|
||||
return client.retrieveFile(filepath, protocol)\
|
||||
.addCallbacks(callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,))
|
||||
return client.retrieveFile(filepath, protocol).addCallbacks(
|
||||
callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,),
|
||||
)
|
||||
|
||||
def _build_response(self, result, request, protocol):
|
||||
self.result = result
|
||||
|
|
|
|||
|
|
@ -230,8 +230,7 @@ class ExecutionEngine:
|
|||
|
||||
def _downloaded(self, response, slot, request, spider):
|
||||
slot.remove_request(request)
|
||||
return self.download(response, spider) \
|
||||
if isinstance(response, Request) else response
|
||||
return self.download(response, spider) if isinstance(response, Request) else response
|
||||
|
||||
def _download(self, request, spider):
|
||||
slot = self.slot
|
||||
|
|
|
|||
|
|
@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
Handle redirection of requests based on response status
|
||||
and meta-refresh html tag.
|
||||
"""
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if (request.meta.get('dont_redirect', False) or
|
||||
response.status in getattr(spider, 'handle_httpstatus_list', []) or
|
||||
response.status in request.meta.get('handle_httpstatus_list', []) or
|
||||
request.meta.get('handle_httpstatus_all', False)):
|
||||
if (
|
||||
request.meta.get('dont_redirect', False)
|
||||
or response.status in getattr(spider, 'handle_httpstatus_list', [])
|
||||
or response.status in request.meta.get('handle_httpstatus_list', [])
|
||||
or request.meta.get('handle_httpstatus_all', False)
|
||||
):
|
||||
return response
|
||||
|
||||
allowed_status = (301, 302, 303, 307, 308)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class CloseSpider:
|
|||
'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
|
||||
'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
|
||||
'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
|
||||
}
|
||||
}
|
||||
|
||||
if not any(self.close_on.values()):
|
||||
raise NotConfigured
|
||||
|
|
|
|||
|
|
@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
|
|||
"""An implementation of IPortal"""
|
||||
@defers
|
||||
def login(self_, credentials, mind, *interfaces):
|
||||
if not (credentials.username == self.username.encode('utf8') and
|
||||
credentials.checkPassword(self.password.encode('utf8'))):
|
||||
if not (
|
||||
credentials.username == self.username.encode('utf8')
|
||||
and credentials.checkPassword(self.password.encode('utf8'))
|
||||
):
|
||||
raise ValueError("Invalid credentials")
|
||||
|
||||
protocol = telnet.TelnetBootstrapProtocol(
|
||||
|
|
|
|||
|
|
@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
|
|||
if the latter is given. If not, it returns the first
|
||||
clickable element found
|
||||
"""
|
||||
clickables = [
|
||||
el for el in form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"})
|
||||
]
|
||||
clickables = list(form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"}
|
||||
))
|
||||
if not clickables:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -61,8 +61,7 @@ class FilteringLinkExtractor:
|
|||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
|
||||
if (issubclass(cls, FilteringLinkExtractor) and
|
||||
not issubclass(cls, LxmlLinkExtractor)):
|
||||
if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
|
||||
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
|
||||
'please use scrapy.linkextractors.LinkExtractor instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
|
|||
|
|
@ -230,6 +230,20 @@ class GCSFilesStore:
|
|||
bucket, prefix = uri[5:].split('/', 1)
|
||||
self.bucket = client.bucket(bucket)
|
||||
self.prefix = prefix
|
||||
permissions = self.bucket.test_iam_permissions(
|
||||
['storage.objects.get', 'storage.objects.create']
|
||||
)
|
||||
if 'storage.objects.get' not in permissions:
|
||||
logger.warning(
|
||||
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
if 'storage.objects.create' not in permissions:
|
||||
logger.error(
|
||||
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _onsuccess(blob):
|
||||
|
|
|
|||
|
|
@ -43,8 +43,7 @@ class MediaPipeline:
|
|||
if allow_redirects:
|
||||
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
|
||||
|
||||
def _key_for_pipe(self, key, base_class_name=None,
|
||||
settings=None):
|
||||
def _key_for_pipe(self, key, base_class_name=None, settings=None):
|
||||
"""
|
||||
>>> MediaPipeline()._key_for_pipe("IMAGES")
|
||||
'IMAGES'
|
||||
|
|
@ -55,8 +54,11 @@ class MediaPipeline:
|
|||
"""
|
||||
class_name = self.__class__.__name__
|
||||
formatted_key = "{}_{}".format(class_name.upper(), key)
|
||||
if class_name == base_class_name or not base_class_name \
|
||||
or (settings and not settings.get(formatted_key)):
|
||||
if (
|
||||
not base_class_name
|
||||
or class_name == base_class_name
|
||||
or settings and not settings.get(formatted_key)
|
||||
):
|
||||
return key
|
||||
return formatted_key
|
||||
|
||||
|
|
|
|||
|
|
@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
|
|||
name = POLICY_STRICT_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
if ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
if (
|
||||
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
|
||||
or not self.tls_protected(response_url)
|
||||
):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
|
|
@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
|
|||
origin = self.origin(response_url)
|
||||
if origin == self.origin(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
elif ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
elif (
|
||||
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
|
||||
or not self.tls_protected(response_url)
|
||||
):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
|
|||
stacklevel=3)
|
||||
|
||||
|
||||
def create_deprecated_class(name, new_class, clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from "
|
||||
"deprecated class {old}, please inherit "
|
||||
"from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, "
|
||||
"instantiate {new} instead."):
|
||||
def create_deprecated_class(
|
||||
name,
|
||||
new_class,
|
||||
clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, instantiate {new} instead."
|
||||
):
|
||||
"""
|
||||
Return a "deprecated" class that causes its subclasses to issue a warning.
|
||||
Subclasses of ``new_class`` are considered subclasses of this class.
|
||||
|
|
|
|||
|
|
@ -52,8 +52,7 @@ def is_gzipped(response):
|
|||
"""Return True if the response is gzipped, or False otherwise"""
|
||||
ctype = response.headers.get('Content-Type', b'')
|
||||
cenc = response.headers.get('Content-Encoding', b'').lower()
|
||||
return (_is_gzipped(ctype) or
|
||||
(_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
|
||||
return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
|
||||
|
||||
|
||||
def gzip_magic_number(response):
|
||||
|
|
|
|||
|
|
@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
|||
|
||||
"""
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower())
|
||||
for h in sorted(include_headers))
|
||||
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
|
|
|
|||
|
|
@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
|
|||
if self.fail_yielding:
|
||||
2 / 0
|
||||
|
||||
assert self.seedsseen, \
|
||||
'All start requests consumed before any download happened'
|
||||
assert self.seedsseen, 'All start requests consumed before any download happened'
|
||||
|
||||
def parse(self, response):
|
||||
self.seedsseen.append(response.meta.get('seed'))
|
||||
|
|
|
|||
|
|
@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
|
|||
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_errorcount')
|
||||
key = 'spider_exceptions/{name}'\
|
||||
.format(name=crawler.spider.exception_cls.__name__)
|
||||
key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
|
||||
errorcount = crawler.stats.get_value(key)
|
||||
self.assertTrue(errorcount >= close_on)
|
||||
|
||||
|
|
|
|||
|
|
@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
def test_ipv6_alternative_name_resolver(self):
|
||||
log = self.run_script('alternative_name_resolver.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
self.assertTrue(any([
|
||||
"twisted.internet.error.ConnectionRefusedError" in log,
|
||||
"twisted.internet.error.ConnectError" in log,
|
||||
]))
|
||||
self.assertTrue(any([
|
||||
"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
|
||||
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
|
||||
]))
|
||||
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
|
||||
|
||||
def test_reactor_select(self):
|
||||
log = self.run_script("twisted_reactor_select.py")
|
||||
|
|
|
|||
|
|
@ -825,11 +825,15 @@ class S3TestCase(unittest.TestCase):
|
|||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 21:15:45 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg',
|
||||
method='PUT',
|
||||
headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
},
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
|
|
@ -909,11 +913,10 @@ class S3TestCase(unittest.TestCase):
|
|||
# ensure that spaces are quoted properly before signing
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request(
|
||||
("s3://johnsmith/photos/my puppy.jpg"
|
||||
"?response-content-disposition=my puppy.jpg"),
|
||||
"s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
|
||||
method='GET',
|
||||
headers={'Date': date},
|
||||
)
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(
|
||||
|
|
@ -1093,8 +1096,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype_encoding(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, 'A brief note')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "US-ASCII")
|
||||
|
||||
request = Request("data:,A%20brief%20note")
|
||||
|
|
@ -1103,8 +1105,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "iso-8859-7")
|
||||
|
||||
request = Request("data:;charset=iso-8859-7,%be%d3%be")
|
||||
|
|
@ -1122,8 +1123,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_mediatype_parameters(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
|
||||
request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
|
|||
for fmt in self.test_formats:
|
||||
rsp = self.test_responses[fmt]
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert isinstance(new, XmlResponse), \
|
||||
'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
assert isinstance(new, XmlResponse), error_msg
|
||||
assert_samelines(self, new.body, self.uncompressed_body, fmt)
|
||||
|
||||
def test_plain_response(self):
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
|
|||
SAMPLEDIR = join(tests_datadir, 'compressed')
|
||||
|
||||
FORMAT = {
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br')
|
||||
}
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br'),
|
||||
}
|
||||
|
||||
|
||||
class HttpCompressionTest(TestCase):
|
||||
|
|
@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
|
|||
body = sample.read()
|
||||
|
||||
headers = {
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
|
||||
response = Response('http://scrapytest.org/', body=body, headers=headers)
|
||||
response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
|
||||
|
|
|
|||
|
|
@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
rsp = Response(url, headers={'Location': url2}, status=301, request=req)
|
||||
r = self.mw.process_response(req, rsp, self.spider)
|
||||
self.assertIs(r, rsp)
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list':
|
||||
[404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
|
||||
|
||||
def test_latin1_location(self):
|
||||
|
|
|
|||
|
|
@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_bytes(self):
|
||||
data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'latin1')
|
||||
self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
|
||||
|
|
@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_textual_data(self):
|
||||
data = {'price': u'£ 100'}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r3.encoding, 'latin1')
|
||||
self.assertEqual(r3.body, b'price=%A3+100')
|
||||
|
||||
|
|
@ -469,7 +467,7 @@ class FormRequestTest(RequestTest):
|
|||
</form>""",
|
||||
url="http://www.example.com/this/list.html",
|
||||
encoding='latin1',
|
||||
)
|
||||
)
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ class MockCrawler(Crawler):
|
|||
def __init__(self, priority_queue_cls, jobdir):
|
||||
|
||||
settings = dict(
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
|
||||
)
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
|
||||
)
|
||||
super(MockCrawler, self).__init__(Spider, settings)
|
||||
self.engine = MockEngine(downloader=MockDownloader())
|
||||
|
||||
|
|
@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
|
|||
class TestIntegrationWithDownloaderAwareInMemory(TestCase):
|
||||
def setUp(self):
|
||||
self.crawler = get_crawler(
|
||||
StartUrlsSpider,
|
||||
{'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
|
||||
)
|
||||
spidercls=StartUrlsSpider,
|
||||
settings_dict={
|
||||
'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
|
||||
},
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
|
|
@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
|
|||
|
||||
def _incompatible(self):
|
||||
settings = dict(
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1
|
||||
)
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1,
|
||||
)
|
||||
crawler = Crawler(Spider, settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = Spider(name='spider')
|
||||
|
|
|
|||
|
|
@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
|
|||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = Selector(response)
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
|
||||
[u'\xa3'])
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
|
|
|
|||
|
|
@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
|
|||
self.mw.process_spider_input(self.res402, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
self.mw.process_spider_input(self.res404, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
|
|||
|
|
@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase):
|
|||
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
|
|||
chunked_body += "8\r\n" + "sequence\r\n"
|
||||
chunked_body += "0\r\n\r\n"
|
||||
body = decode_chunked_transfer(chunked_body)
|
||||
self.assertEqual(body,
|
||||
"This is the data in the first chunk\r\n" +
|
||||
"and this is the second one\r\n" +
|
||||
"consequence")
|
||||
self.assertEqual(
|
||||
body,
|
||||
"This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
class AddHttpIfNoScheme(unittest.TestCase):
|
||||
|
||||
def test_add_scheme(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
|
||||
|
||||
def test_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_http(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_preserve_http_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_preserve_http_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_preserve_http_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_preserve_http_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_preserve_http_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_preserve_http_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_protocol_relative(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_protocol_relative_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//example.com'), 'http://example.com')
|
||||
|
||||
def test_protocol_relative_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_protocol_relative_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_protocol_relative_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_protocol_relative_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_protocol_relative_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_https(self):
|
||||
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
|
||||
def test_preserve_ftp(self):
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
|
||||
'ftp://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
|
||||
|
||||
|
||||
class GuessSchemeTest(unittest.TestCase):
|
||||
|
|
@ -202,41 +218,49 @@ def create_skipped_scheme_t(args):
|
|||
return do_expected
|
||||
|
||||
|
||||
for k, args in enumerate([
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
for k, args in enumerate(
|
||||
[
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
|
||||
], start=1):
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_guess_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
# TODO: the following tests do not pass with current implementation
|
||||
for k, args in enumerate([
|
||||
(r'C:\absolute\path\to\a\file.html', 'file://',
|
||||
'Windows filepath are not supported for scrapy shell'),
|
||||
], start=1):
|
||||
for k, args in enumerate(
|
||||
[
|
||||
(
|
||||
r'C:\absolute\path\to\a\file.html',
|
||||
'file://',
|
||||
'Windows filepath are not supported for scrapy shell',
|
||||
),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_skipped_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_skipped_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
|
@ -272,7 +296,7 @@ class StripUrl(unittest.TestCase):
|
|||
('http://www.example.com',
|
||||
True,
|
||||
'http://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
|
||||
|
||||
def test_credentials(self):
|
||||
|
|
@ -285,7 +309,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_credentials_encoded_delims(self):
|
||||
|
|
@ -304,7 +328,7 @@ class StripUrl(unittest.TestCase):
|
|||
# password: "user@domain.com"
|
||||
('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_default_ports_creds_off(self):
|
||||
|
|
@ -332,7 +356,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i), o)
|
||||
|
||||
def test_default_ports(self):
|
||||
|
|
@ -360,7 +384,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
|
|
@ -388,7 +412,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
|
||||
|
||||
def test_origin_only(self):
|
||||
|
|
@ -404,7 +428,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, origin_only=True), o)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
},
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})))
|
||||
}),
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
|
|||
Loading…
Reference in New Issue