mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into deprecate-baseitem
This commit is contained in:
Eugenio Lacuesta
commit
6160370927
14
.travis.yml
14
.travis.yml
|
|
@ -15,19 +15,25 @@ matrix:
|
|||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
|
||||
- env: TOXENV=pypy3
|
||||
- python: 3.5
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5.2
|
||||
- python: 3.6
|
||||
- python: 3.7
|
||||
- env: PYPI_RELEASE_JOB=true
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
|
|
|
|||
|
|
@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
|
|||
.. note::
|
||||
|
||||
HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
|
||||
unless you use Twisted<11.1, or if you really want to use HTTP/1.0
|
||||
and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
|
||||
accordingly, i.e. to
|
||||
``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
|
||||
unless you really want to use HTTP/1.0 and override
|
||||
:setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
|
||||
i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
|
||||
|
||||
|
|
@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
|
|||
Scrapy also has another context factory class that you can set,
|
||||
``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
|
||||
which uses the platform's certificates to validate remote endpoints.
|
||||
**This is only available if you use Twisted>=14.0.**
|
||||
|
||||
If you do use a custom ContextFactory, make sure its ``__init__`` method
|
||||
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
|
||||
|
|
@ -494,10 +492,6 @@ This setting must be one of these string values:
|
|||
- ``'TLSv1.2'``: forces TLS version 1.2
|
||||
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
|
||||
or above (Twisted>=14.0 if you can).
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
||||
|
|
@ -660,8 +654,6 @@ If you want to disable it set to 0.
|
|||
spider attribute and per-request using :reqmeta:`download_maxsize`
|
||||
Request.meta key.
|
||||
|
||||
This feature needs Twisted >= 11.1.
|
||||
|
||||
.. setting:: DOWNLOAD_WARNSIZE
|
||||
|
||||
DOWNLOAD_WARNSIZE
|
||||
|
|
@ -679,8 +671,6 @@ If you want to disable it set to 0.
|
|||
spider attribute and per-request using :reqmeta:`download_warnsize`
|
||||
Request.meta key.
|
||||
|
||||
This feature needs Twisted >= 11.1.
|
||||
|
||||
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
||||
DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
|
|
|||
98
pytest.ini
98
pytest.ini
|
|
@ -35,62 +35,62 @@ flake8-ignore =
|
|||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128
|
||||
scrapy/commands/fetch.py E501 E128
|
||||
scrapy/commands/genspider.py E128 E501
|
||||
scrapy/commands/parse.py E128 E501
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501
|
||||
scrapy/commands/startproject.py E127 E501 E128
|
||||
scrapy/commands/startproject.py E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/__init__.py E501
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E501 E128 E127
|
||||
scrapy/core/engine.py E501 E128
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E128 W504
|
||||
scrapy/core/spidermw.py E501 E126
|
||||
scrapy/core/scraper.py E501 E128
|
||||
scrapy/core/spidermw.py E501
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/contextfactory.py E501 E128
|
||||
scrapy/core/downloader/middleware.py E501
|
||||
scrapy/core/downloader/tls.py E501
|
||||
scrapy/core/downloader/webclient.py E501 E128 E126
|
||||
scrapy/core/downloader/webclient.py E501 E128
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcache.py E501
|
||||
scrapy/downloadermiddlewares/httpcompression.py E501 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/redirect.py E501
|
||||
scrapy/downloadermiddlewares/retry.py E501
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E128 E123
|
||||
scrapy/extensions/closespider.py E501 E128
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/telnet.py E501
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/form.py E501
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128
|
||||
scrapy/http/response/text.py E501 E128 E124
|
||||
scrapy/http/response/text.py E501 E128
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E501 E402 W504
|
||||
scrapy/linkextractors/__init__.py E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E128
|
||||
|
|
@ -99,7 +99,7 @@ flake8-ignore =
|
|||
scrapy/pipelines/__init__.py E501
|
||||
scrapy/pipelines/files.py E116 E501
|
||||
scrapy/pipelines/images.py E501
|
||||
scrapy/pipelines/media.py E125 E501
|
||||
scrapy/pipelines/media.py E501
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
|
|
@ -110,7 +110,7 @@ flake8-ignore =
|
|||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py E501 E129 W504
|
||||
scrapy/spidermiddlewares/referer.py E501
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
|
|
@ -124,8 +124,8 @@ flake8-ignore =
|
|||
scrapy/utils/datatypes.py E501
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127
|
||||
scrapy/utils/gz.py E501 W504
|
||||
scrapy/utils/deprecate.py E501
|
||||
scrapy/utils/gz.py E501
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501
|
||||
|
|
@ -137,7 +137,7 @@ flake8-ignore =
|
|||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E501
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/request.py E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
|
|
@ -164,15 +164,15 @@ flake8-ignore =
|
|||
scrapy/robotstxt.py E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py F841 E501 E126
|
||||
scrapy/spiderloader.py F841 E501
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/mockserver.py E501
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/spiders.py E501
|
||||
tests/test_closespider.py E501
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128
|
||||
tests/test_command_shell.py E501 E128
|
||||
|
|
@ -181,37 +181,37 @@ flake8-ignore =
|
|||
tests/test_crawl.py E501 E741
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
|
||||
tests/test_downloader_handlers.py E128 E501
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_cookies.py E741 E501 E128
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
|
||||
tests/test_downloadermiddleware_httpcompression.py E501
|
||||
tests/test_downloadermiddleware_decompression.py E501
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128 E127
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E126
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128
|
||||
tests/test_downloadermiddleware_retry.py E501 E128
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E501 E741 E128 E124
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E128 E124
|
||||
tests/test_dupefilters.py E501 E741 E128
|
||||
tests/test_engine.py E501 E128
|
||||
tests/test_exporters.py E501 E128
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
|
||||
tests/test_http_request.py E402 E501 E128 E128
|
||||
tests/test_http_response.py E501 E128
|
||||
tests/test_item.py E128 F841 E501
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_linkextractors.py E501 E128
|
||||
tests/test_loader.py E501 E741 E128 E117
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_logformatter.py E128 E501
|
||||
tests/test_mail.py E128 E501
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E501 E128 E126
|
||||
tests/test_pipeline_crawl.py E501 E128
|
||||
tests/test_pipeline_files.py E501
|
||||
tests/test_pipeline_images.py F841 E501
|
||||
tests/test_pipeline_media.py E501 E741 E128
|
||||
|
|
@ -219,14 +219,14 @@ flake8-ignore =
|
|||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501
|
||||
tests/test_robotstxt_interface.py E501 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_scheduler.py E501
|
||||
tests/test_selector.py E501
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E128
|
||||
|
|
@ -234,17 +234,17 @@ flake8-ignore =
|
|||
tests/test_utils_datatypes.py E402 E501
|
||||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129
|
||||
tests/test_utils_http.py E501 E128
|
||||
tests/test_utils_iterators.py E501 E128
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E123 E126
|
||||
tests/test_utils_sitemap.py E128 E501
|
||||
tests/test_utils_url.py E501 E501
|
||||
tests/test_webclient.py E501 E128 E402
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
|
|
|
|||
|
|
@ -17,10 +17,10 @@ class ContractsManager:
|
|||
self.contracts[contract.name] = contract
|
||||
|
||||
def tested_methods_from_spidercls(self, spidercls):
|
||||
is_method = re.compile(r"^\s*@", re.MULTILINE).search
|
||||
methods = []
|
||||
for key, value in getmembers(spidercls):
|
||||
if (callable(value) and value.__doc__ and
|
||||
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
|
||||
if callable(value) and value.__doc__ and is_method(value.__doc__):
|
||||
methods.append(key)
|
||||
|
||||
return methods
|
||||
|
|
|
|||
|
|
@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
#
|
||||
# This means that a website like https://www.cacert.org will be rejected
|
||||
# by default, since CAcert.org CA certificate is seldom shipped.
|
||||
return optionsForClientTLS(hostname.decode("ascii"),
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={
|
||||
'method': self._ssl_method,
|
||||
})
|
||||
return optionsForClientTLS(
|
||||
hostname=hostname.decode("ascii"),
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={'method': self._ssl_method},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -94,11 +94,12 @@ class FTPDownloadHandler:
|
|||
def gotClient(self, client, request, filepath):
|
||||
self.client = client
|
||||
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
|
||||
return client.retrieveFile(filepath, protocol)\
|
||||
.addCallbacks(callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,))
|
||||
return client.retrieveFile(filepath, protocol).addCallbacks(
|
||||
callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,),
|
||||
)
|
||||
|
||||
def _build_response(self, result, request, protocol):
|
||||
self.result = result
|
||||
|
|
|
|||
|
|
@ -100,11 +100,12 @@ class S3DownloadHandler:
|
|||
url=url, headers=awsrequest.headers.items())
|
||||
else:
|
||||
signed_headers = self.conn.make_request(
|
||||
method=request.method,
|
||||
bucket=bucket,
|
||||
key=unquote(p.path),
|
||||
query_args=unquote(p.query),
|
||||
headers=request.headers,
|
||||
data=request.body)
|
||||
method=request.method,
|
||||
bucket=bucket,
|
||||
key=unquote(p.path),
|
||||
query_args=unquote(p.query),
|
||||
headers=request.headers,
|
||||
data=request.body,
|
||||
)
|
||||
request = request.replace(url=url, headers=signed_headers)
|
||||
return self._download_http(request, spider)
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
self.transport.stopProducing()
|
||||
|
||||
self.factory.noPage(
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." %
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds."
|
||||
% (self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
|
|
|
|||
|
|
@ -230,8 +230,7 @@ class ExecutionEngine:
|
|||
|
||||
def _downloaded(self, response, slot, request, spider):
|
||||
slot.remove_request(request)
|
||||
return self.download(response, spider) \
|
||||
if isinstance(response, Request) else response
|
||||
return self.download(response, spider) if isinstance(response, Request) else response
|
||||
|
||||
def _download(self, request, spider):
|
||||
slot = self.slot
|
||||
|
|
|
|||
|
|
@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
Handle redirection of requests based on response status
|
||||
and meta-refresh html tag.
|
||||
"""
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if (request.meta.get('dont_redirect', False) or
|
||||
response.status in getattr(spider, 'handle_httpstatus_list', []) or
|
||||
response.status in request.meta.get('handle_httpstatus_list', []) or
|
||||
request.meta.get('handle_httpstatus_all', False)):
|
||||
if (
|
||||
request.meta.get('dont_redirect', False)
|
||||
or response.status in getattr(spider, 'handle_httpstatus_list', [])
|
||||
or response.status in request.meta.get('handle_httpstatus_list', [])
|
||||
or request.meta.get('handle_httpstatus_all', False)
|
||||
):
|
||||
return response
|
||||
|
||||
allowed_status = (301, 302, 303, 307, 308)
|
||||
|
|
|
|||
|
|
@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
|
|||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class CloseSpider:
|
|||
'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
|
||||
'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
|
||||
'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
|
||||
}
|
||||
}
|
||||
|
||||
if not any(self.close_on.values()):
|
||||
raise NotConfigured
|
||||
|
|
|
|||
|
|
@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
|
|||
"""An implementation of IPortal"""
|
||||
@defers
|
||||
def login(self_, credentials, mind, *interfaces):
|
||||
if not (credentials.username == self.username.encode('utf8') and
|
||||
credentials.checkPassword(self.password.encode('utf8'))):
|
||||
if not (
|
||||
credentials.username == self.username.encode('utf8')
|
||||
and credentials.checkPassword(self.password.encode('utf8'))
|
||||
):
|
||||
raise ValueError("Invalid credentials")
|
||||
|
||||
protocol = telnet.TelnetBootstrapProtocol(
|
||||
|
|
|
|||
|
|
@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
|
|||
if the latter is given. If not, it returns the first
|
||||
clickable element found
|
||||
"""
|
||||
clickables = [
|
||||
el for el in form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"})
|
||||
]
|
||||
clickables = list(form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"}
|
||||
))
|
||||
if not clickables:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -61,8 +61,7 @@ class FilteringLinkExtractor:
|
|||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
|
||||
if (issubclass(cls, FilteringLinkExtractor) and
|
||||
not issubclass(cls, LxmlLinkExtractor)):
|
||||
if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
|
||||
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
|
||||
'please use scrapy.linkextractors.LinkExtractor instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
|
|||
|
|
@ -230,6 +230,20 @@ class GCSFilesStore:
|
|||
bucket, prefix = uri[5:].split('/', 1)
|
||||
self.bucket = client.bucket(bucket)
|
||||
self.prefix = prefix
|
||||
permissions = self.bucket.test_iam_permissions(
|
||||
['storage.objects.get', 'storage.objects.create']
|
||||
)
|
||||
if 'storage.objects.get' not in permissions:
|
||||
logger.warning(
|
||||
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
if 'storage.objects.create' not in permissions:
|
||||
logger.error(
|
||||
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _onsuccess(blob):
|
||||
|
|
|
|||
|
|
@ -43,8 +43,7 @@ class MediaPipeline:
|
|||
if allow_redirects:
|
||||
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
|
||||
|
||||
def _key_for_pipe(self, key, base_class_name=None,
|
||||
settings=None):
|
||||
def _key_for_pipe(self, key, base_class_name=None, settings=None):
|
||||
"""
|
||||
>>> MediaPipeline()._key_for_pipe("IMAGES")
|
||||
'IMAGES'
|
||||
|
|
@ -55,8 +54,11 @@ class MediaPipeline:
|
|||
"""
|
||||
class_name = self.__class__.__name__
|
||||
formatted_key = "{}_{}".format(class_name.upper(), key)
|
||||
if class_name == base_class_name or not base_class_name \
|
||||
or (settings and not settings.get(formatted_key)):
|
||||
if (
|
||||
not base_class_name
|
||||
or class_name == base_class_name
|
||||
or settings and not settings.get(formatted_key)
|
||||
):
|
||||
return key
|
||||
return formatted_key
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from collections import defaultdict
|
||||
import traceback
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
|
||||
from zope.interface import implementer
|
||||
|
||||
|
|
@ -16,6 +16,7 @@ class SpiderLoader:
|
|||
SpiderLoader is a class which locates and loads spiders
|
||||
in a Scrapy project.
|
||||
"""
|
||||
|
||||
def __init__(self, settings):
|
||||
self.spider_modules = settings.getlist('SPIDER_MODULES')
|
||||
self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
|
||||
|
|
@ -24,16 +25,21 @@ class SpiderLoader:
|
|||
self._load_all_spiders()
|
||||
|
||||
def _check_name_duplicates(self):
|
||||
dupes = ["\n".join(" {cls} named {name!r} (in {module})".format(
|
||||
module=mod, cls=cls, name=name)
|
||||
for (mod, cls) in locations)
|
||||
for name, locations in self._found.items()
|
||||
if len(locations) > 1]
|
||||
dupes = []
|
||||
for name, locations in self._found.items():
|
||||
dupes.extend([
|
||||
" {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
|
||||
for mod, cls in locations
|
||||
if len(locations) > 1
|
||||
])
|
||||
|
||||
if dupes:
|
||||
msg = ("There are several spiders with the same name:\n\n"
|
||||
"{}\n\n This can cause unexpected behavior.".format(
|
||||
"\n\n".join(dupes)))
|
||||
warnings.warn(msg, UserWarning)
|
||||
dupes_string = "\n\n".join(dupes)
|
||||
warnings.warn(
|
||||
"There are several spiders with the same name:\n\n"
|
||||
"{}\n\n This can cause unexpected behavior.".format(dupes_string),
|
||||
category=UserWarning,
|
||||
)
|
||||
|
||||
def _load_spiders(self, module):
|
||||
for spcls in iter_spider_classes(module):
|
||||
|
|
@ -45,12 +51,15 @@ class SpiderLoader:
|
|||
try:
|
||||
for module in walk_modules(name):
|
||||
self._load_spiders(module)
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
if self.warn_only:
|
||||
msg = ("\n{tb}Could not load spiders from module '{modname}'. "
|
||||
"See above traceback for details.".format(
|
||||
modname=name, tb=traceback.format_exc()))
|
||||
warnings.warn(msg, RuntimeWarning)
|
||||
warnings.warn(
|
||||
"\n{tb}Could not load spiders from module '{modname}'. "
|
||||
"See above traceback for details.".format(
|
||||
modname=name, tb=traceback.format_exc()
|
||||
),
|
||||
category=RuntimeWarning,
|
||||
)
|
||||
else:
|
||||
raise
|
||||
self._check_name_duplicates()
|
||||
|
|
@ -73,8 +82,10 @@ class SpiderLoader:
|
|||
"""
|
||||
Return the list of spider names that can handle the given request.
|
||||
"""
|
||||
return [name for name, cls in self._spiders.items()
|
||||
if cls.handles_request(request)]
|
||||
return [
|
||||
name for name, cls in self._spiders.items()
|
||||
if cls.handles_request(request)
|
||||
]
|
||||
|
||||
def list(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
|
|||
name = POLICY_STRICT_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
if ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
if (
|
||||
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
|
||||
or not self.tls_protected(response_url)
|
||||
):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
|
|
@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
|
|||
origin = self.origin(response_url)
|
||||
if origin == self.origin(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
elif ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
elif (
|
||||
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
|
||||
or not self.tls_protected(response_url)
|
||||
):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
|
|||
stacklevel=3)
|
||||
|
||||
|
||||
def create_deprecated_class(name, new_class, clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from "
|
||||
"deprecated class {old}, please inherit "
|
||||
"from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, "
|
||||
"instantiate {new} instead."):
|
||||
def create_deprecated_class(
|
||||
name,
|
||||
new_class,
|
||||
clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, instantiate {new} instead."
|
||||
):
|
||||
"""
|
||||
Return a "deprecated" class that causes its subclasses to issue a warning.
|
||||
Subclasses of ``new_class`` are considered subclasses of this class.
|
||||
|
|
|
|||
|
|
@ -52,8 +52,7 @@ def is_gzipped(response):
|
|||
"""Return True if the response is gzipped, or False otherwise"""
|
||||
ctype = response.headers.get('Content-Type', b'')
|
||||
cenc = response.headers.get('Content-Encoding', b'').lower()
|
||||
return (_is_gzipped(ctype) or
|
||||
(_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
|
||||
return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
|
||||
|
||||
|
||||
def gzip_magic_number(response):
|
||||
|
|
|
|||
|
|
@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
|||
|
||||
"""
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower())
|
||||
for h in sorted(include_headers))
|
||||
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
|
|
|
|||
|
|
@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
|
|||
if self.fail_yielding:
|
||||
2 / 0
|
||||
|
||||
assert self.seedsseen, \
|
||||
'All start requests consumed before any download happened'
|
||||
assert self.seedsseen, 'All start requests consumed before any download happened'
|
||||
|
||||
def parse(self, response):
|
||||
self.seedsseen.append(response.meta.get('seed'))
|
||||
|
|
|
|||
|
|
@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
|
|||
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_errorcount')
|
||||
key = 'spider_exceptions/{name}'\
|
||||
.format(name=crawler.spider.exception_cls.__name__)
|
||||
key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
|
||||
errorcount = crawler.stats.get_value(key)
|
||||
self.assertTrue(errorcount >= close_on)
|
||||
|
||||
|
|
|
|||
|
|
@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
def test_ipv6_alternative_name_resolver(self):
|
||||
log = self.run_script('alternative_name_resolver.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
self.assertTrue(any([
|
||||
"twisted.internet.error.ConnectionRefusedError" in log,
|
||||
"twisted.internet.error.ConnectError" in log,
|
||||
]))
|
||||
self.assertTrue(any([
|
||||
"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
|
||||
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
|
||||
]))
|
||||
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
|
||||
|
||||
def test_reactor_select(self):
|
||||
log = self.run_script("twisted_reactor_select.py")
|
||||
|
|
|
|||
|
|
@ -822,11 +822,15 @@ class S3TestCase(unittest.TestCase):
|
|||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 21:15:45 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg',
|
||||
method='PUT',
|
||||
headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
},
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
|
|
@ -906,11 +910,10 @@ class S3TestCase(unittest.TestCase):
|
|||
# ensure that spaces are quoted properly before signing
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request(
|
||||
("s3://johnsmith/photos/my puppy.jpg"
|
||||
"?response-content-disposition=my puppy.jpg"),
|
||||
"s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
|
||||
method='GET',
|
||||
headers={'Date': date},
|
||||
)
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(
|
||||
|
|
@ -1090,8 +1093,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype_encoding(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, 'A brief note')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "US-ASCII")
|
||||
|
||||
request = Request("data:,A%20brief%20note")
|
||||
|
|
@ -1100,8 +1102,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "iso-8859-7")
|
||||
|
||||
request = Request("data:;charset=iso-8859-7,%be%d3%be")
|
||||
|
|
@ -1119,8 +1120,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_mediatype_parameters(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
|
||||
request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
|
||||
|
|
|
|||
|
|
@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):
|
|||
|
||||
def test_complex_cookies(self):
|
||||
# merge some cookies into jar
|
||||
cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
|
||||
cookies = [
|
||||
{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
|
||||
]
|
||||
|
||||
req = Request('http://scrapytest.org/', cookies=cookies)
|
||||
self.mw.process_request(req, self.spider)
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
|
|||
for fmt in self.test_formats:
|
||||
rsp = self.test_responses[fmt]
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert isinstance(new, XmlResponse), \
|
||||
'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
assert isinstance(new, XmlResponse), error_msg
|
||||
assert_samelines(self, new.body, self.uncompressed_body, fmt)
|
||||
|
||||
def test_plain_response(self):
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
|
|||
SAMPLEDIR = join(tests_datadir, 'compressed')
|
||||
|
||||
FORMAT = {
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br')
|
||||
}
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br'),
|
||||
}
|
||||
|
||||
|
||||
class HttpCompressionTest(TestCase):
|
||||
|
|
@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
|
|||
body = sample.read()
|
||||
|
||||
headers = {
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
|
||||
response = Response('http://scrapytest.org/', body=body, headers=headers)
|
||||
response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
|
||||
|
|
|
|||
|
|
@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
rsp = Response(url, headers={'Location': url2}, status=301, request=req)
|
||||
r = self.mw.process_response(req, rsp, self.spider)
|
||||
self.assertIs(r, rsp)
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list':
|
||||
[404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
|
||||
|
||||
def test_latin1_location(self):
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import unittest
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.downloadermiddlewares.retry import RetryMiddleware
|
||||
|
|
|
|||
|
|
@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html',
|
||||
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
|
||||
)
|
||||
headers={'Referer': 'http://scrapytest.org/INDEX.html'})
|
||||
|
||||
dupefilter.log(r1, spider)
|
||||
dupefilter.log(r2, spider)
|
||||
|
|
|
|||
|
|
@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
i2 = dict(name=u'bar', age=i1)
|
||||
i3 = TestItem(name=u'buz', age=i2)
|
||||
|
||||
self.assertExportResult(i3,
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>'
|
||||
b'<item>'
|
||||
b'<age>'
|
||||
b'<age>'
|
||||
b'<age>22</age>'
|
||||
b'<name>foo\xc2\xa3hoo</name>'
|
||||
b'</age>'
|
||||
b'<name>bar</name>'
|
||||
b'</age>'
|
||||
b'<name>buz</name>'
|
||||
b'</item>'
|
||||
b'</items>'
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<age>
|
||||
<age>
|
||||
<age>22</age>
|
||||
<name>foo\xc2\xa3hoo</name>
|
||||
</age>
|
||||
<name>bar</name>
|
||||
</age>
|
||||
<name>buz</name>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
def test_nested_list_item(self):
|
||||
|
|
@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
|
||||
i3 = TestItem(name=u'buz', age=[i1, i2])
|
||||
|
||||
self.assertExportResult(i3,
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>'
|
||||
b'<item>'
|
||||
b'<age>'
|
||||
b'<value><name>foo</name></value>'
|
||||
b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
|
||||
b'</age>'
|
||||
b'<name>buz</name>'
|
||||
b'</item>'
|
||||
b'</items>'
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<age>
|
||||
<value><name>foo</name></value>
|
||||
<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
|
||||
</age>
|
||||
<name>buz</name>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
def test_nonstring_types_item(self):
|
||||
item = self._get_nonstring_types_item()
|
||||
self.assertExportResult(item,
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>'
|
||||
b'<item>'
|
||||
b'<float>3.14</float>'
|
||||
b'<boolean>False</boolean>'
|
||||
b'<number>22</number>'
|
||||
b'<time>2015-01-01 01:01:01</time>'
|
||||
b'</item>'
|
||||
b'</items>'
|
||||
self.assertExportResult(
|
||||
item,
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<float>3.14</float>
|
||||
<boolean>False</boolean>
|
||||
<number>22</number>
|
||||
<time>2015-01-01 01:01:01</time>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_bytes(self):
|
||||
data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'latin1')
|
||||
self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
|
||||
|
|
@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_textual_data(self):
|
||||
data = {'price': u'£ 100'}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r3.encoding, 'latin1')
|
||||
self.assertEqual(r3.body, b'price=%A3+100')
|
||||
|
||||
|
|
@ -469,7 +467,7 @@ class FormRequestTest(RequestTest):
|
|||
</form>""",
|
||||
url="http://www.example.com/this/list.html",
|
||||
encoding='latin1',
|
||||
)
|
||||
)
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
|
||||
|
|
@ -504,11 +502,13 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_from_response_duplicate_form_key(self):
|
||||
response = _buildresponse(
|
||||
'<form></form>',
|
||||
url='http://www.example.com')
|
||||
req = self.request_class.from_response(response,
|
||||
method='GET',
|
||||
formdata=(('foo', 'bar'), ('foo', 'baz')))
|
||||
'<form></form>',
|
||||
url='http://www.example.com')
|
||||
req = self.request_class.from_response(
|
||||
response=response,
|
||||
method='GET',
|
||||
formdata=(('foo', 'bar'), ('foo', 'baz')),
|
||||
)
|
||||
self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
|
||||
self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
|
||||
|
||||
|
|
@ -532,9 +532,11 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="test" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'},
|
||||
headers={"Accept-Encoding": "gzip,deflate"})
|
||||
req = self.request_class.from_response(
|
||||
response=response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'},
|
||||
headers={"Accept-Encoding": "gzip,deflate"},
|
||||
)
|
||||
self.assertEqual(req.method, 'POST')
|
||||
self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
|
||||
self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
|
||||
|
|
@ -582,9 +584,9 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_from_response_override_method(self):
|
||||
response = _buildresponse(
|
||||
'''<html><body>
|
||||
<form action="/app"></form>
|
||||
</body></html>''')
|
||||
'''<html><body>
|
||||
<form action="/app"></form>
|
||||
</body></html>''')
|
||||
request = FormRequest.from_response(response)
|
||||
self.assertEqual(request.method, 'GET')
|
||||
request = FormRequest.from_response(response, method='POST')
|
||||
|
|
@ -592,9 +594,9 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_from_response_override_url(self):
|
||||
response = _buildresponse(
|
||||
'''<html><body>
|
||||
<form action="/app"></form>
|
||||
</body></html>''')
|
||||
'''<html><body>
|
||||
<form action="/app"></form>
|
||||
</body></html>''')
|
||||
request = FormRequest.from_response(response)
|
||||
self.assertEqual(request.url, 'http://example.com/app')
|
||||
request = FormRequest.from_response(response, url='http://foo.bar/absolute')
|
||||
|
|
|
|||
|
|
@ -413,24 +413,30 @@ class Base:
|
|||
response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
|
||||
)
|
||||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
|
||||
)
|
||||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = b"""
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
|
|||
name = 'redirectedmedia'
|
||||
|
||||
def _process_url(self, url):
|
||||
return add_or_replace_parameter(
|
||||
self.mockserver.url('/redirect-to'),
|
||||
'goto', url)
|
||||
return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
|
||||
|
||||
|
||||
class FileDownloadCrawlTestCase(TestCase):
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ class MockCrawler(Crawler):
|
|||
def __init__(self, priority_queue_cls, jobdir):
|
||||
|
||||
settings = dict(
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
|
||||
)
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
|
||||
)
|
||||
super(MockCrawler, self).__init__(Spider, settings)
|
||||
self.engine = MockEngine(downloader=MockDownloader())
|
||||
|
||||
|
|
@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
|
|||
class TestIntegrationWithDownloaderAwareInMemory(TestCase):
|
||||
def setUp(self):
|
||||
self.crawler = get_crawler(
|
||||
StartUrlsSpider,
|
||||
{'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
|
||||
)
|
||||
spidercls=StartUrlsSpider,
|
||||
settings_dict={
|
||||
'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
|
||||
},
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
|
|
@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
|
|||
|
||||
def _incompatible(self):
|
||||
settings = dict(
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1
|
||||
)
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1,
|
||||
)
|
||||
crawler = Crawler(Spider, settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = Spider(name='spider')
|
||||
|
|
|
|||
|
|
@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
|
|||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = Selector(response)
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
|
||||
[u'\xa3'])
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
|
|
|
|||
|
|
@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
|
|||
self.mw.process_spider_input(self.res402, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
self.mw.process_spider_input(self.res404, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
|
|||
|
|
@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase):
|
|||
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
|
|||
chunked_body += "8\r\n" + "sequence\r\n"
|
||||
chunked_body += "0\r\n\r\n"
|
||||
body = decode_chunked_transfer(chunked_body)
|
||||
self.assertEqual(body,
|
||||
"This is the data in the first chunk\r\n" +
|
||||
"and this is the second one\r\n" +
|
||||
"consequence")
|
||||
self.assertEqual(
|
||||
body,
|
||||
"This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase):
|
|||
# with bytes
|
||||
XmlResponse(url="http://example.com", body=body.encode('utf-8')),
|
||||
# Unicode body needs encoding information
|
||||
XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
|
||||
|
||||
XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
|
||||
):
|
||||
attrs = []
|
||||
for x in self.xmliter(r, u'þingflokkur'):
|
||||
attrs.append((x.attrib['id'],
|
||||
|
|
|
|||
|
|
@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</url>
|
||||
</urlset>
|
||||
""")
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_wrong_ns(self):
|
||||
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
|
||||
|
|
@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</url>
|
||||
</urlset>
|
||||
""")
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_wrong_ns2(self):
|
||||
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
|
||||
|
|
@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</urlset>
|
||||
""")
|
||||
assert s.type == 'urlset'
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_urls_from_robots(self):
|
||||
robots = """User-agent: *
|
||||
|
|
@ -195,11 +204,19 @@ Disallow: /forum/active/
|
|||
</url>
|
||||
</urlset>""")
|
||||
|
||||
self.assertEqual(list(s), [
|
||||
{'loc': 'http://www.example.com/english/',
|
||||
'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
|
||||
}
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{
|
||||
'loc': 'http://www.example.com/english/',
|
||||
'alternate': [
|
||||
'http://www.example.com/deutsch/',
|
||||
'http://www.example.com/schweiz-deutsch/',
|
||||
'http://www.example.com/english/',
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
def test_xml_entity_expansion(self):
|
||||
s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
|
|
|
|||
|
|
@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
class AddHttpIfNoScheme(unittest.TestCase):
|
||||
|
||||
def test_add_scheme(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
|
||||
|
||||
def test_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_http(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_preserve_http_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_preserve_http_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_preserve_http_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_preserve_http_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_preserve_http_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_preserve_http_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_protocol_relative(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_protocol_relative_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//example.com'), 'http://example.com')
|
||||
|
||||
def test_protocol_relative_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_protocol_relative_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_protocol_relative_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_protocol_relative_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_protocol_relative_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_https(self):
|
||||
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
|
||||
def test_preserve_ftp(self):
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
|
||||
'ftp://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
|
||||
|
||||
|
||||
class GuessSchemeTest(unittest.TestCase):
|
||||
|
|
@ -202,41 +218,49 @@ def create_skipped_scheme_t(args):
|
|||
return do_expected
|
||||
|
||||
|
||||
for k, args in enumerate([
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
for k, args in enumerate(
|
||||
[
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
|
||||
], start=1):
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_guess_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
# TODO: the following tests do not pass with current implementation
|
||||
for k, args in enumerate([
|
||||
(r'C:\absolute\path\to\a\file.html', 'file://',
|
||||
'Windows filepath are not supported for scrapy shell'),
|
||||
], start=1):
|
||||
for k, args in enumerate(
|
||||
[
|
||||
(
|
||||
r'C:\absolute\path\to\a\file.html',
|
||||
'file://',
|
||||
'Windows filepath are not supported for scrapy shell',
|
||||
),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_skipped_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_skipped_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
|
@ -272,7 +296,7 @@ class StripUrl(unittest.TestCase):
|
|||
('http://www.example.com',
|
||||
True,
|
||||
'http://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
|
||||
|
||||
def test_credentials(self):
|
||||
|
|
@ -285,7 +309,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_credentials_encoded_delims(self):
|
||||
|
|
@ -304,7 +328,7 @@ class StripUrl(unittest.TestCase):
|
|||
# password: "user@domain.com"
|
||||
('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_default_ports_creds_off(self):
|
||||
|
|
@ -332,7 +356,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i), o)
|
||||
|
||||
def test_default_ports(self):
|
||||
|
|
@ -360,7 +384,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
|
|
@ -388,7 +412,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
|
||||
|
||||
def test_origin_only(self):
|
||||
|
|
@ -404,7 +428,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, origin_only=True), o)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,14 @@ except ImportError:
|
|||
from twisted.python.filepath import FilePath
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.web.test.test_webclient import (
|
||||
ForeverTakingResource,
|
||||
ErrorResource,
|
||||
NoLengthResource,
|
||||
HostHeaderResource,
|
||||
PayloadResource,
|
||||
BrokenDownloadResource,
|
||||
)
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
|
||||
|
|
@ -53,29 +61,29 @@ class ParseUrlTestCase(unittest.TestCase):
|
|||
def testParse(self):
|
||||
lip = '127.0.0.1'
|
||||
tests = (
|
||||
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
|
||||
|
||||
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
|
||||
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
|
||||
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
|
||||
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
|
||||
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
|
||||
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
|
||||
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
|
||||
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
|
||||
|
||||
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
|
||||
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
|
||||
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
|
||||
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
|
||||
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
|
||||
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
|
||||
|
||||
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
|
||||
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
|
||||
)
|
||||
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
|
||||
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
|
||||
)
|
||||
|
||||
for url, test in tests:
|
||||
test = tuple(
|
||||
|
|
@ -149,7 +157,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
},
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -165,7 +174,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})))
|
||||
}),
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -200,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
Headers({'Hello': ['World'], 'Foo': ['Bar']}))
|
||||
|
||||
|
||||
from twisted.web.test.test_webclient import ForeverTakingResource, \
|
||||
ErrorResource, NoLengthResource, HostHeaderResource, \
|
||||
PayloadResource, BrokenDownloadResource
|
||||
|
||||
|
||||
class EncodingResource(resource.Resource):
|
||||
out_encoding = 'cp1251'
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue