mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into pypy
This commit is contained in:
Adrián Chaves
GitHub
commit
6ef0ed9a50
14
.travis.yml
14
.travis.yml
|
|
@ -15,20 +15,26 @@ matrix:
|
|||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
|
||||
- python: 3.5
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5.2
|
||||
- python: 3.6
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
|
||||
- python: 3.7
|
||||
- env: PYPI_RELEASE_JOB=true
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
|
|
|
|||
64
pytest.ini
64
pytest.ini
|
|
@ -35,19 +35,19 @@ flake8-ignore =
|
|||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128
|
||||
scrapy/commands/fetch.py E501 E128
|
||||
scrapy/commands/genspider.py E128 E501
|
||||
scrapy/commands/parse.py E128 E501
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501
|
||||
scrapy/commands/startproject.py E127 E501 E128
|
||||
scrapy/commands/startproject.py E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E501 E128 E127
|
||||
scrapy/core/engine.py E501 E128
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E128
|
||||
scrapy/core/spidermw.py E501 E126
|
||||
|
|
@ -57,7 +57,7 @@ flake8-ignore =
|
|||
scrapy/core/downloader/tls.py E501
|
||||
scrapy/core/downloader/webclient.py E501 E128 E126
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
|
|
@ -73,7 +73,7 @@ flake8-ignore =
|
|||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E128 E123
|
||||
scrapy/extensions/closespider.py E501 E128
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501
|
||||
|
|
@ -85,10 +85,10 @@ flake8-ignore =
|
|||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/form.py E501
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128
|
||||
scrapy/http/response/text.py E501 E128 E124
|
||||
scrapy/http/response/text.py E501 E128
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501
|
||||
|
|
@ -99,7 +99,7 @@ flake8-ignore =
|
|||
scrapy/pipelines/__init__.py E501
|
||||
scrapy/pipelines/files.py E116 E501
|
||||
scrapy/pipelines/images.py E501
|
||||
scrapy/pipelines/media.py E125 E501
|
||||
scrapy/pipelines/media.py E501
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
|
|
@ -110,7 +110,7 @@ flake8-ignore =
|
|||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py E501 E129
|
||||
scrapy/spidermiddlewares/referer.py E501
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
|
|
@ -124,7 +124,7 @@ flake8-ignore =
|
|||
scrapy/utils/datatypes.py E501
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127
|
||||
scrapy/utils/deprecate.py E501
|
||||
scrapy/utils/gz.py E501
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/httpobj.py E501
|
||||
|
|
@ -137,7 +137,7 @@ flake8-ignore =
|
|||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E501
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/request.py E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
|
|
@ -169,10 +169,10 @@ flake8-ignore =
|
|||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/mockserver.py E501 E126
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/spiders.py E501
|
||||
tests/test_closespider.py E501
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128
|
||||
tests/test_command_shell.py E501 E128
|
||||
|
|
@ -181,34 +181,34 @@ flake8-ignore =
|
|||
tests/test_crawl.py E501 E741
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
|
||||
tests/test_downloader_handlers.py E128 E501 E126
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126
|
||||
tests/test_downloadermiddleware_decompression.py E501
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128 E127
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E501 E741 E128 E124
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E128 E124
|
||||
tests/test_dupefilters.py E501 E741 E128
|
||||
tests/test_engine.py E501 E128
|
||||
tests/test_exporters.py E501 E128
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
|
||||
tests/test_http_request.py E402 E501 E128 E128 E126
|
||||
tests/test_http_response.py E501 E128
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_linkextractors.py E501 E128
|
||||
tests/test_loader.py E501 E741 E128 E117
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_logformatter.py E128 E501
|
||||
tests/test_mail.py E128 E501
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E501 E128 E126
|
||||
|
|
@ -219,14 +219,14 @@ flake8-ignore =
|
|||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501
|
||||
tests/test_robotstxt_interface.py E501 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_scheduler.py E501 E126
|
||||
tests/test_selector.py E501
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E128
|
||||
|
|
@ -235,16 +235,16 @@ flake8-ignore =
|
|||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128
|
||||
tests/test_utils_iterators.py E501 E128 E129
|
||||
tests/test_utils_iterators.py E501 E128
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E123 E126
|
||||
tests/test_utils_sitemap.py E128 E501
|
||||
tests/test_utils_url.py E501 E501 E126
|
||||
tests/test_webclient.py E501 E128 E402 E126
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
|
|
|
|||
|
|
@ -94,11 +94,12 @@ class FTPDownloadHandler:
|
|||
def gotClient(self, client, request, filepath):
|
||||
self.client = client
|
||||
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
|
||||
return client.retrieveFile(filepath, protocol)\
|
||||
.addCallbacks(callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,))
|
||||
return client.retrieveFile(filepath, protocol).addCallbacks(
|
||||
callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,),
|
||||
)
|
||||
|
||||
def _build_response(self, result, request, protocol):
|
||||
self.result = result
|
||||
|
|
|
|||
|
|
@ -230,8 +230,7 @@ class ExecutionEngine:
|
|||
|
||||
def _downloaded(self, response, slot, request, spider):
|
||||
slot.remove_request(request)
|
||||
return self.download(response, spider) \
|
||||
if isinstance(response, Request) else response
|
||||
return self.download(response, spider) if isinstance(response, Request) else response
|
||||
|
||||
def _download(self, request, spider):
|
||||
slot = self.slot
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class CloseSpider:
|
|||
'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
|
||||
'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
|
||||
'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
|
||||
}
|
||||
}
|
||||
|
||||
if not any(self.close_on.values()):
|
||||
raise NotConfigured
|
||||
|
|
|
|||
|
|
@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
|
|||
if the latter is given. If not, it returns the first
|
||||
clickable element found
|
||||
"""
|
||||
clickables = [
|
||||
el for el in form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"})
|
||||
]
|
||||
clickables = list(form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"}
|
||||
))
|
||||
if not clickables:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -230,6 +230,20 @@ class GCSFilesStore:
|
|||
bucket, prefix = uri[5:].split('/', 1)
|
||||
self.bucket = client.bucket(bucket)
|
||||
self.prefix = prefix
|
||||
permissions = self.bucket.test_iam_permissions(
|
||||
['storage.objects.get', 'storage.objects.create']
|
||||
)
|
||||
if 'storage.objects.get' not in permissions:
|
||||
logger.warning(
|
||||
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
if 'storage.objects.create' not in permissions:
|
||||
logger.error(
|
||||
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _onsuccess(blob):
|
||||
|
|
|
|||
|
|
@ -43,8 +43,7 @@ class MediaPipeline:
|
|||
if allow_redirects:
|
||||
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
|
||||
|
||||
def _key_for_pipe(self, key, base_class_name=None,
|
||||
settings=None):
|
||||
def _key_for_pipe(self, key, base_class_name=None, settings=None):
|
||||
"""
|
||||
>>> MediaPipeline()._key_for_pipe("IMAGES")
|
||||
'IMAGES'
|
||||
|
|
@ -55,8 +54,11 @@ class MediaPipeline:
|
|||
"""
|
||||
class_name = self.__class__.__name__
|
||||
formatted_key = "{}_{}".format(class_name.upper(), key)
|
||||
if class_name == base_class_name or not base_class_name \
|
||||
or (settings and not settings.get(formatted_key)):
|
||||
if (
|
||||
not base_class_name
|
||||
or class_name == base_class_name
|
||||
or settings and not settings.get(formatted_key)
|
||||
):
|
||||
return key
|
||||
return formatted_key
|
||||
|
||||
|
|
|
|||
|
|
@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
|
|||
stacklevel=3)
|
||||
|
||||
|
||||
def create_deprecated_class(name, new_class, clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from "
|
||||
"deprecated class {old}, please inherit "
|
||||
"from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, "
|
||||
"instantiate {new} instead."):
|
||||
def create_deprecated_class(
|
||||
name,
|
||||
new_class,
|
||||
clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, instantiate {new} instead."
|
||||
):
|
||||
"""
|
||||
Return a "deprecated" class that causes its subclasses to issue a warning.
|
||||
Subclasses of ``new_class`` are considered subclasses of this class.
|
||||
|
|
|
|||
|
|
@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
|||
|
||||
"""
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower())
|
||||
for h in sorted(include_headers))
|
||||
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
|
|
|
|||
|
|
@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
|
|||
if self.fail_yielding:
|
||||
2 / 0
|
||||
|
||||
assert self.seedsseen, \
|
||||
'All start requests consumed before any download happened'
|
||||
assert self.seedsseen, 'All start requests consumed before any download happened'
|
||||
|
||||
def parse(self, response):
|
||||
self.seedsseen.append(response.meta.get('seed'))
|
||||
|
|
|
|||
|
|
@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
|
|||
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_errorcount')
|
||||
key = 'spider_exceptions/{name}'\
|
||||
.format(name=crawler.spider.exception_cls.__name__)
|
||||
key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
|
||||
errorcount = crawler.stats.get_value(key)
|
||||
self.assertTrue(errorcount >= close_on)
|
||||
|
||||
|
|
|
|||
|
|
@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
def test_ipv6_alternative_name_resolver(self):
|
||||
log = self.run_script('alternative_name_resolver.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
self.assertTrue(any([
|
||||
"twisted.internet.error.ConnectionRefusedError" in log,
|
||||
"twisted.internet.error.ConnectError" in log,
|
||||
]))
|
||||
self.assertTrue(any([
|
||||
"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
|
||||
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
|
||||
]))
|
||||
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
|
||||
|
||||
def test_reactor_select(self):
|
||||
log = self.run_script("twisted_reactor_select.py")
|
||||
|
|
|
|||
|
|
@ -822,11 +822,15 @@ class S3TestCase(unittest.TestCase):
|
|||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 21:15:45 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg',
|
||||
method='PUT',
|
||||
headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
},
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
|
|
@ -906,11 +910,10 @@ class S3TestCase(unittest.TestCase):
|
|||
# ensure that spaces are quoted properly before signing
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request(
|
||||
("s3://johnsmith/photos/my puppy.jpg"
|
||||
"?response-content-disposition=my puppy.jpg"),
|
||||
"s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
|
||||
method='GET',
|
||||
headers={'Date': date},
|
||||
)
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(
|
||||
|
|
@ -1090,8 +1093,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype_encoding(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, 'A brief note')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "US-ASCII")
|
||||
|
||||
request = Request("data:,A%20brief%20note")
|
||||
|
|
@ -1100,8 +1102,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "iso-8859-7")
|
||||
|
||||
request = Request("data:;charset=iso-8859-7,%be%d3%be")
|
||||
|
|
@ -1119,8 +1120,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_mediatype_parameters(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
|
||||
request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
|
|||
for fmt in self.test_formats:
|
||||
rsp = self.test_responses[fmt]
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert isinstance(new, XmlResponse), \
|
||||
'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
assert isinstance(new, XmlResponse), error_msg
|
||||
assert_samelines(self, new.body, self.uncompressed_body, fmt)
|
||||
|
||||
def test_plain_response(self):
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
|
|||
SAMPLEDIR = join(tests_datadir, 'compressed')
|
||||
|
||||
FORMAT = {
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br')
|
||||
}
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br'),
|
||||
}
|
||||
|
||||
|
||||
class HttpCompressionTest(TestCase):
|
||||
|
|
@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
|
|||
body = sample.read()
|
||||
|
||||
headers = {
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
|
||||
response = Response('http://scrapytest.org/', body=body, headers=headers)
|
||||
response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
|
||||
|
|
|
|||
|
|
@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
rsp = Response(url, headers={'Location': url2}, status=301, request=req)
|
||||
r = self.mw.process_response(req, rsp, self.spider)
|
||||
self.assertIs(r, rsp)
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list':
|
||||
[404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
|
||||
|
||||
def test_latin1_location(self):
|
||||
|
|
|
|||
|
|
@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html',
|
||||
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
|
||||
)
|
||||
headers={'Referer': 'http://scrapytest.org/INDEX.html'})
|
||||
|
||||
dupefilter.log(r1, spider)
|
||||
dupefilter.log(r2, spider)
|
||||
|
|
|
|||
|
|
@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
i2 = dict(name=u'bar', age=i1)
|
||||
i3 = TestItem(name=u'buz', age=i2)
|
||||
|
||||
self.assertExportResult(i3,
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>'
|
||||
b'<item>'
|
||||
b'<age>'
|
||||
b'<age>'
|
||||
b'<age>22</age>'
|
||||
b'<name>foo\xc2\xa3hoo</name>'
|
||||
b'</age>'
|
||||
b'<name>bar</name>'
|
||||
b'</age>'
|
||||
b'<name>buz</name>'
|
||||
b'</item>'
|
||||
b'</items>'
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<age>
|
||||
<age>
|
||||
<age>22</age>
|
||||
<name>foo\xc2\xa3hoo</name>
|
||||
</age>
|
||||
<name>bar</name>
|
||||
</age>
|
||||
<name>buz</name>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
def test_nested_list_item(self):
|
||||
|
|
@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
|
||||
i3 = TestItem(name=u'buz', age=[i1, i2])
|
||||
|
||||
self.assertExportResult(i3,
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>'
|
||||
b'<item>'
|
||||
b'<age>'
|
||||
b'<value><name>foo</name></value>'
|
||||
b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
|
||||
b'</age>'
|
||||
b'<name>buz</name>'
|
||||
b'</item>'
|
||||
b'</items>'
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<age>
|
||||
<value><name>foo</name></value>
|
||||
<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
|
||||
</age>
|
||||
<name>buz</name>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
def test_nonstring_types_item(self):
|
||||
item = self._get_nonstring_types_item()
|
||||
self.assertExportResult(item,
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>'
|
||||
b'<item>'
|
||||
b'<float>3.14</float>'
|
||||
b'<boolean>False</boolean>'
|
||||
b'<number>22</number>'
|
||||
b'<time>2015-01-01 01:01:01</time>'
|
||||
b'</item>'
|
||||
b'</items>'
|
||||
self.assertExportResult(
|
||||
item,
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<float>3.14</float>
|
||||
<boolean>False</boolean>
|
||||
<number>22</number>
|
||||
<time>2015-01-01 01:01:01</time>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_bytes(self):
|
||||
data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'latin1')
|
||||
self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
|
||||
|
|
@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_textual_data(self):
|
||||
data = {'price': u'£ 100'}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r3.encoding, 'latin1')
|
||||
self.assertEqual(r3.body, b'price=%A3+100')
|
||||
|
||||
|
|
@ -469,7 +467,7 @@ class FormRequestTest(RequestTest):
|
|||
</form>""",
|
||||
url="http://www.example.com/this/list.html",
|
||||
encoding='latin1',
|
||||
)
|
||||
)
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
|
||||
|
|
|
|||
|
|
@ -413,24 +413,30 @@ class Base:
|
|||
response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
|
||||
)
|
||||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
|
||||
)
|
||||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = b"""
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ class MockCrawler(Crawler):
|
|||
def __init__(self, priority_queue_cls, jobdir):
|
||||
|
||||
settings = dict(
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
|
||||
)
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
|
||||
)
|
||||
super(MockCrawler, self).__init__(Spider, settings)
|
||||
self.engine = MockEngine(downloader=MockDownloader())
|
||||
|
||||
|
|
@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
|
|||
class TestIntegrationWithDownloaderAwareInMemory(TestCase):
|
||||
def setUp(self):
|
||||
self.crawler = get_crawler(
|
||||
StartUrlsSpider,
|
||||
{'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
|
||||
)
|
||||
spidercls=StartUrlsSpider,
|
||||
settings_dict={
|
||||
'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
|
||||
},
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
|
|
@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
|
|||
|
||||
def _incompatible(self):
|
||||
settings = dict(
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1
|
||||
)
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1,
|
||||
)
|
||||
crawler = Crawler(Spider, settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = Spider(name='spider')
|
||||
|
|
|
|||
|
|
@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
|
|||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = Selector(response)
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
|
||||
[u'\xa3'])
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
|
|
|
|||
|
|
@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
|
|||
self.mw.process_spider_input(self.res402, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
self.mw.process_spider_input(self.res404, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
|
|||
|
|
@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase):
|
|||
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase):
|
|||
# with bytes
|
||||
XmlResponse(url="http://example.com", body=body.encode('utf-8')),
|
||||
# Unicode body needs encoding information
|
||||
XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
|
||||
|
||||
XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
|
||||
):
|
||||
attrs = []
|
||||
for x in self.xmliter(r, u'þingflokkur'):
|
||||
attrs.append((x.attrib['id'],
|
||||
|
|
|
|||
|
|
@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</url>
|
||||
</urlset>
|
||||
""")
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_wrong_ns(self):
|
||||
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
|
||||
|
|
@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</url>
|
||||
</urlset>
|
||||
""")
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_wrong_ns2(self):
|
||||
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
|
||||
|
|
@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
|
|||
</urlset>
|
||||
""")
|
||||
assert s.type == 'urlset'
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
]
|
||||
)
|
||||
|
||||
def test_sitemap_urls_from_robots(self):
|
||||
robots = """User-agent: *
|
||||
|
|
@ -195,11 +204,19 @@ Disallow: /forum/active/
|
|||
</url>
|
||||
</urlset>""")
|
||||
|
||||
self.assertEqual(list(s), [
|
||||
{'loc': 'http://www.example.com/english/',
|
||||
'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
|
||||
}
|
||||
])
|
||||
self.assertEqual(
|
||||
list(s),
|
||||
[
|
||||
{
|
||||
'loc': 'http://www.example.com/english/',
|
||||
'alternate': [
|
||||
'http://www.example.com/deutsch/',
|
||||
'http://www.example.com/schweiz-deutsch/',
|
||||
'http://www.example.com/english/',
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
def test_xml_entity_expansion(self):
|
||||
s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
|
|
|
|||
|
|
@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
class AddHttpIfNoScheme(unittest.TestCase):
|
||||
|
||||
def test_add_scheme(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
|
||||
|
||||
def test_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_http(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_preserve_http_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_preserve_http_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_preserve_http_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_preserve_http_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_preserve_http_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_preserve_http_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_protocol_relative(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_protocol_relative_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//example.com'), 'http://example.com')
|
||||
|
||||
def test_protocol_relative_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_protocol_relative_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_protocol_relative_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_protocol_relative_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_protocol_relative_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_https(self):
|
||||
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
|
||||
def test_preserve_ftp(self):
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
|
||||
'ftp://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
|
||||
|
||||
|
||||
class GuessSchemeTest(unittest.TestCase):
|
||||
|
|
@ -202,41 +218,49 @@ def create_skipped_scheme_t(args):
|
|||
return do_expected
|
||||
|
||||
|
||||
for k, args in enumerate([
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
for k, args in enumerate(
|
||||
[
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
|
||||
], start=1):
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_guess_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
# TODO: the following tests do not pass with current implementation
|
||||
for k, args in enumerate([
|
||||
(r'C:\absolute\path\to\a\file.html', 'file://',
|
||||
'Windows filepath are not supported for scrapy shell'),
|
||||
], start=1):
|
||||
for k, args in enumerate(
|
||||
[
|
||||
(
|
||||
r'C:\absolute\path\to\a\file.html',
|
||||
'file://',
|
||||
'Windows filepath are not supported for scrapy shell',
|
||||
),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_skipped_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_skipped_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
|
@ -272,7 +296,7 @@ class StripUrl(unittest.TestCase):
|
|||
('http://www.example.com',
|
||||
True,
|
||||
'http://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
|
||||
|
||||
def test_credentials(self):
|
||||
|
|
@ -285,7 +309,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_credentials_encoded_delims(self):
|
||||
|
|
@ -304,7 +328,7 @@ class StripUrl(unittest.TestCase):
|
|||
# password: "user@domain.com"
|
||||
('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_default_ports_creds_off(self):
|
||||
|
|
@ -332,7 +356,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i), o)
|
||||
|
||||
def test_default_ports(self):
|
||||
|
|
@ -360,7 +384,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
|
|
@ -388,7 +412,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
|
||||
|
||||
def test_origin_only(self):
|
||||
|
|
@ -404,7 +428,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, origin_only=True), o)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -53,29 +53,29 @@ class ParseUrlTestCase(unittest.TestCase):
|
|||
def testParse(self):
|
||||
lip = '127.0.0.1'
|
||||
tests = (
|
||||
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
|
||||
|
||||
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
|
||||
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
|
||||
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
|
||||
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
|
||||
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
|
||||
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
|
||||
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
|
||||
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
|
||||
|
||||
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
|
||||
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
|
||||
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
|
||||
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
|
||||
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
|
||||
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
|
||||
|
||||
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
|
||||
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
|
||||
)
|
||||
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
|
||||
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
|
||||
)
|
||||
|
||||
for url, test in tests:
|
||||
test = tuple(
|
||||
|
|
@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
},
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})))
|
||||
}),
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
|
|||
Loading…
Reference in New Issue