mirror of https://github.com/scrapy/scrapy.git
1118 lines
37 KiB
Python
1118 lines
37 KiB
Python
from __future__ import annotations
|
|
|
|
import warnings
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
import pytest
|
|
|
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
from scrapy.http import Request, Response
|
|
from scrapy.settings import Settings
|
|
from scrapy.spidermiddlewares.referer import (
|
|
POLICY_NO_REFERRER,
|
|
POLICY_NO_REFERRER_WHEN_DOWNGRADE,
|
|
POLICY_ORIGIN,
|
|
POLICY_ORIGIN_WHEN_CROSS_ORIGIN,
|
|
POLICY_SAME_ORIGIN,
|
|
POLICY_SCRAPY_DEFAULT,
|
|
POLICY_STRICT_ORIGIN,
|
|
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
|
|
POLICY_UNSAFE_URL,
|
|
DefaultReferrerPolicy,
|
|
NoReferrerPolicy,
|
|
NoReferrerWhenDowngradePolicy,
|
|
OriginPolicy,
|
|
OriginWhenCrossOriginPolicy,
|
|
RefererMiddleware,
|
|
ReferrerPolicy,
|
|
SameOriginPolicy,
|
|
StrictOriginPolicy,
|
|
StrictOriginWhenCrossOriginPolicy,
|
|
UnsafeUrlPolicy,
|
|
)
|
|
from scrapy.utils.misc import build_from_crawler
|
|
from scrapy.utils.test import get_crawler
|
|
from tests.utils.decorators import coroutine_test
|
|
|
|
|
|
class TestRefererMiddleware:
|
|
req_meta: dict[str, Any] = {}
|
|
resp_headers: dict[str, str] = {}
|
|
settings: dict[str, Any] = {}
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
|
|
]
|
|
|
|
@pytest.fixture
|
|
def mw(self) -> RefererMiddleware:
|
|
settings = Settings(self.settings)
|
|
return RefererMiddleware(settings)
|
|
|
|
def get_request(self, target: str) -> Request:
|
|
return Request(target, meta=self.req_meta)
|
|
|
|
def get_response(self, origin: str) -> Response:
|
|
return Response(origin, headers=self.resp_headers)
|
|
|
|
def test(self, mw: RefererMiddleware) -> None:
|
|
for origin, target, referrer in self.scenarii:
|
|
response = self.get_response(origin)
|
|
request = self.get_request(target)
|
|
out = list(mw.process_spider_output(response, [request]))
|
|
assert out[0].headers.get("Referer") == referrer
|
|
|
|
|
|
class MixinDefault:
|
|
"""
|
|
Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
|
|
|
|
with some additional filtering of s3://
|
|
"""
|
|
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
("https://example.com/", "https://scrapy.org/", b"https://example.com/"),
|
|
("http://example.com/", "http://scrapy.org/", b"http://example.com/"),
|
|
("http://example.com/", "https://scrapy.org/", b"http://example.com/"),
|
|
("https://example.com/", "http://scrapy.org/", None),
|
|
# no credentials leak
|
|
(
|
|
"http://user:password@example.com/",
|
|
"https://scrapy.org/",
|
|
b"http://example.com/",
|
|
),
|
|
# no referrer leak for local schemes
|
|
("file:///home/path/to/somefile.html", "https://scrapy.org/", None),
|
|
("file:///home/path/to/somefile.html", "http://scrapy.org/", None),
|
|
# no referrer leak for s3 origins
|
|
("s3://mybucket/path/to/data.csv", "https://scrapy.org/", None),
|
|
("s3://mybucket/path/to/data.csv", "http://scrapy.org/", None),
|
|
]
|
|
|
|
|
|
class MixinNoReferrer:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
("https://example.com/page.html", "https://example.com/", None),
|
|
("http://www.example.com/", "https://scrapy.org/", None),
|
|
("http://www.example.com/", "http://scrapy.org/", None),
|
|
("https://www.example.com/", "http://scrapy.org/", None),
|
|
("file:///home/path/to/somefile.html", "http://scrapy.org/", None),
|
|
]
|
|
|
|
|
|
class MixinNoReferrerWhenDowngrade:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# TLS to TLS: send non-empty referrer
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://not.example.com/",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://scrapy.org/",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"https://example.com:443/page.html",
|
|
"https://scrapy.org/",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"https://example.com:444/page.html",
|
|
"https://scrapy.org/",
|
|
b"https://example.com:444/page.html",
|
|
),
|
|
(
|
|
"ftps://example.com/urls.zip",
|
|
"https://scrapy.org/",
|
|
b"ftps://example.com/urls.zip",
|
|
),
|
|
# TLS to non-TLS: do not send referrer
|
|
("https://example.com/page.html", "http://not.example.com/", None),
|
|
("https://example.com/page.html", "http://scrapy.org/", None),
|
|
("ftps://example.com/urls.zip", "http://scrapy.org/", None),
|
|
# non-TLS to TLS or non-TLS: send referrer
|
|
(
|
|
"http://example.com/page.html",
|
|
"https://not.example.com/",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"https://scrapy.org/",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:8080/page.html",
|
|
"https://scrapy.org/",
|
|
b"http://example.com:8080/page.html",
|
|
),
|
|
(
|
|
"http://example.com:80/page.html",
|
|
"http://not.example.com/",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://scrapy.org/",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:443/page.html",
|
|
"http://scrapy.org/",
|
|
b"http://example.com:443/page.html",
|
|
),
|
|
(
|
|
"ftp://example.com/urls.zip",
|
|
"http://scrapy.org/",
|
|
b"ftp://example.com/urls.zip",
|
|
),
|
|
(
|
|
"ftp://example.com/urls.zip",
|
|
"https://scrapy.org/",
|
|
b"ftp://example.com/urls.zip",
|
|
),
|
|
# test for user/password stripping
|
|
(
|
|
"http://user:password@example.com/page.html",
|
|
"https://not.example.com/",
|
|
b"http://example.com/page.html",
|
|
),
|
|
]
|
|
|
|
|
|
class MixinSameOrigin:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# Same origin (protocol, host, port): send referrer
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://example.com/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"https://example.com:443/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:80/page.html",
|
|
"http://example.com/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://example.com:80/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:8888/page.html",
|
|
"http://example.com:8888/not-page.html",
|
|
b"http://example.com:8888/page.html",
|
|
),
|
|
# Different host: do NOT send referrer
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://not.example.com/otherpage.html",
|
|
None,
|
|
),
|
|
("http://example.com/page.html", "http://not.example.com/otherpage.html", None),
|
|
("http://example.com/page.html", "http://www.example.com/otherpage.html", None),
|
|
# Different port: do NOT send referrer
|
|
(
|
|
"https://example.com:444/page.html",
|
|
"https://example.com/not-page.html",
|
|
None,
|
|
),
|
|
("http://example.com:81/page.html", "http://example.com/not-page.html", None),
|
|
("http://example.com/page.html", "http://example.com:81/not-page.html", None),
|
|
# Different protocols: do NOT send referrer
|
|
("https://example.com/page.html", "http://example.com/not-page.html", None),
|
|
("https://example.com/page.html", "http://not.example.com/", None),
|
|
("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
|
|
("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),
|
|
("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
|
|
# test for user/password stripping
|
|
(
|
|
"https://user:password@example.com/page.html",
|
|
"http://example.com/not-page.html",
|
|
None,
|
|
),
|
|
(
|
|
"https://user:password@example.com/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
]
|
|
|
|
|
|
class MixinOrigin:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/",
|
|
),
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://scrapy.org",
|
|
b"https://example.com/",
|
|
),
|
|
("https://example.com/page.html", "http://scrapy.org", b"https://example.com/"),
|
|
("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),
|
|
# test for user/password stripping
|
|
(
|
|
"https://user:password@example.com/page.html",
|
|
"http://scrapy.org",
|
|
b"https://example.com/",
|
|
),
|
|
]
|
|
|
|
|
|
class MixinStrictOrigin:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/",
|
|
),
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://scrapy.org",
|
|
b"https://example.com/",
|
|
),
|
|
("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),
|
|
# downgrade: send nothing
|
|
("https://example.com/page.html", "http://scrapy.org", None),
|
|
# upgrade: send origin
|
|
("http://example.com/page.html", "https://scrapy.org", b"http://example.com/"),
|
|
# test for user/password stripping
|
|
(
|
|
"https://user:password@example.com/page.html",
|
|
"https://scrapy.org",
|
|
b"https://example.com/",
|
|
),
|
|
("https://user:password@example.com/page.html", "http://scrapy.org", None),
|
|
]
|
|
|
|
|
|
class MixinOriginWhenCrossOrigin:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# Same origin (protocol, host, port): send referrer
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://example.com/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"https://example.com:443/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:80/page.html",
|
|
"http://example.com/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://example.com:80/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:8888/page.html",
|
|
"http://example.com:8888/not-page.html",
|
|
b"http://example.com:8888/page.html",
|
|
),
|
|
# Different host: send origin as referrer
|
|
(
|
|
"https://example2.com/page.html",
|
|
"https://scrapy.org/otherpage.html",
|
|
b"https://example2.com/",
|
|
),
|
|
(
|
|
"https://example2.com/page.html",
|
|
"https://not.example2.com/otherpage.html",
|
|
b"https://example2.com/",
|
|
),
|
|
(
|
|
"http://example2.com/page.html",
|
|
"http://not.example2.com/otherpage.html",
|
|
b"http://example2.com/",
|
|
),
|
|
# exact match required
|
|
(
|
|
"http://example2.com/page.html",
|
|
"http://www.example2.com/otherpage.html",
|
|
b"http://example2.com/",
|
|
),
|
|
# Different port: send origin as referrer
|
|
(
|
|
"https://example3.com:444/page.html",
|
|
"https://example3.com/not-page.html",
|
|
b"https://example3.com:444/",
|
|
),
|
|
(
|
|
"http://example3.com:81/page.html",
|
|
"http://example3.com/not-page.html",
|
|
b"http://example3.com:81/",
|
|
),
|
|
# Different protocols: send origin as referrer
|
|
(
|
|
"https://example4.com/page.html",
|
|
"http://example4.com/not-page.html",
|
|
b"https://example4.com/",
|
|
),
|
|
(
|
|
"https://example4.com/page.html",
|
|
"http://not.example4.com/",
|
|
b"https://example4.com/",
|
|
),
|
|
(
|
|
"ftps://example4.com/urls.zip",
|
|
"https://example4.com/not-page.html",
|
|
b"ftps://example4.com/",
|
|
),
|
|
(
|
|
"ftp://example4.com/urls.zip",
|
|
"http://example4.com/not-page.html",
|
|
b"ftp://example4.com/",
|
|
),
|
|
(
|
|
"ftps://example4.com/urls.zip",
|
|
"https://example4.com/not-page.html",
|
|
b"ftps://example4.com/",
|
|
),
|
|
# test for user/password stripping
|
|
(
|
|
"https://user:password@example5.com/page.html",
|
|
"https://example5.com/not-page.html",
|
|
b"https://example5.com/page.html",
|
|
),
|
|
# TLS to non-TLS downgrade: send origin
|
|
(
|
|
"https://user:password@example5.com/page.html",
|
|
"http://example5.com/not-page.html",
|
|
b"https://example5.com/",
|
|
),
|
|
]
|
|
|
|
|
|
class MixinStrictOriginWhenCrossOrigin:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# Same origin (protocol, host, port): send referrer
|
|
(
|
|
"https://example.com/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://example.com/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"https://example.com:443/page.html",
|
|
"https://example.com/not-page.html",
|
|
b"https://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:80/page.html",
|
|
"http://example.com/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com/page.html",
|
|
"http://example.com:80/not-page.html",
|
|
b"http://example.com/page.html",
|
|
),
|
|
(
|
|
"http://example.com:8888/page.html",
|
|
"http://example.com:8888/not-page.html",
|
|
b"http://example.com:8888/page.html",
|
|
),
|
|
# Different host: send origin as referrer
|
|
(
|
|
"https://example2.com/page.html",
|
|
"https://scrapy.org/otherpage.html",
|
|
b"https://example2.com/",
|
|
),
|
|
(
|
|
"https://example2.com/page.html",
|
|
"https://not.example2.com/otherpage.html",
|
|
b"https://example2.com/",
|
|
),
|
|
(
|
|
"http://example2.com/page.html",
|
|
"http://not.example2.com/otherpage.html",
|
|
b"http://example2.com/",
|
|
),
|
|
# exact match required
|
|
(
|
|
"http://example2.com/page.html",
|
|
"http://www.example2.com/otherpage.html",
|
|
b"http://example2.com/",
|
|
),
|
|
# Different port: send origin as referrer
|
|
(
|
|
"https://example3.com:444/page.html",
|
|
"https://example3.com/not-page.html",
|
|
b"https://example3.com:444/",
|
|
),
|
|
(
|
|
"http://example3.com:81/page.html",
|
|
"http://example3.com/not-page.html",
|
|
b"http://example3.com:81/",
|
|
),
|
|
# downgrade
|
|
("https://example4.com/page.html", "http://example4.com/not-page.html", None),
|
|
("https://example4.com/page.html", "http://not.example4.com/", None),
|
|
# non-TLS to non-TLS
|
|
(
|
|
"ftp://example4.com/urls.zip",
|
|
"http://example4.com/not-page.html",
|
|
b"ftp://example4.com/",
|
|
),
|
|
# upgrade
|
|
(
|
|
"http://example4.com/page.html",
|
|
"https://example4.com/not-page.html",
|
|
b"http://example4.com/",
|
|
),
|
|
(
|
|
"http://example4.com/page.html",
|
|
"https://not.example4.com/",
|
|
b"http://example4.com/",
|
|
),
|
|
# Different protocols: send origin as referrer
|
|
(
|
|
"ftps://example4.com/urls.zip",
|
|
"https://example4.com/not-page.html",
|
|
b"ftps://example4.com/",
|
|
),
|
|
(
|
|
"ftps://example4.com/urls.zip",
|
|
"https://example4.com/not-page.html",
|
|
b"ftps://example4.com/",
|
|
),
|
|
# test for user/password stripping
|
|
(
|
|
"https://user:password@example5.com/page.html",
|
|
"https://example5.com/not-page.html",
|
|
b"https://example5.com/page.html",
|
|
),
|
|
# TLS to non-TLS downgrade: send nothing
|
|
(
|
|
"https://user:password@example5.com/page.html",
|
|
"http://example5.com/not-page.html",
|
|
None,
|
|
),
|
|
]
|
|
|
|
|
|
class MixinUnsafeUrl:
|
|
scenarii: list[tuple[str, str, bytes | None]] = [
|
|
# TLS to TLS: send referrer
|
|
(
|
|
"https://example.com/sekrit.html",
|
|
"http://not.example.com/",
|
|
b"https://example.com/sekrit.html",
|
|
),
|
|
(
|
|
"https://example1.com/page.html",
|
|
"https://not.example1.com/",
|
|
b"https://example1.com/page.html",
|
|
),
|
|
(
|
|
"https://example1.com/page.html",
|
|
"https://scrapy.org/",
|
|
b"https://example1.com/page.html",
|
|
),
|
|
(
|
|
"https://example1.com:443/page.html",
|
|
"https://scrapy.org/",
|
|
b"https://example1.com/page.html",
|
|
),
|
|
(
|
|
"https://example1.com:444/page.html",
|
|
"https://scrapy.org/",
|
|
b"https://example1.com:444/page.html",
|
|
),
|
|
(
|
|
"ftps://example1.com/urls.zip",
|
|
"https://scrapy.org/",
|
|
b"ftps://example1.com/urls.zip",
|
|
),
|
|
# TLS to non-TLS: send referrer (yes, it's unsafe)
|
|
(
|
|
"https://example2.com/page.html",
|
|
"http://not.example2.com/",
|
|
b"https://example2.com/page.html",
|
|
),
|
|
(
|
|
"https://example2.com/page.html",
|
|
"http://scrapy.org/",
|
|
b"https://example2.com/page.html",
|
|
),
|
|
(
|
|
"ftps://example2.com/urls.zip",
|
|
"http://scrapy.org/",
|
|
b"ftps://example2.com/urls.zip",
|
|
),
|
|
# non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
|
|
(
|
|
"http://example3.com/page.html",
|
|
"https://not.example3.com/",
|
|
b"http://example3.com/page.html",
|
|
),
|
|
(
|
|
"http://example3.com/page.html",
|
|
"https://scrapy.org/",
|
|
b"http://example3.com/page.html",
|
|
),
|
|
(
|
|
"http://example3.com:8080/page.html",
|
|
"https://scrapy.org/",
|
|
b"http://example3.com:8080/page.html",
|
|
),
|
|
(
|
|
"http://example3.com:80/page.html",
|
|
"http://not.example3.com/",
|
|
b"http://example3.com/page.html",
|
|
),
|
|
(
|
|
"http://example3.com/page.html",
|
|
"http://scrapy.org/",
|
|
b"http://example3.com/page.html",
|
|
),
|
|
(
|
|
"http://example3.com:443/page.html",
|
|
"http://scrapy.org/",
|
|
b"http://example3.com:443/page.html",
|
|
),
|
|
(
|
|
"ftp://example3.com/urls.zip",
|
|
"http://scrapy.org/",
|
|
b"ftp://example3.com/urls.zip",
|
|
),
|
|
(
|
|
"ftp://example3.com/urls.zip",
|
|
"https://scrapy.org/",
|
|
b"ftp://example3.com/urls.zip",
|
|
),
|
|
# test for user/password stripping
|
|
(
|
|
"http://user:password@example4.com/page.html",
|
|
"https://not.example4.com/",
|
|
b"http://example4.com/page.html",
|
|
),
|
|
(
|
|
"https://user:password@example4.com/page.html",
|
|
"http://scrapy.org/",
|
|
b"https://example4.com/page.html",
|
|
),
|
|
]
|
|
|
|
|
|
class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
|
|
pass
|
|
|
|
|
|
# --- Tests using settings to set policy using class path
|
|
class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerPolicy"}
|
|
|
|
|
|
class TestSettingsNoReferrerWhenDowngrade(
|
|
MixinNoReferrerWhenDowngrade, TestRefererMiddleware
|
|
):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
|
|
}
|
|
|
|
|
|
class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
|
|
|
|
|
|
class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginPolicy"}
|
|
|
|
|
|
class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginPolicy"
|
|
}
|
|
|
|
|
|
class TestSettingsOriginWhenCrossOrigin(
|
|
MixinOriginWhenCrossOrigin, TestRefererMiddleware
|
|
):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
|
|
}
|
|
|
|
|
|
class TestSettingsStrictOriginWhenCrossOrigin(
|
|
MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware
|
|
):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy"
|
|
}
|
|
|
|
|
|
class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
|
|
|
|
|
|
class CustomPythonOrgPolicy(ReferrerPolicy):
|
|
"""
|
|
A dummy policy that returns referrer as http(s)://python.org
|
|
depending on the scheme of the target URL.
|
|
"""
|
|
|
|
def referrer(self, response, request):
|
|
scheme = urlparse(request).scheme
|
|
if scheme == "https":
|
|
return b"https://python.org/"
|
|
if scheme == "http":
|
|
return b"http://python.org/"
|
|
return None
|
|
|
|
|
|
class TestSettingsCustomPolicy(TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": CustomPythonOrgPolicy}
|
|
scenarii = [
|
|
("https://example.com/", "https://scrapy.org/", b"https://python.org/"),
|
|
("http://example.com/", "http://scrapy.org/", b"http://python.org/"),
|
|
("http://example.com/", "https://scrapy.org/", b"https://python.org/"),
|
|
("https://example.com/", "http://scrapy.org/", b"http://python.org/"),
|
|
(
|
|
"file:///home/path/to/somefile.html",
|
|
"https://scrapy.org/",
|
|
b"https://python.org/",
|
|
),
|
|
(
|
|
"file:///home/path/to/somefile.html",
|
|
"http://scrapy.org/",
|
|
b"http://python.org/",
|
|
),
|
|
]
|
|
|
|
|
|
# --- Tests using Request meta dict to set policy
|
|
class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
|
|
req_meta = {"referrer_policy": POLICY_SCRAPY_DEFAULT}
|
|
|
|
|
|
class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):
|
|
req_meta = {"referrer_policy": POLICY_NO_REFERRER}
|
|
|
|
|
|
class TestRequestMetaNoReferrerWhenDowngrade(
|
|
MixinNoReferrerWhenDowngrade, TestRefererMiddleware
|
|
):
|
|
req_meta = {"referrer_policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE}
|
|
|
|
|
|
class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):
|
|
req_meta = {"referrer_policy": POLICY_SAME_ORIGIN}
|
|
|
|
|
|
class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):
|
|
req_meta = {"referrer_policy": POLICY_ORIGIN}
|
|
|
|
|
|
class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
|
|
req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN}
|
|
|
|
|
|
class TestRequestMetaOriginWhenCrossOrigin(
|
|
MixinOriginWhenCrossOrigin, TestRefererMiddleware
|
|
):
|
|
req_meta = {"referrer_policy": POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
|
|
|
|
|
|
class TestRequestMetaStrictOriginWhenCrossOrigin(
|
|
MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware
|
|
):
|
|
req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
|
|
|
|
|
|
class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
|
|
req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
|
|
|
|
|
|
class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
|
|
req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
|
|
|
|
|
|
class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
|
|
}
|
|
req_meta = {"referrer_policy": POLICY_NO_REFERRER}
|
|
|
|
|
|
class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
|
|
}
|
|
req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
|
|
|
|
|
|
class TestRequestMetaSettingFallback:
|
|
params = [
|
|
(
|
|
# When an unknown policy is referenced in Request.meta
|
|
# (here, a typo error),
|
|
# the policy defined in settings takes precedence
|
|
{
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
|
|
},
|
|
{},
|
|
{"referrer_policy": "ssscrapy-default"},
|
|
OriginWhenCrossOriginPolicy,
|
|
True,
|
|
),
|
|
(
|
|
# same as above but with string value for settings policy
|
|
{"REFERRER_POLICY": "origin-when-cross-origin"},
|
|
{},
|
|
{"referrer_policy": "ssscrapy-default"},
|
|
OriginWhenCrossOriginPolicy,
|
|
True,
|
|
),
|
|
(
|
|
# request meta references a wrong policy but it is set,
|
|
# so the Referrer-Policy header in response is not used,
|
|
# and the settings' policy is applied
|
|
{"REFERRER_POLICY": "origin-when-cross-origin"},
|
|
{"Referrer-Policy": "unsafe-url"},
|
|
{"referrer_policy": "ssscrapy-default"},
|
|
OriginWhenCrossOriginPolicy,
|
|
True,
|
|
),
|
|
(
|
|
# here, request meta does not set the policy
|
|
# so response headers take precedence
|
|
{"REFERRER_POLICY": "origin-when-cross-origin"},
|
|
{"Referrer-Policy": "unsafe-url"},
|
|
{},
|
|
UnsafeUrlPolicy,
|
|
False,
|
|
),
|
|
(
|
|
# here, request meta does not set the policy,
|
|
# but response headers also use an unknown policy,
|
|
# so the settings' policy is used
|
|
{"REFERRER_POLICY": "origin-when-cross-origin"},
|
|
{"Referrer-Policy": "unknown"},
|
|
{},
|
|
OriginWhenCrossOriginPolicy,
|
|
True,
|
|
),
|
|
]
|
|
|
|
def test(self):
|
|
origin = "http://www.scrapy.org"
|
|
target = "http://www.example.com"
|
|
|
|
for (
|
|
settings,
|
|
response_headers,
|
|
request_meta,
|
|
policy_class,
|
|
check_warning,
|
|
) in self.params:
|
|
mw = RefererMiddleware(Settings(settings))
|
|
|
|
response = Response(origin, headers=response_headers)
|
|
request = Request(target, meta=request_meta)
|
|
|
|
if check_warning:
|
|
with pytest.warns(
|
|
RuntimeWarning, match="Could not load referrer policy"
|
|
):
|
|
policy = mw.policy(response, request)
|
|
else:
|
|
policy = mw.policy(response, request)
|
|
assert isinstance(policy, policy_class)
|
|
|
|
|
|
class TestSettingsPolicyByName:
|
|
def test_valid_name(self):
|
|
for s, p in [
|
|
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
|
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
|
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
|
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
|
(POLICY_ORIGIN, OriginPolicy),
|
|
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
|
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
|
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
|
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
|
]:
|
|
settings = Settings({"REFERRER_POLICY": s})
|
|
mw = RefererMiddleware(settings)
|
|
assert mw.default_policy == p
|
|
|
|
def test_valid_name_casevariants(self):
|
|
for s, p in [
|
|
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
|
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
|
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
|
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
|
(POLICY_ORIGIN, OriginPolicy),
|
|
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
|
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
|
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
|
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
|
]:
|
|
settings = Settings({"REFERRER_POLICY": s.upper()})
|
|
mw = RefererMiddleware(settings)
|
|
assert mw.default_policy == p
|
|
|
|
def test_invalid_name(self):
|
|
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
|
|
with pytest.raises(RuntimeError):
|
|
RefererMiddleware(settings)
|
|
|
|
def test_multiple_policy_tokens(self):
|
|
# test parsing without space(s) after the comma
|
|
settings1 = Settings(
|
|
{
|
|
"REFERRER_POLICY": (
|
|
f"some-custom-unknown-policy,"
|
|
f"{POLICY_SAME_ORIGIN},"
|
|
f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN},"
|
|
f"another-custom-unknown-policy"
|
|
)
|
|
}
|
|
)
|
|
mw1 = RefererMiddleware(settings1)
|
|
assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy
|
|
|
|
# test parsing with space(s) after the comma
|
|
settings2 = Settings(
|
|
{
|
|
"REFERRER_POLICY": (
|
|
f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN},"
|
|
f" another-custom-unknown-policy,"
|
|
f" {POLICY_UNSAFE_URL}"
|
|
)
|
|
}
|
|
)
|
|
mw2 = RefererMiddleware(settings2)
|
|
assert mw2.default_policy == UnsafeUrlPolicy
|
|
|
|
def test_multiple_policy_tokens_all_invalid(self):
|
|
settings = Settings(
|
|
{
|
|
"REFERRER_POLICY": (
|
|
"some-custom-unknown-policy,"
|
|
"another-custom-unknown-policy,"
|
|
"yet-another-custom-unknown-policy"
|
|
)
|
|
}
|
|
)
|
|
with pytest.raises(RuntimeError):
|
|
RefererMiddleware(settings)
|
|
|
|
|
|
class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
|
|
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
|
|
resp_headers = {"Referrer-Policy": POLICY_UNSAFE_URL.upper()}
|
|
|
|
|
|
class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
|
|
}
|
|
resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER.swapcase()}
|
|
|
|
|
|
class TestPolicyHeaderPrecedence003(
|
|
MixinNoReferrerWhenDowngrade, TestRefererMiddleware
|
|
):
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
|
|
}
|
|
resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
|
|
|
|
|
|
class TestPolicyHeaderPrecedence004(
|
|
MixinNoReferrerWhenDowngrade, TestRefererMiddleware
|
|
):
|
|
"""
|
|
The empty string means "no-referrer-when-downgrade"
|
|
"""
|
|
|
|
settings = {
|
|
"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
|
|
}
|
|
resp_headers = {"Referrer-Policy": ""}
|
|
|
|
|
|
class TestPolicyMethodResponseParamRename:
|
|
def setup_method(self):
|
|
self.crawler = get_crawler()
|
|
self.mw = build_from_crawler(RefererMiddleware, self.crawler)
|
|
self.request = Request("http://www.example.com")
|
|
self.response = Response("http://www.example.com")
|
|
|
|
def test_pos_string(self):
|
|
with pytest.warns(
|
|
ScrapyDeprecationWarning,
|
|
match=r"Passing a response URL to RefererMiddleware\.policy\(\)",
|
|
):
|
|
self.mw.policy("http://old.com", self.request)
|
|
|
|
def test_pos_response(self):
|
|
with warnings.catch_warnings():
|
|
warnings.filterwarnings(
|
|
"error",
|
|
category=ScrapyDeprecationWarning,
|
|
message=r"Passing 'resp_or_url' is deprecated",
|
|
)
|
|
self.mw.policy(self.response, self.request)
|
|
|
|
def test_key_resp_or_url(self):
|
|
with pytest.warns(
|
|
ScrapyDeprecationWarning, match=r"Passing 'resp_or_url' is deprecated"
|
|
):
|
|
self.mw.policy(resp_or_url=self.response, request=self.request)
|
|
|
|
def test_key_response(self):
|
|
with warnings.catch_warnings():
|
|
warnings.filterwarnings(
|
|
"error",
|
|
category=ScrapyDeprecationWarning,
|
|
message=r"Passing 'resp_or_url' is deprecated",
|
|
)
|
|
self.mw.policy(response=self.response, request=self.request)
|
|
|
|
def test_key_response_string(self):
|
|
with pytest.warns(ScrapyDeprecationWarning, match="Passing a response URL"):
|
|
self.mw.policy(response="http://old.com", request=self.request)
|
|
|
|
def test_both_resp_or_url_and_response(self):
|
|
with pytest.raises(
|
|
TypeError, match="Cannot pass both 'response' and 'resp_or_url'"
|
|
):
|
|
self.mw.policy(
|
|
response=self.response, resp_or_url=self.response, request=self.request
|
|
)
|
|
|
|
|
|
@coroutine_test
|
|
async def test_response_policy_only_supports_policy_names():
|
|
crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})
|
|
mw = build_from_crawler(RefererMiddleware, crawler)
|
|
|
|
async def input_result():
|
|
yield Request("https://example.com/")
|
|
|
|
response = Response(
|
|
"https://example.com/",
|
|
headers={
|
|
"Referrer-Policy": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
|
|
},
|
|
)
|
|
with pytest.warns(
|
|
RuntimeWarning,
|
|
match=r"Could not load referrer policy 'scrapy\.spidermiddlewares\.referer\.NoReferrerWhenDowngradePolicy' \(import paths from the response Referrer-Policy header are not allowed\)",
|
|
):
|
|
output = [
|
|
request
|
|
async for request in mw.process_spider_output_async(
|
|
response, input_result()
|
|
)
|
|
]
|
|
assert len(output) == 1
|
|
assert b"Referer" not in output[0].headers
|
|
|
|
response = Response(
|
|
"https://example.com/",
|
|
headers={"Referrer-Policy": "no-referrer-when-downgrade"},
|
|
)
|
|
output = [
|
|
request
|
|
async for request in mw.process_spider_output_async(response, input_result())
|
|
]
|
|
assert len(output) == 1
|
|
assert output[0].headers == {b"Referer": [b"https://example.com/"]}
|
|
|
|
|
|
@coroutine_test
|
|
async def test_referer_policies_setting():
|
|
crawler = get_crawler(
|
|
settings_dict={
|
|
"REFERRER_POLICY": "no-referrer",
|
|
"REFERRER_POLICIES": {
|
|
"no-referrer-when-downgrade": None,
|
|
"custom-policy": CustomPythonOrgPolicy,
|
|
"": CustomPythonOrgPolicy,
|
|
},
|
|
}
|
|
)
|
|
mw = build_from_crawler(RefererMiddleware, crawler)
|
|
|
|
async def input_result():
|
|
yield Request("https://example.com/")
|
|
|
|
# "no-referrer-when-downgrade": None,
|
|
response = Response(
|
|
"https://example.com/",
|
|
headers={"Referrer-Policy": "no-referrer-when-downgrade"},
|
|
)
|
|
with pytest.warns(
|
|
RuntimeWarning,
|
|
match=r"Could not load referrer policy 'no-referrer-when-downgrade'",
|
|
):
|
|
output = [
|
|
request
|
|
async for request in mw.process_spider_output_async(
|
|
response, input_result()
|
|
)
|
|
]
|
|
assert len(output) == 1
|
|
assert b"Referer" not in output[0].headers
|
|
|
|
# "custom-policy": CustomPythonOrgPolicy,
|
|
response = Response(
|
|
"https://example.com/",
|
|
headers={"Referrer-Policy": "custom-policy"},
|
|
)
|
|
output = [
|
|
request
|
|
async for request in mw.process_spider_output_async(response, input_result())
|
|
]
|
|
assert len(output) == 1
|
|
assert output[0].headers == {b"Referer": [b"https://python.org/"]}
|
|
|
|
# "": CustomPythonOrgPolicy,
|
|
response = Response(
|
|
"https://example.com/",
|
|
headers={"Referrer-Policy": ""},
|
|
)
|
|
output = [
|
|
request
|
|
async for request in mw.process_spider_output_async(response, input_result())
|
|
]
|
|
assert len(output) == 1
|
|
assert output[0].headers == {b"Referer": [b"https://python.org/"]}
|