diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 598e8060e..d9c919db9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -73,7 +73,7 @@ class ScrapyCommand(ABC): def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted - by optparser which removes newlines and wraps text. + by argparse which removes newlines and wraps text. """ return self.short_desc() diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c1796c723..a934cbbc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): instance. The purpose of this custom class is to provide a ``creatorForNetloc()`` - method that returns a ``_ScrapyClientTLSOptions`` instance configured based - on TLS settings provided to the factory. + method that returns: + + - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance + configured based on TLS settings provided to the factory (when the + certificate verification is disabled); + - a result of ``optionsForClientTLS()`` called with those TLS settings + (when the certificate verification is enabled). """ def __init__( diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1fc504c59..17dca5acb 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._disconnect_timeout: int = 1 async def download_request(self, request: Request) -> Response: - """Return a deferred for the HTTP download""" if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE") if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover @@ -283,7 +282,7 @@ def _tunnel_request_data( class _TunnelingAgent(Agent): - """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS + """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is transparent to the client; thus the agent should behave like there is no diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7137a0f2b..aa55e29a0 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -114,11 +114,7 @@ class H2ConnectionPool: d.errback(ResponseFailed(errors)) def close_connections(self) -> None: - """Close all the HTTP/2 connections and remove them from pool - - Returns: - Deferred that fires when all connections have been closed - """ + """Close all the HTTP/2 connections and remove them from pool.""" for conn in self._connections.values(): assert conn.transport is not None # typing conn.transport.abortConnection() diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 39703f976..7136e829e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): uri is used to verify that incoming client requests have correct base URL. settings -- Scrapy project settings - conn_lost_deferred -- Deferred fires with the reason: Failure to notify + conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify that connection was lost tls_verbose_logging -- Whether to log TLS details """ @@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events - received from the HTTP/2 data and IH2EventsHandler + received from the HTTP/2 data and the handlers in this class. Arguments: events -- A list of events that the remote peer triggered by sending data diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 7217da942..d807fe347 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -209,8 +209,8 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + :setting:`CONCURRENT_REQUESTS` or + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 466ce656d..58e37ce5e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -441,7 +441,7 @@ class Scraper: self, output: Any, response: Response | Failure ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned - from the given spider. + from the spider. Items are sent to the item pipelines, requests are scheduled. """ @@ -451,7 +451,7 @@ class Scraper: self, output: Any, response: Response | Failure ) -> None: """Process each Request/Item (given in the output parameter) returned - from the given spider. + from the spider. Items are sent to the item pipelines, requests are scheduled. """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9828fe6f5..c72752915 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): """ Run a crawler with the provided arguments. - It will call the given Crawler's :meth:`~Crawler.crawl` method, while - keeping track of it so it can be stopped later. + It will call the given Crawler's :meth:`~Crawler.crawl_async` method, + while keeping track of it so it can be stopped later. If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as @@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS - resolver based on :setting:`DNSCACHE_ENABLED`. + resolver based on :setting:`TWISTED_DNS_RESOLVER`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. @@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): When using a reactor it adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based - on :setting:`DNSCACHE_ENABLED`. + on :setting:`TWISTED_DNS_RESOLVER`. - If ``stop_after_crawl`` is True, the reactor will be stopped after all - crawlers have finished, using :meth:`join`. + If ``stop_after_crawl`` is True, the reactor/event loop will be stopped + after all crawlers have finished, using :meth:`join`. :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 821f41699..45af5c67a 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -196,10 +196,7 @@ class BaseRedirectMiddleware: class RedirectMiddleware(BaseRedirectMiddleware): - """ - Handle redirection of requests based on response status - and meta-refresh html tag. - """ + """Handle redirection of requests based on response status.""" @_warn_spider_arg def process_response( @@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): + """Handle redirection of requests based on meta-refresh html tag.""" + enabled_setting = "METAREFRESH_ENABLED" def __init__(self, settings: BaseSettings): diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 204132973..5330eab48 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -115,7 +115,7 @@ class UsageError(Exception): class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default - DeprecationWarning is silenced on Python 2.7+ + :exc:`DeprecationWarning` is silenced. """ diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 3d7674905..6c94d947e 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) class LogStats: """Log basic scraping stats periodically like: - * RPM - Requests per Minute + * RPM - Responses per Minute * IPM - Items per Minute """ diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 599f20947..e2c0cf864 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -136,9 +136,9 @@ class _DummyLock: class WrappedRequest: - """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class - - see http://docs.python.org/library/urllib2.html#urllib2.Request + """Wraps a :class:`scrapy.Request` class with methods defined by + :class:`urllib.request.Request` class to interact with + :class:`http.cookiejar.CookieJar` class. """ def __init__(self, request: Request): diff --git a/scrapy/item.py b/scrapy/item.py index 1cc0ae584..d5adc1efb 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -1,7 +1,7 @@ """ Scrapy Item -See documentation in docs/topics/item.rst +See documentation in docs/topics/items.rst """ from __future__ import annotations diff --git a/scrapy/mail.py b/scrapy/mail.py index fbd11ad1d..97123e63c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -1,7 +1,5 @@ """ Mail sending helpers - -See documentation in docs/topics/email.rst """ from __future__ import annotations diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 84fb5f85e..383c461e6 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -1,7 +1,7 @@ """ Item pipeline -See documentation in docs/item-pipeline.rst +See documentation in docs/topics/item-pipeline.rst """ from __future__ import annotations diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 0066fd38f..f6e6072ad 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -423,7 +423,7 @@ class FTPFilesStore: class FilesPipeline(MediaPipeline): - """Abstract pipeline that implement the file downloading + """Pipeline that implements the file downloading. This pipeline tries to minimize network transfers and file processing, doing stat of the files and determining if file is new, up-to-date or diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 762b0fdf1..79b6c4f27 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -47,7 +47,7 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): - """Abstract pipeline that implement the image thumbnail generation logic""" + """Pipeline that implements the handling logic specific to images.""" MEDIA_NAME: str = "image" diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index be298e9b1..932463ba9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]): """ Store a key/value attribute with a given priority. - Settings should be populated *before* configuring the Crawler object - (through the :meth:`~scrapy.crawler.Crawler.configure` method), - otherwise they won't have any effect. + Settings should be populated *before* the Crawler object applies them + (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or + :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't + have any effect. :param name: the setting name :type name: str @@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]): """ Make a deep copy of current settings. - This method returns a new instance of the :class:`Settings` class, + This method returns a new instance of this class, populated with the same values and their priorities. Modifications to the new object won't be reflected on the original @@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]): Make a copy of current settings and convert to a dict. This method returns a new dict populated with the same values - and their priorities as the current settings. + as the current settings. Modifications to the returned dict won't be reflected on the original settings. diff --git a/scrapy/shell.py b/scrapy/shell.py index 44dcd880e..f7ec8bea0 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -188,7 +188,8 @@ class Shell: async def _schedule(self, request: Request, spider: Spider | None) -> Response: """Send the request to the engine, wait for the result. - Runs in the reactor thread. + Runs in the reactor thread when using the reactor, or in the asyncio + event loop thread otherwise. """ if not self.spider: await self._open_spider(spider) diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index e09f2d10e..6c62dccb8 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -75,7 +75,7 @@ class BaseSpiderMiddleware: ) -> Request | None: """Return a processed request from the spider output. - This method is called with a single request from the start seeds or the + This method is called with a single request from ``start()`` or the spider output. It should return the same or a different request, or ``None`` to ignore it. @@ -84,7 +84,7 @@ class BaseSpiderMiddleware: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object or ``None`` for - start seeds + start requests :return: the processed request or ``None`` """ @@ -93,7 +93,7 @@ class BaseSpiderMiddleware: def get_processed_item(self, item: Any, response: Response | None) -> Any: """Return a processed item from the spider output. - This method is called with a single item from the start seeds or the + This method is called with a single item from ``start()`` or the spider output. It should return the same or a different item, or ``None`` to ignore it. @@ -102,7 +102,7 @@ class BaseSpiderMiddleware: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object or ``None`` for - start seeds + start items :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 264f685c1..1305874d5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -92,7 +92,7 @@ class ReferrerPolicy(ABC): ) def origin(self, url: str) -> str | None: - """Return serialized origin (scheme, host, path) for a request or response URL.""" + """Return serialized origin (scheme, host, port) for a request or response URL.""" return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url: str) -> bool: diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 395183613..925f31ede 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -54,17 +54,21 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response: Response, selector: Selector) -> Any: - """This method must be overridden with your custom spider functionality""" + """This method is called for the nodes matching the provided tag name + (itertag). Receives the response and an Selector for each node. + + This method must return either an item, a request, or a list + containing any of them. + + This method must be overridden with your custom spider functionality. + """ if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an Selector for each node. - Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either an item, a request, or a list - containing any of them. + (itertag). Receives the response and an iterable of Selectors. """ for selector in nodes: @@ -113,6 +117,9 @@ class CSVFeedSpider(Spider): It receives a CSV file in a response; iterates through each of its rows, and calls parse_row with a dict containing each field's data. + This spider also gives the opportunity to override adapt_response and + process_results methods for pre and post-processing purposes. + You can set some options regarding the CSV file, such as the delimiter, quotechar and the file's headers. """ @@ -136,16 +143,14 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response: Response, row: dict[str, str]) -> Any: - """This method must be overridden with your custom spider functionality""" + """Receives a response and a dict (representing each row) with a key for + each provided (or detected) header of the CSV file. + + This method must be overridden with your custom spider functionality. + """ raise NotImplementedError def parse_rows(self, response: Response) -> Any: - """Receives a response and a dict (representing each row) with a key for - each provided (or detected) header of the CSV file. This spider also - gives the opportunity to override adapt_response and - process_results methods for pre and post-processing purposes. - """ - for row in csviter( response, self.delimiter, self.headers, quotechar=self.quotechar ): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index dd0e062d0..c020ff4b9 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -1,8 +1,6 @@ """ This module contains data types used by Scrapy which are not included in the Python Standard Library. - -This module must not depend on any module outside the Standard Library. """ from __future__ import annotations diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 29a34d4ef..935ac8bc1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None: def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover warnings.warn( "scrapy.utils.defer.defer_result() is deprecated, use" - " twisted.internet.defer.success() and twisted.internet.defer.fail()," + " twisted.internet.defer.succeed() and twisted.internet.defer.fail()," " plus an explicit sleep if needed, or explicit reactor.callLater().", category=ScrapyDeprecationWarning, stacklevel=2, @@ -469,22 +469,22 @@ def _maybeDeferred_coro( def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """Return an :class:`asyncio.Future` object that wraps *d*. - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. + This function requires an installed asyncio reactor or a running asyncio + event loop, see :ref:`using-asyncio`. - When :ref:`using the asyncio reactor `, you cannot await - on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy - callables defined as coroutines `, you can only await on - ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects - allows you to wait on them:: + In this state you cannot await on :class:`~twisted.internet.defer.Deferred` + objects from :ref:`Scrapy callables defined as coroutines + `, you can only await on ``Future`` objects. Wrapping + ``Deferred`` objects into ``Future`` objects allows you to wait on them: + + .. code-block:: python class MySpider(Spider): ... + async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await deferred_to_future(deferred) + deferred = some_dfd_helper() + result = await deferred_to_future(deferred) .. versionchanged:: 2.14 This function no longer installs an asyncio loop if called before the @@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: in this case. """ if not is_asyncio_available(): - raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + raise RuntimeError( + "deferred_to_future() requires an installed asyncio reactor" + " or a running asyncio event loop." + ) return d.asFuture(asyncio.get_event_loop()) @@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: defined as a coroutine `. What you can await in Scrapy callables defined as coroutines depends on the - value of :setting:`TWISTED_REACTOR`: + value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`: - - When :ref:`using the asyncio reactor `, you can only - await on :class:`asyncio.Future` objects. + - When :ref:`using the asyncio reactor `, or :ref:`not + using a reactor at all `, you can only await + on :class:`asyncio.Future` objects. - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. + - When :ref:`using a non-asyncio reactor `, you can only + await on :class:`~twisted.internet.defer.Deferred` objects. - If you want to write code that uses ``Deferred`` objects but works with any - reactor, use this function on all ``Deferred`` objects:: + If you want to write code that uses ``Deferred`` objects but works in both + of these states, use this function on all ``Deferred`` objects: + + .. code-block:: python class MySpider(Spider): ... + async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) + deferred = some_dfd_helper() + result = await deferred_to_future(deferred) """ if not is_asyncio_available(): return d diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index aa77e692a..7645b235e 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -248,8 +248,7 @@ def logformatter_adapter( ) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: """ Helper that takes the dictionary output from the methods in LogFormatter - and adapts it into a tuple of positional arguments for logger.log calls, - handling backward compatibility as well. + and adapts it into a tuple of positional arguments for logger.log calls. """ level = logkws.get("level", logging.INFO) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c068d9b1e..7747a7b9b 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -88,7 +88,7 @@ def open_in_browser( def parse_details(self, response): - if "item name" not in response.body: + if "item name" not in response.text: open_in_browser(response) """ # circular imports diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 919f67240..c65d0c73b 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -39,7 +39,7 @@ def send_catch_log( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns + """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) @@ -172,9 +172,7 @@ async def _send_catch_log_asyncio( Returns a coroutine that completes once all signal handlers have finished. - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. + This function an installed asyncio reactor or a running asyncio event loop. .. versionadded:: 2.14 """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 87df10a02..22f9eadd0 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -4,9 +4,7 @@ references to live object instances. If you want live objects for a particular class to be tracked, you only have to subclass from object_ref (instead of object). -About performance: This library has a minimal performance impact when enabled, -and no performance penalty at all when disabled (as object_ref becomes just an -alias to object in that case). +This library has a minimal performance impact. .. note:: PyPy uses a tracing garbage collector, so objects may remain in the ``live_refs`` longer than expected, even after they diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d2bbdda2..e4967c86a 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -118,7 +118,7 @@ def strip_url( - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - ``origin_only`` replaces path component with "/", also dropping - query and fragment components ; it also strips credentials + query component; it also strips credentials - ``strip_fragment`` drops any #fragment component """