Merge 96437be1b5 into b3670369b8

docs/topics/download-handlers.rst

@@ -378,6 +378,27 @@ This handler supports ``ftp://host/path`` FTP URIs.
 
 It's implemented using :mod:`twisted.protocols.ftp`.
 
+.. _topics-ftp-request-meta:
+
+FTP connection parameters can be passed using :attr:`Request.meta
+<scrapy.Request.meta>`:
+
+* ``ftp_user`` (falls back to :setting:`FTP_USER`)
+* ``ftp_password`` (falls back to :setting:`FTP_PASSWORD`)
+* ``ftp_passive`` (falls back to :setting:`FTP_PASSIVE_MODE`)
+* ``ftp_local_filename``: if not given, file data is returned in
+  ``response.body``. If given, file data is saved to that local path (useful
+  for large files) and the local filename is also returned in ``response.body``.
+
+The built response uses HTTP-like status codes by default: ``200`` on success,
+``404`` when the file is not found (FTP code 550), or the corresponding FTP
+exception otherwise. The mapping from FTP return codes to response status codes
+is defined in the :attr:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler.CODE_MAPPING`
+class attribute. The ``default`` key is used for unmapped codes.
+
+On successful requests (status 200), ``response.headers`` includes
+``Local Filename`` (when ``ftp_local_filename`` is used) and ``Size``.
+
 S3DownloadHandler
 -----------------
 

docs/topics/request-response.rst

@@ -721,6 +721,8 @@ Those are:
 * :reqmeta:`download_warnsize`
 * :reqmeta:`download_timeout`
 * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
+* ``ftp_passive`` (See :setting:`FTP_PASSIVE_MODE` for more info)
+* ``ftp_local_filename`` (See :ref:`topics-ftp-request-meta` for more info)
 * ``ftp_user`` (See :setting:`FTP_USER` for more info)
 * :reqmeta:`give_up_log_level`
 * :reqmeta:`handle_httpstatus_all`

scrapy/__init__.py

@@ -1,7 +1,3 @@
-"""
-Scrapy - a web crawling and web scraping framework written for Python
-"""
-
 import pkgutil
 import sys
 import warnings

scrapy/commands/__init__.py

@@ -1,7 +1,3 @@
-"""
-Base class for Scrapy commands
-"""
-
 from __future__ import annotations
 
 import argparse

scrapy/commands/shell.py

@@ -1,9 +1,3 @@
-"""
-Scrapy Shell
-
-See documentation in docs/topics/shell.rst
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/core/__init__.py

@@ -1,3 +0,0 @@
-"""
-Scrapy core library classes and functions.
-"""

scrapy/core/downloader/handlers/__init__.py

@@ -1,5 +1,3 @@
-"""Download handlers for different schemes"""
-
 from __future__ import annotations
 
 import inspect

scrapy/core/downloader/handlers/_httpx.py

@@ -1,5 +1,3 @@
-"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use."""
-
 from __future__ import annotations
 
 import ipaddress

scrapy/core/downloader/handlers/ftp.py

@@ -1,33 +1,3 @@
-"""
-An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
-
-FTP connection parameters are passed using the request meta field:
-- ftp_user (optional, falls back to FTP_USER)
-- ftp_password (optional, falls back to FTP_PASSWORD)
-- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
-- ftp_local_filename
-        - If not given, file data will come in the response.body, as a normal scrapy Response,
-        which will imply that the entire file will be on memory.
-        - if given, file data will be saved in a local file with the given name
-        This helps when downloading very big files to avoid memory issues. In addition, for
-        convenience the local file name will also be given in the response body.
-
-The status of the built html response will be, by default
-- 200 in case of success
-- 404 in case specified file was not found in the server (ftp code 550)
-
-or raise corresponding ftp exception otherwise
-
-The matching from server ftp command return codes to html response codes is defined in the
-CODE_MAPPING attribute of the handler class. The key 'default' is used for any code
-that is not explicitly present among the map keys. You may need to overwrite this
-mapping if want a different behaviour than default.
-
-In case of status 200 request, response.headers will come with two keys:
-    'Local Filename' - with the value of the local filename if given
-    'Size' - with size of the downloaded data
-"""
-
 from __future__ import annotations
 
 import re

scrapy/core/downloader/handlers/http11.py

@@ -1,5 +1,3 @@
-"""Download handlers for http and https schemes"""
-
 from __future__ import annotations
 
 import ipaddress

scrapy/core/downloader/middleware.py

@@ -1,9 +1,3 @@
-"""
-Downloader Middleware manager
-
-See documentation in docs/topics/downloader-middleware.rst
-"""
-
 from __future__ import annotations
 
 import warnings

scrapy/core/engine.py

@@ -1,10 +1,3 @@
-"""
-This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
-
-For more information see docs/topics/architecture.rst
-
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/core/scraper.py

@@ -1,6 +1,3 @@
-"""This module implements the Scraper component which parses responses and
-extracts information from them"""
-
 from __future__ import annotations
 
 import logging

scrapy/core/spidermw.py

@@ -1,9 +1,3 @@
-"""
-Spider Middleware manager
-
-See documentation in docs/topics/spider-middleware.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/downloadermiddlewares/defaultheaders.py

@@ -1,9 +1,3 @@
-"""
-DefaultHeaders downloader middleware
-
-See documentation in docs/topics/downloader-middleware.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING

scrapy/downloadermiddlewares/downloadtimeout.py

@@ -1,9 +1,3 @@
-"""
-Download timeout middleware
-
-See documentation in docs/topics/downloader-middleware.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING

scrapy/downloadermiddlewares/httpauth.py

@@ -1,9 +1,3 @@
-"""
-HTTP basic auth downloader middleware
-
-See documentation in docs/topics/downloader-middleware.rst
-"""
-
 from __future__ import annotations
 
 import warnings

scrapy/downloadermiddlewares/retry.py

@@ -1,12 +1,3 @@
-"""
-An extension to retry failed requests that are potentially caused by temporary
-problems such as a connection timeout or HTTP 500 error.
-
-You can change the behaviour of this middleware by modifying the scraping settings:
-RETRY_TIMES - how many times to retry a failed page
-RETRY_HTTP_CODES - which HTTP response codes to retry
-"""
-
 from __future__ import annotations
 
 from logging import Logger, getLevelName, getLogger

scrapy/downloadermiddlewares/robotstxt.py

@@ -1,9 +1,3 @@
-"""
-This is a middleware to respect robots.txt policies. To activate it you must
-enable this middleware and enable the ROBOTSTXT_OBEY setting.
-
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/downloadermiddlewares/useragent.py

@@ -1,5 +1,3 @@
-"""Set User-Agent header per spider or use a default value from settings"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING

scrapy/exceptions.py

@@ -1,9 +1,5 @@
-"""
-Scrapy core exceptions
-
-These exceptions are documented in docs/topics/exceptions.rst. Please don't add
-new exceptions here without documenting them there.
-"""
+# These exceptions are documented in docs/topics/exceptions.rst. Please don't add
+# new exceptions here without documenting them there.
 
 from __future__ import annotations
 

scrapy/exporters.py

@@ -1,7 +1,3 @@
-"""
-Item Exporters are used to export/serialize items into different formats.
-"""
-
 from __future__ import annotations
 
 import csv

scrapy/extension.py

@@ -1,9 +1,3 @@
-"""
-The Extension Manager
-
-See documentation in docs/topics/extensions.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any

scrapy/extensions/closespider.py

@@ -1,9 +1,3 @@
-"""CloseSpider is an extension that forces spiders to be closed after certain
-conditions are met.
-
-See documentation in docs/topics/extensions.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/extensions/corestats.py

@@ -1,7 +1,3 @@
-"""
-Extension for collecting core stats like items scraped and start/finish times
-"""
-
 from __future__ import annotations
 
 from datetime import datetime, timezone

scrapy/extensions/debug.py

@@ -1,9 +1,3 @@
-"""
-Extensions for debugging Scrapy
-
-See documentation in docs/topics/extensions.rst
-"""
-
 from __future__ import annotations
 
 import contextlib

scrapy/extensions/feedexport.py

@@ -1,9 +1,3 @@
-"""
-Feed Exports extension
-
-See documentation in docs/topics/feed-exports.rst
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/extensions/memdebug.py

@@ -1,9 +1,3 @@
-"""
-MemoryDebugger extension
-
-See documentation in docs/topics/extensions.rst
-"""
-
 from __future__ import annotations
 
 import gc

scrapy/extensions/memusage.py

@@ -1,9 +1,3 @@
-"""
-MemoryUsage extension
-
-See documentation in docs/topics/extensions.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/extensions/postprocessing.py

@@ -1,7 +1,3 @@
-"""
-Extension for processing data before they are exported to feeds.
-"""
-
 from bz2 import BZ2File
 from gzip import GzipFile
 from io import IOBase

scrapy/extensions/statsmailer.py

@@ -1,9 +1,3 @@
-"""
-StatsMailer extension sends an email when a spider finishes scraping.
-
-Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
-"""
-
 from __future__ import annotations
 
 import warnings

scrapy/extensions/telnet.py

@@ -1,9 +1,3 @@
-"""
-Scrapy Telnet Console extension
-
-See documentation in docs/topics/telnetconsole.rst
-"""
-
 from __future__ import annotations
 
 import binascii

scrapy/http/__init__.py

@@ -1,10 +1,3 @@
-"""
-Module containing all HTTP related classes
-
-Use this module (instead of the more specific ones) when importing Headers,
-Request and Response outside this module.
-"""
-
 from scrapy.http.headers import Headers
 from scrapy.http.request import Request
 from scrapy.http.request.form import FormRequest

scrapy/http/request/__init__.py

@@ -1,10 +1,3 @@
-"""
-This module implements the Request class which is used to represent HTTP
-requests in Scrapy.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from __future__ import annotations
 
 import inspect

scrapy/http/request/form.py

@@ -1,10 +1,3 @@
-"""
-This module implements the FormRequest class which is a more convenient class
-(than Request) to generate Requests based on form data.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from __future__ import annotations
 
 from collections.abc import Iterable

scrapy/http/request/json_request.py

@@ -1,10 +1,3 @@
-"""
-This module implements the JsonRequest class which is a more convenient class
-(than Request) to generate JSON Requests.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from __future__ import annotations
 
 import copy

scrapy/http/request/rpc.py

@@ -1,10 +1,3 @@
-"""
-This module implements the XmlRpcRequest class which is a more convenient class
-(that Request) to generate xml-rpc requests.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from __future__ import annotations
 
 import xmlrpc.client as xmlrpclib

scrapy/http/response/__init__.py

@@ -1,10 +1,3 @@
-"""
-This module implements the Response class which is used to represent HTTP
-responses in Scrapy.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload

scrapy/http/response/html.py

@@ -1,11 +1,3 @@
-"""
-This module implements the :class:`HtmlResponse` class which is used as a
-content type marker by :class:`~scrapy.selector.Selector` and can be used in
-``isinstance()`` checks.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from scrapy.http.response.text import TextResponse
 
 

scrapy/http/response/json.py

@@ -1,10 +1,3 @@
-"""
-This module implements the JsonResponse class that is used when the response
-has a JSON MIME type in its Content-Type header.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from scrapy.http.response.text import TextResponse
 
 

scrapy/http/response/text.py

@@ -1,10 +1,3 @@
-"""
-This module implements the TextResponse class which adds encoding handling and
-discovering (through HTTP headers) to base Response class.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from __future__ import annotations
 
 import json

scrapy/http/response/xml.py

@@ -1,11 +1,3 @@
-"""
-This module implements the :class:`XmlResponse` class which is used as a
-content type marker by :class:`~scrapy.selector.Selector` and can be used in
-``isinstance()`` checks.
-
-See documentation in docs/topics/request-response.rst
-"""
-
 from scrapy.http.response.text import TextResponse
 
 

scrapy/item.py

@@ -1,9 +1,3 @@
-"""
-Scrapy Item
-
-See documentation in docs/topics/items.rst
-"""
-
 from __future__ import annotations
 
 from abc import ABCMeta

scrapy/link.py

@@ -1,11 +1,3 @@
-"""
-This module defines the Link object used in Link extractors.
-
-For actual link extractors implementation see scrapy.linkextractors, or
-its documentation in: docs/topics/link-extractors.rst
-"""
-
-
 class Link:
     """Link objects represent an extracted link by the LinkExtractor.
 

scrapy/linkextractors/__init__.py

@@ -1,11 +1,3 @@
-"""
-scrapy.linkextractors
-
-This package contains a collection of Link Extractors.
-
-For more info see docs/topics/link-extractors.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING

scrapy/linkextractors/lxmlhtml.py

@@ -1,7 +1,3 @@
-"""
-Link extractor based on lxml.html
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/loader/__init__.py

@@ -1,9 +1,3 @@
-"""
-Item Loader
-
-See documentation in docs/topics/loaders.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any

scrapy/mail.py

@@ -1,7 +1,3 @@
-"""
-Mail sending helpers
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/pipelines/__init__.py

@@ -1,9 +1,3 @@
-"""
-Item pipeline
-
-See documentation in docs/topics/item-pipeline.rst
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/pipelines/files.py

@@ -1,9 +1,3 @@
-"""
-Files Pipeline
-
-See documentation in topics/media-pipeline.rst
-"""
-
 from __future__ import annotations
 
 import base64

scrapy/pipelines/images.py

@@ -1,9 +1,3 @@
-"""
-Images Pipeline
-
-See documentation in topics/media-pipeline.rst
-"""
-
 from __future__ import annotations
 
 import functools

scrapy/responsetypes.py

@@ -1,8 +1,3 @@
-"""
-This module implements a class which returns the appropriate Response class
-based on different criteria.
-"""
-
 from __future__ import annotations
 
 from io import StringIO

scrapy/selector/__init__.py

@@ -1,7 +1,3 @@
-"""
-Selectors
-"""
-
 # top-level imports
 from scrapy.selector.unified import Selector, SelectorList
 

scrapy/settings/default_settings.py

@@ -1,17 +1,10 @@
-"""This module contains the default values for all settings used by Scrapy.
-
-For more information about these settings you can read the settings
-documentation in docs/topics/settings.rst
-
-Scrapy developers, if you add a setting here remember to:
-
-* add it in alphabetical order, with the exception that enabling flags and
-  other high-level settings for a group should come first in their group
-  and pairs like host/port and user/password should be in the usual order
-* group similar settings without leaving blank lines
-* add its documentation to the available settings documentation
-  (docs/topics/settings.rst)
-"""
+# Scrapy developers, if you add a setting here remember to:
+#
+# * add it in alphabetical order, with the exception that enabling flags and
+#   other high-level settings for a group should come first in their group
+#   and pairs like host/port and user/password should be in the usual order
+# * group similar settings without leaving blank lines
+# * add its documentation to docs/topics/settings.rst
 
 import sys
 from importlib import import_module

scrapy/shell.py

@@ -1,9 +1,3 @@
-"""Scrapy Shell
-
-See documentation in docs/topics/shell.rst
-
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/signals.py

@@ -1,9 +1,5 @@
-"""
-Scrapy signals
-
-These signals are documented in docs/topics/signals.rst. Please don't add new
-signals here without documenting them there.
-"""
+# These signals are documented in docs/topics/signals.rst. Please don't add new
+# signals here without documenting them there.
 
 engine_started = object()
 engine_stopped = object()

scrapy/spidermiddlewares/depth.py

@@ -1,9 +1,3 @@
-"""
-Depth Spider Middleware
-
-See documentation in docs/topics/spider-middleware.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/httperror.py

@@ -1,9 +1,3 @@
-"""
-HttpError Spider Middleware
-
-See documentation in docs/topics/spider-middleware.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/referer.py

@@ -1,8 +1,3 @@
-"""
-RefererMiddleware: populates Request referer field, based on the Response which
-originated it.
-"""
-
 from __future__ import annotations
 
 import warnings

scrapy/spidermiddlewares/urllength.py

@@ -1,9 +1,3 @@
-"""
-Url Length Spider Middleware
-
-See documentation in docs/topics/spider-middleware.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/spiders/__init__.py

@@ -1,9 +1,3 @@
-"""
-Base class for Scrapy spiders
-
-See documentation in docs/topics/spiders.rst
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/spiders/crawl.py

@@ -1,10 +1,3 @@
-"""
-This modules implements the CrawlSpider which is the recommended spider to use
-for scraping typical websites that requires crawling pages.
-
-See documentation in docs/topics/spiders.rst
-"""
-
 from __future__ import annotations
 
 import copy

scrapy/spiders/feed.py

@@ -1,10 +1,3 @@
-"""
-This module implements the XMLFeedSpider which is the recommended spider to use
-for scraping from an XML feed.
-
-See documentation in docs/topics/spiders.rst
-"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any

scrapy/squeues.py

@@ -1,7 +1,3 @@
-"""
-Scheduler queues
-"""
-
 from __future__ import annotations
 
 import marshal

scrapy/statscollectors.py

@@ -1,7 +1,3 @@
-"""
-Scrapy extension for collecting scraping stats
-"""
-
 from __future__ import annotations
 
 import logging

scrapy/utils/_download_handlers.py

@@ -1,5 +1,3 @@
-"""Utils for built-in HTTP download handlers."""
-
 from __future__ import annotations
 
 from contextlib import contextmanager

scrapy/utils/asyncio.py

@@ -1,5 +1,3 @@
-"""Utilities related to asyncio and its support in Scrapy."""
-
 from __future__ import annotations
 
 import asyncio

scrapy/utils/boto.py

@@ -1,5 +1,3 @@
-"""Boto/botocore helpers"""
-
 
 def is_botocore_available() -> bool:
     try:

scrapy/utils/datatypes.py

@@ -1,8 +1,3 @@
-"""
-This module contains data types used by Scrapy which are not included in the
-Python Standard Library.
-"""
-
 from __future__ import annotations
 
 import collections

scrapy/utils/defer.py

@@ -1,7 +1,3 @@
-"""
-Helper functions for dealing with Twisted deferreds
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/utils/deprecate.py

@@ -1,5 +1,3 @@
-"""Some helpers for deprecation messages"""
-
 from __future__ import annotations
 
 import inspect

scrapy/utils/display.py

@@ -1,7 +1,3 @@
-"""
-pprint and pformat wrappers with colorization support
-"""
-
 import ctypes
 import platform
 import sys

scrapy/utils/engine.py

@@ -1,5 +1,3 @@
-"""Some debugging functions for working with the Scrapy engine"""
-
 from __future__ import annotations
 
 # used in global tests code

scrapy/utils/httpobj.py

@@ -1,5 +1,3 @@
-"""Helper functions for scrapy.http objects (Request, Response)"""
-
 from __future__ import annotations
 
 from typing import TYPE_CHECKING

scrapy/utils/misc.py

@@ -1,5 +1,3 @@
-"""Helper functions which don't fit anywhere else"""
-
 from __future__ import annotations
 
 import ast

scrapy/utils/python.py

@@ -1,7 +1,3 @@
-"""
-This module contains essential stuff that should've come with Python itself ;)
-"""
-
 from __future__ import annotations
 
 import gc

scrapy/utils/request.py

@@ -1,8 +1,3 @@
-"""
-This module provides some useful functions for working with
-scrapy.Request objects
-"""
-
 from __future__ import annotations
 
 import hashlib

scrapy/utils/response.py

@@ -1,8 +1,3 @@
-"""
-This module provides some useful functions for working with
-scrapy.http.Response objects
-"""
-
 from __future__ import annotations
 
 import os

scrapy/utils/signal.py

@@ -1,5 +1,3 @@
-"""Helper functions for working with signals"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/utils/sitemap.py

@@ -1,9 +1,5 @@
-"""
-Module for processing Sitemaps.
-
-Note: The main purpose of this module is to provide support for the
-SitemapSpider, its API is subject to change without notice.
-"""
+# The main purpose of this module is to provide support for the SitemapSpider,
+# its API is subject to change without notice.
 
 from __future__ import annotations
 

scrapy/utils/template.py

@@ -1,5 +1,3 @@
-"""Helper functions for working with templates"""
-
 from __future__ import annotations
 
 import re

scrapy/utils/test.py

@@ -1,7 +1,3 @@
-"""
-This module contains some assorted functions used in tests
-"""
-
 from __future__ import annotations
 
 import asyncio

scrapy/utils/trackref.py

@@ -1,16 +1,7 @@
-"""This module provides some functions and classes to record and report
-references to live object instances.
-
-If you want live objects for a particular class to be tracked, you only have to
-subclass from object_ref (instead of object).
-
-This library has a minimal performance impact.
-
-.. note:: PyPy uses a tracing garbage collector, so objects may
-    remain in the ``live_refs`` longer than expected, even after they
-    go out of scope. If deterministic behavior is required, you may need
-    to explicitly trigger garbage collection or call ``trackref.live_refs.clear()``.
-"""
+# PyPy uses a tracing garbage collector, so objects may remain in live_refs
+# longer than expected, even after they go out of scope. If deterministic
+# behavior is required, you may need to explicitly trigger garbage collection
+# or call trackref.live_refs.clear().
 
 from __future__ import annotations
 

scrapy/utils/url.py

@@ -1,8 +1,3 @@
-"""
-This module contains general purpose URL functions not found in the standard
-library.
-"""
-
 from __future__ import annotations
 
 import re