diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6e7be142e..8cecb7ad4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.5.0 +current_version = 1.6.0 commit = True tag = True tag_name = {new_version} diff --git a/.gitignore b/.gitignore index 7392ed31e..ff6e2ea65 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ htmlcov/ .pytest_cache/ .coverage.* .cache/ -.pytest_cache/ # Windows Thumbs.db diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a11b05d2..a05d07aee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ The guidelines for contributing are available here: -https://doc.scrapy.org/en/master/contributing.html +https://docs.scrapy.org/en/master/contributing.html Please do not abuse the issue tracker for support questions. If your issue topic can be rephrased to "How to ...?", please use the diff --git a/INSTALL b/INSTALL index a3c7899c6..06e812936 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ For information about installing Scrapy see: * docs/intro/install.rst (local file) -* https://doc.scrapy.org/en/latest/intro/install.html (online version) +* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/README.rst b/README.rst index 1361eac26..c28d217ff 100644 --- a/README.rst +++ b/README.rst @@ -51,18 +51,18 @@ The quick way:: pip install scrapy For more details see the install section in the documentation: -https://doc.scrapy.org/en/latest/intro/install.html +https://docs.scrapy.org/en/latest/intro/install.html Documentation ============= -Documentation is available online at https://doc.scrapy.org/ and in the ``docs`` +Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` directory. Releases ======== -You can find release notes at https://doc.scrapy.org/en/latest/news.html +You can find release notes at https://docs.scrapy.org/en/latest/news.html Community (blog, twitter, mail list, IRC) ========================================= @@ -72,7 +72,7 @@ See https://scrapy.org/community/ Contributing ============ -See https://doc.scrapy.org/en/master/contributing.html +See https://docs.scrapy.org/en/master/contributing.html Code of Conduct --------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index 2369c3436..9b508e418 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,7 +7,7 @@ Contributing to Scrapy .. important:: Double check that you are reading the most recent version of this document at - https://doc.scrapy.org/en/master/contributing.html + https://docs.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: @@ -55,7 +55,7 @@ guidelines when you're going to report a new bug. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has been discussed there, or if you're not sure if what you're seeing is a bug. - You can also ask in the `#scrapy` IRC channel. + You can also ask in the ``#scrapy`` IRC channel. * write **complete, reproducible, specific bug reports**. The smaller the test case, the better. Remember that other developers won't have your project to diff --git a/docs/index.rst b/docs/index.rst index 0a96aa88e..cedde8f38 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,13 @@ Scrapy |version| documentation ============================== -This documentation contains everything you need to know about Scrapy. +Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used +to crawl websites and extract structured data from their pages. It can be used +for a wide range of purposes, from data mining to monitoring and automated +testing. + +.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler +.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping Getting help ============ diff --git a/docs/news.rst b/docs/news.rst index 01016e2e6..75e3eef91 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,15 +3,222 @@ Release notes ============= -Scrapy 1.6.0 (unreleased) +.. _release-1.6.0: + +Scrapy 1.6.0 (2019-01-30) ------------------------- -Cleanups -~~~~~~~~ +Highlights: -* Remove deprecated ``CrawlerSettings`` class. -* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. +* better Windows support; +* Python 3.7 compatibility; +* big documentation improvements, including a switch + from ``.extract_first()`` + ``.extract()`` API to ``.get()`` + ``.getall()`` + API; +* feed exports, FilePipeline and MediaPipeline improvements; +* better extensibility: :signal:`item_error` and + :signal:`request_reached_downloader` signals; ``from_crawler`` support + for feed exporters, feed storages and dupefilters. +* ``scrapy.contracts`` fixes and new features; +* telnet console security improvements, first released as a + backport in :ref:`release-1.5.2`; +* clean-up of the deprecated code; +* various bug fixes, small new features and usability improvements across + the codebase. +Selector API changes +~~~~~~~~~~~~~~~~~~~~ + +While these are not changes in Scrapy itself, but rather in the parsel_ +library which Scrapy uses for xpath/css selectors, these changes are +worth mentioning here. Scrapy now depends on parsel >= 1.5, and +Scrapy documentation is updated to follow recent ``parsel`` API conventions. + +Most visible change is that ``.get()`` and ``.getall()`` selector +methods are now preferred over ``.extract_first()`` and ``.extract()``. +We feel that these new methods result in a more concise and readable code. +See :ref:`old-extraction-api` for more details. + +.. note:: + There are currently **no plans** to deprecate ``.extract()`` + and ``.extract_first()`` methods. + +Another useful new feature is the introduction of ``Selector.attrib`` and +``SelectorList.attrib`` properties, which make it easier to get +attributes of HTML elements. See :ref:`selecting-attributes`. + +CSS selectors are cached in parsel >= 1.5, which makes them faster +when the same CSS path is used many times. This is very common in +case of Scrapy spiders: callbacks are usually called several times, +on different pages. + +If you're using custom ``Selector`` or ``SelectorList`` subclasses, +a **backward incompatible** change in parsel may affect your code. +See `parsel changelog`_ for a detailed description, as well as for the +full list of improvements. + +.. _parsel changelog: https://parsel.readthedocs.io/en/latest/history.html + +Telnet console +~~~~~~~~~~~~~~ + +**Backward incompatible**: Scrapy's telnet console now requires username +and password. See :ref:`topics-telnetconsole` for more details. This change +fixes a **security issue**; see :ref:`release-1.5.2` release notes for details. + +New extensibility features +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``from_crawler`` support is added to feed exporters and feed storages. This, + among other things, allows to access Scrapy settings from custom feed + storages and exporters (:issue:`1605`, :issue:`3348`). +* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows + to access e.g. settings or a spider from a dupefilter. +* :signal:`item_error` is fired when an error happens in a pipeline + (:issue:`3256`); +* :signal:`request_reached_downloader` is fired when Downloader gets + a new Request; this signal can be useful e.g. for custom Schedulers + (:issue:`3393`). +* new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows + to select sitemap entries based on their attributes in SitemapSpider + subclasses (:issue:`3512`). +* Lazy loading of Downloader Handlers is now optional; this enables better + initialization error handling in custom Downloader Handlers (:issue:`3394`). + +New FilePipeline and MediaPipeline features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`, + :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`. + For example, this allows to use alternative or self-hosted + AWS-compatible providers (:issue:`2609`, :issue:`3548`). +* ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and + :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`). + +``scrapy.contracts`` improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Exceptions in contracts code are handled better (:issue:`3377`); +* ``dont_filter=True`` is used for contract requests, which allows to test + different callbacks with the same URL (:issue:`3381`); +* ``request_cls`` attribute in Contract subclasses allow to use different + Request classes in contracts, for example FormRequest (:issue:`3383`). +* Fixed errback handling in contracts, e.g. for cases where a contract + is executed for URL which returns non-200 response (:issue:`3371`). + +Usability improvements +~~~~~~~~~~~~~~~~~~~~~~ + +* more stats for RobotsTxtMiddleware (:issue:`3100`) +* INFO log level is used to show telnet host/port (:issue:`3115`) +* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) +* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) +* non-zero exit code is returned from Scrapy commands when error happens + on spider inititalization (:issue:`3226`) +* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + "flv" is added to common video extensions (:issue:`3165`) +* better error message when an exporter is disabled (:issue:`3358`); +* ``scrapy shell --help`` mentions syntax required for local files + (``./file.html``) - :issue:`3496`. +* Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) + +Bug fixes +~~~~~~~~~ + +* fixed issue with extra blank lines in .csv exports under Windows + (:issue:`3039`); +* proper handling of pickling errors in Python 3 when serializing objects + for disk queues (:issue:`3082`) +* flags are now preserved when copying Requests (:issue:`3342`); +* FormRequest.from_response clickdata shouldn't ignore elements with + ``input[type=image]`` (:issue:`3153`). +* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) + +Documentation improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Docs are re-written to suggest .get/.getall API instead of + .extract/.extract_first. Also, :ref:`topics-selectors` docs are updated + and re-structured to match latest parsel docs; they now contain more topics, + such as :ref:`selecting-attributes` or :ref:`topics-selectors-css-extensions` + (:issue:`3390`). +* :ref:`topics-developer-tools` is a new tutorial which replaces + old Firefox and Firebug tutorials (:issue:`3400`). +* SCRAPY_PROJECT environment variable is documented (:issue:`3518`); +* troubleshooting section is added to install instructions (:issue:`3517`); +* improved links to beginner resources in the tutorial + (:issue:`3367`, :issue:`3468`); +* fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); +* remove unused ``DEPTH_STATS`` option from docs (:issue:`3245`); +* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`, + :issue:`3605`). + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +Compatibility shims for pre-1.0 Scrapy module names are removed +(:issue:`3318`): + +* ``scrapy.command`` +* ``scrapy.contrib`` (with all submodules) +* ``scrapy.contrib_exp`` (with all submodules) +* ``scrapy.dupefilter`` +* ``scrapy.linkextractor`` +* ``scrapy.project`` +* ``scrapy.spider`` +* ``scrapy.spidermanager`` +* ``scrapy.squeue`` +* ``scrapy.stats`` +* ``scrapy.statscol`` +* ``scrapy.utils.decorator`` + +See :ref:`module-relocations` for more information, or use suggestions +from Scrapy 1.5.x deprecation warnings to update your code. + +Other deprecation removals: + +* Deprecated scrapy.interfaces.ISpiderManager is removed; please use + scrapy.interfaces.ISpiderLoader. +* Deprecated ``CrawlerSettings`` class is removed (:issue:`3327`). +* Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes + are removed (:issue:`3327`, :issue:`3359`). + +Other improvements, cleanups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) +* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" + optional arguments (:issue:`3231`). +* additional files are included to sdist (:issue:`3495`); +* code style fixes (:issue:`3405`, :issue:`3304`); +* unneeded .strip() call is removed (:issue:`3519`); +* collections.deque is used to store MiddlewareManager methods instead + of a list (:issue:`3476`) + +.. _release-1.5.2: + +Scrapy 1.5.2 (2019-01-22) +------------------------- + +* *Security bugfix*: Telnet console extension can be easily exploited by rogue + websites POSTing content to http://localhost:6023, we haven't found a way to + exploit it from Scrapy, but it is very easy to trick a browser to do so and + elevates the risk for local development environment. + + *The fix is backward incompatible*, it enables telnet user-password + authentication by default with a random generated password. If you can't + upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT` + out of its default value. + + See :ref:`telnet console ` documentation for more info + +* Backport CI build failure under GCE environemnt due to boto import error. + +.. _release-1.5.1: Scrapy 1.5.1 (2018-07-12) ------------------------- @@ -28,6 +235,9 @@ This is a maintenance release with important bug fixes, but no new features: :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`, :issue:`3294`). + +.. _release-1.5.0: + Scrapy 1.5.0 (2017-12-29) ------------------------- @@ -46,15 +256,15 @@ Some highlights: * Better default handling of HTTP 308, 522 and 524 status codes. * Documentation is improved, as usual. -Backwards Incompatible Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backward Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Scrapy 1.5 drops support for Python 3.3. * Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). - **This is technically backwards-incompatible**; override + **This is technically backward-incompatible**; override :setting:`USER_AGENT` if you relied on old value. * Logging of settings overridden by ``custom_settings`` is fixed; - **this is technically backwards-incompatible** because the logger + **this is technically backward-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're parsing Scrapy logs, please update your log parsers (:issue:`1343`). * LinkExtractor now ignores ``m4v`` extension by default, this is change @@ -91,11 +301,11 @@ Bug fixes ~~~~~~~~~ - Fix logging of settings overridden by ``custom_settings``; - **this is technically backwards-incompatible** because the logger + **this is technically backward-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please update your log parsers if needed (:issue:`1343`) - Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). - **This is technically backwards-incompatible**; override + **This is technically backward-incompatible**; override :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, @@ -139,6 +349,7 @@ Docs - Document ``from_crawler`` methods for spider and downloader middlewares (:issue:`3019`) +.. _release-1.4.0: Scrapy 1.4.0 (2017-05-18) ------------------------- @@ -204,18 +415,18 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting. Enjoy! (Or read on for the rest of changes in this release.) -Deprecations and Backwards Incompatible Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Deprecations and Backward Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): - **warning, this is technically backwards-incompatible** + **warning, this is technically backward-incompatible** - Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`); - **this is technically backwards-incompatible** so please check if you have + **this is technically backward-incompatible** so please check if you have any non-default ``MEMUSAGE_***`` options set. - ``EDITOR`` environment variable now takes precedence over ``EDITOR`` option defined in settings.py (:issue:`1829`); Scrapy default settings - no longer depend on environment variables. **This is technically a backwards + no longer depend on environment variables. **This is technically a backward incompatible change**. - ``Spider.make_requests_from_url`` is deprecated (:issue:`1728`, fixes :issue:`1495`). @@ -325,6 +536,8 @@ Documentation - Clarify ``allowed_domains`` example (:issue:`2670`) +.. _release-1.3.3: + Scrapy 1.3.3 (2017-03-10) ------------------------- @@ -337,6 +550,7 @@ Bug fixes A new setting is introduced to toggle between warning or exception if needed ; see :setting:`SPIDER_LOADER_WARN_ONLY` for details. +.. _release-1.3.2: Scrapy 1.3.2 (2017-02-13) ------------------------- @@ -348,6 +562,8 @@ Bug fixes - Use consistent selectors for author field in tutorial (:issue:`2551`). - Fix TLS compatibility in Twisted 17+ (:issue:`2558`) +.. _release-1.3.1: + Scrapy 1.3.1 (2017-02-08) ------------------------- @@ -396,6 +612,8 @@ Cleanups - Remove dead code supporting old Twisted versions (:issue:`2544`). +.. _release-1.3.0: + Scrapy 1.3.0 (2016-12-21) ------------------------- @@ -418,10 +636,10 @@ New Features scrapy shell now follow HTTP redirections by default (:issue:`2290`); See :command:`fetch` and :command:`shell` for details. - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; - this is technically **backwards incompatible** so please check your log parsers. + this is technically **backward incompatible** so please check your log parsers. - By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); - this is **backwards incompatible** if you have log parsers expecting the short + this is **backward incompatible** if you have log parsers expecting the short logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` set to ``True``. @@ -435,6 +653,7 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. +.. _release-1.2.3: Scrapy 1.2.3 (2017-03-03) ------------------------- @@ -442,6 +661,8 @@ Scrapy 1.2.3 (2017-03-03) - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.2.2: + Scrapy 1.2.2 (2016-12-06) ------------------------- @@ -477,6 +698,8 @@ Other changes .. _conda-forge: https://anaconda.org/conda-forge/scrapy +.. _release-1.2.1: + Scrapy 1.2.1 (2016-10-21) ------------------------- @@ -501,6 +724,8 @@ Other changes - Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`). +.. _release-1.2.0: + Scrapy 1.2.0 (2016-10-03) ------------------------- @@ -525,11 +750,11 @@ Bug fixes ~~~~~~~~~ - DefaultRequestHeaders middleware now runs before UserAgent middleware - (:issue:`2088`). **Warning: this is technically backwards incompatible**, + (:issue:`2088`). **Warning: this is technically backward incompatible**, though we consider this a bug fix. - HTTP cache extension and plugins that use the ``.scrapy`` data directory now work outside projects (:issue:`1581`). **Warning: this is technically - backwards incompatible**, though we consider this a bug fix. + backward incompatible**, though we consider this a bug fix. - ``Selector`` does not allow passing both ``response`` and ``text`` anymore (:issue:`2153`). - Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`). @@ -569,12 +794,14 @@ Documentation - Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`). - Add StackOverflow as a support channel (:issue:`2257`). +.. _release-1.1.4: Scrapy 1.1.4 (2017-03-03) ------------------------- - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.1.3: Scrapy 1.1.3 (2016-09-22) ------------------------- @@ -592,6 +819,7 @@ Documentation rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`, :issue:`2252`). +.. _release-1.1.2: Scrapy 1.1.2 (2016-08-18) ------------------------- @@ -606,6 +834,7 @@ Bug fixes - :setting:`IMAGES_EXPIRES` default value set back to 90 (the regression was introduced in 1.1.1) +.. _release-1.1.1: Scrapy 1.1.1 (2016-07-13) ------------------------- @@ -658,6 +887,7 @@ Tests - Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`) +.. _release-1.1.0: Scrapy 1.1.0 (2016-05-11) ------------------------- @@ -704,13 +934,13 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Accept XML node names containing dots as valid (:issue:`1533`). - When uploading files or images to S3 (with ``FilesPipeline`` or ``ImagesPipeline``), the default ACL policy is now "private" instead - of "public" **Warning: backwards incompatible!**. + of "public" **Warning: backward incompatible!**. You can use :setting:`FILES_STORE_S3_ACL` to change it. - We've reimplemented ``canonicalize_url()`` for more correct output, especially for URLs with non-ASCII characters (:issue:`1947`). This could change link extractors output compared to previous scrapy versions. This may also invalidate some cache entries you could still have from pre-1.1 runs. - **Warning: backwards incompatible!**. + **Warning: backward incompatible!**. Keep reading for more details on other improvements and bug fixes. @@ -743,7 +973,7 @@ Additional New Features and Enhancements - Support for bpython and configure preferred Python shell via ``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`). - Support URLs without scheme (:issue:`1498`) - **Warning: backwards incompatible!** + **Warning: backward incompatible!** - Bring back support for relative file path (:issue:`1710`, :issue:`1550`). - Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check @@ -826,7 +1056,7 @@ Bugfixes ~~~~~~~~ - Scrapy does not retry requests that got a ``HTTP 400 Bad Request`` - response anymore (:issue:`1289`). **Warning: backwards incompatible!** + response anymore (:issue:`1289`). **Warning: backward incompatible!** - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). @@ -847,12 +1077,14 @@ Bugfixes - HTTPS+CONNECT tunnels could get mixed up when using multiple proxies to same remote host (:issue:`1912`). +.. _release-1.0.7: Scrapy 1.0.7 (2017-03-03) ------------------------- - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.0.6: Scrapy 1.0.6 (2016-05-04) ------------------------- @@ -862,6 +1094,7 @@ Scrapy 1.0.6 (2016-05-04) - DOC: Support for Sphinx 1.4+ (:issue:`1893`) - DOC: Consistency in selectors examples (:issue:`1869`) +.. _release-1.0.5: Scrapy 1.0.5 (2016-02-04) ------------------------- @@ -871,6 +1104,7 @@ Scrapy 1.0.5 (2016-02-04) - DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`) - DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`) +.. _release-1.0.4: Scrapy 1.0.4 (2015-12-30) ------------------------- @@ -924,12 +1158,16 @@ Scrapy 1.0.4 (2015-12-30) - Small grammatical change (:commit:`8752294`) - Add openssl version to version command (:commit:`13c45ac`) +.. _release-1.0.3: + Scrapy 1.0.3 (2015-08-11) ------------------------- - add service_identity to scrapy install_requires (:commit:`cbc2501`) - Workaround for travis#296 (:commit:`66af9cd`) +.. _release-1.0.2: + Scrapy 1.0.2 (2015-08-06) ------------------------- @@ -940,6 +1178,8 @@ Scrapy 1.0.2 (2015-08-06) - Fixed typos (:commit:`a9ae7b0`) - Fix doc reference. (:commit:`7c8a4fe`) +.. _release-1.0.1: + Scrapy 1.0.1 (2015-07-01) ------------------------- @@ -950,6 +1190,8 @@ Scrapy 1.0.1 (2015-07-01) - DOC remove version suffix from ubuntu package (:commit:`5303c66`) - DOC Update release date for 1.0 (:commit:`c89fa29`) +.. _release-1.0.0: + Scrapy 1.0.0 (2015-06-19) ------------------------- @@ -1064,12 +1306,14 @@ until it reaches a stable status. See more examples for scripts running Scrapy: :ref:`topics-practices` +.. _module-relocations: + Module Relocations ~~~~~~~~~~~~~~~~~~ There’s been a large rearrangement of modules trying to improve the general structure of Scrapy. Main changes were separating various subpackages into -new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp` +new projects and dissolving both ``scrapy.contrib`` and ``scrapy.contrib_exp`` into top level packages. Backward compatibility was kept among internal relocations, while importing deprecated modules expect warnings indicating their new place. @@ -1100,7 +1344,7 @@ Outsourced packages | | /scrapy-plugins/scrapy-jsonrpc>`_ | +-------------------------------------+-------------------------------------+ -`scrapy.contrib_exp` and `scrapy.contrib` dissolutions +``scrapy.contrib_exp`` and ``scrapy.contrib`` dissolutions +-------------------------------------+-------------------------------------+ | Old location | New location | @@ -1312,7 +1556,7 @@ Code refactoring (:issue:`1078`) - Pydispatch pep8 (:issue:`992`) - Removed unused 'load=False' parameter from walk_modules() (:issue:`871`) -- For consistency, use `job_dir` helper in `SpiderState` extension. +- For consistency, use ``job_dir`` helper in ``SpiderState`` extension. (:issue:`805`) - rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`) @@ -1402,7 +1646,7 @@ Scrapy 0.24.2 (2014-07-08) Scrapy 0.24.1 (2014-06-27) -------------------------- -- Fix deprecated CrawlerSettings and increase backwards compatibility with +- Fix deprecated CrawlerSettings and increase backward compatibility with .defaults attribute (:commit:`8e3f20a`) @@ -1425,10 +1669,10 @@ Enhancements cache middleware (:issue:`541`, :issue:`500`, :issue:`571`) - Expose current crawler in Scrapy shell (:issue:`557`) - Improve testsuite comparing CSV and XML exporters (:issue:`570`) -- New `offsite/filtered` and `offsite/domains` stats (:issue:`566`) +- New ``offsite/filtered`` and ``offsite/domains`` stats (:issue:`566`) - Support process_links as generator in CrawlSpider (:issue:`555`) - Verbose logging and new stats counters for DupeFilter (:issue:`553`) -- Add a mimetype parameter to `MailSender.send()` (:issue:`602`) +- Add a mimetype parameter to ``MailSender.send()`` (:issue:`602`) - Generalize file pipeline log messages (:issue:`622`) - Replace unencodeable codepoints with html entities in SGMLLinkExtractor (:issue:`565`) - Converted SEP documents to rst format (:issue:`629`, :issue:`630`, @@ -1447,20 +1691,20 @@ Enhancements - Make scrapy.version_info a tuple of integers (:issue:`681`, :issue:`692`) - Infer exporter's output format from filename extensions (:issue:`546`, :issue:`659`, :issue:`760`) -- Support case-insensitive domains in `url_is_from_any_domain()` (:issue:`693`) +- Support case-insensitive domains in ``url_is_from_any_domain()`` (:issue:`693`) - Remove pep8 warnings in project and spider templates (:issue:`698`) -- Tests and docs for `request_fingerprint` function (:issue:`597`) -- Update SEP-19 for GSoC project `per-spider settings` (:issue:`705`) +- Tests and docs for ``request_fingerprint`` function (:issue:`597`) +- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`) - Set exit code to non-zero when contracts fails (:issue:`727`) - Add a setting to control what class is instanciated as Downloader component (:issue:`738`) -- Pass response in `item_dropped` signal (:issue:`724`) -- Improve `scrapy check` contracts command (:issue:`733`, :issue:`752`) -- Document `spider.closed()` shortcut (:issue:`719`) -- Document `request_scheduled` signal (:issue:`746`) +- Pass response in ``item_dropped`` signal (:issue:`724`) +- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`) +- Document ``spider.closed()`` shortcut (:issue:`719`) +- Document ``request_scheduled`` signal (:issue:`746`) - Add a note about reporting security issues (:issue:`697`) - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`) -- Sort spider list output of `scrapy list` command (:issue:`742`) +- Sort spider list output of ``scrapy list`` command (:issue:`742`) - Multiple documentation enhancemens and fixes (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`, :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`, @@ -1528,23 +1772,23 @@ Scrapy 0.22.0 (released 2014-01-17) Enhancements ~~~~~~~~~~~~ -- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) - To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage` +- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) + To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage`` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`) - Selectors register EXSLT namespaces by default (:issue:`472`) - Unify item loaders similar to selectors renaming (:issue:`461`) -- Make `RFPDupeFilter` class easily subclassable (:issue:`533`) +- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in `get_func_args` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) - Allow running indiviual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) - Improve offsite middleware tests (:issue:`478`) - Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`) -- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`) +- Do not send ``x-gzip`` in default ``Accept-Encoding`` header (:issue:`469`) - Support defining http error handling using settings (:issue:`466`) - Use modern python idioms wherever you find legacies (:issue:`497`) - Improve and correct documentation @@ -1555,14 +1799,14 @@ Fixes ~~~~~ - Update Selector class imports in CrawlSpider template (:issue:`484`) -- Fix unexistent reference to `engine.slots` (:issue:`464`) -- Do not try to call `body_as_unicode()` on a non-TextResponse instance (:issue:`462`) +- Fix unexistent reference to ``engine.slots`` (:issue:`464`) +- Do not try to call ``body_as_unicode()`` on a non-TextResponse instance (:issue:`462`) - Warn when subclassing XPathItemLoader, previously it only warned on instantiation. (:issue:`523`) - Warn when subclassing XPathSelector, previously it only warned on instantiation. (:issue:`537`) - Multiple fixes to memory stats (:issue:`531`, :issue:`530`, :issue:`529`) -- Fix overriding url in `FormRequest.from_response()` (:issue:`507`) +- Fix overriding url in ``FormRequest.from_response()`` (:issue:`507`) - Fix tests runner under pip 1.5 (:issue:`513`) - Fix logging error when spider name is unicode (:issue:`479`) @@ -1589,7 +1833,7 @@ Enhancements (modifying them had been deprecated for a long time) - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) - Sitemap spider can fetch alternate URLs (:issue:`360`) -- `Selector.remove_namespaces()` now remove namespaces from element's attributes. (:issue:`416`) +- ``Selector.remove_namespaces()`` now remove namespaces from element's attributes. (:issue:`416`) - Paved the road for Python 3.3+ (:issue:`435`, :issue:`436`, :issue:`431`, :issue:`452`) - New item exporter using native python types with nesting support (:issue:`366`) - Tune HTTP1.1 pool size so it matches concurrency defined by settings (:commit:`b43b5f575`) @@ -1600,13 +1844,13 @@ Enhancements - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`) - Remove multi spider support from multiple core components (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`) -- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages. +- Travis-CI now tests Scrapy changes against development versions of ``w3lib`` and ``queuelib`` python packages. - Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`) - Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`) - Use importlib for parametric imports (:issue:`445`) - Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`) - Bugfix crawling shutdown on SIGINT (:issue:`450`) -- Do not submit `reset` type inputs in FormRequest.from_response (:commit:`b326b87`) +- Do not submit ``reset`` type inputs in FormRequest.from_response (:commit:`b326b87`) - Do not silence download errors when request errback raises an exception (:commit:`684cfc0`) Bugfixes @@ -1621,8 +1865,8 @@ Bugfixes - Improve request-response docs (:issue:`391`) - Improve best practices docs (:issue:`399`, :issue:`400`, :issue:`401`, :issue:`402`) - Improve django integration docs (:issue:`404`) -- Document `bindaddress` request meta (:commit:`37c24e01d7`) -- Improve `Request` class documentation (:issue:`226`) +- Document ``bindaddress`` request meta (:commit:`37c24e01d7`) +- Improve ``Request`` class documentation (:issue:`226`) Other ~~~~~ @@ -1631,7 +1875,7 @@ Other - Add `cssselect`_ python package as install dependency - Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on. - Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support. -- Running test suite now requires `mock` python library (:issue:`390`) +- Running test suite now requires ``mock`` python library (:issue:`390`) Thanks @@ -1685,7 +1929,7 @@ Scrapy 0.18.3 (released 2013-10-03) Scrapy 0.18.2 (released 2013-09-03) ----------------------------------- -- Backport `scrapy check` command fixes and backward compatible multi +- Backport ``scrapy check`` command fixes and backward compatible multi crawler process(:issue:`339`) Scrapy 0.18.1 (released 2013-08-27) @@ -1714,31 +1958,31 @@ Scrapy 0.18.0 (released 2013-08-09) - Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) - Use lxml recover option to parse sitemaps (:issue:`347`) - Bugfix cookie merging by hostname and not by netloc (:issue:`352`) -- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`) -- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`) -- Support `dont_cache` request meta flag (:issue:`19`) -- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`) -- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`) -- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`) +- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`) +- Support xml namespaces using ``iternodes`` parser in ``XMLFeedSpider`` (:issue:`12`) +- Support ``dont_cache`` request meta flag (:issue:`19`) +- Bugfix ``scrapy.utils.gz.gunzip`` broken by changes in python 2.7.4 (:commit:`4dc76e`) +- Bugfix url encoding on ``SgmlLinkExtractor`` (:issue:`24`) +- Bugfix ``TakeFirst`` processor shouldn't discard zero (0) value (:issue:`59`) - Support nested items in xml exporter (:issue:`66`) - Improve cookies handling performance (:issue:`77`) - Log dupe filtered requests once (:issue:`105`) - Split redirection middleware into status and meta based middlewares (:issue:`78`) - Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`) -- Support xpath form selection on `FormRequest.from_response` (:issue:`185`) -- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`) +- Support xpath form selection on ``FormRequest.from_response`` (:issue:`185`) +- Bugfix unicode decoding error on ``SgmlLinkExtractor`` (:issue:`199`) - Bugfix signal dispatching on pypi interpreter (:issue:`205`) - Improve request delay and concurrency handling (:issue:`206`) -- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`) +- Add RFC2616 cache policy to ``HttpCacheMiddleware`` (:issue:`212`) - Allow customization of messages logged by engine (:issue:`214`) -- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`) +- Multiples improvements to ``DjangoItem`` (:issue:`217`, :issue:`218`, :issue:`221`) - Extend Scrapy commands using setuptools entry points (:issue:`260`) -- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`) -- Support `settings.getdict` (:issue:`269`) -- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`) -- Added `Item.copy` (:issue:`290`) +- Allow spider ``allowed_domains`` value to be set/tuple (:issue:`261`) +- Support ``settings.getdict`` (:issue:`269`) +- Simplify internal ``scrapy.core.scraper`` slot handling (:issue:`271`) +- Added ``Item.copy`` (:issue:`290`) - Collect idle downloader slots (:issue:`297`) -- Add `ftp://` scheme downloader handler (:issue:`329`) +- Add ``ftp://`` scheme downloader handler (:issue:`329`) - Added downloader benchmark webserver and spider tools :ref:`benchmarking` - Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on - Add scrapy commands using external libraries (:issue:`260`) @@ -1848,7 +2092,7 @@ Scrapy 0.16.1 (released 2012-10-26) ----------------------------------- - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`) -- better backwards compatibility for scrapy.conf.settings (:commit:`3403089`) +- better backward compatibility for scrapy.conf.settings (:commit:`3403089`) - extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`) - removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`) - fix dashes under rst headers (:commit:`fa4f7f9`) @@ -1863,13 +2107,13 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` -- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backwards compatibility is kept on the Stats Collector API and signals. +- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` -- `lxml` is now the default selectors backend instead of `libxml2` +- ``lxml`` is now the default selectors backend instead of ``libxml2`` - ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_ - removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm`` - SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`) @@ -1962,16 +2206,16 @@ New features and settings - New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`) - Add boto 2.0 support for S3 downloader handler (:rev:`2763`) - Added `marshal`_ to formats supported by feed exports (:rev:`2744`) -- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`) +- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) -- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`) -- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`) -- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) +- Moved spider queues to scrapyd: ``scrapy.spiderqueue`` -> ``scrapyd.spiderqueue`` (:rev:`2708`) +- Moved sqlite utils to scrapyd: ``scrapy.utils.sqlite`` -> ``scrapyd.sqlite`` (:rev:`2781`) +- Real support for returning iterators on ``start_requests()`` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) - Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`) - Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`) - Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`) @@ -1979,19 +2223,19 @@ New features and settings - Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`) - Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`) - Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`) -- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`. +- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an ``IOError``. - Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`) -- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`) +- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and ``-e`` flag to ``genspider`` command that uses it (:rev:`2653`) - Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines. - Added :signal:`spider_error` signal (:rev:`2628`) - Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`) -- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. +- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to ``True``). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. - Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`) - Added new DBM HTTP cache storage backend (:rev:`2576`) - Added ``listjobs.json`` API to Scrapyd (:rev:`2571`) - ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`) - Added namespace support to ``xmliter_lxml`` (:rev:`2552`) -- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`) +- Improved cookies middleware by making ``COOKIES_DEBUG`` nicer and documenting it (:rev:`2579`) - Several improvements to Scrapyd and Link extractors Code rearranged and removed @@ -2005,17 +2249,17 @@ Code rearranged and removed - Reduced Scrapy codebase by striping part of Scrapy code into two new libraries: - `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`) - `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`) -- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`) -- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot +- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`) +- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on github: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) - removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`) - Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`) -- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept. +- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backward compatibility kept. Scrapy 0.12 ----------- @@ -2045,13 +2289,13 @@ Scrapyd changes - Scrapyd now uses one process per spider - It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default -- There is now a `scrapy server` command to start a Scrapyd server of the current project +- There is now a ``scrapy server`` command to start a Scrapyd server of the current project Changes to settings ~~~~~~~~~~~~~~~~~~~ -- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware -- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire". +- added ``HTTPCACHE_ENABLED`` setting (False by default) to enable HTTP cache middleware +- changed ``HTTPCACHE_EXPIRATION_SECS`` semantics: now zero means "never expire". Deprecated/obsoleted functionality ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2082,17 +2326,17 @@ New features and improvements - Splitted Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) -- Added `dont_redirect` request.meta key for avoiding redirects (#233) -- Added `dont_retry` request.meta key for avoiding retries (#234) +- Added ``dont_redirect`` request.meta key for avoiding redirects (#233) +- Added ``dont_retry`` request.meta key for avoiding retries (#234) Command-line tool changes ~~~~~~~~~~~~~~~~~~~~~~~~~ -- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199) - - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project - - Added `scrapy.bat` script for running more conveniently from Windows +- New ``scrapy`` command which replaces the old ``scrapy-ctl.py`` (#199) + - there is only one global ``scrapy`` command now, instead of one ``scrapy-ctl.py`` per project + - Added ``scrapy.bat`` script for running more conveniently from Windows - Added bash completion to command-line tool (#210) -- Renamed command `start` to `runserver` (#209) +- Renamed command ``start`` to ``runserver`` (#209) API changes ~~~~~~~~~~~ @@ -2112,11 +2356,11 @@ API changes - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` - default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201) - changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - moved ``scrapy.core.signals`` module to ``scrapy.signals`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - added ``handles_request()`` class method to ``BaseSpider`` - dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead) - dropped ``component`` argument of ``scrapy.log.msg()`` function @@ -2187,8 +2431,8 @@ New features - Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`) - Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`) -Backwards-incompatible changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`) - Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 985cc0433..ba832ab5d 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -94,7 +94,7 @@ how you :ref:`configure the downloader middlewares .. method:: crawl(\*args, \**kwargs) Starts the crawler by instantiating its spider class with the given - `args` and `kwargs` arguments, while setting the execution engine in + ``args`` and ``kwargs`` arguments, while setting the execution engine in motion. Returns a deferred that is fired when the crawl is finished. @@ -180,7 +180,7 @@ SpiderLoader API .. method:: load(spider_name) Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name `spider_name` and will raise + loaded spiders for a spider class with name ``spider_name`` and will raise a KeyError if not found. :param spider_name: spider class name diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4ac39ad2d..2effe94dc 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -172,5 +172,5 @@ links: .. _Twisted: https://twistedmatrix.com/trac/ .. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/ +.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c1976258d..82857c9da 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -233,7 +233,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1] + start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..0d976077b 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it -in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { @@ -55,8 +55,12 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each middleware component is a Python class that defines one or -more of the following methods: +Each downloader middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.downloadermiddlewares @@ -357,7 +361,7 @@ HttpCacheMiddleware .. reqmeta:: dont_cache - You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`. + You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``. .. _httpcache-policy-dummy: @@ -390,17 +394,17 @@ runs to avoid downloading unmodified data (to save bandwidth and speed up crawls what is implemented: -* Do not attempt to store responses/requests with `no-store` cache-control directive set -* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses -* Compute freshness lifetime from `max-age` cache-control directive -* Compute freshness lifetime from `Expires` response header -* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox) -* Compute current age from `Age` response header -* Compute current age from `Date` header -* Revalidate stale responses based on `Last-Modified` response header -* Revalidate stale responses based on `ETag` response header -* Set `Date` header for any received response missing it -* Support `max-stale` cache-control directive in requests +* Do not attempt to store responses/requests with ``no-store`` cache-control directive set +* Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses +* Compute freshness lifetime from ``max-age`` cache-control directive +* Compute freshness lifetime from ``Expires`` response header +* Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox) +* Compute current age from ``Age`` response header +* Compute current age from ``Date`` header +* Revalidate stale responses based on ``Last-Modified`` response header +* Revalidate stale responses based on ``ETag`` response header +* Set ``Date`` header for any received response missing it +* Support ``max-stale`` cache-control directive in requests This allows spiders to be configured with the full RFC2616 cache policy, but avoid revalidation on a request-by-request basis, while remaining @@ -408,15 +412,15 @@ what is implemented: Example: - Add `Cache-Control: max-stale=600` to Request headers to accept responses that + Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that have exceeded their expiration time by no more than 600 seconds. See also: RFC2616, 14.9.3 what is missing: -* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 -* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 +* ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 +* ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 * ... probably others .. @@ -626,12 +630,12 @@ Default: ``False`` If enabled, will cache pages unconditionally. A spider may wish to have all responses available in the cache, for -future use with `Cache-Control: max-stale`, for instance. The +future use with ``Cache-Control: max-stale``, for instance. The DummyPolicy caches all responses but never revalidates them, and sometimes a more nuanced policy is desirable. -This setting still respects `Cache-Control: no-store` directives in responses. -If you don't want that, filter `no-store` out of the Cache-Control headers in +This setting still respects ``Cache-Control: no-store`` directives in responses. +If you don't want that, filter ``no-store`` out of the Cache-Control headers in responses you feedto the cache middleware. .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS @@ -834,8 +838,6 @@ RetryMiddleware Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. -Once there are no more failed pages to retry, this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. The :class:`RetryMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -940,7 +942,7 @@ UserAgentMiddleware Middleware that allows spiders to override the default user agent. - In order for a spider to override the default user agent, its `user_agent` + In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. .. _ajaxcrawl-middleware: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 95f7920f8..f5048d2da 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -303,7 +303,7 @@ CsvItemExporter The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor, and the leftover arguments to the - `csv.writer`_ constructor, so you can use any `csv.writer` constructor + `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor argument to customize this exporter. A typical output of this exporter would be:: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index c421a5e05..d6e7452a1 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -19,7 +19,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like -`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. +``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps @@ -368,7 +368,7 @@ Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues running normally. -For more info see `Debugging in Python`. +For more info see `Debugging in Python`_. This extension only works on POSIX-compliant platforms (ie. not Windows). diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0957a5997..9f837221e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,6 +184,10 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` + +You can also define a custom ACL for exported feeds using this setting: + + * :setting:`FEED_STORAGE_S3_ACL` .. _topics-feed-storage-gcs: @@ -226,6 +230,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_URI` (mandatory) * :setting:`FEED_FORMAT` * :setting:`FEED_STORAGES` + * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` @@ -323,6 +328,17 @@ Default: ``{}`` A dict containing additional feed storage backends supported by your project. The keys are URI schemes and the values are paths to storage classes. +.. setting:: FEED_STORAGE_S3_ACL + +FEED_STORAGE_S3_ACL +------------------- + +Default: ``''`` (empty string) + +A string containing a custom ACL for feeds exported to Amazon S3 by your project. + +For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs. + .. setting:: FEED_STORAGES_BASE FEED_STORAGES_BASE @@ -387,4 +403,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore -.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file +.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 38265b474..fae18200a 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -87,8 +87,8 @@ contain a price:: vat_factor = 1.15 def process_item(self, item, spider): - if item['price']: - if item['price_excludes_vat']: + if item.get('price'): + if item.get('price_excludes_vat'): item['price'] = item['price'] * self.vat_factor return item else: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index ae44aecd3..d744fd9ea 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -40,6 +40,7 @@ objects. Here is an example:: name = scrapy.Field() price = scrapy.Field() stock = scrapy.Field() + tags = scrapy.Field() last_updated = scrapy.Field(serializer=str) .. note:: Those familiar with `Django`_ will notice that Scrapy Items are @@ -155,19 +156,42 @@ To access all populated values, just use the typical `dict API`_:: >>> product.items() [('price', 1000), ('name', 'Desktop PC')] + +Copying items +------------- + +To copy an item, you must first decide whether you want a shallow copy or a +deep copy. + +If your item contains mutable_ values like lists or dictionaries, a shallow +copy will keep references to the same mutable values across all different +copies. + +.. _mutable: https://docs.python.org/glossary.html#term-mutable + +For example, if you have an item with a list of tags, and you create a shallow +copy of that item, both the original item and the copy have the same list of +tags. Adding a tag to the list of one of the items will add the tag to the +other item as well. + +If that is not the desired behavior, use a deep copy instead. + +See the `documentation of the copy module`_ for more information. + +.. _documentation of the copy module: https://docs.python.org/library/copy.html + +To create a shallow copy of an item, you can either call +:meth:`~scrapy.item.Item.copy` on an existing item +(``product2 = product.copy()``) or instantiate your item class from an existing +item (``product2 = Product(product)``). + +To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +(``product2 = product.deepcopy()``). + + Other common tasks ------------------ -Copying items:: - - >>> product2 = Product(product) - >>> print(product2) - Product(name='Desktop PC', price=1000) - - >>> product3 = product2.copy() - >>> print(product3) - Product(name='Desktop PC', price=1000) - Creating dicts from items:: >>> dict(product) # create a dict from all populated values diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 8e1574376..1a5d52487 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -30,7 +30,7 @@ a *single* job. How to use it ============= -To start a spider with persistence supported enabled, run it like this:: +To start a spider with persistence support enabled, run it like this:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 @@ -71,7 +71,7 @@ on cookies. Request serialization --------------------- -Requests must be serializable by the `pickle` module, in order for persistence +Requests must be serializable by the ``pickle`` module, in order for persistence to work, so you should make sure that your requests are serializable. The most common issue here is to use ``lambda`` functions on request callbacks that diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f40a36d31..713a94e10 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -93,6 +93,12 @@ LxmlLinkExtractor Has the same behaviour as ``restrict_xpaths``. :type restrict_css: str or list + :param restrict_text: a single regular expression (or list of regular expressions) + that the link's text must match in order to be extracted. If not + given (or empty), it will match all links. If a list of regular expressions is + given, the link will be extracted if it matches at least one. + :type restrict_text: a regular expression (or list of) + :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. :type tags: str or list diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index f3b6aa4a1..1c2f1da4d 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -286,7 +286,7 @@ ItemLoader objects given, one is instantiated automatically using the class in :attr:`default_item_class`. - When instantiated with a `selector` or a `response` parameters + When instantiated with a ``selector`` or a ``response`` parameters the :class:`ItemLoader` class provides convenient mechanisms for extracting data from web pages using :ref:`selectors `. diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 0986929ad..8e280d929 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -243,7 +243,7 @@ scrapy.utils.log module case, its usage is not required but it's recommended. If you plan on configuring the handlers yourself is still recommended you - call this function, passing `install_root_handler=False`. Bear in mind + call this function, passing ``install_root_handler=False``. Bear in mind there won't be any log output set by default in that case. To get you started on manually configuring logging's output, you can use diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index a1f518cbd..381a2988a 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -132,7 +132,7 @@ For example, the following image URL:: http://www.example.com/image.jpg -Whose `SHA1 hash` is:: +Whose ``SHA1 hash`` is:: 3afec3b4765f8f0a07b78f98c07b83f013567a0a @@ -310,7 +310,7 @@ images. .. setting:: IMAGES_THUMBS -In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary +In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary where the keys are the thumbnail names and the values are their dimensions. For example:: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 02cfa9b05..298a078a7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -80,7 +80,7 @@ returned by the :meth:`CrawlerRunner.crawl ` method. Here's an example of its usage, along with a callback to manually stop the -reactor after `MySpider` has finished running. +reactor after ``MySpider`` has finished running. :: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e29914dbf..ca59b46d8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -50,7 +50,7 @@ Request objects :type meta: dict :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the `encoding` passed (which defaults to ``utf-8``). If + ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). @@ -489,6 +489,11 @@ method for this job. Here's an example spider which uses it:: import scrapy + def authentication_failed(response): + # TODO: Check the contents of the response and return True if it failed + # or False if it succeeded. + pass + class LoginSpider(scrapy.Spider): name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] @@ -501,13 +506,50 @@ method for this job. Here's an example spider which uses it:: ) def after_login(self, response): - # check login succeed before going on - if "authentication failed" in response.body: + if authentication_failed(response): self.logger.error("Login failed") return # continue scraping with authenticated session... +JSONRequest +----------- + +The JSONRequest class extends the base :class:`Request` class with functionality for +dealing with JSON requests. + +.. class:: JSONRequest(url, [... data, dumps_kwargs]) + + The :class:`JSONRequest` class adds two new argument to the constructor. The + remaining arguments are the same as for the :class:`Request` class and are + not documented here. + + Using the :class:`JSONRequest` will set the ``Content-Type`` header to ``application/json`` + and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01`` + + :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. + if :attr:`Request.body` argument is provided this parameter will be ignored. + if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + set to ``'POST'`` automatically. + :type data: JSON serializable object + + :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize + data into JSON format. + :type dumps_kwargs: dict + +.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps + +JSONRequest usage example +------------------------- + +Sending a JSON POST request with a JSON payload:: + + data = { + 'name1': 'value1', + 'name2': 'value2', + } + yield JSONRequest(url='http://www.example.com/post/action', data=data) + Response objects ================ @@ -606,7 +648,7 @@ Response objects .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for - tagging Responses. For example: `'cached'`, `'redirected`', etc. And + tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. @@ -678,7 +720,7 @@ TextResponse objects ``unicode(response.body)`` is not a correct way to convert response body to unicode: you would be using the system default encoding - (typically `ascii`) instead of the response encoding. + (typically ``ascii``) instead of the response encoding. .. attribute:: TextResponse.encoding @@ -686,7 +728,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the constructor `encoding` argument + 1. the encoding passed in the constructor ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (ie. unknown), it is ignored and the next @@ -724,7 +766,7 @@ TextResponse objects .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is - kept for backwards compatibility; please prefer ``response.text``. + kept for backward compatibility; please prefer ``response.text``. HtmlResponse objects diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9dced7473..282a585d4 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -96,11 +96,11 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of Using selectors --------------- -To explain how to use the selectors we'll use the `Scrapy shell` (which +To explain how to use the selectors we'll use the ``Scrapy shell`` (which provides interactive testing) and an example page located in the Scrapy documentation server: - https://doc.scrapy.org/en/latest/_static/selectors-sample1.html + https://docs.scrapy.org/en/latest/_static/selectors-sample1.html .. _topics-selectors-htmlcode: @@ -113,7 +113,7 @@ For the sake of completeness, here's its full HTML code: First, let's open the shell:: - scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html + scrapy shell https://docs.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have the response available as ``response`` shell variable, and its attached selector in ``response.selector`` attribute. @@ -436,7 +436,7 @@ The following examples show how these methods map to each other. >>> response.css('a::attr(href)').extract() ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] -2. ``Selector.get()`` is the same as ``Selector.extract()``:: +3. ``Selector.get()`` is the same as ``Selector.extract()``:: >>> response.css('a::attr(href)')[0].get() 'image1.html' diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90ae8fd93..b62851679 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -331,16 +331,16 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the request priority based on its depth: +An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of +a :class:`~scrapy.http.Request` based on its depth. -- if zero (default), no priority adjustment is made from depth -- **a positive value will decrease the priority, i.e. higher depth - requests will be processed later** ; this is commonly used when doing - breadth-first crawls (BFO) -- a negative value will increase priority, i.e., higher depth requests - will be processed sooner (DFO) +The priority of a request is adjusted as follows:: -See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. + request.priority = request.priority - ( depth * DEPTH_PRIORITY ) + +As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request +priority (BFO), while negative values increase request priority (DFO). See +also :ref:`faq-bfo-dfo`. .. note:: @@ -599,7 +599,7 @@ The amount of time (in secs) that the downloader will wait before timing out. DOWNLOAD_MAXSIZE ---------------- -Default: `1073741824` (1024MB) +Default: ``1073741824`` (1024MB) The maximum response size (in bytes) that downloader will download. @@ -620,7 +620,7 @@ If you want to disable it set to 0. DOWNLOAD_WARNSIZE ----------------- -Default: `33554432` (32MB) +Default: ``33554432`` (32MB) The response size (in bytes) that downloader will start to warn. @@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. .. setting:: FTP_PASSIVE_MODE @@ -1388,4 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search -.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file +.. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..dfc4d8921 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied. If you want to disable a builtin middleware (the ones defined in :setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it -in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its +in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the off-site middleware:: SPIDER_MIDDLEWARES = { @@ -57,8 +57,12 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each middleware component is a Python class that defines one or more of the -following methods: +Each spider middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.spidermiddlewares @@ -200,7 +204,7 @@ DepthMiddleware .. class:: DepthMiddleware DepthMiddleware is used for tracking the depth of each Request inside the - site being scraped. It works by setting `request.meta['depth'] = 0` whenever + site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever there is no value previously set (usually just the first Request) and incrementing it by 1 otherwise. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..1bf5874a0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -129,7 +129,7 @@ scrapy.Spider You probably won't need to override this directly because the default implementation acts as a proxy to the :meth:`__init__` method, calling - it with the given arguments `args` and named arguments `kwargs`. + it with the given arguments ``args`` and named arguments ``kwargs``. Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` attributes in the new instance so they can be accessed later inside the @@ -190,7 +190,7 @@ scrapy.Spider .. method:: log(message, [level, component]) Wrapper that sends a log message through the Spider's :attr:`logger`, - kept for backwards compatibility. For more information see + kept for backward compatibility. For more information see :ref:`topics-logging-from-spiders`. .. method:: closed(reason) @@ -298,13 +298,13 @@ The above example can also be written as follows:: Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. -If you were to set the `start_urls` attribute from the command line, +If you were to set the ``start_urls`` attribute from the command line, you would have to parse it on your own into a list using something like `ast.literal_eval `_ or `json.loads `_ and then set it as an attribute. -Otherwise, you would cause iteration over a `start_urls` string +Otherwise, you would cause iteration over a ``start_urls`` string (a very common python pitfall) resulting in each character being seen as a separate url. diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 81ce800aa..6c993a970 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -22,7 +22,7 @@ To use the packages: sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7 -2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command:: +2. Create ``/etc/apt/sources.list.d/scrapy.list`` file using the following command:: echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list @@ -34,7 +34,7 @@ To use the packages: .. note:: Repeat step 3 if you are trying to upgrade Scrapy. -.. warning:: `python-scrapy` is a different package provided by official debian +.. warning:: ``python-scrapy`` is a different package provided by official debian repositories, it's very outdated and it isn't supported by Scrapy team. .. _Scrapinghub: https://scrapinghub.com/ diff --git a/docs/versioning.rst b/docs/versioning.rst index 0421ba544..227085f02 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very large changes. * *B* is the release number. This will include many changes including features - and things that possibly break backwards compatibility, although we strive to + and things that possibly break backward compatibility, although we strive to keep theses cases at a minimum. * *C* is the bugfix release number. diff --git a/scrapy/VERSION b/scrapy/VERSION index bc80560fa..dc1e644a1 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.5.0 +1.6.0 diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index dc6b59fe0..fa2506eb0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -99,7 +99,7 @@ def execute(argv=None, settings=None): if argv is None: argv = sys.argv - # --- backwards compatibility for scrapy.conf.settings singleton --- + # --- backward compatibility for scrapy.conf.settings singleton --- if settings is None and 'scrapy.conf' in sys.modules: from scrapy import conf if hasattr(conf, 'settings'): @@ -116,7 +116,7 @@ def execute(argv=None, settings=None): settings['EDITOR'] = editor check_deprecated_settings(settings) - # --- backwards compatibility for scrapy.conf.settings singleton --- + # --- backward compatibility for scrapy.conf.settings singleton --- import warnings from scrapy.exceptions import ScrapyDeprecationWarning with warnings.catch_warnings(): diff --git a/scrapy/conf.py b/scrapy/conf.py index 23efc6ffd..6c40edcdd 100644 --- a/scrapy/conf.py +++ b/scrapy/conf.py @@ -1,4 +1,4 @@ -# This module is kept for backwards compatibility, so users can import +# This module is kept for backward compatibility, so users can import # scrapy.conf.settings and get the settings they expect import sys diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index e4a7d8564..e76823623 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -3,7 +3,7 @@ from .http10 import HTTP10DownloadHandler from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler -# backwards compatibility +# backward compatibility class HttpDownloadHandler(HTTP10DownloadHandler): def __init__(self, *args, **kwargs): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04aee18ed..99bf48385 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -7,6 +7,7 @@ import sys from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement +from scrapy import Spider from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader @@ -27,6 +28,10 @@ logger = logging.getLogger(__name__) class Crawler(object): def __init__(self, spidercls, settings=None): + if isinstance(spidercls, Spider): + raise ValueError( + 'The spidercls argument must be a class, not an object') + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -153,7 +158,7 @@ class CrawlerRunner(object): It will call the given Crawler's :meth:`~Crawler.crawl` method, while keeping track of it so it can be stopped later. - If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler` + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as the spider class given to it. @@ -168,6 +173,10 @@ class CrawlerRunner(object): :param dict kwargs: keyword arguments to initialize the spider """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + 'The crawler_or_spidercls argument cannot be a spider object, ' + 'it must be a spider class (or a Crawler object)') crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) @@ -188,13 +197,17 @@ class CrawlerRunner(object): """ Return a :class:`~scrapy.crawler.Crawler` object. - * If `crawler_or_spidercls` is a Crawler, it is returned as-is. - * If `crawler_or_spidercls` is a Spider subclass, a new Crawler + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler is constructed for it. - * If `crawler_or_spidercls` is a string, this function finds + * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + 'The crawler_or_spidercls argument cannot be a spider object, ' + 'it must be a spider class (or a Crawler object)') if isinstance(crawler_or_spidercls, Crawler): return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) @@ -273,7 +286,7 @@ class CrawlerProcess(CrawlerRunner): :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. - If `stop_after_crawl` is True, the reactor will be stopped after all + If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 07e979628..dbc605a4c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,9 +7,7 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. Once -there is no more failed pages to retry this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. +once the spider has finished crawling all regular (non failed) pages. """ import logging diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 9d8966b9c..0bcdd3495 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -3,8 +3,7 @@ import os import logging from scrapy.utils.job import job_dir -from scrapy.utils.request import request_fingerprint - +from scrapy.utils.request import referer_str, request_fingerprint class BaseDupeFilter(object): @@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter): def log(self, request, spider): if self.debug: - msg = "Filtered duplicate request: %(request)s" - self.logger.debug(msg, {'request': request}, extra={'spider': spider}) + msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" + args = {'request': request, 'referer': referer_str(request) } + self.logger.debug(msg, args, extra={'spider': spider}) elif self.logdupes: msg = ("Filtered duplicate request: %(request)s" " - no more duplicates will be shown" diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1e982c684..3c9c6a3b5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -93,8 +93,8 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None): - # BEGIN Backwards compatibility for initialising without keys (and + def __init__(self, uri, access_key=None, secret_key=None, acl=None): + # BEGIN Backward compatibility for initialising without keys (and # without using from_crawler) no_defaults = access_key is None and secret_key is None if no_defaults: @@ -111,13 +111,14 @@ class S3FeedStorage(BlockingFeedStorage): ) access_key = settings['AWS_ACCESS_KEY_ID'] secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backwards compatibility + # END Backward compatibility u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" + self.acl = acl if self.is_botocore: import botocore.session session = botocore.session.get_session() @@ -130,19 +131,26 @@ class S3FeedStorage(BlockingFeedStorage): @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], - crawler.settings['AWS_SECRET_ACCESS_KEY']) + return cls( + uri=uri, + access_key=crawler.settings['AWS_ACCESS_KEY_ID'], + secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None + ) def _store_in_thread(self, file): file.seek(0) if self.is_botocore: + kwargs = {'ACL': self.acl} if self.acl else {} self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file) + Bucket=self.bucketname, Key=self.keyname, Body=file, + **kwargs) else: conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - key.set_contents_from_file(file) + kwargs = {'policy': self.acl} if self.acl else {} + key.set_contents_from_file(file, **kwargs) key.close() diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index dcf73eb88..26b214ee2 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -112,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index f04a9d3e5..4b2f7b33f 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -10,6 +10,7 @@ from scrapy.http.headers import Headers from scrapy.http.request import Request from scrapy.http.request.form import FormRequest from scrapy.http.request.rpc import XmlRpcRequest +from scrapy.http.request.json_request import JSONRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py new file mode 100644 index 000000000..8f7a61a6d --- /dev/null +++ b/scrapy/http/request/json_request.py @@ -0,0 +1,53 @@ +""" +This module implements the JSONRequest class which is a more convenient class +(than Request) to generate JSON Requests. + +See documentation in docs/topics/request-response.rst +""" + +import copy +import json +import warnings + +from scrapy.http.request import Request + + +class JSONRequest(Request): + def __init__(self, *args, **kwargs): + dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + dumps_kwargs.setdefault('sort_keys', True) + self._dumps_kwargs = dumps_kwargs + + body_passed = kwargs.get('body', None) is not None + data = kwargs.pop('data', None) + data_passed = data is not None + + if body_passed and data_passed: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data_passed: + kwargs['body'] = self._dumps(data) + + if 'method' not in kwargs: + kwargs['method'] = 'POST' + + super(JSONRequest, self).__init__(*args, **kwargs) + self.headers.setdefault('Content-Type', 'application/json') + self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + + def replace(self, *args, **kwargs): + body_passed = kwargs.get('body', None) is not None + data = kwargs.pop('data', None) + data_passed = data is not None + + if body_passed and data_passed: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data_passed: + kwargs['body'] = self._dumps(data) + + return super(JSONRequest, self).replace(*args, **kwargs) + + def _dumps(self, data): + """Convert to JSON """ + return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/item.py b/scrapy/item.py index aa05e9c69..031b80a2d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst from pprint import pformat from collections import MutableMapping +from copy import deepcopy from abc import ABCMeta import six @@ -96,6 +97,13 @@ class DictItem(MutableMapping, BaseItem): def copy(self): return self.__class__(self) + def deepcopy(self): + """Return a `deep copy`_ of this item. + + .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy + """ + return deepcopy(self) + @six.add_metaclass(ItemMeta) class Item(DictItem): diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 97e8c0af1..ebf3cd7d8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -50,7 +50,7 @@ class FilteringLinkExtractor(object): _csstranslator = HTMLTranslator() def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css): + restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): self.link_extractor = link_extractor @@ -70,6 +70,8 @@ class FilteringLinkExtractor(object): if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] def _link_allowed(self, link): if not _is_valid_url(link.url): @@ -85,6 +87,8 @@ class FilteringLinkExtractor(object): return False if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False return True def matches(self, url): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index a7092f9b8..8f6f93a44 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -97,7 +97,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs @@ -111,9 +111,10 @@ class LxmlLinkExtractor(FilteringLinkExtractor): ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = get_base_url(response) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 5fa6b771c..8940a4d77 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -113,7 +113,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=()): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -127,13 +127,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip, - canonicalized=canonicalize) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = None diff --git a/scrapy/log.py b/scrapy/log.py index 719fceaad..777bd6dc4 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -17,7 +17,7 @@ warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on " ScrapyDeprecationWarning, stacklevel=2) -# Imports and level_names variable kept for backwards-compatibility +# Imports and level_names variable kept for backward-compatibility DEBUG = logging.DEBUG INFO = logging.INFO diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 075a6d862..65f347dcf 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -13,21 +13,21 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - All methods must return a dictionary listing the parameters `level`, `msg` - and `args` which are going to be used for constructing the log message when - calling logging.log. + All methods must return a dictionary listing the parameters ``level``, + ``msg`` and ``args`` which are going to be used for constructing the log + message when calling logging.log. Dictionary keys for the method outputs: - * `level` should be the log level for that action, you can use those + * ``level`` should be the log level for that action, you can use those from the python logging library: logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR and logging.CRITICAL. - * `msg` should be a string that can contain different formatting - placeholders. This string, formatted with the provided `args`, is going - to be the log message for that action. + * ``msg`` should be a string that can contain different formatting + placeholders. This string, formatted with the provided ``args``, is + going to be the log message for that action. - * `args` should be a tuple or dict with the formatting placeholders for - `msg`. The final log message is computed as output['msg'] % + * ``args`` should be a tuple or dict with the formatting placeholders + for ``msg``. The final log message is computed as output['msg'] % output['args']. """ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 510cc23c7..2d8091f5b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -255,13 +255,13 @@ class FilesPipeline(MediaPipeline): doing stat of the files and determining if file is new, uptodate or expired. - `new` files are those that pipeline never processed and needs to be + ``new`` files are those that pipeline never processed and needs to be downloaded from supplier site the first time. - `uptodate` files are the ones that the pipeline processed and are still + ``uptodate`` files are the ones that the pipeline processed and are still valid files. - `expired` files are those that pipeline already processed but the last + ``expired`` files are those that pipeline already processed but the last modification was made long time ago, so a reprocessing is recommended to refresh it in case of change. diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 95323c613..a1457c7e9 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -89,6 +89,10 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs'] diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 45257a61c..367a2eb43 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -160,6 +160,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FEED_STORAGE_GCS_ACL = '' +FEED_STORAGE_S3_ACL = '' FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/scrapy/signals.py b/scrapy/signals.py index c0e4bb74e..6b9125302 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -20,7 +20,7 @@ item_scraped = object() item_dropped = object() item_error = object() -# for backwards compatibility +# for backward compatibility stats_spider_opened = spider_opened stats_spider_closing = spider_closed stats_spider_closed = spider_closed diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 7d766f4fc..a12d08414 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your scraped items # # See documentation in: -# https://doc.scrapy.org/en/latest/topics/items.html +# https://docs.scrapy.org/en/latest/topics/items.html import scrapy diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index c5b542bd6..5debe1cd2 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your spider middleware # # See documentation in: -# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index e58dab089..fb641d447 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -3,7 +3,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class ${ProjectName}Pipeline(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a0557473e..cb220eafc 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,9 +5,9 @@ # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # -# https://doc.scrapy.org/en/latest/topics/settings.html -# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html -# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' @@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) -# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: @@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True #} # Enable or disable spider middlewares -# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares -# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions -# See https://doc.scrapy.org/en/latest/topics/extensions.html +# See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines -# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) -# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 @@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) -# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 435e9a6b3..fbd297340 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -42,14 +42,14 @@ def build_component_list(compdict, custom=None, convert=update_classpath): raise ValueError('Invalid value {} for component {}, please provide ' \ 'a real number or None instead'.format(value, name)) - # BEGIN Backwards compatibility for old (base, custom) call signature + # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) return type(custom)(convert(c) for c in custom) if custom is not None: compdict.update(custom) - # END Backwards compatibility + # END Backward compatibility _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f255d436f..9eca6a4da 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -2,7 +2,7 @@ from ftplib import error_perm from posixpath import dirname def ftp_makedirs_cwd(ftp, path, first_call=True): - """Set the current directory of the FTP connection given in the `ftp` + """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. """ diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 828880709..e07fb8698 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -32,7 +32,7 @@ class TopLevelFormatter(logging.Filter): Since it can't be set for just one logger (it won't propagate for its children), it's going to be set in the root handler, with a parametrized - `loggers` list where it should act. + ``loggers`` list where it should act. """ def __init__(self, loggers=None): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ccfdcd72..6de36d45c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -116,7 +116,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return True if rel is not None and 'nofollow' in rel.split() else False + return rel is not None and 'nofollow' in rel.split() def create_instance(objcls, settings, crawler, *args, **kwargs): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 732ca13a0..aade3d9ac 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -97,8 +97,8 @@ def unicode_to_str(text, encoding=None, errors='strict'): def to_unicode(text, encoding=None, errors='strict'): - """Return the unicode representation of a bytes object `text`. If `text` - is already an unicode object, return it as-is.""" + """Return the unicode representation of a bytes object ``text``. If + ``text`` is already an unicode object, return it as-is.""" if isinstance(text, six.text_type): return text if not isinstance(text, (bytes, six.text_type)): @@ -110,7 +110,7 @@ def to_unicode(text, encoding=None, errors='strict'): def to_bytes(text, encoding=None, errors='strict'): - """Return the binary representation of `text`. If `text` + """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text @@ -123,7 +123,7 @@ def to_bytes(text, encoding=None, errors='strict'): def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of `text` + """ Return str representation of ``text`` (bytes in Python 2.x and unicode in Python 3.x). """ if six.PY2: return to_bytes(text, encoding, errors) @@ -189,7 +189,7 @@ def isbinarytext(text): def binary_is_text(data): - """ Returns `True` if the given ``data`` argument (a ``bytes`` object) + """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ if not isinstance(data, bytes): @@ -314,7 +314,7 @@ class WeakKeyCache(object): @deprecated def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """Return a (new) dict with unicode keys (and values when "keys_only" is - False) of the given dict converted to strings. `dct_or_tuples` can be a + False) of the given dict converted to strings. ``dct_or_tuples`` can be a dict or a list of tuples, like any dict constructor supports. """ d = {} @@ -357,10 +357,10 @@ def retry_on_eintr(function, *args, **kw): def without_none_values(iterable): - """Return a copy of `iterable` with all `None` entries removed. + """Return a copy of ``iterable`` with all ``None`` entries removed. - If `iterable` is a mapping, return a dictionary where all pairs that have - value `None` have been removed. + If ``iterable`` is a mapping, return a dictionary where all pairs that have + value ``None`` have been removed. """ try: return {k: v for k, v in six.iteritems(iterable) if v is not None} diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 657c53815..b3a4be007 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -109,12 +109,12 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= """Strip URL string from some of its components: - - `strip_credentials` removes "user:password@" - - `strip_default_port` removes ":80" (resp. ":443", ":21") + - ``strip_credentials`` removes "user:password@" + - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - - `origin_only` replaces path component with "/", also dropping + - ``origin_only`` replaces path component with "/", also dropping query and fragment components ; it also strips credentials - - `strip_fragment` drops any #fragment component + - ``strip_fragment`` drops any #fragment component """ parsed_url = urlparse(url) diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 3766f38fc..2a66f9802 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -61,7 +61,7 @@ ItemForm -------- Pros: -- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html) +- same API used for Items (see https://docs.scrapy.org/en/latest/topics/items.html) - some people consider setitem API more elegant than methods API Cons: diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 7425c0930..eb362e945 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -10,13 +10,14 @@ Status Obsolete (discarded) SEP-006: Rename of Selectors to Extractors ========================================== -This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and their `x` method. +This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and +their ``x`` method. Motivation ========== When you use Selectors in Scrapy, your final goal is to "extract" the data that -you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html +you've selected, as the [https://docs.scrapy.org/en/latest/topics/selectors.html XPath Selectors documentation] says (bolding by me): When you’re scraping web pages, the most common task you need to perform is @@ -57,7 +58,7 @@ Additional changes As the name of the method for performing selection (the ``x`` method) is not descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method (x sounds like a short for extract in english), we propose to rename it to -`select`, `sel` (is shortness if required), or `xpath` after `lxml's +``select``, ``sel`` (is shortness if required), or ``xpath`` after `lxml's `_ ``xpath`` method. Bonus (ItemBuilder) @@ -71,5 +72,5 @@ webpage or set of pages. References ========== - 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) + 1. XPath Selectors (https://docs.scrapy.org/topics/selectors.html) 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index aca7ac342..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -211,7 +211,7 @@ spider methods on each event such as: - call additional spider middlewares defined in the ``Spider.middlewares`` attribute - call ``Spider.next_request()`` and ``Spider.start_requests()`` on - ``next_request()`` middleware method (this would implicitly support backwards + ``next_request()`` middleware method (this would implicitly support backward compatibility) Differences with Spider middleware v1 diff --git a/tests/__init__.py b/tests/__init__.py index 55b1ecde8..a54367f8c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ """ tests: this package contains all Scrapy unittests -see https://doc.scrapy.org/en/latest/contributing.html#running-tests +see https://docs.scrapy.org/en/latest/contributing.html#running-tests """ import os diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 40d9293b2..431428331 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -16,7 +16,7 @@ _DATABASES = collections.defaultdict(DummyDB) def open(file, flag='r', mode=0o666): """Open or create a dummy database compatible. - Arguments `flag` and `mode` are ignored. + Arguments ``flag`` and ``mode`` are ignored. """ # return same instance for same file argument return _DATABASES[file] diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 36baacfbd..d664b6ade 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -61,7 +61,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_fetch_redirect_follow_302(self): - """Test that calling `fetch(url)` follows HTTP redirects by default.""" + """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') code = "fetch('{0}')" errcode, out, errout = yield self.execute(['-c', code.format(url)]) @@ -71,7 +71,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_fetch_redirect_not_follow_302(self): - """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') code = "fetch('{0}', redirect=False)" errcode, out, errout = yield self.execute(['-c', code.format(url)]) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 268948a70..37cea3ad3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import warnings from twisted.internet import defer from twisted.trial import unittest +from pytest import raises import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess @@ -66,6 +67,10 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') + def test_crawler_rejects_spider_objects(self): + with raises(ValueError): + Crawler(DefaultSpider()) + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -177,6 +182,14 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertEqual(len(w), 1) self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message)) + def test_crawl_rejects_spider_objects(self): + with raises(ValueError): + CrawlerRunner().crawl(DefaultSpider()) + + def test_create_crawler_rejects_spider_objects(self): + with raises(ValueError): + CrawlerRunner().create_crawler(DefaultSpider()) + class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0d0829793..81235a16f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -50,7 +50,7 @@ class DummyDH(object): class DummyLazyDH(object): - # Default is lazy for backwards compatibility + # Default is lazy for backward compatibility def __init__(self, crawler): pass diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 537126613..30920b2da 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -25,7 +25,7 @@ class TestHttpProxyMiddleware(TestCase): def test_not_enabled(self): settings = Settings({'HTTPPROXY_ENABLED': False}) - crawler = Crawler(spider, settings) + crawler = Crawler(Spider, settings) self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) def test_no_environment_proxies(self): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index db69597a2..d7eb98c97 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,6 +2,7 @@ import hashlib import tempfile import unittest import shutil +from testfixtures import LogCapture from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request @@ -9,7 +10,7 @@ from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler - +from tests.spiders import SimpleSpider class FromCrawlerRFPDupeFilter(RFPDupeFilter): @@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase): assert case_insensitive_dupefilter.request_seen(r2) case_insensitive_dupefilter.close('finished') + + def test_log(self): + with LogCapture() as l: + settings = {'DUPEFILTER_DEBUG': False, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + spider = SimpleSpider.from_crawler(crawler) + + dupefilter = scheduler.df + dupefilter.open() + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html') + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' - no more duplicates will be shown' + ' (see DUPEFILTER_DEBUG to show all duplicates)'))) + + dupefilter.close('finished') + + def test_log_debug(self): + with LogCapture() as l: + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + spider = SimpleSpider.from_crawler(crawler) + + dupefilter = scheduler.df + dupefilter.open() + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html', + headers={'Referer': 'http://scrapytest.org/INDEX.html'} + ) + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' (referer: None)'))) + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' (referer: http://scrapytest.org/INDEX.html)'))) + + dupefilter.close('finished') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 69f144d07..9b8e2660a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -162,7 +162,7 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backwards compatibility for initialising without settings + # Backward compatibility for initialising without settings with warnings.catch_warnings(record=True) as w: storage = S3FeedStorage('s3://mybucket/export.csv') self.assertEqual(storage.access_key, 'conf_key') @@ -187,6 +187,151 @@ class S3FeedStorageTest(unittest.TestCase): content = get_s3_content_and_delete(u.hostname, u.path[1:]) self.assertEqual(content, expected_content) + def test_init_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_init_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + def test_from_crawler_without_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_from_crawler_with_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_STORAGE_S3_ACL': 'custom-acl', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + @defer.inlineCallbacks + def test_store_botocore_without_acl(self): + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') + + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1]) + + @defer.inlineCallbacks + def test_store_botocore_with_acl(self): + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') + + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertEqual( + storage.s3_client.put_object.call_args[1].get('ACL'), + 'custom-acl' + ) + + @defer.inlineCallbacks + def test_store_not_botocore_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + yield storage.store(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertNotIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + + @defer.inlineCallbacks + def test_store_not_botocore_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + yield storage.store(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + class GCSFeedStorageTest(unittest.TestCase): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..49f148016 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -2,6 +2,8 @@ import cgi import unittest import re +import json +import warnings import six from six.moves import xmlrpc_client as xmlrpclib @@ -9,9 +11,11 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote if six.PY3: from urllib.parse import unquote_to_bytes -from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse +from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str +from tests import mock + class RequestTest(unittest.TestCase): @@ -1147,5 +1151,169 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=(u'pas£',), encoding='latin1') +class JSONRequestTest(RequestTest): + request_class = JSONRequest + default_method = 'GET' + default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} + + def setUp(self): + warnings.simplefilter("always") + super(JSONRequestTest, self).setUp() + + def test_data(self): + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.body, b'') + + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.body, body) + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.body, to_bytes(json.dumps(data))) + + # empty data + r4 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r4.body, to_bytes(json.dumps([]))) + + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.method, 'GET') + + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.method, 'GET') + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.method, 'POST') + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method='GET') + self.assertEqual(r4.method, 'GET') + + r5 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r5.method, 'POST') + + def test_body_data(self): + """ passing both body and data should result a warning """ + body = b'body' + data = { + 'name': 'value', + } + with warnings.catch_warnings(record=True) as _warnings: + r5 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r5.body, body) + self.assertEqual(r5.method, 'GET') + self.assertEqual(len(_warnings), 1) + self.assertIn('data will be ignored', str(_warnings[0].message)) + + def test_empty_body_data(self): + """ passing any body value and data should result a warning """ + data = { + 'name': 'value', + } + with warnings.catch_warnings(record=True) as _warnings: + r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) + self.assertEqual(r6.body, b'') + self.assertEqual(r6.method, 'GET') + self.assertEqual(len(_warnings), 1) + self.assertIn('data will be ignored', str(_warnings[0].message)) + + def test_body_none_data(self): + data = { + 'name': 'value', + } + with warnings.catch_warnings(record=True) as _warnings: + r7 = self.request_class(url="http://www.example.com/", body=None, data=data) + self.assertEqual(r7.body, to_bytes(json.dumps(data))) + self.assertEqual(r7.method, 'POST') + self.assertEqual(len(_warnings), 0) + + def test_body_data_none(self): + with warnings.catch_warnings(record=True) as _warnings: + r8 = self.request_class(url="http://www.example.com/", body=None, data=None) + self.assertEqual(r8.method, 'GET') + self.assertEqual(len(_warnings), 0) + + def test_dumps_sort_keys(self): + """ Test that sort_keys=True is passed to json.dumps by default """ + data = { + 'name': 'value', + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_dumps_kwargs(self): + """ Test that dumps_kwargs are passed to json.dumps """ + data = { + 'name': 'value', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + + def test_replace_data(self): + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + + def test_replace_sort_keys(self): + """ Test that replace provides sort_keys=True to json.dumps """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_replace_dumps_kwargs(self): + """ Test that dumps_kwargs are provided to json.dumps when replace is called """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + + def tearDown(self): + warnings.resetwarnings() + super(JSONRequestTest, self).tearDown() + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_item.py b/tests/test_item.py index 2c1eb0dd3..010d3b141 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -249,6 +249,14 @@ class ItemTest(unittest.TestCase): copied_item['name'] = copied_item['name'].upper() self.assertNotEqual(item['name'], copied_item['name']) + def test_deepcopy(self): + class TestItem(Item): + tags = Field() + item = TestItem({'tags': ['tag1']}) + copied_item = item.deepcopy() + item['tags'].append('tag2') + assert item['tags'] != copied_item['tags'] + class ItemMetaTest(unittest.TestCase): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 903032b52..c9cd629f4 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -479,6 +479,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_link_restrict_text(self): + html = b""" + Pic of a cat + Pic of a dog + Pic of a cow + """ + response = HtmlResponse("http://example.org/index.html", body=html) + # Simple text inclusion test + lx = self.extractor_cls(restrict_text='dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Unique regex test + lx = self.extractor_cls(restrict_text=r'of.*dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Multiple regex test + lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f203c32ef..29937c189 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -11,7 +11,7 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ['one', 'four', 'three']) - def test_backwards_compatible_build_dict(self): + def test_backward_compatible_build_dict(self): base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} custom = {'two': None, 'three': 8, 'four': 4} self.assertEqual(build_component_list(base, custom,