mirror of https://github.com/scrapy/scrapy.git
Merge pull request #2 from scrapy/master
LOG_DICT_CONFIG for custom logging
This commit is contained in:
w-495
GitHub
commit
f80beea6f8
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 1.2.1
|
||||
current_version = 1.3.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -12,3 +12,4 @@ prune docs/build
|
|||
recursive-include extras *
|
||||
recursive-include bin *
|
||||
recursive-include tests *
|
||||
global-exclude __pycache__ *.py[cod]
|
||||
|
|
|
|||
|
|
@ -317,6 +317,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items
|
|||
You may need to remove namespaces. See :ref:`removing-namespaces`.
|
||||
|
||||
.. _user agents: https://en.wikipedia.org/wiki/User_agent
|
||||
.. _LIFO: https://en.wikipedia.org/wiki/LIFO
|
||||
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
|
||||
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
|
||||
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
|
||||
|
|
|
|||
|
|
@ -130,15 +130,15 @@ will send some requests for the ``quotes.toscrape.com`` domain. You will get an
|
|||
similar to this::
|
||||
|
||||
... (omitted for brevity)
|
||||
2016-09-20 14:48:00 [scrapy] INFO: Spider opened
|
||||
2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
...
|
||||
|
||||
Now, check the files in the current directory. You should notice that two new
|
||||
|
|
@ -212,7 +212,7 @@ using the shell :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
You will see something like::
|
||||
|
||||
[ ... Scrapy log here ... ]
|
||||
2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
|
||||
|
|
@ -429,9 +429,9 @@ in the callback, as you can see below::
|
|||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
||||
2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
|
||||
2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,81 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
Scrapy 1.3.0 (2016-12-21)
|
||||
-------------------------
|
||||
|
||||
This release comes rather soon after 1.2.2 for one main reason:
|
||||
it was found out that releases since 0.18 up to 1.2.2 (included) use
|
||||
some backported code from Twisted (``scrapy.xlib.tx.*``),
|
||||
even if newer Twisted modules are available.
|
||||
Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly.
|
||||
(See also cleanups below.)
|
||||
|
||||
As it is a major change, we wanted to get the bug fix out quickly
|
||||
while not breaking any projects using the 1.2 series.
|
||||
|
||||
New Features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- ``MailSender`` now accepts single strings as values for ``to`` and ``cc``
|
||||
arguments (:issue:`2272`)
|
||||
- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside
|
||||
scrapy shell now follow HTTP redirections by default (:issue:`2290`);
|
||||
See :command:`fetch` and :command:`shell` for details.
|
||||
- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
|
||||
this is technically **backwards incompatible** so please check your log parsers.
|
||||
- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
|
||||
instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
|
||||
this is **backwards incompatible** if you have log parsers expecting the short
|
||||
logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
|
||||
set to ``True``.
|
||||
|
||||
Dependencies & Cleanups
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Scrapy now requires Twisted >= 13.1 which is the case for many Linux
|
||||
distributions already.
|
||||
- As a consequence, we got rid of ``scrapy.xlib.tx.*`` modules, which
|
||||
copied some of Twisted code for users stuck with an "old" Twisted version
|
||||
- ``ChunkedTransferMiddleware`` is deprecated and removed from the default
|
||||
downloader middlewares.
|
||||
|
||||
|
||||
Scrapy 1.2.2 (2016-12-06)
|
||||
-------------------------
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Fix a cryptic traceback when a pipeline fails on ``open_spider()`` (:issue:`2011`)
|
||||
- Fix embedded IPython shell variables (fixing :issue:`396` that re-appeared
|
||||
in 1.2.0, fixed in :issue:`2418`)
|
||||
- A couple of patches when dealing with robots.txt:
|
||||
|
||||
- handle (non-standard) relative sitemap URLs (:issue:`2390`)
|
||||
- handle non-ASCII URLs and User-Agents in Python 2 (:issue:`2373`)
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`)
|
||||
- Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`)
|
||||
- A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`)
|
||||
and clarifications (:issue:`2354`, :issue:`2325`, :issue:`2414`)
|
||||
|
||||
Other changes
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Advertize `conda-forge`_ as Scrapy's official conda channel (:issue:`2387`)
|
||||
- More helpful error messages when trying to use ``.css()`` or ``.xpath()``
|
||||
on non-Text Responses (:issue:`2264`)
|
||||
- ``startproject`` command now generates a sample ``middlewares.py`` file (:issue:`2335`)
|
||||
- Add more dependencies' version info in ``scrapy version`` verbose output (:issue:`2404`)
|
||||
- Remove all ``*.pyc`` files from source distribution (:issue:`2386`)
|
||||
|
||||
.. _conda-forge: https://anaconda.org/conda-forge/scrapy
|
||||
|
||||
|
||||
Scrapy 1.2.1 (2016-10-21)
|
||||
-------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -171,7 +171,8 @@ SpiderLoader API
|
|||
|
||||
This class method is used by Scrapy to create an instance of the class.
|
||||
It's called with the current project settings, and it loads the spiders
|
||||
found in the modules of the :setting:`SPIDER_MODULES` setting.
|
||||
found recursively in the modules of the :setting:`SPIDER_MODULES`
|
||||
setting.
|
||||
|
||||
:param settings: project settings
|
||||
:type settings: :class:`~scrapy.settings.Settings` instance
|
||||
|
|
|
|||
|
|
@ -18,40 +18,66 @@ To run it use::
|
|||
|
||||
You should see an output like this::
|
||||
|
||||
2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot)
|
||||
2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened
|
||||
2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout)
|
||||
2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats:
|
||||
{'downloader/request_bytes': 231508,
|
||||
'downloader/request_count': 682,
|
||||
'downloader/request_method_count/GET': 682,
|
||||
'downloader/response_bytes': 1172802,
|
||||
'downloader/response_count': 682,
|
||||
'downloader/response_status_count/200': 682,
|
||||
'finish_reason': 'closespider_timeout',
|
||||
'finish_time': datetime.datetime(2013, 5, 16, 16, 8, 57, 985539),
|
||||
'log_count/INFO': 14,
|
||||
'request_depth_max': 34,
|
||||
'response_received_count': 682,
|
||||
'scheduler/dequeued': 682,
|
||||
'scheduler/dequeued/memory': 682,
|
||||
'scheduler/enqueued': 12767,
|
||||
'scheduler/enqueued/memory': 12767,
|
||||
'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)}
|
||||
2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout)
|
||||
2016-12-16 21:18:48 [scrapy.utils.log] INFO: Scrapy 1.2.2 started (bot: quotesbot)
|
||||
2016-12-16 21:18:48 [scrapy.utils.log] INFO: Overridden settings: {'CLOSESPIDER_TIMEOUT': 10, 'ROBOTSTXT_OBEY': True, 'SPIDER_MODULES': ['quotesbot.spiders'], 'LOGSTATS_INTERVAL': 1, 'BOT_NAME': 'quotesbot', 'LOG_LEVEL': 'INFO', 'NEWSPIDER_MODULE': 'quotesbot.spiders'}
|
||||
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled extensions:
|
||||
['scrapy.extensions.closespider.CloseSpider',
|
||||
'scrapy.extensions.logstats.LogStats',
|
||||
'scrapy.extensions.telnet.TelnetConsole',
|
||||
'scrapy.extensions.corestats.CoreStats']
|
||||
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares:
|
||||
['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
|
||||
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
|
||||
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
|
||||
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
|
||||
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
|
||||
'scrapy.downloadermiddlewares.retry.RetryMiddleware',
|
||||
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
|
||||
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
|
||||
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
|
||||
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
|
||||
'scrapy.downloadermiddlewares.stats.DownloaderStats']
|
||||
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares:
|
||||
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
|
||||
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
|
||||
'scrapy.spidermiddlewares.referer.RefererMiddleware',
|
||||
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
|
||||
'scrapy.spidermiddlewares.depth.DepthMiddleware']
|
||||
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled item pipelines:
|
||||
[]
|
||||
2016-12-16 21:18:49 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:18:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:50 [scrapy.extensions.logstats] INFO: Crawled 70 pages (at 4200 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:51 [scrapy.extensions.logstats] INFO: Crawled 134 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:52 [scrapy.extensions.logstats] INFO: Crawled 198 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:53 [scrapy.extensions.logstats] INFO: Crawled 254 pages (at 3360 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:54 [scrapy.extensions.logstats] INFO: Crawled 302 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:55 [scrapy.extensions.logstats] INFO: Crawled 358 pages (at 3360 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:56 [scrapy.extensions.logstats] INFO: Crawled 406 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:57 [scrapy.extensions.logstats] INFO: Crawled 438 pages (at 1920 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:58 [scrapy.extensions.logstats] INFO: Crawled 470 pages (at 1920 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:18:59 [scrapy.core.engine] INFO: Closing spider (closespider_timeout)
|
||||
2016-12-16 21:18:59 [scrapy.extensions.logstats] INFO: Crawled 518 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:19:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
|
||||
{'downloader/request_bytes': 229995,
|
||||
'downloader/request_count': 534,
|
||||
'downloader/request_method_count/GET': 534,
|
||||
'downloader/response_bytes': 1565504,
|
||||
'downloader/response_count': 534,
|
||||
'downloader/response_status_count/200': 534,
|
||||
'finish_reason': 'closespider_timeout',
|
||||
'finish_time': datetime.datetime(2016, 12, 16, 16, 19, 0, 647725),
|
||||
'log_count/INFO': 17,
|
||||
'request_depth_max': 19,
|
||||
'response_received_count': 534,
|
||||
'scheduler/dequeued': 533,
|
||||
'scheduler/dequeued/memory': 533,
|
||||
'scheduler/enqueued': 10661,
|
||||
'scheduler/enqueued/memory': 10661,
|
||||
'start_time': datetime.datetime(2016, 12, 16, 16, 18, 49, 799869)}
|
||||
2016-12-16 21:19:00 [scrapy.core.engine] INFO: Spider closed (closespider_timeout)
|
||||
|
||||
That tells you that Scrapy is able to crawl about 3900 pages per minute in the
|
||||
That tells you that Scrapy is able to crawl about 3000 pages per minute in the
|
||||
hardware where you run it. Note that this is a very simple spider intended to
|
||||
follow links, any custom spider you write will probably do more stuff which
|
||||
results in slower crawl rates. How slower depends on how much your spider does
|
||||
|
|
|
|||
|
|
@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page.
|
|||
If used outside a project, no particular per-spider behaviour would be applied
|
||||
and it will just use the default Scrapy downloader settings.
|
||||
|
||||
Supported options:
|
||||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``--headers``: print the response's HTTP headers instead of the response's body
|
||||
|
||||
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
|
||||
|
||||
Usage examples::
|
||||
|
||||
$ scrapy fetch --nolog http://www.example.com/some/page.html
|
||||
|
|
@ -368,11 +376,34 @@ given. Also supports UNIX-style local file paths, either relative with
|
|||
``./`` or ``../`` prefixes or absolute file paths.
|
||||
See :ref:`topics-shell` for more info.
|
||||
|
||||
Supported options:
|
||||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``-c code``: evaluate the code in the shell, print the result and exit
|
||||
|
||||
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them);
|
||||
this only affects the URL you may pass as argument on the command line;
|
||||
once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default.
|
||||
|
||||
Usage example::
|
||||
|
||||
$ scrapy shell http://www.example.com/some/page.html
|
||||
[ ... scrapy shell starts ... ]
|
||||
|
||||
$ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)'
|
||||
(200, 'http://www.example.com/')
|
||||
|
||||
# shell follows HTTP redirects by default
|
||||
$ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
|
||||
(200, 'http://example.com/')
|
||||
|
||||
# you can disable this with --no-redirect
|
||||
# (only for the URL passed as command line argument)
|
||||
$ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
|
||||
(302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F')
|
||||
|
||||
|
||||
.. command:: parse
|
||||
|
||||
parse
|
||||
|
|
|
|||
|
|
@ -238,14 +238,14 @@ header) and all cookies received in responses (ie. ``Set-Cookie`` header).
|
|||
|
||||
Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
|
||||
|
||||
2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened
|
||||
2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
|
||||
2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened
|
||||
2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
|
||||
Cookie: clientlanguage_nl=en_EN
|
||||
2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
|
||||
2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
|
||||
Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
|
||||
Set-Cookie: ip_isocode=US
|
||||
Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
|
||||
2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
|
||||
2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
|
||||
[...]
|
||||
|
||||
|
||||
|
|
@ -657,16 +657,6 @@ Default: ``True``
|
|||
Whether the Compression middleware will be enabled.
|
||||
|
||||
|
||||
ChunkedTransferMiddleware
|
||||
-------------------------
|
||||
|
||||
.. module:: scrapy.downloadermiddlewares.chunked
|
||||
:synopsis: Chunked Transfer Middleware
|
||||
|
||||
.. class:: ChunkedTransferMiddleware
|
||||
|
||||
This middleware adds support for `chunked transfer encoding`_
|
||||
|
||||
HttpProxyMiddleware
|
||||
-------------------
|
||||
|
||||
|
|
@ -970,4 +960,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
|
|||
|
||||
.. _DBM: https://en.wikipedia.org/wiki/Dbm
|
||||
.. _anydbm: https://docs.python.org/2/library/anydbm.html
|
||||
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
|
|
|
|||
|
|
@ -35,12 +35,6 @@ And here is how to use it to send an e-mail (without attachments)::
|
|||
|
||||
mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
|
||||
|
||||
.. note::
|
||||
As shown in the example above, ``to`` and ``cc`` need to be lists
|
||||
of email addresses, not single addresses, and even for one recipient,
|
||||
i.e. ``to="someone@example.com"`` will not work.
|
||||
|
||||
|
||||
MailSender class reference
|
||||
==========================
|
||||
|
||||
|
|
@ -87,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
|
|||
Send email to the given recipients.
|
||||
|
||||
:param to: the e-mail recipients
|
||||
:type to: list
|
||||
:type to: str or list of str
|
||||
|
||||
:param subject: the subject of the e-mail
|
||||
:type subject: str
|
||||
|
||||
:param cc: the e-mails to CC
|
||||
:type cc: list
|
||||
:type cc: str or list of str
|
||||
|
||||
:param body: the e-mail body
|
||||
:type body: str
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ Logging
|
|||
about the new logging system.
|
||||
|
||||
Scrapy uses `Python's builtin logging system
|
||||
<https://docs.python.org/2/library/logging.html>`_ for event logging. We'll
|
||||
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
|
||||
provide some simple examples to get you started, but for more advanced
|
||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||
|
||||
|
|
@ -150,6 +150,7 @@ These settings can be used to configure the logging:
|
|||
* :setting:`LOG_FORMAT`
|
||||
* :setting:`LOG_DATEFORMAT`
|
||||
* :setting:`LOG_STDOUT`
|
||||
* :setting:`LOG_SHORT_NAMES`
|
||||
|
||||
The first couple of settings define a destination for log messages. If
|
||||
:setting:`LOG_FILE` is set, messages sent through the root logger will be
|
||||
|
|
@ -170,6 +171,10 @@ listed in `logging's logrecord attributes docs
|
|||
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
|
||||
respectively.
|
||||
|
||||
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy
|
||||
component that prints the log. It is unset by default, hence logs contain the
|
||||
scrapy component responsible for that log output.
|
||||
|
||||
Command-line options
|
||||
--------------------
|
||||
|
||||
|
|
@ -188,6 +193,43 @@ to override some of the Scrapy settings regarding logging.
|
|||
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
|
||||
Further documentation on available handlers
|
||||
|
||||
Advanced customization
|
||||
----------------------
|
||||
|
||||
Because Scrapy uses stdlib logging module, you can customize logging using
|
||||
all features of stdlib logging.
|
||||
|
||||
For example, let's say you're scraping a website which returns many
|
||||
HTTP 404 and 500 responses, and you want to hide all messages like this::
|
||||
|
||||
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
|
||||
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
is not handled or not allowed
|
||||
|
||||
The first thing to note is a logger name - it is in brackets:
|
||||
``[scrapy.spidermiddlewares.httperror]``. If you get just ``[scrapy]`` then
|
||||
:setting:`LOG_SHORT_NAMES` is likely set to True; set it to False and re-run
|
||||
the crawl.
|
||||
|
||||
Next, we can see that the message has INFO level. To hide it
|
||||
we should set logging level for ``scrapy.spidermiddlewares.httperror``
|
||||
higher than INFO; next level after INFO is WARNING. It could be done
|
||||
e.g. in the spider's ``__init__`` method::
|
||||
|
||||
import logging
|
||||
import scrapy
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# ...
|
||||
def __init__(self, *args, **kwargs):
|
||||
logger = logging.getLogger('scrapy.spidermiddlewares.httperror')
|
||||
logger.setLevel(logging.WARNING)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
If you run this spider again then INFO messages from
|
||||
``scrapy.spidermiddlewares.httperror`` logger will be gone.
|
||||
|
||||
scrapy.utils.log module
|
||||
=======================
|
||||
|
||||
|
|
|
|||
|
|
@ -468,7 +468,6 @@ Default::
|
|||
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
|
||||
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
|
||||
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
|
||||
'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
|
||||
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
|
||||
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
|
||||
}
|
||||
|
|
@ -789,6 +788,16 @@ If ``True``, all standard output (and error) of your process will be redirected
|
|||
to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
|
||||
log.
|
||||
|
||||
.. setting:: LOG_SHORT_NAMES
|
||||
|
||||
LOG_SHORT_NAMES
|
||||
---------------
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If ``True``, the logs will just contain the root path. If it is set to ``False``
|
||||
then it displays the component responsible for the log output
|
||||
|
||||
.. setting:: MEMDEBUG_ENABLED
|
||||
|
||||
MEMDEBUG_ENABLED
|
||||
|
|
@ -1028,11 +1037,40 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap
|
|||
|
||||
Example entry in logs::
|
||||
|
||||
1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request:
|
||||
1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request:
|
||||
<GET http://example.com> - reason: cannot serialize <Request at 0x9a7c7ec>
|
||||
(type Request)> - no more unserializable requests will be logged
|
||||
(see 'scheduler/unserializable' stats counter)
|
||||
|
||||
|
||||
.. setting:: SCHEDULER_DISK_QUEUE
|
||||
|
||||
SCHEDULER_DISK_QUEUE
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.squeues.PickleLifoDiskQueue'``
|
||||
|
||||
Type of disk queue that will be used by scheduler. Other available types are
|
||||
``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``,
|
||||
``scrapy.squeues.MarshalLifoDiskQueue``.
|
||||
|
||||
.. setting:: SCHEDULER_MEMORY_QUEUE
|
||||
|
||||
SCHEDULER_MEMORY_QUEUE
|
||||
----------------------
|
||||
Default: ``'scrapy.squeues.LifoMemoryQueue'``
|
||||
|
||||
Type of in-memory queue used by scheduler. Other available type is:
|
||||
``scrapy.squeues.FifoMemoryQueue``.
|
||||
|
||||
.. setting:: SCHEDULER_PRIORITY_QUEUE
|
||||
|
||||
SCHEDULER_PRIORITY_QUEUE
|
||||
------------------------
|
||||
Default: ``'queuelib.PriorityQueue'``
|
||||
|
||||
Type of priority queue used by scheduler.
|
||||
|
||||
.. setting:: SPIDER_CONTRACTS
|
||||
|
||||
SPIDER_CONTRACTS
|
||||
|
|
|
|||
|
|
@ -97,8 +97,12 @@ Available Shortcuts
|
|||
|
||||
* ``shelp()`` - print a help with the list of available objects and shortcuts
|
||||
|
||||
* ``fetch(request_or_url)`` - fetch a new response from the given request or
|
||||
URL and update all related objects accordingly.
|
||||
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
|
||||
URL and update all related objects accordingly. You can optionaly ask for
|
||||
HTTP 3xx redirections to not be followed by passing ``redirect=False``
|
||||
|
||||
* ``fetch(request)`` - fetch a new response from the given request and
|
||||
update all related objects accordingly.
|
||||
|
||||
* ``view(response)`` - open the given response in your local web browser, for
|
||||
inspection. This will add a `\<base\> tag`_ to the response body in order
|
||||
|
|
@ -157,49 +161,65 @@ list of available objects and useful shortcuts (you'll notice that these lines
|
|||
all start with the ``[s]`` prefix)::
|
||||
|
||||
[s] Available Scrapy objects:
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x1e16b50>
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7f07395dd690>
|
||||
[s] item {}
|
||||
[s] request <GET http://scrapy.org>
|
||||
[s] response <200 http://scrapy.org>
|
||||
[s] settings <scrapy.settings.Settings object at 0x2bfd650>
|
||||
[s] spider <Spider 'default' at 0x20c6f50>
|
||||
[s] response <200 https://scrapy.org/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7f07395dd710>
|
||||
[s] spider <DefaultSpider 'default' at 0x7f0735891690>
|
||||
[s] Useful shortcuts:
|
||||
[s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed)
|
||||
[s] fetch(req) Fetch a scrapy.Request and update local objects
|
||||
[s] shelp() Shell help (print this help)
|
||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||
[s] view(response) View response in a browser
|
||||
|
||||
>>>
|
||||
|
||||
|
||||
After that, we can start playing with the objects::
|
||||
|
||||
>>> response.xpath('//title/text()').extract_first()
|
||||
u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
|
||||
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
|
||||
|
||||
>>> fetch("http://reddit.com")
|
||||
[s] Available Scrapy objects:
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fb3ed9c9c90>
|
||||
[s] item {}
|
||||
[s] request <GET http://reddit.com>
|
||||
[s] response <200 https://www.reddit.com/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7fb3ed9c9c10>
|
||||
[s] spider <DefaultSpider 'default' at 0x7fb3ecdd3390>
|
||||
[s] Useful shortcuts:
|
||||
[s] shelp() Shell help (print this help)
|
||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||
[s] view(response) View response in a browser
|
||||
|
||||
>>> response.xpath('//title/text()').extract()
|
||||
[u'reddit: the front page of the internet']
|
||||
['reddit: the front page of the internet']
|
||||
|
||||
>>> request = request.replace(method="POST")
|
||||
|
||||
>>> fetch(request)
|
||||
[s] Available Scrapy objects:
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x1e16b50>
|
||||
...
|
||||
|
||||
>>> response.status
|
||||
404
|
||||
|
||||
>>> from pprint import pprint
|
||||
|
||||
>>> pprint(response.headers)
|
||||
{'Accept-Ranges': ['bytes'],
|
||||
'Cache-Control': ['max-age=0, must-revalidate'],
|
||||
'Content-Type': ['text/html; charset=UTF-8'],
|
||||
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
|
||||
'Server': ['snooserv'],
|
||||
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
|
||||
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
|
||||
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
|
||||
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
|
||||
'Vary': ['accept-encoding'],
|
||||
'Via': ['1.1 varnish'],
|
||||
'X-Cache': ['MISS'],
|
||||
'X-Cache-Hits': ['0'],
|
||||
'X-Content-Type-Options': ['nosniff'],
|
||||
'X-Frame-Options': ['SAMEORIGIN'],
|
||||
'X-Moose': ['majestic'],
|
||||
'X-Served-By': ['cache-cdg8730-CDG'],
|
||||
'X-Timer': ['S1481214079.394283,VS0,VE159'],
|
||||
'X-Ua-Compatible': ['IE=edge'],
|
||||
'X-Xss-Protection': ['1; mode=block']}
|
||||
>>>
|
||||
|
||||
|
||||
.. _topics-shell-inspect-response:
|
||||
|
||||
Invoking the shell from spiders to inspect responses
|
||||
|
|
@ -234,8 +254,8 @@ Here's an example of how you would call it from your spider::
|
|||
|
||||
When you run the spider, you will get something similar to this::
|
||||
|
||||
2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
|
||||
2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
|
||||
2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
|
||||
2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x1e16b50>
|
||||
...
|
||||
|
|
@ -258,7 +278,7 @@ Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
|
|||
crawling::
|
||||
|
||||
>>> ^D
|
||||
2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
|
||||
2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
|
||||
...
|
||||
|
||||
Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Twisted>=10.0.0
|
||||
Twisted>=13.1.0
|
||||
lxml
|
||||
pyOpenSSL
|
||||
cssselect>=0.9
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
1.2.1
|
||||
1.3.0
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from w3lib.url import is_url
|
|||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
|
@ -27,6 +28,8 @@ class Command(ScrapyCommand):
|
|||
help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true", \
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
@ -50,7 +53,12 @@ class Command(ScrapyCommand):
|
|||
raise UsageError()
|
||||
cb = lambda x: self._print_response(x, opts)
|
||||
request = Request(args[0], callback=cb, dont_filter=True)
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
# by default, let the framework handle redirects,
|
||||
# i.e. command handles all codes expect 3xx
|
||||
if not opts.no_redirect:
|
||||
request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
|
||||
else:
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
|
||||
spidercls = DefaultSpider
|
||||
spider_loader = self.crawler_process.spider_loader
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ class Command(ScrapyCommand):
|
|||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
@ -68,7 +70,7 @@ class Command(ScrapyCommand):
|
|||
self._start_crawler_thread()
|
||||
|
||||
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
|
||||
shell.start(url=url)
|
||||
shell.start(url=url, redirect=not opts.no_redirect)
|
||||
|
||||
def _start_crawler_thread(self):
|
||||
t = Thread(target=self.crawler_process.start,
|
||||
|
|
|
|||
|
|
@ -26,12 +26,25 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
import cssselect
|
||||
import parsel
|
||||
import lxml.etree
|
||||
import w3lib
|
||||
|
||||
lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
|
||||
libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
|
||||
|
||||
try:
|
||||
w3lib_version = w3lib.__version__
|
||||
except AttributeError:
|
||||
w3lib_version = "<1.14.3"
|
||||
|
||||
print("Scrapy : %s" % scrapy.__version__)
|
||||
print("lxml : %s" % lxml_version)
|
||||
print("libxml2 : %s" % libxml2_version)
|
||||
print("cssselect : %s" % cssselect.__version__)
|
||||
print("parsel : %s" % parsel.__version__)
|
||||
print("w3lib : %s" % w3lib_version)
|
||||
print("Twisted : %s" % twisted.version.short())
|
||||
print("Python : %s" % sys.version.replace("\n", "- "))
|
||||
print("pyOpenSSL : %s" % self._get_openssl_version())
|
||||
|
|
|
|||
|
|
@ -13,8 +13,9 @@ from twisted.web.http_headers import Headers as TxHeaders
|
|||
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
|
||||
from twisted.internet.error import TimeoutError
|
||||
from twisted.web.http import PotentialDataLoss
|
||||
from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
|
||||
HTTPConnectionPool, TCP4ClientEndpoint
|
||||
from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
|
||||
HTTPConnectionPool
|
||||
from twisted.internet.endpoints import TCP4ClientEndpoint
|
||||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
|
@ -318,14 +319,13 @@ class ScrapyAgent(object):
|
|||
expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
|
||||
|
||||
if maxsize and expected_size > maxsize:
|
||||
error_message = ("Cancelling download of {url}: expected response "
|
||||
"size ({size}) larger than "
|
||||
"download max size ({maxsize})."
|
||||
).format(url=request.url, size=expected_size, maxsize=maxsize)
|
||||
error_msg = ("Cancelling download of %(url)s: expected response "
|
||||
"size (%(size)s) larger than download max size (%(maxsize)s).")
|
||||
error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
|
||||
|
||||
logger.error(error_message)
|
||||
logger.error(error_msg, error_args)
|
||||
txresponse._transport._producer.loseConnection()
|
||||
raise defer.CancelledError(error_message)
|
||||
raise defer.CancelledError(error_msg % error_args)
|
||||
|
||||
if warnsize and expected_size > warnsize:
|
||||
logger.warning("Expected response size (%(size)s) larger than "
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.http import decode_chunked_transfer
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
|
||||
"chunked transfers are supported by default.",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
|
||||
class ChunkedTransferMiddleware(object):
|
||||
"""This middleware adds support for chunked transfer encoding, as
|
||||
documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ from twisted.internet import defer
|
|||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.web.client import ResponseFailed
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.xlib.tx import ResponseFailed
|
||||
|
||||
|
||||
class HttpCacheMiddleware(object):
|
||||
|
|
|
|||
|
|
@ -17,10 +17,10 @@ from twisted.internet import defer
|
|||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.response import response_status_message
|
||||
from scrapy.xlib.tx import ResponseFailed
|
||||
from scrapy.core.downloader.handlers.http11 import TunnelError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
|
|||
from scrapy.http import Request
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.python import to_native_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -40,7 +41,8 @@ class RobotsTxtMiddleware(object):
|
|||
return d
|
||||
|
||||
def process_request_2(self, rp, request, spider):
|
||||
if rp is not None and not rp.can_fetch(self._useragent, request.url):
|
||||
if rp is not None and not rp.can_fetch(
|
||||
to_native_str(self._useragent), request.url):
|
||||
logger.debug("Forbidden by robots.txt: %(request)s",
|
||||
{'request': request}, extra={'spider': spider})
|
||||
raise IgnoreRequest()
|
||||
|
|
@ -94,7 +96,9 @@ class RobotsTxtMiddleware(object):
|
|||
# Running rp.parse() will set rp state from
|
||||
# 'disallow all' to 'allow any'.
|
||||
pass
|
||||
rp.parse(body.splitlines())
|
||||
# stdlib's robotparser expects native 'str' ;
|
||||
# with unicode input, non-ASCII encoded bytes decoding fails in Python2
|
||||
rp.parse(to_native_str(body).splitlines())
|
||||
|
||||
rp_dfd = self._parsers[netloc]
|
||||
self._parsers[netloc] = rp
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ else:
|
|||
|
||||
from twisted.internet import defer, reactor, ssl
|
||||
|
||||
from .utils.misc import arg_to_iter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -48,6 +50,10 @@ class MailSender(object):
|
|||
msg = MIMEMultipart()
|
||||
else:
|
||||
msg = MIMENonMultipart(*mimetype.split('/', 1))
|
||||
|
||||
to = list(arg_to_iter(to))
|
||||
cc = list(arg_to_iter(cc))
|
||||
|
||||
msg['From'] = self.mailfrom
|
||||
msg['To'] = COMMASPACE.join(to)
|
||||
msg['Date'] = formatdate(localtime=True)
|
||||
|
|
|
|||
|
|
@ -102,7 +102,6 @@ DOWNLOADER_MIDDLEWARES_BASE = {
|
|||
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
|
||||
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
|
||||
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
|
||||
'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
|
||||
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
|
||||
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
|
||||
# Downloader side
|
||||
|
|
@ -192,6 +191,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
|
|||
LOG_STDOUT = False
|
||||
LOG_LEVEL = 'DEBUG'
|
||||
LOG_FILE = None
|
||||
LOG_SHORT_NAMES = False
|
||||
|
||||
SCHEDULER_DEBUG = False
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from scrapy.item import BaseItem
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.console import start_python_console
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.response import open_in_browser
|
||||
from scrapy.utils.conf import get_config
|
||||
|
|
@ -40,11 +41,11 @@ class Shell(object):
|
|||
self.code = code
|
||||
self.vars = {}
|
||||
|
||||
def start(self, url=None, request=None, response=None, spider=None):
|
||||
def start(self, url=None, request=None, response=None, spider=None, redirect=True):
|
||||
# disable accidental Ctrl-C key press from shutting down the engine
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
if url:
|
||||
self.fetch(url, spider)
|
||||
self.fetch(url, spider, redirect=redirect)
|
||||
elif request:
|
||||
self.fetch(request, spider)
|
||||
elif response:
|
||||
|
|
@ -98,14 +99,16 @@ class Shell(object):
|
|||
self.spider = spider
|
||||
return spider
|
||||
|
||||
def fetch(self, request_or_url, spider=None):
|
||||
def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
|
||||
if isinstance(request_or_url, Request):
|
||||
request = request_or_url
|
||||
url = request.url
|
||||
else:
|
||||
url = any_to_uri(request_or_url)
|
||||
request = Request(url, dont_filter=True)
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
request = Request(url, dont_filter=True, **kwargs)
|
||||
if redirect:
|
||||
request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
|
||||
else:
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
response = None
|
||||
try:
|
||||
response, spider = threads.blockingCallFromThread(
|
||||
|
|
@ -144,10 +147,13 @@ class Shell(object):
|
|||
if self._is_relevant(v):
|
||||
b.append(" %-10s %s" % (k, v))
|
||||
b.append("Useful shortcuts:")
|
||||
b.append(" shelp() Shell help (print this help)")
|
||||
if self.inthread:
|
||||
b.append(" fetch(req_or_url) Fetch request (or URL) and "
|
||||
"update local objects")
|
||||
b.append(" fetch(url[, redirect=True]) "
|
||||
"Fetch URL and update local objects "
|
||||
"(by default, redirects are followed)")
|
||||
b.append(" fetch(req) "
|
||||
"Fetch a scrapy.Request and update local objects ")
|
||||
b.append(" shelp() Shell help (print this help)")
|
||||
b.append(" view(response) View response in a browser")
|
||||
|
||||
return "\n".join("[s] %s" % l for l in b)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import absolute_import
|
||||
import traceback
|
||||
import warnings
|
||||
|
||||
from zope.interface import implementer
|
||||
|
||||
|
|
@ -18,15 +20,21 @@ class SpiderLoader(object):
|
|||
self.spider_modules = settings.getlist('SPIDER_MODULES')
|
||||
self._spiders = {}
|
||||
self._load_all_spiders()
|
||||
|
||||
|
||||
def _load_spiders(self, module):
|
||||
for spcls in iter_spider_classes(module):
|
||||
self._spiders[spcls.name] = spcls
|
||||
|
||||
def _load_all_spiders(self):
|
||||
for name in self.spider_modules:
|
||||
for module in walk_modules(name):
|
||||
self._load_spiders(module)
|
||||
try:
|
||||
for module in walk_modules(name):
|
||||
self._load_spiders(module)
|
||||
except ImportError as e:
|
||||
msg = ("\n{tb}Could not load spiders from module '{modname}'. "
|
||||
"Check SPIDER_MODULES setting".format(
|
||||
modname=name, tb=traceback.format_exc()))
|
||||
warnings.warn(msg, RuntimeWarning)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class HttpErrorMiddleware(object):
|
|||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
if isinstance(exception, HttpError):
|
||||
logger.debug(
|
||||
logger.info(
|
||||
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
|
||||
{'response': response}, extra={'spider': spider},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class SitemapSpider(Spider):
|
|||
|
||||
def _parse_sitemap(self, response):
|
||||
if response.url.endswith('/robots.txt'):
|
||||
for url in sitemap_urls_from_robots(response.text):
|
||||
for url in sitemap_urls_from_robots(response.text, base_url=response.url):
|
||||
yield Request(url, callback=self._parse_sitemap)
|
||||
else:
|
||||
body = self._get_sitemap_body(response)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ def _embed_ipython_shell(namespace={}, banner=''):
|
|||
config = load_default_config()
|
||||
# Always use .instace() to ensure _instance propagation to all parents
|
||||
# this is needed for <TAB> completion works well for new imports
|
||||
# and clear the instance to always have the fresh env
|
||||
# on repeated breaks like with inspect_response()
|
||||
InteractiveShellEmbed.clear_instance()
|
||||
shell = InteractiveShellEmbed.instance(
|
||||
banner1=banner, user_ns=namespace, config=config)
|
||||
shell()
|
||||
|
|
|
|||
|
|
@ -304,3 +304,13 @@ class LocalCache(OrderedDict):
|
|||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
super(LocalCache, self).__setitem__(key, value)
|
||||
|
||||
|
||||
class SequenceExclude(object):
|
||||
"""Object to test if an item is NOT within some sequence."""
|
||||
|
||||
def __init__(self, seq):
|
||||
self.seq = seq
|
||||
|
||||
def __contains__(self, item):
|
||||
return item not in self.seq
|
||||
|
|
|
|||
|
|
@ -119,7 +119,8 @@ def _get_handler(settings):
|
|||
)
|
||||
handler.setFormatter(formatter)
|
||||
handler.setLevel(settings.get('LOG_LEVEL'))
|
||||
handler.addFilter(TopLevelFormatter(['scrapy']))
|
||||
if settings.getbool('LOG_SHORT_NAMES'):
|
||||
handler.addFilter(TopLevelFormatter(['scrapy']))
|
||||
return handler
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ Module for processing Sitemaps.
|
|||
Note: The main purpose of this module is to provide support for the
|
||||
SitemapSpider, its API is subject to change without notice.
|
||||
"""
|
||||
|
||||
import lxml.etree
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
|
||||
class Sitemap(object):
|
||||
|
|
@ -34,10 +36,11 @@ class Sitemap(object):
|
|||
yield d
|
||||
|
||||
|
||||
def sitemap_urls_from_robots(robots_text):
|
||||
def sitemap_urls_from_robots(robots_text, base_url=None):
|
||||
"""Return an iterator over all sitemap urls contained in the given
|
||||
robots.txt file
|
||||
"""
|
||||
for line in robots_text.splitlines():
|
||||
if line.lstrip().lower().startswith('sitemap:'):
|
||||
yield line.split(':', 1)[1].strip()
|
||||
url = line.split(':', 1)[1].strip()
|
||||
yield urljoin(base_url, url)
|
||||
|
|
|
|||
|
|
@ -20,12 +20,20 @@ class SiteTest(object):
|
|||
return urljoin(self.baseurl, path)
|
||||
|
||||
|
||||
class NoMetaRefreshRedirect(util.Redirect):
|
||||
def render(self, request):
|
||||
content = util.Redirect.render(self, request)
|
||||
return content.replace(b'http-equiv=\"refresh\"',
|
||||
b'http-no-equiv=\"do-not-refresh-me\"')
|
||||
|
||||
|
||||
def test_site():
|
||||
r = resource.Resource()
|
||||
r.putChild(b"text", static.Data(b"Works", "text/plain"))
|
||||
r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
|
||||
r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
|
||||
r.putChild(b"redirect", util.Redirect(b"/redirected"))
|
||||
r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
|
||||
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
|
||||
return server.Site(r)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
from twisted.web import client
|
||||
from twisted.internet import endpoints
|
||||
|
||||
Agent = client.Agent # since < 11.1
|
||||
ProxyAgent = client.ProxyAgent # since 11.1
|
||||
ResponseDone = client.ResponseDone # since 11.1
|
||||
ResponseFailed = client.ResponseFailed # since 11.1
|
||||
HTTPConnectionPool = client.HTTPConnectionPool # since 12.1
|
||||
TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1
|
||||
|
||||
warnings.warn("Importing from scrapy.xlib.tx is deprecated and will"
|
||||
" no longer be supported in future Scrapy versions."
|
||||
" Update your code to import from twisted proper.",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
Copyright (c) 2001-2013
|
||||
Allen Short
|
||||
Andy Gayton
|
||||
Andrew Bennetts
|
||||
Antoine Pitrou
|
||||
Apple Computer, Inc.
|
||||
Benjamin Bruheim
|
||||
Bob Ippolito
|
||||
Canonical Limited
|
||||
Christopher Armstrong
|
||||
David Reid
|
||||
Donovan Preston
|
||||
Eric Mangold
|
||||
Eyal Lotem
|
||||
Itamar Turner-Trauring
|
||||
James Knight
|
||||
Jason A. Mobarak
|
||||
Jean-Paul Calderone
|
||||
Jessica McKellar
|
||||
Jonathan Jacobs
|
||||
Jonathan Lange
|
||||
Jonathan D. Simms
|
||||
Jürgen Hermann
|
||||
Kevin Horn
|
||||
Kevin Turner
|
||||
Mary Gardiner
|
||||
Matthew Lefkowitz
|
||||
Massachusetts Institute of Technology
|
||||
Moshe Zadka
|
||||
Paul Swartz
|
||||
Pavel Pergamenshchik
|
||||
Ralph Meijer
|
||||
Sean Riley
|
||||
Software Freedom Conservancy
|
||||
Travis B. Hartwell
|
||||
Thijs Triemstra
|
||||
Thomas Herve
|
||||
Timothy Allen
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
This source files are adapted copies from Twisted trunk to support HTTP1.1
|
||||
handler under Twisted >= 11.1 and Twisted <= 13.0.0
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
from scrapy import twisted_version
|
||||
if twisted_version > (13, 0, 0):
|
||||
from twisted.web import client
|
||||
from twisted.internet import endpoints
|
||||
if twisted_version >= (11, 1, 0):
|
||||
from . import client, endpoints
|
||||
else:
|
||||
from scrapy.exceptions import NotSupported
|
||||
class _Mocked(object):
|
||||
def __init__(self, *args, **kw):
|
||||
raise NotSupported('HTTP1.1 not supported')
|
||||
class _Mock(object):
|
||||
def __getattr__(self, name):
|
||||
return _Mocked
|
||||
client = endpoints = _Mock()
|
||||
|
||||
|
||||
Agent = client.Agent
|
||||
ProxyAgent = client.ProxyAgent
|
||||
ResponseDone = client.ResponseDone
|
||||
ResponseFailed = client.ResponseFailed
|
||||
HTTPConnectionPool = client.HTTPConnectionPool
|
||||
TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,587 +0,0 @@
|
|||
# -*- test-case-name: twisted.web.test -*-
|
||||
# Copyright (c) Twisted Matrix Laboratories.
|
||||
# See LICENSE for details.
|
||||
|
||||
"""
|
||||
Interface definitions for L{twisted.web}.
|
||||
|
||||
@var UNKNOWN_LENGTH: An opaque object which may be used as the value of
|
||||
L{IBodyProducer.length} to indicate that the length of the entity
|
||||
body is not known in advance.
|
||||
"""
|
||||
|
||||
from zope.interface import Interface, Attribute
|
||||
|
||||
from twisted.internet.interfaces import IPushProducer
|
||||
|
||||
|
||||
class IRequest(Interface):
|
||||
"""
|
||||
An HTTP request.
|
||||
|
||||
@since: 9.0
|
||||
"""
|
||||
|
||||
method = Attribute("A C{str} giving the HTTP method that was used.")
|
||||
uri = Attribute(
|
||||
"A C{str} giving the full encoded URI which was requested (including "
|
||||
"query arguments).")
|
||||
path = Attribute(
|
||||
"A C{str} giving the encoded query path of the request URI.")
|
||||
args = Attribute(
|
||||
"A mapping of decoded query argument names as C{str} to "
|
||||
"corresponding query argument values as C{list}s of C{str}. "
|
||||
"For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} "
|
||||
"for its query part, C{args} will be C{{'foo': ['bar', 'baz'], "
|
||||
"'quux': ['spam']}}.")
|
||||
|
||||
received_headers = Attribute(
|
||||
"Backwards-compatibility access to C{requestHeaders}. Use "
|
||||
"C{requestHeaders} instead. C{received_headers} behaves mostly "
|
||||
"like a C{dict} and does not provide access to all header values.")
|
||||
|
||||
requestHeaders = Attribute(
|
||||
"A L{http_headers.Headers} instance giving all received HTTP request "
|
||||
"headers.")
|
||||
|
||||
content = Attribute(
|
||||
"A file-like object giving the request body. This may be a file on "
|
||||
"disk, a C{StringIO}, or some other type. The implementation is free "
|
||||
"to decide on a per-request basis.")
|
||||
|
||||
headers = Attribute(
|
||||
"Backwards-compatibility access to C{responseHeaders}. Use"
|
||||
"C{responseHeaders} instead. C{headers} behaves mostly like a "
|
||||
"C{dict} and does not provide access to all header values nor "
|
||||
"does it allow multiple values for one header to be set.")
|
||||
|
||||
responseHeaders = Attribute(
|
||||
"A L{http_headers.Headers} instance holding all HTTP response "
|
||||
"headers to be sent.")
|
||||
|
||||
def getHeader(key):
|
||||
"""
|
||||
Get an HTTP request header.
|
||||
|
||||
@type key: C{str}
|
||||
@param key: The name of the header to get the value of.
|
||||
|
||||
@rtype: C{str} or C{NoneType}
|
||||
@return: The value of the specified header, or C{None} if that header
|
||||
was not present in the request.
|
||||
"""
|
||||
|
||||
|
||||
def getCookie(key):
|
||||
"""
|
||||
Get a cookie that was sent from the network.
|
||||
"""
|
||||
|
||||
|
||||
def getAllHeaders():
|
||||
"""
|
||||
Return dictionary mapping the names of all received headers to the last
|
||||
value received for each.
|
||||
|
||||
Since this method does not return all header information,
|
||||
C{requestHeaders.getAllRawHeaders()} may be preferred.
|
||||
"""
|
||||
|
||||
|
||||
def getRequestHostname():
|
||||
"""
|
||||
Get the hostname that the user passed in to the request.
|
||||
|
||||
This will either use the Host: header (if it is available) or the
|
||||
host we are listening on if the header is unavailable.
|
||||
|
||||
@returns: the requested hostname
|
||||
@rtype: C{str}
|
||||
"""
|
||||
|
||||
|
||||
def getHost():
|
||||
"""
|
||||
Get my originally requesting transport's host.
|
||||
|
||||
@return: An L{IAddress<twisted.internet.interfaces.IAddress>}.
|
||||
"""
|
||||
|
||||
|
||||
def getClientIP():
|
||||
"""
|
||||
Return the IP address of the client who submitted this request.
|
||||
|
||||
@returns: the client IP address or C{None} if the request was submitted
|
||||
over a transport where IP addresses do not make sense.
|
||||
@rtype: L{str} or C{NoneType}
|
||||
"""
|
||||
|
||||
|
||||
def getClient():
|
||||
"""
|
||||
Return the hostname of the IP address of the client who submitted this
|
||||
request, if possible.
|
||||
|
||||
This method is B{deprecated}. See L{getClientIP} instead.
|
||||
|
||||
@rtype: C{NoneType} or L{str}
|
||||
@return: The canonical hostname of the client, as determined by
|
||||
performing a name lookup on the IP address of the client.
|
||||
"""
|
||||
|
||||
|
||||
def getUser():
|
||||
"""
|
||||
Return the HTTP user sent with this request, if any.
|
||||
|
||||
If no user was supplied, return the empty string.
|
||||
|
||||
@returns: the HTTP user, if any
|
||||
@rtype: C{str}
|
||||
"""
|
||||
|
||||
|
||||
def getPassword():
|
||||
"""
|
||||
Return the HTTP password sent with this request, if any.
|
||||
|
||||
If no password was supplied, return the empty string.
|
||||
|
||||
@returns: the HTTP password, if any
|
||||
@rtype: C{str}
|
||||
"""
|
||||
|
||||
|
||||
def isSecure():
|
||||
"""
|
||||
Return True if this request is using a secure transport.
|
||||
|
||||
Normally this method returns True if this request's HTTPChannel
|
||||
instance is using a transport that implements ISSLTransport.
|
||||
|
||||
This will also return True if setHost() has been called
|
||||
with ssl=True.
|
||||
|
||||
@returns: True if this request is secure
|
||||
@rtype: C{bool}
|
||||
"""
|
||||
|
||||
|
||||
def getSession(sessionInterface=None):
|
||||
"""
|
||||
Look up the session associated with this request or create a new one if
|
||||
there is not one.
|
||||
|
||||
@return: The L{Session} instance identified by the session cookie in
|
||||
the request, or the C{sessionInterface} component of that session
|
||||
if C{sessionInterface} is specified.
|
||||
"""
|
||||
|
||||
|
||||
def URLPath():
|
||||
"""
|
||||
@return: A L{URLPath} instance which identifies the URL for which this
|
||||
request is.
|
||||
"""
|
||||
|
||||
|
||||
def prePathURL():
|
||||
"""
|
||||
@return: At any time during resource traversal, a L{str} giving an
|
||||
absolute URL to the most nested resource which has yet been
|
||||
reached.
|
||||
"""
|
||||
|
||||
|
||||
def rememberRootURL():
|
||||
"""
|
||||
Remember the currently-processed part of the URL for later
|
||||
recalling.
|
||||
"""
|
||||
|
||||
|
||||
def getRootURL():
|
||||
"""
|
||||
Get a previously-remembered URL.
|
||||
"""
|
||||
|
||||
|
||||
# Methods for outgoing response
|
||||
def finish():
|
||||
"""
|
||||
Indicate that the response to this request is complete.
|
||||
"""
|
||||
|
||||
|
||||
def write(data):
|
||||
"""
|
||||
Write some data to the body of the response to this request. Response
|
||||
headers are written the first time this method is called, after which
|
||||
new response headers may not be added.
|
||||
"""
|
||||
|
||||
|
||||
def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
|
||||
"""
|
||||
Set an outgoing HTTP cookie.
|
||||
|
||||
In general, you should consider using sessions instead of cookies, see
|
||||
L{twisted.web.server.Request.getSession} and the
|
||||
L{twisted.web.server.Session} class for details.
|
||||
"""
|
||||
|
||||
|
||||
def setResponseCode(code, message=None):
|
||||
"""
|
||||
Set the HTTP response code.
|
||||
"""
|
||||
|
||||
|
||||
def setHeader(k, v):
|
||||
"""
|
||||
Set an HTTP response header. Overrides any previously set values for
|
||||
this header.
|
||||
|
||||
@type name: C{str}
|
||||
@param name: The name of the header for which to set the value.
|
||||
|
||||
@type value: C{str}
|
||||
@param value: The value to set for the named header.
|
||||
"""
|
||||
|
||||
|
||||
def redirect(url):
|
||||
"""
|
||||
Utility function that does a redirect.
|
||||
|
||||
The request should have finish() called after this.
|
||||
"""
|
||||
|
||||
|
||||
def setLastModified(when):
|
||||
"""
|
||||
Set the C{Last-Modified} time for the response to this request.
|
||||
|
||||
If I am called more than once, I ignore attempts to set Last-Modified
|
||||
earlier, only replacing the Last-Modified time if it is to a later
|
||||
value.
|
||||
|
||||
If I am a conditional request, I may modify my response code to
|
||||
L{NOT_MODIFIED<http.NOT_MODIFIED>} if appropriate for the time given.
|
||||
|
||||
@param when: The last time the resource being returned was modified, in
|
||||
seconds since the epoch.
|
||||
@type when: L{int}, L{long} or L{float}
|
||||
|
||||
@return: If I am a C{If-Modified-Since} conditional request and the time
|
||||
given is not newer than the condition, I return
|
||||
L{CACHED<http.CACHED>} to indicate that you should write no body.
|
||||
Otherwise, I return a false value.
|
||||
"""
|
||||
|
||||
|
||||
def setETag(etag):
|
||||
"""
|
||||
Set an C{entity tag} for the outgoing response.
|
||||
|
||||
That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for
|
||||
comparing two or more entities from the same requested resource."
|
||||
|
||||
If I am a conditional request, I may modify my response code to
|
||||
L{NOT_MODIFIED<http.NOT_MODIFIED>} or
|
||||
L{PRECONDITION_FAILED<http.PRECONDITION_FAILED>}, if appropriate for the
|
||||
tag given.
|
||||
|
||||
@param etag: The entity tag for the resource being returned.
|
||||
@type etag: C{str}
|
||||
|
||||
@return: If I am a C{If-None-Match} conditional request and the tag
|
||||
matches one in the request, I return L{CACHED<http.CACHED>} to
|
||||
indicate that you should write no body. Otherwise, I return a
|
||||
false value.
|
||||
"""
|
||||
|
||||
|
||||
def setHost(host, port, ssl=0):
|
||||
"""
|
||||
Change the host and port the request thinks it's using.
|
||||
|
||||
This method is useful for working with reverse HTTP proxies (e.g. both
|
||||
Squid and Apache's mod_proxy can do this), when the address the HTTP
|
||||
client is using is different than the one we're listening on.
|
||||
|
||||
For example, Apache may be listening on https://www.example.com, and
|
||||
then forwarding requests to http://localhost:8080, but we don't want
|
||||
HTML produced by Twisted to say 'http://localhost:8080', they should
|
||||
say 'https://www.example.com', so we do::
|
||||
|
||||
request.setHost('www.example.com', 443, ssl=1)
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class ICredentialFactory(Interface):
|
||||
"""
|
||||
A credential factory defines a way to generate a particular kind of
|
||||
authentication challenge and a way to interpret the responses to these
|
||||
challenges. It creates
|
||||
L{ICredentials<twisted.cred.credentials.ICredentials>} providers from
|
||||
responses. These objects will be used with L{twisted.cred} to authenticate
|
||||
an authorize requests.
|
||||
"""
|
||||
scheme = Attribute(
|
||||
"A C{str} giving the name of the authentication scheme with which "
|
||||
"this factory is associated. For example, C{'basic'} or C{'digest'}.")
|
||||
|
||||
|
||||
def getChallenge(request):
|
||||
"""
|
||||
Generate a new challenge to be sent to a client.
|
||||
|
||||
@type peer: L{twisted.web.http.Request}
|
||||
@param peer: The request the response to which this challenge will be
|
||||
included.
|
||||
|
||||
@rtype: C{dict}
|
||||
@return: A mapping from C{str} challenge fields to associated C{str}
|
||||
values.
|
||||
"""
|
||||
|
||||
|
||||
def decode(response, request):
|
||||
"""
|
||||
Create a credentials object from the given response.
|
||||
|
||||
@type response: C{str}
|
||||
@param response: scheme specific response string
|
||||
|
||||
@type request: L{twisted.web.http.Request}
|
||||
@param request: The request being processed (from which the response
|
||||
was taken).
|
||||
|
||||
@raise twisted.cred.error.LoginFailed: If the response is invalid.
|
||||
|
||||
@rtype: L{twisted.cred.credentials.ICredentials} provider
|
||||
@return: The credentials represented by the given response.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class IBodyProducer(IPushProducer):
|
||||
"""
|
||||
Objects which provide L{IBodyProducer} write bytes to an object which
|
||||
provides L{IConsumer<twisted.internet.interfaces.IConsumer>} by calling its
|
||||
C{write} method repeatedly.
|
||||
|
||||
L{IBodyProducer} providers may start producing as soon as they have an
|
||||
L{IConsumer<twisted.internet.interfaces.IConsumer>} provider. That is, they
|
||||
should not wait for a C{resumeProducing} call to begin writing data.
|
||||
|
||||
L{IConsumer.unregisterProducer<twisted.internet.interfaces.IConsumer.unregisterProducer>}
|
||||
must not be called. Instead, the
|
||||
L{Deferred<twisted.internet.defer.Deferred>} returned from C{startProducing}
|
||||
must be fired when all bytes have been written.
|
||||
|
||||
L{IConsumer.write<twisted.internet.interfaces.IConsumer.write>} may
|
||||
synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or
|
||||
C{stopProducing}. These methods must be implemented with this in mind.
|
||||
|
||||
@since: 9.0
|
||||
"""
|
||||
|
||||
# Despite the restrictions above and the additional requirements of
|
||||
# stopProducing documented below, this interface still needs to be an
|
||||
# IPushProducer subclass. Providers of it will be passed to IConsumer
|
||||
# providers which only know about IPushProducer and IPullProducer, not
|
||||
# about this interface. This interface needs to remain close enough to one
|
||||
# of those interfaces for consumers to work with it.
|
||||
|
||||
length = Attribute(
|
||||
"""
|
||||
C{length} is a C{int} indicating how many bytes in total this
|
||||
L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}
|
||||
if this is not known in advance.
|
||||
""")
|
||||
|
||||
def startProducing(consumer):
|
||||
"""
|
||||
Start producing to the given
|
||||
L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.
|
||||
|
||||
@return: A L{Deferred<twisted.internet.defer.Deferred>} which fires with
|
||||
C{None} when all bytes have been produced or with a
|
||||
L{Failure<twisted.python.failure.Failure>} if there is any problem
|
||||
before all bytes have been produced.
|
||||
"""
|
||||
|
||||
|
||||
def stopProducing():
|
||||
"""
|
||||
In addition to the standard behavior of
|
||||
L{IProducer.stopProducing<twisted.internet.interfaces.IProducer.stopProducing>}
|
||||
(stop producing data), make sure the
|
||||
L{Deferred<twisted.internet.defer.Deferred>} returned by
|
||||
C{startProducing} is never fired.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class IRenderable(Interface):
|
||||
"""
|
||||
An L{IRenderable} is an object that may be rendered by the
|
||||
L{twisted.web.template} templating system.
|
||||
"""
|
||||
|
||||
def lookupRenderMethod(name):
|
||||
"""
|
||||
Look up and return the render method associated with the given name.
|
||||
|
||||
@type name: C{str}
|
||||
@param name: The value of a render directive encountered in the
|
||||
document returned by a call to L{IRenderable.render}.
|
||||
|
||||
@return: A two-argument callable which will be invoked with the request
|
||||
being responded to and the tag object on which the render directive
|
||||
was encountered.
|
||||
"""
|
||||
|
||||
|
||||
def render(request):
|
||||
"""
|
||||
Get the document for this L{IRenderable}.
|
||||
|
||||
@type request: L{IRequest} provider or C{NoneType}
|
||||
@param request: The request in response to which this method is being
|
||||
invoked.
|
||||
|
||||
@return: An object which can be flattened.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class ITemplateLoader(Interface):
|
||||
"""
|
||||
A loader for templates; something usable as a value for
|
||||
L{twisted.web.template.Element}'s C{loader} attribute.
|
||||
"""
|
||||
|
||||
def load():
|
||||
"""
|
||||
Load a template suitable for rendering.
|
||||
|
||||
@return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and
|
||||
other L{IRenderable} providers.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class IResponse(Interface):
|
||||
"""
|
||||
An object representing an HTTP response received from an HTTP server.
|
||||
|
||||
@since: 11.1
|
||||
"""
|
||||
|
||||
version = Attribute(
|
||||
"A three-tuple describing the protocol and protocol version "
|
||||
"of the response. The first element is of type C{str}, the second "
|
||||
"and third are of type C{int}. For example, C{('HTTP', 1, 1)}.")
|
||||
|
||||
|
||||
code = Attribute("The HTTP status code of this response, as a C{int}.")
|
||||
|
||||
|
||||
phrase = Attribute(
|
||||
"The HTTP reason phrase of this response, as a C{str}.")
|
||||
|
||||
|
||||
headers = Attribute("The HTTP response L{Headers} of this response.")
|
||||
|
||||
|
||||
length = Attribute(
|
||||
"The C{int} number of bytes expected to be in the body of this "
|
||||
"response or L{UNKNOWN_LENGTH} if the server did not indicate how "
|
||||
"many bytes to expect. For I{HEAD} responses, this will be 0; if "
|
||||
"the response includes a I{Content-Length} header, it will be "
|
||||
"available in C{headers}.")
|
||||
|
||||
|
||||
def deliverBody(protocol):
|
||||
"""
|
||||
Register an L{IProtocol<twisted.internet.interfaces.IProtocol>} provider
|
||||
to receive the response body.
|
||||
|
||||
The protocol will be connected to a transport which provides
|
||||
L{IPushProducer}. The protocol's C{connectionLost} method will be
|
||||
called with:
|
||||
|
||||
- ResponseDone, which indicates that all bytes from the response
|
||||
have been successfully delivered.
|
||||
|
||||
- PotentialDataLoss, which indicates that it cannot be determined
|
||||
if the entire response body has been delivered. This only occurs
|
||||
when making requests to HTTP servers which do not set
|
||||
I{Content-Length} or a I{Transfer-Encoding} in the response.
|
||||
|
||||
- ResponseFailed, which indicates that some bytes from the response
|
||||
were lost. The C{reasons} attribute of the exception may provide
|
||||
more specific indications as to why.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class _IRequestEncoder(Interface):
|
||||
"""
|
||||
An object encoding data passed to L{IRequest.write}, for example for
|
||||
compression purpose.
|
||||
|
||||
@since: 12.3
|
||||
"""
|
||||
|
||||
def encode(data):
|
||||
"""
|
||||
Encode the data given and return the result.
|
||||
|
||||
@param data: The content to encode.
|
||||
@type data: C{str}
|
||||
|
||||
@return: The encoded data.
|
||||
@rtype: C{str}
|
||||
"""
|
||||
|
||||
|
||||
def finish():
|
||||
"""
|
||||
Callback called when the request is closing.
|
||||
|
||||
@return: If necessary, the pending data accumulated from previous
|
||||
C{encode} calls.
|
||||
@rtype: C{str}
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class _IRequestEncoderFactory(Interface):
|
||||
"""
|
||||
A factory for returing L{_IRequestEncoder} instances.
|
||||
|
||||
@since: 12.3
|
||||
"""
|
||||
|
||||
def encoderForRequest(request):
|
||||
"""
|
||||
If applicable, returns a L{_IRequestEncoder} instance which will encode
|
||||
the request.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
|
||||
|
||||
__all__ = [
|
||||
"ICredentialFactory", "IRequest",
|
||||
"IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",
|
||||
"_IRequestEncoderFactory",
|
||||
|
||||
"UNKNOWN_LENGTH"]
|
||||
4
setup.py
4
setup.py
|
|
@ -41,14 +41,14 @@ setup(
|
|||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
install_requires=[
|
||||
'Twisted>=10.0.0',
|
||||
'Twisted>=13.1.0',
|
||||
'w3lib>=1.15.0',
|
||||
'queuelib',
|
||||
'lxml',
|
||||
'pyOpenSSL',
|
||||
'cssselect>=0.9',
|
||||
'six>=1.5.2',
|
||||
'parsel>=0.9.3',
|
||||
'parsel>=0.9.5',
|
||||
'PyDispatcher>=2.0.5',
|
||||
'service_identity',
|
||||
],
|
||||
|
|
|
|||
|
|
@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
_, out, _ = yield self.execute([self.url('/text')])
|
||||
self.assertEqual(out.strip(), b'Works')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_redirect_default(self):
|
||||
_, out, _ = yield self.execute([self.url('/redirect')])
|
||||
self.assertEqual(out.strip(), b'Redirected here')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_redirect_disabled(self):
|
||||
_, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')])
|
||||
err = err.strip()
|
||||
self.assertIn(b'downloader/response_status_count/302', err, err)
|
||||
self.assertNotIn(b'downloader/response_status_count/200', err, err)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_headers(self):
|
||||
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
|
||||
|
|
|
|||
|
|
@ -49,6 +49,35 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
|
||||
assert out.strip().endswith(b'/redirected')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_redirect_follow_302(self):
|
||||
_, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
|
||||
assert out.strip().endswith(b'200')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_redirect_not_follow_302(self):
|
||||
_, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
|
||||
assert out.strip().endswith(b'302')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_fetch_redirect_follow_302(self):
|
||||
"""Test that calling `fetch(url)` follows HTTP redirects by default."""
|
||||
url = self.url('/redirect-no-meta-refresh')
|
||||
code = "fetch('{0}')"
|
||||
errcode, out, errout = yield self.execute(['-c', code.format(url)])
|
||||
self.assertEqual(errcode, 0, out)
|
||||
assert b'Redirecting (302)' in errout
|
||||
assert b'Crawled (200)' in errout
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_fetch_redirect_not_follow_302(self):
|
||||
"""Test that calling `fetch(url, redirect=False)` disables automatic redirects."""
|
||||
url = self.url('/redirect-no-meta-refresh')
|
||||
code = "fetch('{0}', redirect=False)"
|
||||
errcode, out, errout = yield self.execute(['-c', code.format(url)])
|
||||
self.assertEqual(errcode, 0, out)
|
||||
assert b'Crawled (302)' in errout
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_request_replace(self):
|
||||
url = self.url('/text')
|
||||
|
|
|
|||
|
|
@ -25,5 +25,7 @@ class VersionTest(ProcessTest, unittest.TestCase):
|
|||
_, out, _ = yield self.execute(['-v'])
|
||||
headers = [l.partition(":")[0].strip()
|
||||
for l in out.strip().decode(encoding).splitlines()]
|
||||
self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'Twisted',
|
||||
'Python', 'pyOpenSSL', 'Platform'])
|
||||
self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
|
||||
'cssselect', 'parsel', 'w3lib',
|
||||
'Twisted', 'Python', 'pyOpenSSL',
|
||||
'Platform'])
|
||||
|
|
|
|||
|
|
@ -38,11 +38,11 @@ class ProjectTest(unittest.TestCase):
|
|||
return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd,
|
||||
env=self.env, **kwargs)
|
||||
|
||||
def proc(self, *new_args, **kwargs):
|
||||
def proc(self, *new_args, **popen_kwargs):
|
||||
args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
|
||||
p = subprocess.Popen(args, cwd=self.cwd, env=self.env,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
**kwargs)
|
||||
**popen_kwargs)
|
||||
|
||||
waited = 0
|
||||
interval = 0.2
|
||||
|
|
@ -182,6 +182,17 @@ class MiscCommandsTest(CommandTest):
|
|||
|
||||
class RunSpiderCommandTest(CommandTest):
|
||||
|
||||
debug_log_spider = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug("It Works!")
|
||||
return []
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
def _create_file(self, content, name):
|
||||
tmpdir = self.mktemp()
|
||||
|
|
@ -194,32 +205,44 @@ class RunSpiderCommandTest(CommandTest):
|
|||
finally:
|
||||
rmtree(tmpdir)
|
||||
|
||||
def runspider(self, code, name='myspider.py'):
|
||||
def runspider(self, code, name='myspider.py', args=()):
|
||||
with self._create_file(code, name) as fname:
|
||||
return self.proc('runspider', fname)
|
||||
return self.proc('runspider', fname, *args)
|
||||
|
||||
def get_log(self, code, name='myspider.py', args=()):
|
||||
p = self.runspider(code, name=name, args=args)
|
||||
return to_native_str(p.stderr.read())
|
||||
|
||||
def test_runspider(self):
|
||||
spider = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug("It Works!")
|
||||
return []
|
||||
"""
|
||||
p = self.runspider(spider)
|
||||
log = to_native_str(p.stderr.read())
|
||||
|
||||
log = self.get_log(self.debug_log_spider)
|
||||
self.assertIn("DEBUG: It Works!", log)
|
||||
self.assertIn("INFO: Spider opened", log)
|
||||
self.assertIn("INFO: Closing spider (finished)", log)
|
||||
self.assertIn("INFO: Spider closed (finished)", log)
|
||||
|
||||
def test_runspider_log_level(self):
|
||||
log = self.get_log(self.debug_log_spider,
|
||||
args=('-s', 'LOG_LEVEL=INFO'))
|
||||
self.assertNotIn("DEBUG: It Works!", log)
|
||||
self.assertIn("INFO: Spider opened", log)
|
||||
|
||||
def test_runspider_log_short_names(self):
|
||||
log1 = self.get_log(self.debug_log_spider,
|
||||
args=('-s', 'LOG_SHORT_NAMES=1'))
|
||||
print(log1)
|
||||
self.assertIn("[myspider] DEBUG: It Works!", log1)
|
||||
self.assertIn("[scrapy]", log1)
|
||||
self.assertNotIn("[scrapy.core.engine]", log1)
|
||||
|
||||
log2 = self.get_log(self.debug_log_spider,
|
||||
args=('-s', 'LOG_SHORT_NAMES=0'))
|
||||
print(log2)
|
||||
self.assertIn("[myspider] DEBUG: It Works!", log2)
|
||||
self.assertNotIn("[scrapy]", log2)
|
||||
self.assertIn("[scrapy.core.engine]", log2)
|
||||
|
||||
def test_runspider_no_spider_found(self):
|
||||
p = self.runspider("from scrapy.spiders import Spider\n")
|
||||
log = to_native_str(p.stderr.read())
|
||||
log = self.get_log("from scrapy.spiders import Spider\n")
|
||||
self.assertIn("No spider found in file", log)
|
||||
|
||||
def test_runspider_file_not_found(self):
|
||||
|
|
@ -228,12 +251,11 @@ class MySpider(scrapy.Spider):
|
|||
self.assertIn("File not found: some_non_existent_file", log)
|
||||
|
||||
def test_runspider_unable_to_load(self):
|
||||
p = self.runspider('', 'myspider.txt')
|
||||
log = to_native_str(p.stderr.read())
|
||||
log = self.get_log('', name='myspider.txt')
|
||||
self.assertIn('Unable to load', log)
|
||||
|
||||
def test_start_requests_errors(self):
|
||||
p = self.runspider("""
|
||||
log = self.get_log("""
|
||||
import scrapy
|
||||
|
||||
class BadSpider(scrapy.Spider):
|
||||
|
|
@ -241,11 +263,11 @@ class BadSpider(scrapy.Spider):
|
|||
def start_requests(self):
|
||||
raise Exception("oops!")
|
||||
""", name="badspider.py")
|
||||
log = to_native_str(p.stderr.read())
|
||||
print(log)
|
||||
self.assertIn("start_requests", log)
|
||||
self.assertIn("badspider.py", log)
|
||||
|
||||
|
||||
class BenchCommandTest(CommandTest):
|
||||
|
||||
def test_run(self):
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ from twisted.internet import defer
|
|||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy import twisted_version
|
||||
from scrapy.downloadermiddlewares.retry import RetryMiddleware
|
||||
from scrapy.xlib.tx import ResponseFailed
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import absolute_import
|
||||
import re
|
||||
from twisted.internet import reactor, error
|
||||
|
|
@ -30,11 +31,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
|
|||
def _get_successful_crawler(self):
|
||||
crawler = self.crawler
|
||||
crawler.settings.set('ROBOTSTXT_OBEY', True)
|
||||
ROBOTS = re.sub(b'^\s+(?m)', b'', b'''
|
||||
ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
|
||||
User-Agent: *
|
||||
Disallow: /admin/
|
||||
Disallow: /static/
|
||||
''')
|
||||
|
||||
# taken from https://en.wikipedia.org/robots.txt
|
||||
Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
|
||||
Disallow: /wiki/Käyttäjä:
|
||||
|
||||
User-Agent: UnicödeBöt
|
||||
Disallow: /some/randome/page.html
|
||||
'''.encode('utf-8'))
|
||||
response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
|
||||
def return_response(request, spider):
|
||||
deferred = Deferred()
|
||||
|
|
@ -48,7 +56,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
|
|||
return DeferredList([
|
||||
self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
|
||||
self.assertIgnored(Request('http://site.local/admin/main'), middleware),
|
||||
self.assertIgnored(Request('http://site.local/static/'), middleware)
|
||||
self.assertIgnored(Request('http://site.local/static/'), middleware),
|
||||
self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
|
||||
self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
|
||||
], fireOnOneErrback=True)
|
||||
|
||||
def test_robotstxt_ready_parser(self):
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ class MailSenderTest(unittest.TestCase):
|
|||
|
||||
def test_send(self):
|
||||
mailsender = MailSender(debug=True)
|
||||
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', _callback=self._catch_mail_sent)
|
||||
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
|
||||
_callback=self._catch_mail_sent)
|
||||
|
||||
assert self.catched_msg
|
||||
|
||||
|
|
@ -24,9 +25,16 @@ class MailSenderTest(unittest.TestCase):
|
|||
self.assertEqual(msg.get_payload(), 'body')
|
||||
self.assertEqual(msg.get('Content-Type'), 'text/plain')
|
||||
|
||||
def test_send_single_values_to_and_cc(self):
|
||||
mailsender = MailSender(debug=True)
|
||||
mailsender.send(to='test@scrapy.org', subject='subject', body='body',
|
||||
cc='test@scrapy.org', _callback=self._catch_mail_sent)
|
||||
|
||||
def test_send_html(self):
|
||||
mailsender = MailSender(debug=True)
|
||||
mailsender.send(to=['test@scrapy.org'], subject='subject', body='<p>body</p>', mimetype='text/html', _callback=self._catch_mail_sent)
|
||||
mailsender.send(to=['test@scrapy.org'], subject='subject',
|
||||
body='<p>body</p>', mimetype='text/html',
|
||||
_callback=self._catch_mail_sent)
|
||||
|
||||
msg = self.catched_msg['msg']
|
||||
self.assertEqual(msg.get_payload(), '<p>body</p>')
|
||||
|
|
@ -90,7 +98,8 @@ class MailSenderTest(unittest.TestCase):
|
|||
|
||||
mailsender = MailSender(debug=True)
|
||||
mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
|
||||
attachs=attachs, charset='utf-8', _callback=self._catch_mail_sent)
|
||||
attachs=attachs, charset='utf-8',
|
||||
_callback=self._catch_mail_sent)
|
||||
|
||||
assert self.catched_msg
|
||||
self.assertEqual(self.catched_msg['subject'], subject)
|
||||
|
|
@ -99,7 +108,8 @@ class MailSenderTest(unittest.TestCase):
|
|||
msg = self.catched_msg['msg']
|
||||
self.assertEqual(msg['subject'], subject)
|
||||
self.assertEqual(msg.get_charset(), Charset('utf-8'))
|
||||
self.assertEqual(msg.get('Content-Type'), 'multipart/mixed; charset="utf-8"')
|
||||
self.assertEqual(msg.get('Content-Type'),
|
||||
'multipart/mixed; charset="utf-8"')
|
||||
|
||||
payload = msg.get_payload()
|
||||
assert isinstance(payload, list)
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
|
||||
|
||||
settings = {
|
||||
"FILES_EXPIRES": random.randint(1, 1000),
|
||||
"FILES_EXPIRES": random.randint(100, 1000),
|
||||
"FILES_URLS_FIELD": random_string(),
|
||||
"FILES_RESULT_FIELD": random_string(),
|
||||
"FILES_STORE": self.tempdir
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
|
||||
|
||||
settings = {
|
||||
"IMAGES_EXPIRES": random.randint(1, 1000),
|
||||
"IMAGES_EXPIRES": random.randint(100, 1000),
|
||||
"IMAGES_STORE": self.tempdir,
|
||||
"IMAGES_RESULT_FIELD": random_string(),
|
||||
"IMAGES_URLS_FIELD": random_string(),
|
||||
|
|
|
|||
|
|
@ -332,13 +332,17 @@ class SitemapSpiderTest(SpiderTest):
|
|||
robots = b"""# Sitemap files
|
||||
Sitemap: http://example.com/sitemap.xml
|
||||
Sitemap: http://example.com/sitemap-product-index.xml
|
||||
Sitemap: HTTP://example.com/sitemap-uppercase.xml
|
||||
Sitemap: /sitemap-relative-url.xml
|
||||
"""
|
||||
|
||||
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
|
||||
spider = self.spider_class("example.com")
|
||||
self.assertEqual([req.url for req in spider._parse_sitemap(r)],
|
||||
['http://example.com/sitemap.xml',
|
||||
'http://example.com/sitemap-product-index.xml'])
|
||||
'http://example.com/sitemap-product-index.xml',
|
||||
'http://example.com/sitemap-uppercase.xml',
|
||||
'http://www.example.com/sitemap-relative-url.xml'])
|
||||
|
||||
|
||||
class BaseSpiderDeprecationTest(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import sys
|
||||
import os
|
||||
import shutil
|
||||
import warnings
|
||||
|
||||
from zope.interface.verify import verifyObject
|
||||
from twisted.trial import unittest
|
||||
|
|
@ -9,6 +10,7 @@ from twisted.trial import unittest
|
|||
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
|
||||
# alone
|
||||
import scrapy
|
||||
import tempfile
|
||||
from scrapy.interfaces import ISpiderLoader
|
||||
from scrapy.spiderloader import SpiderLoader
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -22,8 +24,7 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
|
||||
def setUp(self):
|
||||
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
|
||||
self.tmpdir = self.mktemp()
|
||||
os.mkdir(self.tmpdir)
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
|
||||
shutil.copytree(orig_spiders_dir, self.spiders_dir)
|
||||
sys.path.append(self.tmpdir)
|
||||
|
|
@ -40,7 +41,7 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_list(self):
|
||||
self.assertEqual(set(self.spider_loader.list()),
|
||||
set(['spider1', 'spider2', 'spider3']))
|
||||
set(['spider1', 'spider2', 'spider3', 'spider4']))
|
||||
|
||||
def test_load(self):
|
||||
spider1 = self.spider_loader.load("spider1")
|
||||
|
|
@ -66,7 +67,7 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
self.spider_loader = SpiderLoader.from_settings(settings)
|
||||
assert len(self.spider_loader._spiders) == 1
|
||||
|
||||
def test_load_spider_module(self):
|
||||
def test_load_spider_module_multiple(self):
|
||||
prefix = 'tests.test_spiderloader.test_spiders.'
|
||||
module = ','.join(prefix + s for s in ('spider1', 'spider2'))
|
||||
settings = Settings({'SPIDER_MODULES': module})
|
||||
|
|
@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
crawler = runner.create_crawler('spider1')
|
||||
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
|
||||
self.assertEqual(crawler.spidercls.name, 'spider1')
|
||||
|
||||
def test_bad_spider_modules_warning(self):
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
module = 'tests.test_spiderloader.test_spiders.doesnotexist'
|
||||
settings = Settings({'SPIDER_MODULES': [module]})
|
||||
spider_loader = SpiderLoader.from_settings(settings)
|
||||
self.assertIn("Could not load spiders from module", str(w[0].message))
|
||||
|
||||
spiders = spider_loader.list()
|
||||
self.assertEqual(spiders, [])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
from scrapy.spiders import Spider
|
||||
|
||||
class Spider4(Spider):
|
||||
name = "spider4"
|
||||
allowed_domains = ['spider4.com']
|
||||
|
||||
@classmethod
|
||||
def handles_request(cls, request):
|
||||
return request.url == 'http://spider4.com/onlythis'
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import logging
|
||||
from unittest import TestCase
|
||||
|
||||
from testfixtures import LogCapture
|
||||
|
|
@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
self.assertIn('Ignoring response <500', str(log))
|
||||
self.assertNotIn('Ignoring response <200', str(log))
|
||||
self.assertNotIn('Ignoring response <402', str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_logging_level(self):
|
||||
# HttpError logs ignored responses with level INFO
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.INFO) as log:
|
||||
yield crawler.crawl()
|
||||
self.assertEqual(crawler.spider.parsed, {'200'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
|
||||
|
||||
self.assertIn('Ignoring response <402', str(log))
|
||||
self.assertIn('Ignoring response <404', str(log))
|
||||
self.assertIn('Ignoring response <500', str(log))
|
||||
self.assertNotIn('Ignoring response <200', str(log))
|
||||
|
||||
# with level WARNING, we shouldn't capture anything from HttpError
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.WARNING) as log:
|
||||
yield crawler.crawl()
|
||||
self.assertEqual(crawler.spider.parsed, {'200'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
|
||||
|
||||
self.assertNotIn('Ignoring response <402', str(log))
|
||||
self.assertNotIn('Ignoring response <404', str(log))
|
||||
self.assertNotIn('Ignoring response <500', str(log))
|
||||
self.assertNotIn('Ignoring response <200', str(log))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import copy
|
||||
import unittest
|
||||
|
||||
from scrapy.utils.datatypes import CaselessDict
|
||||
from scrapy.utils.datatypes import CaselessDict, SequenceExclude
|
||||
|
||||
__doctests__ = ['scrapy.utils.datatypes']
|
||||
|
||||
|
|
@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase):
|
|||
assert isinstance(h2, CaselessDict)
|
||||
|
||||
|
||||
class SequenceExcludeTest(unittest.TestCase):
|
||||
|
||||
def test_list(self):
|
||||
seq = [1, 2, 3]
|
||||
d = SequenceExclude(seq)
|
||||
self.assertIn(0, d)
|
||||
self.assertIn(4, d)
|
||||
self.assertNotIn(2, d)
|
||||
|
||||
def test_range(self):
|
||||
seq = range(10, 20)
|
||||
d = SequenceExclude(seq)
|
||||
self.assertIn(5, d)
|
||||
self.assertIn(20, d)
|
||||
self.assertNotIn(15, d)
|
||||
|
||||
def test_six_range(self):
|
||||
import six.moves
|
||||
seq = six.moves.range(10**3, 10**6)
|
||||
d = SequenceExclude(seq)
|
||||
self.assertIn(10**2, d)
|
||||
self.assertIn(10**7, d)
|
||||
self.assertNotIn(10**4, d)
|
||||
|
||||
def test_range_step(self):
|
||||
seq = range(10, 20, 3)
|
||||
d = SequenceExclude(seq)
|
||||
are_not_in = [v for v in range(10, 20, 3) if v in d]
|
||||
self.assertEquals([], are_not_in)
|
||||
|
||||
are_not_in = [v for v in range(10, 20) if v in d]
|
||||
self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in)
|
||||
|
||||
def test_string_seq(self):
|
||||
seq = "cde"
|
||||
d = SequenceExclude(seq)
|
||||
chars = "".join(v for v in "abcdefg" if v in d)
|
||||
self.assertEquals("abfg", chars)
|
||||
|
||||
def test_stringset_seq(self):
|
||||
seq = set("cde")
|
||||
d = SequenceExclude(seq)
|
||||
chars = "".join(v for v in "abcdefg" if v in d)
|
||||
self.assertEquals("abfg", chars)
|
||||
|
||||
def test_set(self):
|
||||
"""Anything that is not in the supplied sequence will evaluate as 'in' the container."""
|
||||
seq = set([-3, "test", 1.1])
|
||||
d = SequenceExclude(seq)
|
||||
self.assertIn(0, d)
|
||||
self.assertIn("foo", d)
|
||||
self.assertIn(3.14, d)
|
||||
self.assertIn(set("bar"), d)
|
||||
|
||||
# supplied sequence is a set, so checking for list (non)inclusion fails
|
||||
self.assertRaises(TypeError, (0, 1, 2) in d)
|
||||
self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c'])
|
||||
|
||||
for v in [-3, "test", 1.1]:
|
||||
self.assertNotIn(v, d)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
|
|
|||
|
|
@ -119,13 +119,18 @@ Disallow: /s*/*tags
|
|||
# Sitemap files
|
||||
Sitemap: http://example.com/sitemap.xml
|
||||
Sitemap: http://example.com/sitemap-product-index.xml
|
||||
Sitemap: HTTP://example.com/sitemap-uppercase.xml
|
||||
Sitemap: /sitemap-relative-url.xml
|
||||
|
||||
# Forums
|
||||
Disallow: /forum/search/
|
||||
Disallow: /forum/active/
|
||||
"""
|
||||
self.assertEqual(list(sitemap_urls_from_robots(robots)),
|
||||
['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml'])
|
||||
self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')),
|
||||
['http://example.com/sitemap.xml',
|
||||
'http://example.com/sitemap-product-index.xml',
|
||||
'http://example.com/sitemap-uppercase.xml',
|
||||
'http://example.com/sitemap-relative-url.xml'])
|
||||
|
||||
def test_sitemap_blanklines(self):
|
||||
"""Assert we can deal with starting blank lines before <xml> tag"""
|
||||
|
|
|
|||
Loading…
Reference in New Issue