From aca2655c12d806759c6e0821a40d0277d200e0ea Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 14:57:39 +0100 Subject: [PATCH 01/30] [WIP] Run tests for Python 3.7 --- .travis.yml | 2 ++ tox.ini | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..e4df22139 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs + - python: 3.7 + env: TOXENV=py37 install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/tox.ini b/tox.ini index 60ff8c15e..5301624ee 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,10 @@ deps = {[testenv:py34]deps} basepython = python3.6 deps = {[testenv:py34]deps} +[testenv:py37] +basepython = python3.7 +deps = {[testenv:py34]deps} + [testenv:pypy3] basepython = pypy3 deps = {[testenv:py34]deps} From fab68ff6260b9ce4f55ca7b211a1aeb3e8e6df3d Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 17:05:14 +0100 Subject: [PATCH 02/30] Use 3.7-dev version for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e4df22139..aa1a3c4c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7 + - python: 3.7-dev env: TOXENV=py37 install: - | From 4c05441450bc1f8438239af0310ab76777a2dacf Mon Sep 17 00:00:00 2001 From: nctl144 Date: Sat, 3 Mar 2018 00:00:03 -0500 Subject: [PATCH 03/30] add ftp to the scheme list --- scrapy/linkextractors/__init__.py | 3 ++- tests/test_linkextractors.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..cda6ddc7e 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -41,7 +41,8 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \ + 'file', 'ftp'} class FilteringLinkExtractor(object): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 1d7c4f311..903032b52 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -451,6 +451,17 @@ class Base: Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_ftp_links(self): + body = b""" + +
An Item
+ """ + response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + ]) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor @@ -471,4 +482,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() - From ca7d79c29a55be4482bf2d4f704fc145ba801337 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 10:46:51 +0100 Subject: [PATCH 04/30] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..47eddf1fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +git+https://github.com/twisted/twisted.git@dcaf946 lxml pyOpenSSL cssselect>=0.9 From 5d1f5245f2699745e73449b013a29cc370f424c9 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:14:50 +0100 Subject: [PATCH 05/30] [WIP] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47eddf1fc..95cd37772 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/twisted/twisted.git@dcaf946 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml pyOpenSSL cssselect>=0.9 From f10a43d562dee32f324703fddb19bee5266912ce Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:43:39 +0100 Subject: [PATCH 06/30] [WIP] Install Twisted from branch for py3.7 --- requirements-py3.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..c3357e970 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/requirements.txt b/requirements.txt index 95cd37772..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 From 2dfc5d128bba42e2fe2bb24c0326fe47b0d3cd97 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 9 May 2018 11:59:38 -0400 Subject: [PATCH 07/30] Update DEPTH_STATS refs to DEPTH_STATS_VERBOSE --- docs/topics/settings.rst | 11 ----------- docs/topics/spider-middleware.rst | 3 ++- scrapy/settings/default_settings.py | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 076dc6bfd..1f1217770 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -335,17 +335,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` and :setting:`RETRY_PRIORITY_ADJUST`. -.. setting:: DEPTH_STATS - -DEPTH_STATS ------------ - -Default: ``True`` - -Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` - -Whether to collect maximum depth stats. - .. setting:: DEPTH_STATS_VERBOSE DEPTH_STATS_VERBOSE diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..265acdb43 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -212,7 +212,8 @@ DepthMiddleware * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to crawl for any site. If zero, no limit will be imposed. - * :setting:`DEPTH_STATS` - Whether to collect depth stats. + * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of + requests for each depth. * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on their depth. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 36e17ef6b..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -55,7 +55,7 @@ DEFAULT_REQUEST_HEADERS = { } DEPTH_LIMIT = 0 -DEPTH_STATS = True +DEPTH_STATS_VERBOSE = False DEPTH_PRIORITY = 0 DNSCACHE_ENABLED = True From 6a182c955273745daf334033944c82da3aa4eb12 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 9 May 2018 12:00:18 -0400 Subject: [PATCH 08/30] Depth stats are not optional --- scrapy/spidermiddlewares/depth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index e2f039146..34a87f2df 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class DepthMiddleware(object): - def __init__(self, maxdepth, stats=None, verbose_stats=False, prio=1): + def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth self.stats = stats self.verbose_stats = verbose_stats @@ -41,7 +41,7 @@ class DepthMiddleware(object): extra={'spider': spider} ) return False - elif self.stats: + else: if self.verbose_stats: self.stats.inc_value('request_depth_count/%s' % depth, spider=spider) @@ -50,7 +50,7 @@ class DepthMiddleware(object): return True # base case (depth=0) - if self.stats and 'depth' not in response.meta: + if 'depth' not in response.meta: response.meta['depth'] = 0 if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) From e2bb218e9ad89e9b367604f5f11a6fd7106cd79a Mon Sep 17 00:00:00 2001 From: Renne Rocha Date: Wed, 13 Jun 2018 18:11:43 -0300 Subject: [PATCH 09/30] Include Python version indication to each required library used in S3 storage --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 135d05c93..b64dbfbfd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ or `boto`_ + * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only) The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: From 9ad3af9d88bcefa18394c1cb2c833902ac20c533 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Sat, 23 Jun 2018 17:31:54 +0800 Subject: [PATCH 10/30] Update requirements.txt make the version of ipython less than 6.0 in python 2.7 --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index c1576a2e7..790f29d34 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,4 +10,4 @@ brotlipy testfixtures # optional for shell wrapper tests bpython -ipython +ipython<6.0 From fac1b2f3516f3db6ce669664943336c08d95d0aa Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jun 2018 03:23:47 +0500 Subject: [PATCH 11/30] TST remove workaround for old Pillow versions which don't support BytesIO --- tests/test_pipeline_images.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 03c6d8059..a7c652959 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,8 +1,8 @@ -import os +import io import hashlib import random import warnings -from tempfile import mkdtemp, TemporaryFile +from tempfile import mkdtemp from shutil import rmtree from twisted.trial import unittest @@ -401,8 +401,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) + def _create_image(format, *a, **kw): - buf = TemporaryFile() + buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf) From 45f67eb64d54f2ac9fcd69233d7bddbdcec88a37 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jun 2018 14:51:01 +0500 Subject: [PATCH 12/30] TST exclude lxml==4.2.2 from tests, as it doesn't play well with Pillow --- tests/constraints.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/constraints.txt b/tests/constraints.txt index 3bc30de15..e59e68b3f 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1 +1,2 @@ Twisted!=18.4.0 +lxml!=4.2.2 \ No newline at end of file diff --git a/tox.ini b/tox.ini index c2fa9af28..82348eb24 100644 --- a/tox.ini +++ b/tox.ini @@ -67,6 +67,7 @@ commands = [testenv:py34] basepython = python3.4 deps = + -ctests/constraints.txt -rrequirements-py3.txt # Extras Pillow From 8782901fc865ca6f505bcd1a7d17314076507028 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Jun 2018 01:11:15 +0500 Subject: [PATCH 13/30] [MRG+1] TST test agains latest pypy (#3309) pypy3 is not upgraded, as tests segfault with pypy3 6.0 for some reason --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..065f23805 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: install: - | if [ "$TOXENV" = "pypy" ]; then - export PYPY_VERSION="pypy-5.9-linux_x86_64-portable" + export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable" wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" From f11d65f7d66cf2d8560c707c4bb2b76079d45e5f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Jun 2018 18:34:11 +0500 Subject: [PATCH 14/30] TST make it clear which requirements are Python 2-only * rename requirements.txt to requirements-py2.txt, to make it clear they are Python 2-only * make requirements-py3.txt consistent with requirements-py2.txt --- requirements.txt => requirements-py2.txt | 4 ++-- requirements-py3.txt | 3 +++ tests/{requirements.txt => requirements-py2.txt} | 0 tox.ini | 8 ++++---- 4 files changed, 9 insertions(+), 6 deletions(-) rename requirements.txt => requirements-py2.txt (100%) rename tests/{requirements.txt => requirements-py2.txt} (100%) diff --git a/requirements.txt b/requirements-py2.txt similarity index 100% rename from requirements.txt rename to requirements-py2.txt index 2a94d742d..03b33d02d 100644 --- a/requirements.txt +++ b/requirements-py2.txt @@ -2,9 +2,9 @@ Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 -w3lib>=1.17.0 queuelib +w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -service_identity parsel>=1.4 +service_identity diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..d76d9412f 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -4,4 +4,7 @@ pyOpenSSL>=0.13.1 cssselect>=0.9 queuelib>=1.1.1 w3lib>=1.17.0 +six>=1.5.2 +PyDispatcher>=2.0.5 +parsel>=1.4 service_identity diff --git a/tests/requirements.txt b/tests/requirements-py2.txt similarity index 100% rename from tests/requirements.txt rename to tests/requirements-py2.txt diff --git a/tox.ini b/tox.ini index 82348eb24..ee40983de 100644 --- a/tox.ini +++ b/tox.ini @@ -9,13 +9,13 @@ envlist = py27 [testenv] deps = -ctests/constraints.txt - -rrequirements.txt + -rrequirements-py2.txt # Extras botocore google-cloud-storage Pillow != 3.0.0 leveldb - -rtests/requirements.txt + -rtests/requirements-py2.txt passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -35,7 +35,7 @@ deps = Pillow==2.3.0 cssselect==0.9.1 zope.interface==4.0.5 - -rtests/requirements.txt + -rtests/requirements-py2.txt [testenv:jessie] # https://packages.debian.org/en/jessie/python/ @@ -50,7 +50,7 @@ deps = Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 - -rtests/requirements.txt + -rtests/requirements-py2.txt [testenv:trunk] basepython = python2.7 From d05c8677c5079b88db4405fe2bed83dc437c9204 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Wed, 4 Jul 2018 02:58:43 +0800 Subject: [PATCH 15/30] [MRG+1] change the bad smell code (#3304) Change the bad smell code --- scrapy/downloadermiddlewares/httpproxy.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 0d5320bf8..1dd47359f 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,14 +1,13 @@ import base64 +from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -from six.moves.urllib.parse import unquote try: from urllib2 import _parse_proxy except ImportError: from urllib.request import _parse_proxy -from six.moves.urllib.parse import urlunparse -from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -17,8 +16,8 @@ class HttpProxyMiddleware(object): def __init__(self, auth_encoding='latin-1'): self.auth_encoding = auth_encoding self.proxies = {} - for type, url in getproxies().items(): - self.proxies[type] = self._get_proxy(url, type) + for type_, url in getproxies().items(): + self.proxies[type_] = self._get_proxy(url, type_) @classmethod def from_crawler(cls, crawler): From 74ce1561542dac9be5d1363a4a3e623653855e05 Mon Sep 17 00:00:00 2001 From: chainly <1258626769@qq.com> Date: Wed, 4 Jul 2018 03:00:59 +0800 Subject: [PATCH 16/30] add item_error to be catchable (#3256) --- docs/topics/signals.rst | 23 +++++++++++++++++++++++ scrapy/core/scraper.py | 3 +++ scrapy/signals.py | 1 + tests/pipelines.py | 6 ++++++ tests/test_engine.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index cf1588df8..d40c0e1df 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -135,6 +135,29 @@ item_dropped to be dropped :type exception: :exc:`~scrapy.exceptions.DropItem` exception +item_error +------------ + +.. signal:: item_error +.. function:: item_error(item, response, spider, failure) + + Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises + an exception), except :exc:`~scrapy.exceptions.DropItem` exception. + + This signal supports returning deferreds from their handlers. + + :param item: the item dropped from the :ref:`topics-item-pipeline` + :type item: dict or :class:`~scrapy.item.Item` object + + :param response: the response being processed when the exception was raised + :type response: :class:`~scrapy.http.Response` object + + :param spider: the spider which raised the exception + :type spider: :class:`~scrapy.spiders.Spider` object + + :param failure: the exception raised as a Twisted `Failure`_ object + :type failure: `Failure`_ object + spider_closed ------------- diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c08e37367..ee1e95a0c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -232,6 +232,9 @@ class Scraper(object): logger.error('Error processing %(item)s', {'item': item}, exc_info=failure_to_exc_info(output), extra={'spider': spider}) + return self.signals.send_catch_log_deferred( + signal=signals.item_error, item=item, response=response, + spider=spider, failure=output) else: logkws = self.logformatter.scraped(output, response, spider) logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) diff --git a/scrapy/signals.py b/scrapy/signals.py index de0886fb6..e36c27203 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ response_received = object() response_downloaded = object() item_scraped = object() item_dropped = object() +item_error = object() # for backwards compatibility stats_spider_opened = spider_opened diff --git a/tests/pipelines.py b/tests/pipelines.py index ddfbc7a99..7e2895a5c 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -9,3 +9,9 @@ class ZeroDivisionErrorPipeline(object): def process_item(self, item, spider): return item + + +class ProcessWithZeroDivisionErrorPipiline(object): + + def process_item(self, item, spider): + 1/0 diff --git a/tests/test_engine.py b/tests/test_engine.py index 04113ddcf..719c0c60c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -74,6 +74,14 @@ class DictItemsSpider(TestSpider): item_cls = dict +class ItemZeroDivisionErrorSpider(TestSpider): + custom_settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300, + } + } + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -95,6 +103,7 @@ class CrawlerRun(object): self.respplug = [] self.reqplug = [] self.reqdropped = [] + self.itemerror = [] self.itemresp = [] self.signals_catched = {} self.spider_class = spider_class @@ -112,6 +121,7 @@ class CrawlerRun(object): self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.item_error, signals.item_error) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) @@ -136,6 +146,9 @@ class CrawlerRun(object): u = urlparse(url) return u.path + def item_error(self, item, response, spider, failure): + self.itemerror.append((item, response, spider, failure)) + def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) @@ -175,6 +188,10 @@ class EngineTest(unittest.TestCase): self._assert_scheduled_requests(urls_to_visit=7) self._assert_dropped_requests() + self.run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield self.run.run() + self._assert_items_error() + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] @@ -209,6 +226,20 @@ class EngineTest(unittest.TestCase): if self.run.getpath(response.url) == '/redirect': self.assertEqual(302, response.status) + def _assert_items_error(self): + self.assertEqual(2, len(self.run.itemerror)) + for item, response, spider, failure in self.run.itemerror: + self.assertEqual(failure.value.__class__, ZeroDivisionError) + self.assertEqual(spider, self.run.spider) + + self.assertEqual(item['url'], response.url) + if 'item1.html' in item['url']: + self.assertEqual('Item 1 name', item['name']) + self.assertEqual('100', item['price']) + if 'item2.html' in item['url']: + self.assertEqual('Item 2 name', item['name']) + self.assertEqual('200', item['price']) + def _assert_scraped_items(self): self.assertEqual(2, len(self.run.itemresp)) for item, response in self.run.itemresp: From 6f5c39d65f3e5d74c292f2143b4f84c7f43b155a Mon Sep 17 00:00:00 2001 From: Oz T Date: Wed, 4 Jul 2018 00:22:24 +0300 Subject: [PATCH 17/30] Fix for CSV export unnecessary blank lines problem on Windows (#3039) --- scrapy/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 07f43b494..695c74fec 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -214,7 +214,8 @@ class CsvItemExporter(BaseItemExporter): file, line_buffering=False, write_through=True, - encoding=self.encoding + encoding=self.encoding, + newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 ) if six.PY3 else file self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True From d4c7cc848b83c2ec38ea90b76369885daba7375c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:19:43 +0500 Subject: [PATCH 18/30] remove backwards compatibility shims for relocated modules --- .coveragerc | 9 --------- conftest.py | 9 +-------- scrapy/command.py | 7 ------- scrapy/contrib/__init__.py | 0 scrapy/contrib/closespider.py | 7 ------- scrapy/contrib/corestats.py | 7 ------- scrapy/contrib/debug.py | 7 ------- .../contrib/downloadermiddleware/__init__.py | 0 .../contrib/downloadermiddleware/ajaxcrawl.py | 7 ------- scrapy/contrib/downloadermiddleware/chunked.py | 7 ------- scrapy/contrib/downloadermiddleware/cookies.py | 7 ------- .../downloadermiddleware/decompression.py | 7 ------- .../downloadermiddleware/defaultheaders.py | 7 ------- .../downloadermiddleware/downloadtimeout.py | 7 ------- .../contrib/downloadermiddleware/httpauth.py | 7 ------- .../contrib/downloadermiddleware/httpcache.py | 7 ------- .../downloadermiddleware/httpcompression.py | 7 ------- .../contrib/downloadermiddleware/httpproxy.py | 7 ------- .../contrib/downloadermiddleware/redirect.py | 7 ------- scrapy/contrib/downloadermiddleware/retry.py | 7 ------- .../contrib/downloadermiddleware/robotstxt.py | 7 ------- scrapy/contrib/downloadermiddleware/stats.py | 7 ------- .../contrib/downloadermiddleware/useragent.py | 7 ------- scrapy/contrib/exporter/__init__.py | 8 -------- scrapy/contrib/feedexport.py | 7 ------- scrapy/contrib/httpcache.py | 7 ------- scrapy/contrib/linkextractors/__init__.py | 7 ------- scrapy/contrib/linkextractors/htmlparser.py | 7 ------- scrapy/contrib/linkextractors/lxmlhtml.py | 7 ------- scrapy/contrib/linkextractors/regex.py | 7 ------- scrapy/contrib/linkextractors/sgml.py | 7 ------- scrapy/contrib/loader/__init__.py | 7 ------- scrapy/contrib/loader/common.py | 7 ------- scrapy/contrib/loader/processor.py | 7 ------- scrapy/contrib/logstats.py | 7 ------- scrapy/contrib/memdebug.py | 7 ------- scrapy/contrib/memusage.py | 7 ------- scrapy/contrib/pipeline/__init__.py | 7 ------- scrapy/contrib/pipeline/files.py | 7 ------- scrapy/contrib/pipeline/images.py | 7 ------- scrapy/contrib/pipeline/media.py | 7 ------- scrapy/contrib/spidermiddleware/__init__.py | 0 scrapy/contrib/spidermiddleware/depth.py | 7 ------- scrapy/contrib/spidermiddleware/httperror.py | 7 ------- scrapy/contrib/spidermiddleware/offsite.py | 7 ------- scrapy/contrib/spidermiddleware/referer.py | 7 ------- scrapy/contrib/spidermiddleware/urllength.py | 7 ------- scrapy/contrib/spiders/__init__.py | 7 ------- scrapy/contrib/spiders/crawl.py | 7 ------- scrapy/contrib/spiders/feed.py | 7 ------- scrapy/contrib/spiders/init.py | 7 ------- scrapy/contrib/spiders/sitemap.py | 7 ------- scrapy/contrib/spiderstate.py | 7 ------- scrapy/contrib/statsmailer.py | 7 ------- scrapy/contrib/throttle.py | 7 ------- scrapy/contrib_exp/__init__.py | 0 .../downloadermiddleware/__init__.py | 0 .../downloadermiddleware/decompression.py | 7 ------- scrapy/contrib_exp/iterators.py | 6 ------ scrapy/dupefilter.py | 7 ------- scrapy/linkextractor.py | 7 ------- scrapy/spider.py | 7 ------- scrapy/squeue.py | 7 ------- scrapy/statscol.py | 7 ------- scrapy/utils/decorator.py | 7 ------- scrapy/utils/deprecate.py | 18 ------------------ 66 files changed, 1 insertion(+), 441 deletions(-) delete mode 100644 scrapy/command.py delete mode 100644 scrapy/contrib/__init__.py delete mode 100644 scrapy/contrib/closespider.py delete mode 100644 scrapy/contrib/corestats.py delete mode 100644 scrapy/contrib/debug.py delete mode 100644 scrapy/contrib/downloadermiddleware/__init__.py delete mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawl.py delete mode 100644 scrapy/contrib/downloadermiddleware/chunked.py delete mode 100644 scrapy/contrib/downloadermiddleware/cookies.py delete mode 100644 scrapy/contrib/downloadermiddleware/decompression.py delete mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py delete mode 100644 scrapy/contrib/downloadermiddleware/downloadtimeout.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpauth.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpcompression.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpproxy.py delete mode 100644 scrapy/contrib/downloadermiddleware/redirect.py delete mode 100644 scrapy/contrib/downloadermiddleware/retry.py delete mode 100644 scrapy/contrib/downloadermiddleware/robotstxt.py delete mode 100644 scrapy/contrib/downloadermiddleware/stats.py delete mode 100644 scrapy/contrib/downloadermiddleware/useragent.py delete mode 100644 scrapy/contrib/exporter/__init__.py delete mode 100644 scrapy/contrib/feedexport.py delete mode 100644 scrapy/contrib/httpcache.py delete mode 100644 scrapy/contrib/linkextractors/__init__.py delete mode 100644 scrapy/contrib/linkextractors/htmlparser.py delete mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py delete mode 100644 scrapy/contrib/linkextractors/regex.py delete mode 100644 scrapy/contrib/linkextractors/sgml.py delete mode 100644 scrapy/contrib/loader/__init__.py delete mode 100644 scrapy/contrib/loader/common.py delete mode 100644 scrapy/contrib/loader/processor.py delete mode 100644 scrapy/contrib/logstats.py delete mode 100644 scrapy/contrib/memdebug.py delete mode 100644 scrapy/contrib/memusage.py delete mode 100644 scrapy/contrib/pipeline/__init__.py delete mode 100644 scrapy/contrib/pipeline/files.py delete mode 100644 scrapy/contrib/pipeline/images.py delete mode 100644 scrapy/contrib/pipeline/media.py delete mode 100644 scrapy/contrib/spidermiddleware/__init__.py delete mode 100644 scrapy/contrib/spidermiddleware/depth.py delete mode 100644 scrapy/contrib/spidermiddleware/httperror.py delete mode 100644 scrapy/contrib/spidermiddleware/offsite.py delete mode 100644 scrapy/contrib/spidermiddleware/referer.py delete mode 100644 scrapy/contrib/spidermiddleware/urllength.py delete mode 100644 scrapy/contrib/spiders/__init__.py delete mode 100644 scrapy/contrib/spiders/crawl.py delete mode 100644 scrapy/contrib/spiders/feed.py delete mode 100644 scrapy/contrib/spiders/init.py delete mode 100644 scrapy/contrib/spiders/sitemap.py delete mode 100644 scrapy/contrib/spiderstate.py delete mode 100644 scrapy/contrib/statsmailer.py delete mode 100644 scrapy/contrib/throttle.py delete mode 100644 scrapy/contrib_exp/__init__.py delete mode 100644 scrapy/contrib_exp/downloadermiddleware/__init__.py delete mode 100644 scrapy/contrib_exp/downloadermiddleware/decompression.py delete mode 100644 scrapy/contrib_exp/iterators.py delete mode 100644 scrapy/dupefilter.py delete mode 100644 scrapy/linkextractor.py delete mode 100644 scrapy/spider.py delete mode 100644 scrapy/squeue.py delete mode 100644 scrapy/statscol.py delete mode 100644 scrapy/utils/decorator.py diff --git a/.coveragerc b/.coveragerc index 3105409ba..aeadccb25 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,13 +7,4 @@ omit = scrapy/conf.py scrapy/stats.py scrapy/project.py - scrapy/utils/decorator.py - scrapy/statscol.py - scrapy/squeue.py scrapy/log.py - scrapy/dupefilter.py - scrapy/command.py - scrapy/linkextractor.py - scrapy/spider.py - scrapy/contrib/* - scrapy/contrib_exp/* diff --git a/conftest.py b/conftest.py index 8b4faf8fc..c733db646 100644 --- a/conftest.py +++ b/conftest.py @@ -13,19 +13,12 @@ collect_ignore = [ "scrapy/conf.py", "scrapy/stats.py", "scrapy/project.py", - "scrapy/utils/decorator.py", - "scrapy/statscol.py", - "scrapy/squeue.py", "scrapy/log.py", - "scrapy/dupefilter.py", - "scrapy/command.py", - "scrapy/linkextractor.py", - "scrapy/spider.py", # not a test, but looks like a test "scrapy/utils/testsite.py", -] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +] if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): collect_ignore += _py_files("scrapy/xlib/tx") diff --git a/scrapy/command.py b/scrapy/command.py deleted file mode 100644 index 3e1219bbc..000000000 --- a/scrapy/command.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.command` is deprecated, " - "use `scrapy.commands` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.commands import * diff --git a/scrapy/contrib/__init__.py b/scrapy/contrib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py deleted file mode 100644 index 9c52c418f..000000000 --- a/scrapy/contrib/closespider.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.closespider` is deprecated, " - "use `scrapy.extensions.closespider` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.closespider import * diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py deleted file mode 100644 index 2f5354239..000000000 --- a/scrapy/contrib/corestats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.corestats` is deprecated, " - "use `scrapy.extensions.corestats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.corestats import * diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py deleted file mode 100644 index a38f059ce..000000000 --- a/scrapy/contrib/debug.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.debug` is deprecated, " - "use `scrapy.extensions.debug` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.debug import * diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py deleted file mode 100644 index 90ebc46b6..000000000 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, " - "use `scrapy.downloadermiddlewares.ajaxcrawl` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.ajaxcrawl import * diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py deleted file mode 100644 index 1322c9083..000000000 --- a/scrapy/contrib/downloadermiddleware/chunked.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, " - "use `scrapy.downloadermiddlewares.chunked` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.chunked import * diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py deleted file mode 100644 index bad970690..000000000 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, " - "use `scrapy.downloadermiddlewares.cookies` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.cookies import * diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py deleted file mode 100644 index a541aa61e..000000000 --- a/scrapy/contrib/downloadermiddleware/decompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.downloadermiddlewares.decompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.decompression import * diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py deleted file mode 100644 index cf023dc8f..000000000 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, " - "use `scrapy.downloadermiddlewares.defaultheaders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.defaultheaders import * diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py deleted file mode 100644 index 84bd06acf..000000000 --- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, " - "use `scrapy.downloadermiddlewares.downloadtimeout` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.downloadtimeout import * diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py deleted file mode 100644 index a37ffa0dc..000000000 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.httpauth` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpauth import * diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py deleted file mode 100644 index f5f068204..000000000 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, " - "use `scrapy.downloadermiddlewares.httpcache` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpcache import * diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py deleted file mode 100644 index 8a52ec50b..000000000 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, " - "use `scrapy.downloadermiddlewares.httpcompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpcompression import * diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py deleted file mode 100644 index d94d85076..000000000 --- a/scrapy/contrib/downloadermiddleware/httpproxy.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, " - "use `scrapy.downloadermiddlewares.httpproxy` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpproxy import * diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py deleted file mode 100644 index 824eee8ae..000000000 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, " - "use `scrapy.downloadermiddlewares.redirect` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.redirect import * diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py deleted file mode 100644 index aafe0f508..000000000 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, " - "use `scrapy.downloadermiddlewares.retry` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.retry import * diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py deleted file mode 100644 index 408f760a0..000000000 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, " - "use `scrapy.downloadermiddlewares.robotstxt` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.robotstxt import * diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py deleted file mode 100644 index fa84a8206..000000000 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, " - "use `scrapy.downloadermiddlewares.stats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.stats import * diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py deleted file mode 100644 index 893d5241c..000000000 --- a/scrapy/contrib/downloadermiddleware/useragent.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, " - "use `scrapy.downloadermiddlewares.useragent` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.useragent import * diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py deleted file mode 100644 index 12adaaddd..000000000 --- a/scrapy/contrib/exporter/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.exporter` is deprecated, " - "use `scrapy.exporters` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.exporters import * -from scrapy.exporters import PythonItemExporter diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py deleted file mode 100644 index 19651998a..000000000 --- a/scrapy/contrib/feedexport.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, " - "use `scrapy.extensions.feedexport` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.feedexport import * diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py deleted file mode 100644 index 196372fcb..000000000 --- a/scrapy/contrib/httpcache.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, " - "use `scrapy.extensions.httpcache` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.httpcache import * diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py deleted file mode 100644 index 976658df3..000000000 --- a/scrapy/contrib/linkextractors/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, " - "use `scrapy.linkextractors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors import * diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py deleted file mode 100644 index ff03da98f..000000000 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, " - "use `scrapy.linkextractors.htmlparser` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.htmlparser import * diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py deleted file mode 100644 index fc2b7de3c..000000000 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, " - "use `scrapy.linkextractors.lxmlhtml` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.lxmlhtml import * diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py deleted file mode 100644 index 97bda29c1..000000000 --- a/scrapy/contrib/linkextractors/regex.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, " - "use `scrapy.linkextractors.regex` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.regex import * diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py deleted file mode 100644 index a5a598208..000000000 --- a/scrapy/contrib/linkextractors/sgml.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, " - "use `scrapy.linkextractors.sgml` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.sgml import * diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py deleted file mode 100644 index 2b9453e18..000000000 --- a/scrapy/contrib/loader/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader` is deprecated, " - "use `scrapy.loader` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader import * diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py deleted file mode 100644 index a59b2b7b1..000000000 --- a/scrapy/contrib/loader/common.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, " - "use `scrapy.loader.common` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader.common import * diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py deleted file mode 100644 index da7e484a5..000000000 --- a/scrapy/contrib/loader/processor.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, " - "use `scrapy.loader.processors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader.processors import * diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py deleted file mode 100644 index 62bc9b860..000000000 --- a/scrapy/contrib/logstats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.logstats` is deprecated, " - "use `scrapy.extensions.logstats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.logstats import * diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py deleted file mode 100644 index 4f6e4760e..000000000 --- a/scrapy/contrib/memdebug.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, " - "use `scrapy.extensions.memdebug` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.memdebug import * diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py deleted file mode 100644 index e13bd78f3..000000000 --- a/scrapy/contrib/memusage.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.memusage` is deprecated, " - "use `scrapy.extensions.memusage` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.memusage import * diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py deleted file mode 100644 index aedf34a3f..000000000 --- a/scrapy/contrib/pipeline/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, " - "use `scrapy.pipelines` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines import * diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py deleted file mode 100644 index cd1238b5d..000000000 --- a/scrapy/contrib/pipeline/files.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, " - "use `scrapy.pipelines.files` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.files import * diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py deleted file mode 100644 index 4f5ce4c40..000000000 --- a/scrapy/contrib/pipeline/images.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, " - "use `scrapy.pipelines.images` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.images import * diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py deleted file mode 100644 index 4b4fea560..000000000 --- a/scrapy/contrib/pipeline/media.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, " - "use `scrapy.pipelines.media` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.media import * diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py deleted file mode 100644 index 718803148..000000000 --- a/scrapy/contrib/spidermiddleware/depth.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, " - "use `scrapy.spidermiddlewares.depth` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.depth import * diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py deleted file mode 100644 index e39fb3f56..000000000 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, " - "use `scrapy.spidermiddlewares.httperror` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.httperror import * diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py deleted file mode 100644 index a5ed9ea7e..000000000 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, " - "use `scrapy.spidermiddlewares.offsite` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.offsite import * diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py deleted file mode 100644 index fdf8d6659..000000000 --- a/scrapy/contrib/spidermiddleware/referer.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, " - "use `scrapy.spidermiddlewares.referer` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.referer import * diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py deleted file mode 100644 index 5e51add59..000000000 --- a/scrapy/contrib/spidermiddleware/urllength.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, " - "use `scrapy.spidermiddlewares.urllength` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.urllength import * diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py deleted file mode 100644 index 56780533b..000000000 --- a/scrapy/contrib/spiders/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders` is deprecated, " - "use `scrapy.spiders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders import * diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py deleted file mode 100644 index d20a8bb16..000000000 --- a/scrapy/contrib/spiders/crawl.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, " - "use `scrapy.spiders.crawl` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.crawl import * diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py deleted file mode 100644 index 5eea9a062..000000000 --- a/scrapy/contrib/spiders/feed.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, " - "use `scrapy.spiders.feed` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.feed import * diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py deleted file mode 100644 index 6d1ec0aa9..000000000 --- a/scrapy/contrib/spiders/init.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, " - "use `scrapy.spiders.init` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.init import * diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py deleted file mode 100644 index 2ad231fd8..000000000 --- a/scrapy/contrib/spiders/sitemap.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, " - "use `scrapy.spiders.sitemap` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.sitemap import * diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py deleted file mode 100644 index 06afc8bfc..000000000 --- a/scrapy/contrib/spiderstate.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, " - "use `scrapy.extensions.spiderstate` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.spiderstate import * diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py deleted file mode 100644 index f9c9a37f5..000000000 --- a/scrapy/contrib/statsmailer.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, " - "use `scrapy.extensions.statsmailer` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.statsmailer import * diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py deleted file mode 100644 index d5c234871..000000000 --- a/scrapy/contrib/throttle.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.throttle` is deprecated, " - "use `scrapy.extensions.throttle` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.throttle import * diff --git a/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/downloadermiddleware/__init__.py b/scrapy/contrib_exp/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py deleted file mode 100644 index 1f8490587..000000000 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.downloadermiddlewares.decompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py deleted file mode 100644 index c59f47bcc..000000000 --- a/scrapy/contrib_exp/iterators.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib_exp.iterators` is deprecated, use `scrapy.utils.iterators` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.utils.iterators import xmliter_lxml diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py deleted file mode 100644 index 232d96288..000000000 --- a/scrapy/dupefilter.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.dupefilter` is deprecated, " - "use `scrapy.dupefilters` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.dupefilters import * diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py deleted file mode 100644 index b744aff8e..000000000 --- a/scrapy/linkextractor.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.linkextractor` is deprecated, " - "use `scrapy.linkextractors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors import * diff --git a/scrapy/spider.py b/scrapy/spider.py deleted file mode 100644 index 56a5a0a0b..000000000 --- a/scrapy/spider.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.spider` is deprecated, " - "use `scrapy.spiders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders import * diff --git a/scrapy/squeue.py b/scrapy/squeue.py deleted file mode 100644 index a4a3f4238..000000000 --- a/scrapy/squeue.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.squeue` is deprecated, " - "use `scrapy.squeues` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.squeues import * diff --git a/scrapy/statscol.py b/scrapy/statscol.py deleted file mode 100644 index b4ddcce28..000000000 --- a/scrapy/statscol.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.statscol` is deprecated, " - "use `scrapy.statscollectors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.statscollectors import * diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py deleted file mode 100644 index e8c8eae39..000000000 --- a/scrapy/utils/decorator.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.utils.decorator` is deprecated, " - "use `scrapy.utils.decorators` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.utils.decorators import * diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f76161a68..8c72cc556 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -124,25 +124,7 @@ def _clspath(cls, forced=None): DEPRECATION_RULES = [ - ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'), - ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'), - ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'), - ('scrapy.contrib.exporter.', 'scrapy.exporters.'), - ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'), - ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'), - ('scrapy.contrib.loader.', 'scrapy.loader.'), - ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'), - ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'), - ('scrapy.contrib.spiders.', 'scrapy.spiders.'), - ('scrapy.contrib.', 'scrapy.extensions.'), - ('scrapy.command.', 'scrapy.commands.'), - ('scrapy.dupefilter.', 'scrapy.dupefilters.'), - ('scrapy.linkextractor.', 'scrapy.linkextractors.'), ('scrapy.telnet.', 'scrapy.extensions.telnet.'), - ('scrapy.spider.', 'scrapy.spiders.'), - ('scrapy.squeue.', 'scrapy.squeues.'), - ('scrapy.statscol.', 'scrapy.statscollectors.'), - ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'), ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'), ] From 36453348fad9babc96558ab10af9b2942eb5431e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:23:37 +0500 Subject: [PATCH 19/30] remove ancient modules kept only for error messages --- .coveragerc | 2 -- conftest.py | 2 -- scrapy/project.py | 17 ----------------- scrapy/stats.py | 8 -------- 4 files changed, 29 deletions(-) delete mode 100644 scrapy/project.py delete mode 100644 scrapy/stats.py diff --git a/.coveragerc b/.coveragerc index aeadccb25..1fde07e7e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,4 @@ omit = tests/* scrapy/xlib/* scrapy/conf.py - scrapy/stats.py - scrapy/project.py scrapy/log.py diff --git a/conftest.py b/conftest.py index c733db646..2d015f5e9 100644 --- a/conftest.py +++ b/conftest.py @@ -11,8 +11,6 @@ def _py_files(folder): collect_ignore = [ # deprecated or moved modules "scrapy/conf.py", - "scrapy/stats.py", - "scrapy/project.py", "scrapy/log.py", # not a test, but looks like a test diff --git a/scrapy/project.py b/scrapy/project.py deleted file mode 100644 index d8973a6c7..000000000 --- a/scrapy/project.py +++ /dev/null @@ -1,17 +0,0 @@ - -""" -Obsolete module, kept for giving a meaningful error message when trying to -import. -""" - -raise ImportError("""scrapy.project usage has become obsolete. - -If you want to get the Scrapy crawler from your extension, middleware or -pipeline implement the `from_crawler` class method (or look up for extending -components that have already done it, such as spiders). - -For example: - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler)""") diff --git a/scrapy/stats.py b/scrapy/stats.py deleted file mode 100644 index 710601430..000000000 --- a/scrapy/stats.py +++ /dev/null @@ -1,8 +0,0 @@ - -""" -Obsolete module, kept for giving a meaningful error message when trying to -import. -""" - -raise ImportError("scrapy.stats usage has become obsolete, use " - "`crawler.stats` attribute instead") From f531b66822491140740a6d86af2f3f11f0443d38 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:28:01 +0500 Subject: [PATCH 20/30] SpiderManager shim is removed --- scrapy/interfaces.py | 4 ---- scrapy/spidermanager.py | 7 ------- scrapy/utils/deprecate.py | 1 - 3 files changed, 12 deletions(-) delete mode 100644 scrapy/spidermanager.py diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index eb93c6f7e..89ad2b14f 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -16,7 +16,3 @@ class ISpiderLoader(Interface): def find_by_request(request): """Return the list of spiders names that can handle the given request""" - -# ISpiderManager is deprecated, don't use it! -# An alias is kept for backwards compatibility. -ISpiderManager = ISpiderLoader diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py deleted file mode 100644 index 220257bb1..000000000 --- a/scrapy/spidermanager.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Backwards compatibility shim. Use scrapy.spiderloader instead. -""" -from scrapy.spiderloader import SpiderLoader -from scrapy.utils.deprecate import create_deprecated_class - -SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 8c72cc556..2d3db431d 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -125,7 +125,6 @@ def _clspath(cls, forced=None): DEPRECATION_RULES = [ ('scrapy.telnet.', 'scrapy.extensions.telnet.'), - ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'), ] From 722e1afcdb337bf11652167f02435c81fc68ecfb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:21:19 +0300 Subject: [PATCH 21/30] Update ancient pytest on python 3 2.9 gives collection errors on python 3.7 due to PEP 479. --- tests/requirements-py3.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 51a25f5e5..8d9ce5231 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,6 @@ -pytest==2.9.2 +pytest==3.6.3 pytest-twisted -pytest-cov==2.2.1 +pytest-cov==2.5.1 testfixtures jmespath leveldb From 17e9914b8a12e5a96cc40016b74f301fb9835cbf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:09 +0300 Subject: [PATCH 22/30] Catch SyntaxError as well when importing manhole Also give a more detailed reason why telnet is not enabled (for the future). --- requirements-py3.txt | 2 +- scrapy/extensions/telnet.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index b941fd867..b38c4cc09 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index e78afa1fc..7cc8f823a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -12,7 +12,7 @@ try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True -except ImportError: +except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,7 +40,8 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured + raise NotConfigured('TelnetConsole not enabled: failed to import ' + 'required twisted modules.') self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From cf9399acc149cf5eafb2d00d310416ab2ba185e5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:56 +0300 Subject: [PATCH 23/30] Use python 3.7 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29f9f0065..f6ea670ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7-dev + - python: 3.7 env: TOXENV=py37 install: - | From 2773fe09e4b4fac51dbc06725f1fafb2e5d9a271 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:36:58 +0300 Subject: [PATCH 24/30] Make "docs" the last build, even though it still uses python3.6 for now --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6ea670ae..88c72b08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.6 - env: TOXENV=docs - python: 3.7 env: TOXENV=py37 + - python: 3.6 + env: TOXENV=docs install: - | if [ "$TOXENV" = "pypy" ]; then From f4f39057cbbfa4daf66f82061e57101b88d88d05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:46:45 +0300 Subject: [PATCH 25/30] Make csviter work on python 3.7 PEP 479 does not allow for StopIteration in generators. Instead, handle it explicitly, also use a for loop which looks simpler. --- scrapy/utils/iterators.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 73857b410..a12e14005 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): """ encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' - def _getrow(csv_r): - return [to_unicode(field, encoding) for field in next(csv_r)] + + def row_to_unicode(row_): + return [to_unicode(field, encoding) for field in row_] # Python 3 csv reader input object needs to return strings if six.PY3: @@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): csv_r = csv.reader(lines, **kwargs) if not headers: - headers = _getrow(csv_r) + try: + row = next(csv_r) + except StopIteration: + return + headers = row_to_unicode(row) - while True: - row = _getrow(csv_r) + for row in csv_r: + row = row_to_unicode(row) if len(row) != len(headers): logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, " "should be: %(csvheader)d)", From b3cd12dc48592fb8b1d4c6883315fe2b341ca5c2 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:53:40 +0300 Subject: [PATCH 26/30] Try to get python3.7 by using xenial base and sudo See https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-401756442 and https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-402045581 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 88c72b08e..4218d13bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 + dist: xenial + sudo: true - python: 3.6 env: TOXENV=docs install: From 92b504eae5379dadade2d78efba4e54b201cbd93 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 13:43:36 +0300 Subject: [PATCH 27/30] Fix telnet warnings in tests Disable telnet console if it's not available, else we'll get an extra warning about failure to enable it, and tests will fail. --- tests/test_crawler.py | 6 ++++-- tests/test_utils_log.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d3b80f460..6a8e11363 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import tempfile import warnings import unittest @@ -14,8 +13,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle +from scrapy.extensions import telnet + class BaseCrawlerTest(unittest.TestCase): @@ -100,6 +100,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file.name, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 45527b03b..742e04803 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -10,6 +10,7 @@ from twisted.python.failure import Failure from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter, LogCounterHandler, StreamLogger) from scrapy.utils.test import get_crawler +from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): @@ -65,10 +66,14 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): + settings = {'LOG_LEVEL': 'WARNING'} + if not telnet.TWISTED_CONCH_AVAILABLE: + # disable it to avoid the extra warning + settings['TELNETCONSOLE_ENABLED'] = False self.logger = logging.getLogger('test') self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'}) + self.crawler = get_crawler(settings_dict=settings) self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) From 4f6778aa7332aecd15f7672a0852d1f49596969b Mon Sep 17 00:00:00 2001 From: nyov Date: Mon, 9 Jul 2018 17:16:31 +0000 Subject: [PATCH 28/30] Remove deprecated CrawlerSettings class and Settings attributes --- docs/news.rst | 9 +++++ scrapy/settings/__init__.py | 48 -------------------------- tests/test_settings/__init__.py | 60 +-------------------------------- 3 files changed, 10 insertions(+), 107 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1b8d121a1..633e5c72f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,15 @@ Release notes ============= +Scrapy 1.6.0 (unreleased) +------------------------- + +Cleanups +~~~~~~~~ + +* Remove deprecated ``CrawlerSettings`` class. +* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. + Scrapy 1.5.0 (2017-12-29) ------------------------- diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 28446a372..7d6d20164 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -6,7 +6,6 @@ from collections import MutableMapping from importlib import import_module from pprint import pformat -from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning from . import default_settings @@ -405,30 +404,6 @@ class BaseSettings(MutableMapping): else: p.text(pformat(self.copy_to_dict())) - @property - def overrides(self): - warnings.warn("`Settings.overrides` attribute is deprecated and won't " - "be supported in Scrapy 0.26, use " - "`Settings.set(name, value, priority='cmdline')` instead", - category=ScrapyDeprecationWarning, stacklevel=2) - try: - o = self._overrides - except AttributeError: - self._overrides = o = _DictProxy(self, 'cmdline') - return o - - @property - def defaults(self): - warnings.warn("`Settings.defaults` attribute is deprecated and won't " - "be supported in Scrapy 0.26, use " - "`Settings.set(name, value, priority='default')` instead", - category=ScrapyDeprecationWarning, stacklevel=2) - try: - o = self._defaults - except AttributeError: - self._defaults = o = _DictProxy(self, 'default') - return o - class _DictProxy(MutableMapping): @@ -479,29 +454,6 @@ class Settings(BaseSettings): self.update(values, priority) -class CrawlerSettings(Settings): - - def __init__(self, settings_module=None, **kw): - self.settings_module = settings_module - Settings.__init__(self, **kw) - - def __getitem__(self, opt_name): - if opt_name in self.overrides: - return self.overrides[opt_name] - if self.settings_module and hasattr(self.settings_module, opt_name): - return getattr(self.settings_module, opt_name) - if opt_name in self.defaults: - return self.defaults[opt_name] - return Settings.__getitem__(self, opt_name) - - def __str__(self): - return "" % self.settings_module - -CrawlerSettings = create_deprecated_class( - 'CrawlerSettings', CrawlerSettings, - new_class_path='scrapy.settings.Settings') - - def iter_default_settings(): """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 863684075..1dbacbea3 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -3,8 +3,7 @@ import unittest import warnings from scrapy.settings import (BaseSettings, Settings, SettingsAttribute, - CrawlerSettings, SETTINGS_PRIORITIES, - get_settings_priority) + SETTINGS_PRIORITIES, get_settings_priority) from tests import mock from . import default_settings @@ -341,35 +340,6 @@ class BaseSettingsTest(unittest.TestCase): self.assertTrue(frozencopy.frozen) self.assertIsNot(frozencopy, self.settings) - def test_deprecated_attribute_overrides(self): - self.settings.set('BAR', 'fuz', priority='cmdline') - with warnings.catch_warnings(record=True) as w: - self.settings.overrides['BAR'] = 'foo' - self.assertIn("Settings.overrides", str(w[0].message)) - self.assertEqual(self.settings.get('BAR'), 'foo') - self.assertEqual(self.settings.overrides.get('BAR'), 'foo') - self.assertIn('BAR', self.settings.overrides) - - self.settings.overrides.update(BAR='bus') - self.assertEqual(self.settings.get('BAR'), 'bus') - self.assertEqual(self.settings.overrides.get('BAR'), 'bus') - - self.settings.overrides.setdefault('BAR', 'fez') - self.assertEqual(self.settings.get('BAR'), 'bus') - - self.settings.overrides.setdefault('FOO', 'fez') - self.assertEqual(self.settings.get('FOO'), 'fez') - self.assertEqual(self.settings.overrides.get('FOO'), 'fez') - - def test_deprecated_attribute_defaults(self): - self.settings.set('BAR', 'fuz', priority='default') - with warnings.catch_warnings(record=True) as w: - self.settings.defaults['BAR'] = 'foo' - self.assertIn("Settings.defaults", str(w[0].message)) - self.assertEqual(self.settings.get('BAR'), 'foo') - self.assertEqual(self.settings.defaults.get('BAR'), 'foo') - self.assertIn('BAR', self.settings.defaults) - class SettingsTest(unittest.TestCase): @@ -422,33 +392,5 @@ class SettingsTest(unittest.TestCase): self.assertEqual(mydict['key'], 'val') -class CrawlerSettingsTest(unittest.TestCase): - - def test_deprecated_crawlersettings(self): - def _get_settings(settings_dict=None): - settings_module = type('SettingsModuleMock', (object,), settings_dict or {}) - return CrawlerSettings(settings_module) - - with warnings.catch_warnings(record=True) as w: - settings = _get_settings() - self.assertIn("CrawlerSettings is deprecated", str(w[0].message)) - - # test_global_defaults - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180) - - # test_defaults - settings.defaults['DOWNLOAD_TIMEOUT'] = '99' - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99) - - # test_settings_module - settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'}) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3) - - # test_overrides - settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'}) - settings.overrides['DOWNLOAD_TIMEOUT'] = '15' - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15) - - if __name__ == "__main__": unittest.main() From 9428a4a3aa4c679c15eb2c606de7b49ad832ee6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 21:03:26 +0300 Subject: [PATCH 29/30] More visible telnet conch message Capture traceback when trying to import required twisted modules, print it in case telnet is enabled, and mention settings variable that can be used to supress the message. Thanks @kmike! --- scrapy/extensions/telnet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 7cc8f823a..3024ddfaa 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -6,6 +6,7 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging +import traceback from twisted.internet import protocol try: @@ -13,6 +14,7 @@ try: from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True except (ImportError, SyntaxError): + _TWISTED_CONCH_TRACEBACK = traceback.format_exc() TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,8 +42,9 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured('TelnetConsole not enabled: failed to import ' - 'required twisted modules.') + raise NotConfigured( + 'TELNETCONSOLE_ENABLED setting is True but required twisted ' + 'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK) self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From c86213317daf25aec04f3afc69c327a43033987f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 12 Jul 2018 02:10:24 +0500 Subject: [PATCH 30/30] 1.5.1 release notes --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 633e5c72f..01016e2e6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,6 +12,22 @@ Cleanups * Remove deprecated ``CrawlerSettings`` class. * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. + +Scrapy 1.5.1 (2018-07-12) +------------------------- + +This is a maintenance release with important bug fixes, but no new features: + +* ``O(N^2)`` gzip decompression issue which affected Python 3 and PyPy + is fixed (:issue:`3281`); +* skipping of TLS validation errors is improved (:issue:`3166`); +* Ctrl-C handling is fixed in Python 3.5+ (:issue:`3096`); +* testing fixes (:issue:`3092`, :issue:`3263`); +* documentation improvements (:issue:`3058`, :issue:`3059`, :issue:`3089`, + :issue:`3123`, :issue:`3127`, :issue:`3189`, :issue:`3224`, :issue:`3280`, + :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`, + :issue:`3294`). + Scrapy 1.5.0 (2017-12-29) -------------------------