RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:StasDeep/scrapy into feature/issue-3382

This commit is contained in:
Stas Glubokiy 2018-08-19 17:21:28 +03:00
parent b4b1e48343 c7654f7cb1
commit 9ab85fe788
15 changed files with 153 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
appveyor.yml Normal file
View File

@ -0,0 +1,24 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

5
docs/topics/contracts.rst
View File

@ -86,9 +86,8 @@ override three methods:
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
for request object. If ``formdata`` is in ``args``, then
:class:`~scrapy.http.FormRequest` object is created,
otherwise :class:`~scrapy.http.Request` is used.
for request object. :class:`~scrapy.http.Request` is used
if ``request_cls`` is not set on ``args``.
Must return the same or a modified version of it.

4
scrapy/commands/startproject.py
View File

@ -107,8 +107,8 @@ class Command(ScrapyCommand):
string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name,
ProjectName=string_camelcase(project_name))
print("New Scrapy project %r, using template directory %r, created in:" % \
(project_name, self.templates_dir))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))
print("You can start your first spider with:")
print(" cd %s" % project_dir)

20
scrapy/contracts/__init__.py
View File

@ -1,6 +1,7 @@
import sys
import re
from functools import wraps
from inspect import getmembers
from unittest import TestCase
from scrapy import FormRequest
@ -18,7 +19,7 @@ class ContractsManager(object):
def tested_methods_from_spidercls(self, spidercls):
methods = []
for key, value in vars(spidercls).items():
for key, value in getmembers(spidercls):
if (callable(value) and value.__doc__ and
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
methods.append(key)
@ -49,20 +50,19 @@ class ContractsManager(object):
def from_method(self, method, results):
contracts = self.extract_contracts(method)
if contracts:
# calculate request args
args, kwargs = get_spec(Request.__init__)
kwargs['callback'] = method
# prepare request arguments
kwargs = {'callback': method}
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
# create and prepare request
request_cls = kwargs.pop('request_cls', Request)
args, _ = get_spec(request_cls.__init__)
args.remove('self')
# check if all positional arguments are defined in kwargs
if set(args).issubset(set(kwargs)):
if 'formdata' in kwargs:
kwargs['method'] = 'POST'
request = FormRequest(**kwargs)
else:
request = Request(**kwargs)
request = request_cls(**kwargs)
# execute pre and post hooks in order
for contract in reversed(contracts):

2
tests/mockserver.py
View File

@ -209,7 +209,7 @@ class MockServer():
time.sleep(0.2)
def url(self, path, is_secure=False):
host = self.http_address
host = self.http_address.replace('0.0.0.0', '127.0.0.1')
if is_secure:
host = self.https_address
return host + path

3
tests/requirements-py3.txt
View File

@ -3,9 +3,10 @@ pytest-twisted
pytest-cov==2.5.1
testfixtures
jmespath
leveldb
leveldb; sys_platform != "win32"
botocore
# optional for shell wrapper tests
bpython
ipython
brotlipy
pywin32; sys_platform == "win32"

3
tests/test_cmdline/__init__.py
View File

@ -52,7 +52,8 @@ class CmdlineTest(unittest.TestCase):
stats.print_stats()
out.seek(0)
stats = out.read()
self.assertIn('scrapy/commands/version.py', stats)
self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),
stats)
self.assertIn('tottime', stats)
finally:
shutil.rmtree(path)

38
tests/test_command_parse.py
View File

@ -1,3 +1,4 @@
import os
from os.path import join, abspath
from twisted.trial import unittest
from twisted.internet import defer
@ -7,6 +8,11 @@ from scrapy.utils.python import to_native_str
from tests.test_commands import CommandTest
def _textmode(bstr):
"""Normalize input the same as writing to a file
and reading from it in text mode"""
return to_native_str(bstr).replace(os.linesep, '\n')
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
command = 'parse'
@ -97,7 +103,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'-a', 'test_arg=1',
'-c', 'parse',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
def test_request_with_meta(self):
@ -106,13 +112,13 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'--meta', raw_json_string,
'-c', 'parse_request_with_meta',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-m', raw_json_string,
'-c', 'parse_request_with_meta',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
@ -120,7 +126,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-c', 'parse_request_without_meta',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
@ -129,29 +135,29 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'--pipelines',
'-c', 'parse',
self.url('/html')])
self.assertIn("INFO: It Works!", to_native_str(stderr))
self.assertIn("INFO: It Works!", _textmode(stderr))
@defer.inlineCallbacks
def test_parse_items(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
@defer.inlineCallbacks
def test_parse_items_no_callback_passed(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
@defer.inlineCallbacks
def test_wrong_callback_passed(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find callback""", to_native_str(stderr))
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find callback""", _textmode(stderr))
@defer.inlineCallbacks
def test_crawlspider_matching_rule_callback_set(self):
@ -159,7 +165,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
@defer.inlineCallbacks
def test_crawlspider_matching_rule_default_callback(self):
@ -167,7 +173,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
)
self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
@defer.inlineCallbacks
def test_spider_with_no_rules_attribute(self):
@ -175,15 +181,15 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-r', self.url('/html')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr))
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
@defer.inlineCallbacks
def test_crawlspider_missing_callback(self):
status, out, stderr = yield self.execute(
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
@defer.inlineCallbacks
def test_crawlspider_no_matching_rule(self):
@ -191,5 +197,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr))
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))

51
tests/test_commands.py
View File

@ -3,18 +3,17 @@ import os
import sys
import subprocess
import tempfile
from time import sleep
from os.path import exists, join, abspath
from shutil import rmtree, copytree
from tempfile import mkdtemp
from contextlib import contextmanager
from threading import Timer
from twisted.trial import unittest
from twisted.internet import defer
import scrapy
from scrapy.utils.python import to_native_str
from scrapy.utils.python import retry_on_eintr
from scrapy.utils.test import get_testenv
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
@ -46,16 +45,18 @@ class ProjectTest(unittest.TestCase):
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
**popen_kwargs)
waited = 0
interval = 0.2
while p.poll() is None:
sleep(interval)
waited += interval
if waited > 15:
p.kill()
assert False, 'Command took too much time to complete'
def kill_proc():
p.kill()
assert False, 'Command took too much time to complete'
return p
timer = Timer(15, kill_proc)
try:
timer.start()
stdout, stderr = p.communicate()
finally:
timer.cancel()
return p, to_native_str(stdout), to_native_str(stderr)
class StartprojectTest(ProjectTest):
@ -111,9 +112,9 @@ class StartprojectTemplatesTest(ProjectTest):
assert exists(join(self.tmpl_proj, 'root_template'))
args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
p = self.proc('startproject', self.project_name, *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
p, out, err = self.proc('startproject', self.project_name, *args)
self.assertIn("New Scrapy project '%s', using template directory"
% self.project_name, out)
self.assertIn(self.tmpl_proj, out)
assert exists(join(self.proj_path, 'root_template'))
@ -140,12 +141,10 @@ class GenspiderCommandTest(CommandTest):
def test_template(self, tplname='crawl'):
args = ['--template=%s' % tplname] if tplname else []
spname = 'test_spider'
p = self.proc('genspider', spname, 'test.com', *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
p = self.proc('genspider', spname, 'test.com', *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn("Spider %r already exists in module" % spname, out)
def test_template_basic(self):
@ -212,8 +211,8 @@ class MySpider(scrapy.Spider):
return self.proc('runspider', fname, *args)
def get_log(self, code, name='myspider.py', args=()):
p = self.runspider(code, name=name, args=args)
return to_native_str(p.stderr.read())
p, stdout, stderr = self.runspider(code, name=name, args=args)
return stderr
def test_runspider(self):
log = self.get_log(self.debug_log_spider)
@ -223,12 +222,12 @@ class MySpider(scrapy.Spider):
self.assertIn("INFO: Spider closed (finished)", log)
def test_run_fail_spider(self):
proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
ret = proc.returncode
self.assertNotEqual(ret, 0)
def test_run_good_spider(self):
proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
ret = proc.returncode
self.assertEqual(ret, 0)
@ -279,8 +278,7 @@ class MySpider(scrapy.Spider):
self.assertIn("No spider found in file", log)
def test_runspider_file_not_found(self):
p = self.proc('runspider', 'some_non_existent_file')
log = to_native_str(p.stderr.read())
_, _, log = self.proc('runspider', 'some_non_existent_file')
self.assertIn("File not found: some_non_existent_file", log)
def test_runspider_unable_to_load(self):
@ -304,8 +302,7 @@ class BadSpider(scrapy.Spider):
class BenchCommandTest(CommandTest):
def test_run(self):
p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
log = to_native_str(p.stderr.read())
_, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
self.assertIn('INFO: Crawled', log)
self.assertNotIn('Unhandled Error', log)

11
tests/test_contracts.py
View File

@ -29,6 +29,7 @@ class CustomFormContract(Contract):
name = 'custom_form'
def adjust_request_args(self, args):
args['request_cls'] = FormRequest
args['formdata'] = {'name': 'scrapy'}
return args
@ -117,6 +118,10 @@ class TestSpider(Spider):
pass
class InheritsTestSpider(TestSpider):
name = 'inherits_demo_spider'
class ContractsManagerTest(unittest.TestCase):
contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract]
@ -224,3 +229,9 @@ class ContractsManagerTest(unittest.TestCase):
request = self.conman.from_method(spider.custom_form, self.results)
self.assertEqual(request.method, 'POST')
self.assertIsInstance(request, FormRequest)
def test_inherited_contracts(self):
spider = InheritsTestSpider()
requests = self.conman.from_spider(spider, self.results)
self.assertTrue(requests)

51
tests/test_crawler.py
View File

@ -1,10 +1,9 @@
import logging
import tempfile
import warnings
import unittest
from twisted.internet import defer
import twisted.trial.unittest
from twisted.trial import unittest
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@ -94,26 +93,29 @@ class CrawlerLoggingTestCase(unittest.TestCase):
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
with tempfile.NamedTemporaryFile() as log_file:
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file.name,
# disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
log_file = self.mktemp()
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file,
# disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
crawler = Crawler(MySpider, {})
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message')
logging.error('error message')
with open(log_file, 'rb') as fo:
logged = fo.read().decode('utf8')
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
crawler = Crawler(MySpider, {})
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message')
logging.error('error message')
logged = log_file.read().decode('utf8')
self.assertNotIn('debug message', logged)
self.assertIn('info message', logged)
self.assertIn('warning message', logged)
@ -141,9 +143,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
settings = Settings({
'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
})
with warnings.catch_warnings(record=True) as w, \
self.assertRaises(AttributeError):
CrawlerRunner(settings)
with warnings.catch_warnings(record=True) as w:
self.assertRaises(AttributeError, CrawlerRunner, settings)
self.assertEqual(len(w), 1)
self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
@ -203,7 +204,7 @@ class NoRequestsSpider(scrapy.Spider):
return []
class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase):
class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):

10
tests/test_downloader_handlers.py
View File

@ -1,7 +1,8 @@
import os
import six
import contextlib
import shutil
import tempfile
import contextlib
try:
from unittest import mock
except ImportError:
@ -913,7 +914,9 @@ class BaseFTPTestCase(unittest.TestCase):
return self._add_test_callbacks(d, _test)
def test_ftp_local_filename(self):
local_fname = b"/tmp/file.txt"
f, local_fname = tempfile.mkstemp()
local_fname = to_bytes(local_fname)
os.close(f)
meta = {"ftp_local_filename": local_fname}
meta.update(self.req_meta)
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
@ -922,7 +925,8 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(r.body, local_fname)
self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']})
self.assertEqual(r.headers, {b'Local Filename': [local_fname],
b'Size': [b'17']})
self.assertTrue(os.path.exists(local_fname))
with open(local_fname, "rb") as f:
self.assertEqual(f.read(), b"I have the power!")

18
tests/test_feedexport.py
View File

@ -2,11 +2,12 @@ from __future__ import absolute_import
import os
import csv
import json
import warnings
from io import BytesIO
import tempfile
import shutil
from six.moves.urllib.parse import urlparse
import warnings
from six.moves.urllib.parse import urljoin, urlparse
from six.moves.urllib.request import pathname2url
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@ -226,9 +227,10 @@ class FeedExportTest(unittest.TestCase):
def run_and_export(self, spider_cls, settings=None):
""" Run spider with specified settings; return exported data. """
tmpdir = tempfile.mkdtemp()
res_name = tmpdir + '/res'
res_path = os.path.join(tmpdir, 'res')
res_uri = urljoin('file:', pathname2url(res_path))
defaults = {
'FEED_URI': 'file://' + res_name,
'FEED_URI': res_uri,
'FEED_FORMAT': 'csv',
}
defaults.update(settings or {})
@ -238,11 +240,13 @@ class FeedExportTest(unittest.TestCase):
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
with open(res_name, 'rb') as f:
defer.returnValue(f.read())
with open(res_path, 'rb') as f:
content = f.read()
finally:
shutil.rmtree(tmpdir)
shutil.rmtree(tmpdir, ignore_errors=True)
defer.returnValue(content)
@defer.inlineCallbacks
def exported_data(self, items, settings):

2
tests/test_utils_iterators.py
View File

@ -7,7 +7,7 @@ from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
from scrapy.http import XmlResponse, TextResponse, Response
from tests import get_testdata
FOOBAR_NL = u"foo" + os.linesep + u"bar"
FOOBAR_NL = u"foo\nbar"
class XmliterTestCase(unittest.TestCase):

11
tests/test_utils_project.py
View File

@ -25,8 +25,12 @@ def inside_a_project():
class ProjectUtilsTest(unittest.TestCase):
def test_data_path_outside_project(self):
self.assertEqual('.scrapy/somepath', data_path('somepath'))
self.assertEqual('/absolute/path', data_path('/absolute/path'))
self.assertEqual(
os.path.join('.scrapy', 'somepath'),
data_path('somepath')
)
abspath = os.path.join(os.path.sep, 'absolute', 'path')
self.assertEqual(abspath, data_path(abspath))
def test_data_path_inside_project(self):
with inside_a_project() as proj_path:
@ -35,4 +39,5 @@ class ProjectUtilsTest(unittest.TestCase):
os.path.realpath(expected),
os.path.realpath(data_path('somepath'))
)
self.assertEqual('/absolute/path', data_path('/absolute/path'))
abspath = os.path.join(os.path.sep, 'absolute', 'path')
self.assertEqual(abspath, data_path(abspath))