RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3315 from scrapy/test-on-windows 

Enable AppVeyor CI for running test suite on Windows env
This commit is contained in:
Mikhail Korobov 2018-08-15 21:18:37 +05:00 committed by GitHub
parent 8a4e51a19b 38608bc249
commit 91f986ecf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 130 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
appveyor.yml Normal file
View File

@ -0,0 +1,24 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

4
scrapy/commands/startproject.py
View File

@ -107,8 +107,8 @@ class Command(ScrapyCommand):
string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name,
ProjectName=string_camelcase(project_name))
print("New Scrapy project %r, using template directory %r, created in:" % \
(project_name, self.templates_dir))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))
print("You can start your first spider with:")
print(" cd %s" % project_dir)

2
tests/mockserver.py
View File

@ -209,7 +209,7 @@ class MockServer():
time.sleep(0.2)
def url(self, path, is_secure=False):
host = self.http_address
host = self.http_address.replace('0.0.0.0', '127.0.0.1')
if is_secure:
host = self.https_address
return host + path

3
tests/requirements-py3.txt
View File

@ -3,9 +3,10 @@ pytest-twisted
pytest-cov==2.5.1
testfixtures
jmespath
leveldb
leveldb; sys_platform != "win32"
botocore
# optional for shell wrapper tests
bpython
ipython
brotlipy
pywin32; sys_platform == "win32"

3
tests/test_cmdline/__init__.py
View File

@ -52,7 +52,8 @@ class CmdlineTest(unittest.TestCase):
stats.print_stats()
out.seek(0)
stats = out.read()
self.assertIn('scrapy/commands/version.py', stats)
self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),
stats)
self.assertIn('tottime', stats)
finally:
shutil.rmtree(path)

38
tests/test_command_parse.py
View File

@ -1,3 +1,4 @@
import os
from os.path import join, abspath
from twisted.trial import unittest
from twisted.internet import defer
@ -7,6 +8,11 @@ from scrapy.utils.python import to_native_str
from tests.test_commands import CommandTest
def _textmode(bstr):
"""Normalize input the same as writing to a file
and reading from it in text mode"""
return to_native_str(bstr).replace(os.linesep, '\n')
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
command = 'parse'
@ -97,7 +103,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'-a', 'test_arg=1',
'-c', 'parse',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
def test_request_with_meta(self):
@ -106,13 +112,13 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'--meta', raw_json_string,
'-c', 'parse_request_with_meta',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-m', raw_json_string,
'-c', 'parse_request_with_meta',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
@ -120,7 +126,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-c', 'parse_request_without_meta',
self.url('/html')])
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@defer.inlineCallbacks
@ -129,29 +135,29 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'--pipelines',
'-c', 'parse',
self.url('/html')])
self.assertIn("INFO: It Works!", to_native_str(stderr))
self.assertIn("INFO: It Works!", _textmode(stderr))
@defer.inlineCallbacks
def test_parse_items(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
@defer.inlineCallbacks
def test_parse_items_no_callback_passed(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
@defer.inlineCallbacks
def test_wrong_callback_passed(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find callback""", to_native_str(stderr))
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find callback""", _textmode(stderr))
@defer.inlineCallbacks
def test_crawlspider_matching_rule_callback_set(self):
@ -159,7 +165,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
@defer.inlineCallbacks
def test_crawlspider_matching_rule_default_callback(self):
@ -167,7 +173,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
)
self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
@defer.inlineCallbacks
def test_spider_with_no_rules_attribute(self):
@ -175,15 +181,15 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-r', self.url('/html')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr))
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
@defer.inlineCallbacks
def test_crawlspider_missing_callback(self):
status, out, stderr = yield self.execute(
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
@defer.inlineCallbacks
def test_crawlspider_no_matching_rule(self):
@ -191,5 +197,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
)
self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr))
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))

51
tests/test_commands.py
View File

@ -3,18 +3,17 @@ import os
import sys
import subprocess
import tempfile
from time import sleep
from os.path import exists, join, abspath
from shutil import rmtree, copytree
from tempfile import mkdtemp
from contextlib import contextmanager
from threading import Timer
from twisted.trial import unittest
from twisted.internet import defer
import scrapy
from scrapy.utils.python import to_native_str
from scrapy.utils.python import retry_on_eintr
from scrapy.utils.test import get_testenv
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
@ -46,16 +45,18 @@ class ProjectTest(unittest.TestCase):
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
**popen_kwargs)
waited = 0
interval = 0.2
while p.poll() is None:
sleep(interval)
waited += interval
if waited > 15:
p.kill()
assert False, 'Command took too much time to complete'
def kill_proc():
p.kill()
assert False, 'Command took too much time to complete'
return p
timer = Timer(15, kill_proc)
try:
timer.start()
stdout, stderr = p.communicate()
finally:
timer.cancel()
return p, to_native_str(stdout), to_native_str(stderr)
class StartprojectTest(ProjectTest):
@ -111,9 +112,9 @@ class StartprojectTemplatesTest(ProjectTest):
assert exists(join(self.tmpl_proj, 'root_template'))
args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
p = self.proc('startproject', self.project_name, *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
p, out, err = self.proc('startproject', self.project_name, *args)
self.assertIn("New Scrapy project '%s', using template directory"
% self.project_name, out)
self.assertIn(self.tmpl_proj, out)
assert exists(join(self.proj_path, 'root_template'))
@ -140,12 +141,10 @@ class GenspiderCommandTest(CommandTest):
def test_template(self, tplname='crawl'):
args = ['--template=%s' % tplname] if tplname else []
spname = 'test_spider'
p = self.proc('genspider', spname, 'test.com', *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
p = self.proc('genspider', spname, 'test.com', *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn("Spider %r already exists in module" % spname, out)
def test_template_basic(self):
@ -212,8 +211,8 @@ class MySpider(scrapy.Spider):
return self.proc('runspider', fname, *args)
def get_log(self, code, name='myspider.py', args=()):
p = self.runspider(code, name=name, args=args)
return to_native_str(p.stderr.read())
p, stdout, stderr = self.runspider(code, name=name, args=args)
return stderr
def test_runspider(self):
log = self.get_log(self.debug_log_spider)
@ -223,12 +222,12 @@ class MySpider(scrapy.Spider):
self.assertIn("INFO: Spider closed (finished)", log)
def test_run_fail_spider(self):
proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
ret = proc.returncode
self.assertNotEqual(ret, 0)
def test_run_good_spider(self):
proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
ret = proc.returncode
self.assertEqual(ret, 0)
@ -279,8 +278,7 @@ class MySpider(scrapy.Spider):
self.assertIn("No spider found in file", log)
def test_runspider_file_not_found(self):
p = self.proc('runspider', 'some_non_existent_file')
log = to_native_str(p.stderr.read())
_, _, log = self.proc('runspider', 'some_non_existent_file')
self.assertIn("File not found: some_non_existent_file", log)
def test_runspider_unable_to_load(self):
@ -304,8 +302,7 @@ class BadSpider(scrapy.Spider):
class BenchCommandTest(CommandTest):
def test_run(self):
p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
log = to_native_str(p.stderr.read())
_, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
self.assertIn('INFO: Crawled', log)
self.assertNotIn('Unhandled Error', log)

51
tests/test_crawler.py
View File

@ -1,10 +1,9 @@
import logging
import tempfile
import warnings
import unittest
from twisted.internet import defer
import twisted.trial.unittest
from twisted.trial import unittest
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@ -94,26 +93,29 @@ class CrawlerLoggingTestCase(unittest.TestCase):
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
with tempfile.NamedTemporaryFile() as log_file:
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file.name,
# disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
log_file = self.mktemp()
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file,
# disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
crawler = Crawler(MySpider, {})
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message')
logging.error('error message')
with open(log_file, 'rb') as fo:
logged = fo.read().decode('utf8')
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
crawler = Crawler(MySpider, {})
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message')
logging.error('error message')
logged = log_file.read().decode('utf8')
self.assertNotIn('debug message', logged)
self.assertIn('info message', logged)
self.assertIn('warning message', logged)
@ -141,9 +143,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
settings = Settings({
'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
})
with warnings.catch_warnings(record=True) as w, \
self.assertRaises(AttributeError):
CrawlerRunner(settings)
with warnings.catch_warnings(record=True) as w:
self.assertRaises(AttributeError, CrawlerRunner, settings)
self.assertEqual(len(w), 1)
self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
@ -203,7 +204,7 @@ class NoRequestsSpider(scrapy.Spider):
return []
class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase):
class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):

10
tests/test_downloader_handlers.py
View File

@ -1,7 +1,8 @@
import os
import six
import contextlib
import shutil
import tempfile
import contextlib
try:
from unittest import mock
except ImportError:
@ -913,7 +914,9 @@ class BaseFTPTestCase(unittest.TestCase):
return self._add_test_callbacks(d, _test)
def test_ftp_local_filename(self):
local_fname = b"/tmp/file.txt"
f, local_fname = tempfile.mkstemp()
local_fname = to_bytes(local_fname)
os.close(f)
meta = {"ftp_local_filename": local_fname}
meta.update(self.req_meta)
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
@ -922,7 +925,8 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(r.body, local_fname)
self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']})
self.assertEqual(r.headers, {b'Local Filename': [local_fname],
b'Size': [b'17']})
self.assertTrue(os.path.exists(local_fname))
with open(local_fname, "rb") as f:
self.assertEqual(f.read(), b"I have the power!")

18
tests/test_feedexport.py
View File

@ -2,11 +2,12 @@ from __future__ import absolute_import
import os
import csv
import json
import warnings
from io import BytesIO
import tempfile
import shutil
from six.moves.urllib.parse import urlparse
import warnings
from six.moves.urllib.parse import urljoin, urlparse
from six.moves.urllib.request import pathname2url
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@ -226,9 +227,10 @@ class FeedExportTest(unittest.TestCase):
def run_and_export(self, spider_cls, settings=None):
""" Run spider with specified settings; return exported data. """
tmpdir = tempfile.mkdtemp()
res_name = tmpdir + '/res'
res_path = os.path.join(tmpdir, 'res')
res_uri = urljoin('file:', pathname2url(res_path))
defaults = {
'FEED_URI': 'file://' + res_name,
'FEED_URI': res_uri,
'FEED_FORMAT': 'csv',
}
defaults.update(settings or {})
@ -238,11 +240,13 @@ class FeedExportTest(unittest.TestCase):
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
with open(res_name, 'rb') as f:
defer.returnValue(f.read())
with open(res_path, 'rb') as f:
content = f.read()
finally:
shutil.rmtree(tmpdir)
shutil.rmtree(tmpdir, ignore_errors=True)
defer.returnValue(content)
@defer.inlineCallbacks
def exported_data(self, items, settings):

2
tests/test_utils_iterators.py
View File

@ -7,7 +7,7 @@ from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
from scrapy.http import XmlResponse, TextResponse, Response
from tests import get_testdata
FOOBAR_NL = u"foo" + os.linesep + u"bar"
FOOBAR_NL = u"foo\nbar"
class XmliterTestCase(unittest.TestCase):

11
tests/test_utils_project.py
View File

@ -25,8 +25,12 @@ def inside_a_project():
class ProjectUtilsTest(unittest.TestCase):
def test_data_path_outside_project(self):
self.assertEqual('.scrapy/somepath', data_path('somepath'))
self.assertEqual('/absolute/path', data_path('/absolute/path'))
self.assertEqual(
os.path.join('.scrapy', 'somepath'),
data_path('somepath')
)
abspath = os.path.join(os.path.sep, 'absolute', 'path')
self.assertEqual(abspath, data_path(abspath))
def test_data_path_inside_project(self):
with inside_a_project() as proj_path:
@ -35,4 +39,5 @@ class ProjectUtilsTest(unittest.TestCase):
os.path.realpath(expected),
os.path.realpath(data_path('somepath'))
)
self.assertEqual('/absolute/path', data_path('/absolute/path'))
abspath = os.path.join(os.path.sep, 'absolute', 'path')
self.assertEqual(abspath, data_path(abspath))