From 73d78ec99fc622fe9a552c78193c0b4bf652533b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 17:59:20 +0100 Subject: [PATCH 01/42] Add Code of Conduct Version 1.3.0 from http://contributor-covenant.org/ Closes #1645 --- CODE_OF_CONDUCT.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 6 ++++++ 2 files changed, 56 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..95b4a7e3c --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,50 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer at opensource@scrapinghub.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/README.rst b/README.rst index 6cbed75ee..8a7d2c71d 100644 --- a/README.rst +++ b/README.rst @@ -73,6 +73,12 @@ See http://scrapy.org/community/ Contributing ============ +Please note that this project is released with a Contributor Code of Conduct +(see CODE_OF_CONDUCT.md). + +By participating in this project you agree to abide by its terms. +Please report unacceptable behavior to opensource@scrapinghub.com. + See http://doc.scrapy.org/en/master/contributing.html Companies using Scrapy From a76ecd4ef0bd7fa2dbe2e02d5b5721b39ead18c0 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 8 Oct 2015 22:18:14 -0300 Subject: [PATCH 02/42] remove test_exporters from py3 ignores --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index f189a4c86..570287d9d 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,4 +1,3 @@ -tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_mail.py tests/test_pipeline_files.py From b6ef1f19fd768243407206a882d689764624b42c Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 9 Oct 2015 00:19:05 -0300 Subject: [PATCH 03/42] make BaseItemExporter export unicode, pushed down previous behavior for classes that need it --- scrapy/exporters.py | 15 +++++++------ tests/test_exporters.py | 48 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 7e1d01a0a..6f679480d 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -38,7 +38,7 @@ class BaseItemExporter(object): raise NotImplementedError def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._to_str_if_unicode) + serializer = field.get('serializer', lambda x: x) return serializer(value) def start_exporting(self): @@ -47,9 +47,6 @@ class BaseItemExporter(object): def finish_exporting(self): pass - def _to_str_if_unicode(self, value): - return value.encode(self.encoding) if isinstance(value, unicode) else value - def _get_serialized_fields(self, item, default_value=None, include_empty=None): """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -89,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter): self.file.write(self.encoder.encode(itemdict) + '\n') -class JsonItemExporter(JsonLinesItemExporter): +class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) @@ -170,13 +167,17 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued + def serialize_field(self, field, name, value): + serializer = field.get('serializer', self._to_str_if_unicode) + return serializer(value) + def _to_str_if_unicode(self, value): if isinstance(value, (list, tuple)): try: value = self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return super(CsvItemExporter, self)._to_str_if_unicode(value) + return value.encode(self.encoding) if isinstance(value, unicode) else value def export_item(self, item): if self._headers_not_written: @@ -251,7 +252,7 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_dict(value)) if hasattr(value, '__iter__'): return [self._serialize_value(v) for v in value] - return self._to_str_if_unicode(value) + return value.encode(self.encoding) if isinstance(value, unicode) else value def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b24633959..c84fb978a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -23,7 +23,7 @@ class TestItem(Item): class BaseItemExporterTest(unittest.TestCase): def setUp(self): - self.i = TestItem(name=u'John\xa3', age='22') + self.i = TestItem(name=u'John\xa3', age=u'22') self.output = BytesIO() self.ie = self._get_exporter() @@ -55,6 +55,42 @@ class BaseItemExporterTest(unittest.TestCase): self.assertItemExportWorks(dict(self.i)) def test_serialize_field(self): + res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) + self.assertEqual(res, u'John\xa3') + + res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) + self.assertEqual(res, u'22') + + def test_fields_to_export(self): + ie = self._get_exporter(fields_to_export=['name']) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + + def test_field_custom_serializer(self): + def custom_serializer(value): + return str(int(value) + 2) + + class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + i = CustomFieldItem(name=u'John\xa3', age=u'22') + + ie = self._get_exporter() + self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + + +class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): + """Class introduced just to keep old behavior of BaseItemExporterTest for the + test cases that inherit from it while we make changes to exporters one by + one -- a needed refactoring trick because the test cases are quite coupled. + + When we're done with the changes, we'll have ditched this class. + """ + def test_serialize_field(self): + if self.ie.__class__ is BaseItemExporter: + return + res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) self.assertEqual(res, 'John\xc2\xa3') @@ -62,6 +98,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(res, '22') def test_fields_to_export(self): + if self.ie.__class__ is BaseItemExporter: + return + ie = self._get_exporter(fields_to_export=['name']) self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) @@ -71,6 +110,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, 'John\xa3') def test_field_custom_serializer(self): + if self.ie.__class__ is BaseItemExporter: + return + def custom_serializer(value): return str(int(value) + 2) @@ -85,7 +127,7 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class PythonItemExporterTest(BaseItemExporterTest): +class PythonItemExporterTest(MidRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(**kwargs) @@ -152,7 +194,7 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -class CsvItemExporterTest(BaseItemExporterTest): +class CsvItemExporterTest(MidRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) From c76190d491fca9f35b6758bdc06c34d77f5d9be9 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 18:24:06 -0200 Subject: [PATCH 04/42] PY3: ported json(lines), xml exporters --- scrapy/exporters.py | 39 +++++++++++++++------- tests/test_exporters.py | 72 +++++++++++++++++++++-------------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 6f679480d..4138f6192 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,7 +11,10 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder +from scrapy.utils.python import to_bytes, to_unicode from scrapy.item import BaseItem +import warnings + __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', @@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict) + '\n') + self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n')) class JsonItemExporter(BaseItemExporter): @@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter): self.first_item = True def start_exporting(self): - self.file.write("[") + self.file.write(b"[") def finish_exporting(self): - self.file.write("]") + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(',\n') + self.file.write(b',\n') itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict)) + self.file.write(to_bytes(self.encoder.encode(itemdict))) class XmlItemExporter(BaseItemExporter): @@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) - elif hasattr(serialized_value, '__iter__'): - for value in serialized_value: + elif (hasattr(serialized_value, '__iter__') + and not isinstance(serialized_value, six.string_types)): + for value in serialized_value: self._export_xml_field('value', value) else: self._xg_characters(serialized_value) @@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter): # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. if sys.version_info[:3] >= (2, 7, 4): def _xg_characters(self, serialized_value): - if not isinstance(serialized_value, unicode): + if not isinstance(serialized_value, six.text_type): serialized_value = serialized_value.decode(self.encoding) return self.xg.characters(serialized_value) else: @@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter): value = self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return value.encode(self.encoding) if isinstance(value, unicode) else value + return value.encode(self.encoding) if isinstance(value, six.text_type) else value def export_item(self, item): if self._headers_not_written: @@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(pprint.pformat(itemdict) + '\n') + self.file.write(to_bytes(pprint.pformat(itemdict) + '\n')) class PythonItemExporter(BaseItemExporter): @@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter): json, msgpack, binc, etc) can be used on top of it. Its main goal is to seamless support what BaseItemExporter does plus nested items. """ + def _configure(self, options, dont_fail=False): + self.binary = options.pop('binary', True) + super(PythonItemExporter, self)._configure(options, dont_fail) + if self.binary: + warnings.warn( + "PythonItemExporter will drop support for binary export in the future", + PendingDeprecationWarning) def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) @@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) - if hasattr(value, '__iter__'): + if hasattr(value, '__iter__') \ + and not isinstance(value, six.string_types): return [self._serialize_value(v) for v in value] - return value.encode(self.encoding) if isinstance(value, unicode) else value + if self.binary: + return to_bytes(value, encoding=self.encoding) + else: + return to_unicode(value, encoding=self.encoding) def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c84fb978a..05374e617 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,6 +3,7 @@ import re import json import unittest from io import BytesIO +import six from six.moves import cPickle as pickle import lxml.etree @@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): +class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest): """Class introduced just to keep old behavior of BaseItemExporterTest for the test cases that inherit from it while we make changes to exporters one by one -- a needed refactoring trick because the test cases are quite coupled. @@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class PythonItemExporterTest(MidRefactoringBaseItemExporterTest): +class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(**kwargs) + return PythonItemExporter(binary=False, **kwargs) def test_nested_item(self): i1 = TestItem(name=u'Joseph', age='22') @@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -class CsvItemExporterTest(MidRefactoringBaseItemExporterTest): +@unittest.skipUnless(six.PY2, "TODO") +class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = '\n22John\xc2\xa3' + expected_value = u'\n22John\xa3' self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - '\nJohn\xc2\xa3Doe' + u'\nJohn\xa3Doe' ) def test_nested_item(self): @@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=i2) self.assertExportResult(i3, - '\n' - '' - '' - '' - '' - '22' - 'foo\xc2\xa3hoo' - '' - 'bar' - '' - 'buz' - '' - '' + u'\n' + u'' + u'' + u'' + u'' + u'22' + u'foo\xa3hoo' + u'' + u'bar' + u'' + u'buz' + u'' + u'' ) def test_nested_list_item(self): @@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=[i1, i2]) self.assertExportResult(i3, - '\n' - '' - '' - '' - 'foo' - 'barspam' - '' - 'buz' - '' - '' + u'\n' + u'' + u'' + u'' + u'foo' + u'barspam' + u'' + u'buz' + u'' + u'' ) @@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, dict(self.i)) def test_nested_item(self): @@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) def test_extra_keywords(self): @@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): return JsonItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, [dict(self.i)]) def assertTwoItemsExported(self, item): @@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, [dict(item), dict(item)]) def test_two_items(self): @@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} self.assertEqual(exported, [expected]) @@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) From fed7c8b4fca3bb2722eebca97b298b0316ebfbc2 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 18:39:59 -0200 Subject: [PATCH 05/42] fix: use is_listlike --- scrapy/exporters.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 4138f6192..ad14f38b3 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,7 +11,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem import warnings @@ -139,8 +139,7 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) - elif (hasattr(serialized_value, '__iter__') - and not isinstance(serialized_value, six.string_types)): + elif is_listlike(serialized_value): for value in serialized_value: self._export_xml_field('value', value) else: @@ -261,8 +260,7 @@ class PythonItemExporter(BaseItemExporter): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) - if hasattr(value, '__iter__') \ - and not isinstance(value, six.string_types): + if is_listlike(value): return [self._serialize_value(v) for v in value] if self.binary: return to_bytes(value, encoding=self.encoding) From 9f35c286431584e07f9a53b1b2e7b7822a18f7e1 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 18:43:36 -0200 Subject: [PATCH 06/42] fix indentation --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ad14f38b3..c029ac473 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -140,7 +140,7 @@ class XmlItemExporter(BaseItemExporter): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) elif is_listlike(serialized_value): - for value in serialized_value: + for value in serialized_value: self._export_xml_field('value', value) else: self._xg_characters(serialized_value) From b746d85f4ca11f7f0149d06f7a4b58501ad3ee23 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:12:43 -0200 Subject: [PATCH 07/42] PY3 port csv exporter --- scrapy/exporters.py | 17 ++++++----- tests/test_exporters.py | 67 ++++++----------------------------------- 2 files changed, 20 insertions(+), 64 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c029ac473..8d7ffbc71 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats. """ import csv +import io import sys import pprint import marshal @@ -11,7 +12,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, is_listlike +from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike from scrapy.item import BaseItem import warnings @@ -166,21 +167,22 @@ class CsvItemExporter(BaseItemExporter): def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): self._configure(kwargs, dont_fail=True) self.include_headers_line = include_headers_line + file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True) self.csv_writer = csv.writer(file, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._to_str_if_unicode) + serializer = field.get('serializer', self._join_if_needed) return serializer(value) - def _to_str_if_unicode(self, value): + def _join_if_needed(self, value): if isinstance(value, (list, tuple)): try: - value = self._join_multivalued.join(value) + return self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return value.encode(self.encoding) if isinstance(value, six.text_type) else value + return value def export_item(self, item): if self._headers_not_written: @@ -189,7 +191,7 @@ class CsvItemExporter(BaseItemExporter): fields = self._get_serialized_fields(item, default_value='', include_empty=True) - values = [x[1] for x in fields] + values = [to_native_str(x) for _, x in fields] self.csv_writer.writerow(values) def _write_headers_and_set_fields_to_export(self, item): @@ -201,7 +203,8 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - self.csv_writer.writerow(self.fields_to_export) + row = [to_native_str(s) for s in self.fields_to_export] + self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 05374e617..39e996062 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,7 +3,6 @@ import re import json import unittest from io import BytesIO -import six from six.moves import cPickle as pickle import lxml.etree @@ -81,53 +80,6 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest): - """Class introduced just to keep old behavior of BaseItemExporterTest for the - test cases that inherit from it while we make changes to exporters one by - one -- a needed refactoring trick because the test cases are quite coupled. - - When we're done with the changes, we'll have ditched this class. - """ - def test_serialize_field(self): - if self.ie.__class__ is BaseItemExporter: - return - - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) - self.assertEqual(res, 'John\xc2\xa3') - - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) - self.assertEqual(res, '22') - - def test_fields_to_export(self): - if self.ie.__class__ is BaseItemExporter: - return - - ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) - - ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') - name = list(ie._get_serialized_fields(self.i))[0][1] - assert isinstance(name, str) - self.assertEqual(name, 'John\xa3') - - def test_field_custom_serializer(self): - if self.ie.__class__ is BaseItemExporter: - return - - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age='22') - - ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') - - class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(binary=False, **kwargs) @@ -195,19 +147,19 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -@unittest.skipUnless(six.PY2, "TODO") -class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): - +class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): + first = to_unicode(first) + second = to_unicode(second) csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line)) for line in csv.splitlines(True)] return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -221,13 +173,13 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): self.assertExportResult( item=self.i, fields_to_export=self.i.fields.keys(), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( item=dict(self.i), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): @@ -235,7 +187,7 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): self.assertExportResult( item=item, fields_to_export=['age'], - expected='age\r\n22\r\n', + expected=b'age\r\n22\r\n', ) def test_header_export_two_items(self): @@ -246,14 +198,15 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() - self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), + b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): for item in [self.i, dict(self.i)]: self.assertExportResult( item=item, include_headers_line=False, - expected='22,John\xc2\xa3\r\n', + expected=b'22,John\xc2\xa3\r\n', ) def test_join_multivalue(self): From 2514973242e35831fd90493b3db17227c3c0195e Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:22:12 -0200 Subject: [PATCH 08/42] re-enable skipped feed export tests --- tests/test_feedexport.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d6c96ca74..8e1cadc74 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage ) from scrapy.utils.test import assert_aws_environ +from scrapy.utils.python import to_native_str class FileFeedStorageTest(unittest.TestCase): @@ -120,8 +121,6 @@ class StdoutFeedStorageTest(unittest.TestCase): class FeedExportTest(unittest.TestCase): - skip = not six.PY2 - class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -170,7 +169,7 @@ class FeedExportTest(unittest.TestCase): settings.update({'FEED_FORMAT': 'csv'}) data = yield self.exported_data(items, settings) - reader = csv.DictReader(data.splitlines()) + reader = csv.DictReader(to_native_str(data).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -184,7 +183,7 @@ class FeedExportTest(unittest.TestCase): settings = settings or {} settings.update({'FEED_FORMAT': 'jl'}) data = yield self.exported_data(items, settings) - parsed = [json.loads(line) for line in data.splitlines()] + parsed = [json.loads(to_native_str(line)) for line in data.splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) From e938752973b4fc53e0fa0c0bc68a431613b987e4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:51:59 -0200 Subject: [PATCH 09/42] add test for PythonItemExporter binary mode --- scrapy/exporters.py | 6 +++++- tests/test_exporters.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 8d7ffbc71..118df34a7 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -272,7 +272,11 @@ class PythonItemExporter(BaseItemExporter): def _serialize_dict(self, value): for key, val in six.iteritems(value): + key = to_bytes(key) if self.binary else key yield key, self._serialize_value(val) def export_item(self, item): - return dict(self._get_serialized_fields(item)) + result = dict(self._get_serialized_fields(item)) + if self.binary: + result = dict(self._serialize_dict(result)) + return result diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 39e996062..9e57745dc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import re import json import unittest +import warnings from io import BytesIO from six.moves import cPickle as pickle @@ -115,6 +116,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) + def test_export_binary(self): + exporter = PythonItemExporter(binary=True) + value = TestItem(name=u'John\xa3', age=u'22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) + class PprintItemExporterTest(BaseItemExporterTest): From d0955fd08320f8402303bb4424e7dfab384068f4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 22 Jan 2016 10:07:55 -0200 Subject: [PATCH 10/42] add back test for latin-1 encoding --- tests/test_exporters.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 9e57745dc..070624830 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -2,11 +2,11 @@ from __future__ import absolute_import import re import json import unittest -import warnings from io import BytesIO from six.moves import cPickle as pickle import lxml.etree +import six from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -66,6 +66,11 @@ class BaseItemExporterTest(unittest.TestCase): ie = self._get_exporter(fields_to_export=['name']) self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') + _, name = list(ie._get_serialized_fields(self.i))[0] + assert isinstance(name, six.text_type) + self.assertEqual(name, u'John\xa3') + def test_field_custom_serializer(self): def custom_serializer(value): return str(int(value) + 2) From c75f1fe46a8a2a3c471eeef2c023754ee5e6c2f1 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 23 Jan 2016 16:09:57 -0200 Subject: [PATCH 11/42] restore bytes instead of text, for easier reviewing --- tests/test_exporters.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 070624830..00352f61e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -261,13 +261,13 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = u'\n22John\xa3' + expected_value = b'\n22John\xc2\xa3' self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - u'\nJohn\xa3Doe' + b'\nJohn\xc2\xa3Doe' ) def test_nested_item(self): @@ -276,19 +276,19 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=i2) self.assertExportResult(i3, - u'\n' - u'' - u'' - u'' - u'' - u'22' - u'foo\xa3hoo' - u'' - u'bar' - u'' - u'buz' - u'' - u'' + b'\n' + b'' + b'' + b'' + b'' + b'22' + b'foo\xc2\xa3hoo' + b'' + b'bar' + b'' + b'buz' + b'' + b'' ) def test_nested_list_item(self): @@ -297,16 +297,16 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=[i1, i2]) self.assertExportResult(i3, - u'\n' - u'' - u'' - u'' - u'foo' - u'barspam' - u'' - u'buz' - u'' - u'' + b'\n' + b'' + b'' + b'' + b'foo' + b'barspam' + b'' + b'buz' + b'' + b'' ) From 935b1da8c2d1d801f4d7d9ce8f498c1d7a527644 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 23 Jan 2016 16:13:42 -0200 Subject: [PATCH 12/42] uses ScrapyDeprecationWarning instead of silenced PendingDeprecationWarning --- scrapy/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 118df34a7..fa6663ed4 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -14,6 +14,7 @@ from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike from scrapy.item import BaseItem +from scrapy.exceptions import ScrapyDeprecationWarning import warnings @@ -252,7 +253,7 @@ class PythonItemExporter(BaseItemExporter): if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", - PendingDeprecationWarning) + ScrapyDeprecationWarning) def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) From 9fbe6f3e814578f90f206031ae99a344b842e400 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 23 Jan 2016 17:17:40 -0200 Subject: [PATCH 13/42] added feedexport test for xml output --- tests/test_feedexport.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e1cadc74..8db9d589e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,6 @@ import json from io import BytesIO import tempfile import shutil -import six from six.moves.urllib.parse import urlparse from zope.interface.verify import verifyObject @@ -187,10 +186,22 @@ class FeedExportTest(unittest.TestCase): rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'xml'}) + data = yield self.exported_data(items, settings) + rows = [{k: v for k, v in row.items() if v} for row in rows] + import lxml.etree + root = lxml.etree.fromstring(data) + got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] + self.assertEqual(rows, got_rows) + @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) @defer.inlineCallbacks def test_export_items(self): From 9704226ee4c933e1214e029d66005f1bee2fb766 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 13:25:14 +0300 Subject: [PATCH 14/42] py3: fix test_mail - get_payload returns bytes when decode is True --- tests/test_mail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mail.py b/tests/test_mail.py index 58d44bdb3..25dd35099 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -53,8 +53,8 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(len(payload), 2) text, attach = payload - self.assertEqual(text.get_payload(decode=True), 'body') - self.assertEqual(attach.get_payload(decode=True), 'content') + self.assertEqual(text.get_payload(decode=True), b'body') + self.assertEqual(attach.get_payload(decode=True), b'content') def _catch_mail_sent(self, **kwargs): self.catched_msg = dict(**kwargs) From 860353b0c03fffb2fed44942b4389f92bd7f7c09 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 13:27:41 +0300 Subject: [PATCH 15/42] py3: unskip test_mail and scrapy/mail.py --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index f189a4c86..70d3fb905 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,5 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py @@ -22,4 +21,3 @@ scrapy/linkextractors/htmlparser.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py -scrapy/mail.py From 333d4c91fb998b4f7f8a9e184e73715c33a38ce9 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 22:52:50 +0300 Subject: [PATCH 16/42] py3: add boto to py3 test requirements, test_pipeline_files and test_pipeline_images passing now --- tests/py3-ignores.txt | 2 -- tests/requirements-py3.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 70d3fb905..212f40f23 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,7 +1,5 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_pipeline_files.py -tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 5cf786a89..73e73e651 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,6 +4,7 @@ pytest-cov testfixtures jmespath leveldb +boto # optional for shell wrapper tests bpython ipython From 097082cffa0a9f11f72a3cee4d8941b7c2566538 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 23:05:23 +0300 Subject: [PATCH 17/42] reviewed py3 compat in pipelines/images.py and pipelines/files.py --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 212f40f23..eb2cc4f5a 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -11,8 +11,6 @@ scrapy/xlib/tx/_newclient.py scrapy/xlib/tx/__init__.py scrapy/core/downloader/handlers/s3.py scrapy/core/downloader/handlers/ftp.py -scrapy/pipelines/images.py -scrapy/pipelines/files.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py From 4233b3cda4514a364511bc6f35a495cfbe50c4a6 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 23:10:03 +0300 Subject: [PATCH 18/42] py3: reviewed passing test_spidermiddleware_httperror.py --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 70d3fb905..e753b993e 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -3,7 +3,6 @@ tests/test_linkextractors_deprecated.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py -tests/test_spidermiddleware_httperror.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py From 1be90323c27bfda7588d4437a1996c5e92eb452d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 23:44:56 +0300 Subject: [PATCH 19/42] py3: properly skip s3 tests on py3 --- tests/test_downloader_handlers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1eb6192ce..56608bfc6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -437,6 +437,8 @@ class S3AnonTestCase(unittest.TestCase): import boto except ImportError: skip = 'missing boto library' + if six.PY3: + skip = 'S3 not supported on Py3' def setUp(self): self.s3reqh = S3DownloadHandler(Settings(), @@ -459,6 +461,8 @@ class S3TestCase(unittest.TestCase): import boto except ImportError: skip = 'missing boto library' + if six.PY3: + skip = 'S3 not supported on Py3' # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf From 0c44fac2b54e72202787fda9d221f62a234151d4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 24 Jan 2016 19:17:42 -0200 Subject: [PATCH 20/42] added tests for feed export marshal and pickle --- tests/test_feedexport.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8db9d589e..176fd93e3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -197,11 +197,42 @@ class FeedExportTest(unittest.TestCase): got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] self.assertEqual(rows, got_rows) + def _load_until_eof(self, data, load_func): + bytes_output = BytesIO(data) + result = [] + while True: + try: + result.append(load_func(bytes_output)) + except EOFError: + break + return result + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'pickle'}) + data = yield self.exported_data(items, settings) + expected = [{k: v for k, v in row.items() if v} for row in rows] + import pickle + result = self._load_until_eof(data, load_func=pickle.load) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'marshal'}) + data = yield self.exported_data(items, settings) + expected = [{k: v for k, v in row.items() if v} for row in rows] + import marshal + result = self._load_until_eof(data, load_func=marshal.load) + self.assertEqual(expected, result) + @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) @defer.inlineCallbacks def test_export_items(self): From 23b3336c1feb3acb603a935f11e442557d6434f4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Mon, 25 Jan 2016 22:11:04 -0200 Subject: [PATCH 21/42] add test for invalid option --- tests/test_exporters.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 00352f61e..61a0229a4 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -90,6 +90,10 @@ class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(binary=False, **kwargs) + def test_invalid_option(self): + with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"): + PythonItemExporter(invalid_option='something') + def test_nested_item(self): i1 = TestItem(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) From 2dfdde3c79a5be468302a1e825cc5ad77444a8ac Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Mon, 25 Jan 2016 22:24:35 -0200 Subject: [PATCH 22/42] fallback to repr when can't convert to native string --- scrapy/exporters.py | 11 +++++++++-- tests/test_exporters.py | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fa6663ed4..69c180ea4 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -192,9 +192,16 @@ class CsvItemExporter(BaseItemExporter): fields = self._get_serialized_fields(item, default_value='', include_empty=True) - values = [to_native_str(x) for _, x in fields] + values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def _build_row(self, values): + for s in values: + try: + yield to_native_str(s) + except TypeError: + yield to_native_str(repr(s)) + def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: @@ -204,7 +211,7 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - row = [to_native_str(s) for s in self.fields_to_export] + row = list(self._build_row(self.fields_to_export)) self.csv_writer.writerow(row) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 61a0229a4..8930545a6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -237,6 +237,13 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"Mary,Paul",John\r\n', ) + def test_join_multivalue_not_strings(self): + self.assertExportResult( + item=dict(name='John', friends=[4, 8]), + include_headers_line=False, + expected='"[4, 8]",John\r\n', + ) + class XmlItemExporterTest(BaseItemExporterTest): From d0eacfe0f90263035f98beb8fe6b5a8f182d5a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 26 Jan 2016 00:26:27 -0300 Subject: [PATCH 23/42] Add test case for marshal item exporter --- scrapy/exporters.py | 2 +- tests/test_exporters.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 69c180ea4..145468dbe 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -158,7 +158,7 @@ class XmlItemExporter(BaseItemExporter): if not isinstance(serialized_value, six.text_type): serialized_value = serialized_value.decode(self.encoding) return self.xg.characters(serialized_value) - else: + else: # pragma: no cover def _xg_characters(self, serialized_value): return self.xg.characters(serialized_value) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8930545a6..1633e1039 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,6 +1,8 @@ from __future__ import absolute_import import re import json +import marshal +import tempfile import unittest from io import BytesIO from six.moves import cPickle as pickle @@ -12,7 +14,8 @@ from scrapy.item import Item, Field from scrapy.utils.python import to_unicode from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, - XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter + XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, + PythonItemExporter, MarshalItemExporter ) @@ -163,6 +166,17 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) +class MarshalItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() + return MarshalItemExporter(self.output, **kwargs) + + def _check_output(self): + self.output.seek(0) + self._assert_expected_item(marshal.load(self.output)) + + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) From 7070dae48da02fa29aa1650af4df28561e343436 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 26 Jan 2016 13:56:16 +0500 Subject: [PATCH 24/42] deprecate unused and untested scrapy.utils.datatypes.SiteNode --- scrapy/utils/datatypes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 097bd1ac9..2b54982b8 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -137,10 +137,18 @@ class MultiValueDict(dict): for key, value in six.iteritems(kwargs): self.setlistdefault(key, []).append(value) + class SiteNode(object): """Class to represent a site node (page, image or any other file)""" def __init__(self, url): + warnings.warn( + "scrapy.utils.datatypes.SiteNode is deprecated " + "and will be removed in future releases.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + self.url = url self.itemnames = [] self.children = [] From 9c2aa50ea20d2333eb131c6aae7b9842d646e32e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 26 Jan 2016 13:58:20 +0500 Subject: [PATCH 25/42] deprecate unused and untested scrapy.utils.datatypes.MultiValueDict --- scrapy/utils/datatypes.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2b54982b8..d04b43176 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -7,11 +7,22 @@ This module must not depend on any module outside the Standard Library. import copy import six +import warnings from collections import OrderedDict +from scrapy.exceptions import ScrapyDeprecationWarning + class MultiValueDictKeyError(KeyError): - pass + def __init__(self, *args, **kwargs): + warnings.warn( + "scrapy.utils.datatypes.MultiValueDictKeyError is deprecated " + "and will be removed in future releases.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + super(MultiValueDictKeyError, self).__init__(*args, **kwargs) + class MultiValueDict(dict): """ @@ -31,6 +42,10 @@ class MultiValueDict(dict): single name-value pairs. """ def __init__(self, key_to_list_mapping=()): + warnings.warn("scrapy.utils.datatypes.MultiValueDict is deprecated " + "and will be removed in future releases.", + category=ScrapyDeprecationWarning, + stacklevel=2) dict.__init__(self, key_to_list_mapping) def __repr__(self): From 1cffa99e0d524ad6a3f989893de1693042e92f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Tue, 26 Jan 2016 12:35:40 +0200 Subject: [PATCH 26/42] tests+doc for subdomains in offsite middleware --- docs/topics/spider-middleware.rst | 3 +++ docs/topics/spiders.rst | 2 +- tests/test_spidermiddleware_offsite.py | 9 ++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 84daaaa55..ced481c71 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -273,6 +273,9 @@ OffsiteMiddleware This middleware filters out every request whose host names aren't in the spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5fd187e4e..b700ea0ef 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -76,7 +76,7 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names - specified in this list won't be followed if + specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index f88c806d7..37c3a450b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,7 +16,7 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org']) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -24,13 +24,16 @@ class TestOffsiteMiddleware(TestCase): onsite_reqs = [Request('http://scrapytest.org/1'), Request('http://scrapy.org/1'), Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True)] + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/')] offsite_reqs = [Request('http://scrapy2.org'), Request('http://offsite.tld/'), Request('http://offsite.tld/scrapytest.org'), Request('http://offsite.tld/rogue.scrapytest.org'), Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org')] + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/')] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) From 7608da8868baba981fa1d84f7691caaf1e0bd5d8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 13:01:12 +0100 Subject: [PATCH 27/42] Fix logging of enabled middlewares Wrong middlewares list was being pretty-printed (introduced in #1263) --- scrapy/middleware.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2ef5f30e2..6120488e2 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -44,9 +44,11 @@ class MiddlewareManager(object): logger.warning("Disabled %(clsname)s: %(eargs)s", {'clsname': clsname, 'eargs': e.args[0]}, extra={'crawler': crawler}) + + enabled = [x.__class__.__name__ for x in middlewares] logger.info("Enabled %(componentname)ss:\n%(enabledlist)s", {'componentname': cls.component_name, - 'enabledlist': pprint.pformat(mwlist)}, + 'enabledlist': pprint.pformat(enabled)}, extra={'crawler': crawler}) return cls(*middlewares) From 6ee8d8650a5d9041d6eeddddabcd16c0e0e8d9c9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 13:08:42 +0100 Subject: [PATCH 28/42] Disable CloseSpider extension if no CLOSPIDER_* setting set --- scrapy/extensions/closespider.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a5df5e8a7..9ccf356ec 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -9,6 +9,7 @@ from collections import defaultdict from twisted.internet import reactor from scrapy import signals +from scrapy.exceptions import NotConfigured class CloseSpider(object): @@ -23,6 +24,9 @@ class CloseSpider(object): 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), } + if not any(self.close_on.values()): + raise NotConfigured + self.counter = defaultdict(int) if self.close_on.get('errorcount'): From f30758c246ef10dc5ddb2316b747b28e109c9327 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 26 Jan 2016 17:47:46 +0500 Subject: [PATCH 29/42] Enable robots.txt handling by default for new projects. Fixes GH-1668. For backwards compatibility reasons the default value is not changed. --- docs/topics/settings.rst | 14 ++++++++++---- scrapy/templates/project/module/settings.py.tmpl | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cc070d8c0..0959a87a7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -750,8 +750,8 @@ Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` The :ref:`Memory usage extension ` -checks the current memory usage, versus the limits set by -:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, +checks the current memory usage, versus the limits set by +:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, at fixed time intervals. This sets the length of these intervals, in seconds. @@ -877,7 +877,13 @@ Default: ``False`` Scope: ``scrapy.downloadermiddlewares.robotstxt`` If enabled, Scrapy will respect robots.txt policies. For more information see -:ref:`topics-dlmw-robots` +:ref:`topics-dlmw-robots`. + +.. note:: + + While the default value is ``False`` for historical reasons, + this option is enabled by default in settings.py file generated + by ``scrapy startproject`` command. .. setting:: SCHEDULER @@ -1036,7 +1042,7 @@ TEMPLATES_DIR Default: ``templates`` dir inside scrapy module The directory where to look for templates when creating new projects with -:command:`startproject` command and new spiders with :command:`genspider` +:command:`startproject` command and new spiders with :command:`genspider` command. The project name must not conflict with the name of custom files or directories diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 822812c9a..f13e85871 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -18,6 +18,9 @@ NEWSPIDER_MODULE = '$project_name.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = '$project_name (+http://www.yourdomain.com)' +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 From 0349bbf9d3691ad89a5e265db4220fb3e64324ff Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 15:25:15 +0100 Subject: [PATCH 30/42] Disable SpiderState extension if no JOBDIR set --- scrapy/extensions/spiderstate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 3799c7c66..2220cbd8f 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -2,6 +2,7 @@ import os from six.moves import cPickle as pickle from scrapy import signals +from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir class SpiderState(object): @@ -12,7 +13,11 @@ class SpiderState(object): @classmethod def from_crawler(cls, crawler): - obj = cls(job_dir(crawler.settings)) + jobdir = job_dir(crawler.settings) + if not jobdir: + raise NotConfigured + + obj = cls(jobdir) crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed) crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj From 29695375d16ae20e3a97dc78bc12662996a9319b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 16:33:24 +0100 Subject: [PATCH 31/42] Add test for raised exception with SpiderState extension when no JOBDIR used --- tests/test_spiderstate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index d83015bd9..d1d6debec 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -4,6 +4,8 @@ from twisted.trial import unittest from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider +from scrapy.exceptions import NotConfigured +from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): @@ -34,3 +36,7 @@ class SpiderStateTest(unittest.TestCase): ss.spider_opened(spider) self.assertEqual(spider.state, {}) ss.spider_closed(spider) + + def test_not_configured(self): + crawler = get_crawler(Spider) + self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler) From c22a4e3bb84448f778613717c3749f226df7880f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 16:41:16 +0100 Subject: [PATCH 32/42] Use long classes names for enabled middlewares in startup logs --- scrapy/middleware.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 6120488e2..be36f977e 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -28,6 +28,7 @@ class MiddlewareManager(object): def from_settings(cls, settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] + enabled = [] for clspath in mwlist: try: mwcls = load_object(clspath) @@ -38,6 +39,7 @@ class MiddlewareManager(object): else: mw = mwcls() middlewares.append(mw) + enabled.append(clspath) except NotConfigured as e: if e.args: clsname = clspath.split('.')[-1] @@ -45,7 +47,6 @@ class MiddlewareManager(object): {'clsname': clsname, 'eargs': e.args[0]}, extra={'crawler': crawler}) - enabled = [x.__class__.__name__ for x in middlewares] logger.info("Enabled %(componentname)ss:\n%(enabledlist)s", {'componentname': cls.component_name, 'enabledlist': pprint.pformat(enabled)}, From 1c83108893cd3cc05e5b8b16e9c03d4a4786fdd6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 19:24:11 +0100 Subject: [PATCH 33/42] Clarify priority adjust settings docs Fixes #1593 --- docs/topics/settings.rst | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0959a87a7..116a10f83 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -864,8 +864,26 @@ REDIRECT_PRIORITY_ADJUST Default: ``+2`` -Adjust redirect request priority relative to original request. -A negative priority adjust means more priority. +Scope: ``scrapy.downloadermiddlewares.redirect.RedirectMiddleware`` + +Adjust redirect request priority relative to original request: + +- **a positive priority adjust (default) means higher priority.** +- a negative priority adjust means lower priority. + +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** .. setting:: ROBOTSTXT_OBEY From 4bcbb77bcc7d7668340baa064db15f29617cf0cb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jan 2016 01:28:11 +0500 Subject: [PATCH 34/42] response.text. Fixes GH-1729. --- docs/topics/request-response.rst | 42 +++++++++++++---------- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 4 +-- scrapy/http/request/form.py | 4 +-- scrapy/http/response/text.py | 5 +++ scrapy/selector/unified.py | 2 +- scrapy/utils/iterators.py | 2 +- scrapy/utils/response.py | 4 +-- tests/test_engine.py | 5 ++- tests/test_http_response.py | 30 +++++++++------- 10 files changed, 58 insertions(+), 42 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ea64d1599..2e92961a9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -445,10 +445,10 @@ Response objects .. attribute:: Response.body - A str containing the body of this Response. Keep in mind that Response.body - is always a str. If you want the unicode version use - :meth:`TextResponse.body_as_unicode` (only available in - :class:`TextResponse` and subclasses). + The body of this Response. Keep in mind that Response.body + is always a bytes object. If you want the unicode version use + :attr:`TextResponse.txt` (only available in :class:`TextResponse` + and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -542,6 +542,21 @@ TextResponse objects :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: + .. attribute:: TextResponse.text + + Response body, as unicode. + + The same as ``response.body.decode(response.encoding)``, but the + result is cached after the first call, so you can access + ``response.text`` multiple times without extra overhead. + + .. note:: + + ``unicode(response.body)`` is not a correct way to convert response + body to unicode: you would be using the system default encoding + (typically `ascii`) instead of the response encoding. + + .. attribute:: TextResponse.encoding A string with the encoding of this response. The encoding is resolved by @@ -568,20 +583,6 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.body_as_unicode() - - Returns the body of the response as unicode. This is equivalent to:: - - response.body.decode(response.encoding) - - But **not** equivalent to:: - - unicode(response.body) - - Since, in the latter case, you would be using the system default encoding - (typically `ascii`) to convert the body to unicode, instead of the response - encoding. - .. method:: TextResponse.xpath(query) A shortcut to ``TextResponse.selector.xpath(query)``:: @@ -594,6 +595,11 @@ TextResponse objects response.css('p') + .. method:: TextResponse.body_as_unicode() + + The same as :attr:`text`, but available as a method. This method is + kept for backwards compatibility; please prefer ``response.text``. + HtmlResponse objects -------------------- diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 6b543b823..da373eca2 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object): Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - body = response.body_as_unicode()[:self.lookup_bytes] + body = response.text[:self.lookup_bytes] return _has_ajaxcrawlable_meta(body) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..d4a33dc36 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object): def _parse_robots(self, response, netloc): rp = robotparser.RobotFileParser(response.url) body = '' - if hasattr(response, 'body_as_unicode'): - body = response.body_as_unicode() + if hasattr(response, 'text'): + body = response.text else: # last effort try try: body = response.body.decode('utf-8') diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 5501634d3..2862dc096 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -64,8 +64,8 @@ def _urlencode(seq, enc): def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ - text = response.body_as_unicode() - root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response)) + root = create_root_node(response.text, lxml.html.HTMLParser, + base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError("No
element found in %s" % response) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1c416bf82..9c667ab7e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -67,6 +67,11 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody + @property + def text(self): + """ Body as unicode """ + return self.body_as_unicode() + def urljoin(self, url): """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 5d77f7624..15f3d26df 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref): response = _response_from_text(text, st) if response is not None: - text = response.body_as_unicode() + text = response.text kwargs.setdefault('base_url', response.url) self.response = response diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b0688791e..73857b410 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): if not unicode: return obj.body elif isinstance(obj, TextResponse): - return obj.body_as_unicode() + return obj.text else: return obj.body.decode('utf-8') elif isinstance(obj, six.text_type): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c4ad52f14..73db2641e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary() def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: - text = response.body_as_unicode()[0:4096] + text = response.text[0:4096] _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] @@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: - text = response.body_as_unicode()[0:4096] + text = response.text[0:4096] text = _noscript_re.sub(u'', text) text = _script_re.sub(u'', text) _metaref_cache[response] = html.get_meta_refresh(text, response.url, diff --git a/tests/test_engine.py b/tests/test_engine.py index 9f2c02bff..baf6ef1bf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -55,12 +55,11 @@ class TestSpider(Spider): def parse_item(self, response): item = self.item_cls() - body = response.body_as_unicode() - m = self.name_re.search(body) + m = self.name_re.search(response.text) if m: item['name'] = m.group(1) item['url'] = response.url - m = self.price_re.search(body) + m = self.price_re.search(response.text) if m: item['price'] = m.group(1) return item diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 710a5b29d..c7f36687a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase): body_bytes = body assert isinstance(response.body, bytes) + assert isinstance(response.text, six.text_type) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) self.assertEqual(response.body_as_unicode(), body_unicode) + self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): self.assertEqual(response.encoding, resolve_encoding(encoding)) @@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest): self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type)) self.assertEqual(r1.body_as_unicode(), unicode_string) + # check response.text + self.assertTrue(isinstance(r1.text, six.text_type)) + self.assertEqual(r1.text, unicode_string) + def test_encoding(self): r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3") r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") @@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd') + self.assertEqual(r6.text, u'WORD\ufffd\ufffd') def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and - # response.body_as_unicode() in indistint order doesn't affect final + # response.text in indistint order doesn't affect final # values for encoding and decoded body. url = 'http://example.com' body = b"\xef\xbb\xbfWORD" @@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): @@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest): r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) - assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) - assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'\ufffd' in r.text, repr(r.text) + assert u'PREFIX' in r.text, repr(r.text) + assert u'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', \ body=b'\xf0value') - assert u'value' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse - #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') - #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') + #assert u'\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page" From 6ed08d23329bc33142804ea78320993d0c680175 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 27 Jan 2016 11:53:29 +0100 Subject: [PATCH 35/42] Add note for DEPTH_PRIORITY --- docs/topics/settings.rst | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 116a10f83..052be4429 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -276,6 +276,8 @@ DEPTH_LIMIT Default: ``0`` +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` + The maximum depth that will be allowed to crawl for any site. If zero, no limit will be imposed. @@ -286,9 +288,20 @@ DEPTH_PRIORITY Default: ``0`` -An integer that is used to adjust the request priority based on its depth. +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -If zero, no priority adjustment is made from depth. +An integer that is used to adjust the request priority based on its depth: + +- **a positive value will decrease the priority** +- a negative value will increase priority + +If zero (default), no priority adjustment is made from depth. + +.. note:: + + This setting adjusts priority **in the opposite way** compared to + other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` + and :setting:`RETRY_PRIORITY_ADJUST`. .. setting:: DEPTH_STATS @@ -297,6 +310,8 @@ DEPTH_STATS Default: ``True`` +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` + Whether to collect maximum depth stats. .. setting:: DEPTH_STATS_VERBOSE @@ -306,6 +321,8 @@ DEPTH_STATS_VERBOSE Default: ``False`` +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` + Whether to collect verbose depth stats. If this is enabled, the number of requests for each depth is collected in the stats. From d999e3f7a704c5009999573d9514a5a51bd8be13 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 27 Jan 2016 12:57:03 +0100 Subject: [PATCH 36/42] More explicit description of DEPTH_PRIORITY --- docs/faq.rst | 4 +++- docs/topics/settings.rst | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 3d2bd8d4d..b3412211a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -45,7 +45,7 @@ Did Scrapy "steal" X from Django? Probably, but we don't like that word. We think Django_ is a great open source project and an example to follow, so we've used it as an inspiration for -Scrapy. +Scrapy. We believe that, if something is already done well, there's no need to reinvent it. This concept, besides being one of the foundations for open source and free @@ -85,6 +85,8 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. +.. _faq-bfo-dfo: + Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 052be4429..725345f2a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -292,10 +292,14 @@ Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` An integer that is used to adjust the request priority based on its depth: -- **a positive value will decrease the priority** -- a negative value will increase priority +- if zero (default), no priority adjustment is made from depth +- **a positive value will decrease the priority, i.e. higher depth + requests will be processed later** ; this is commonly used when doing + breadth-first crawls (BFO) +- a negative value will increase priority, i.e., higher depth requests + will be processed sooner (DFO) -If zero (default), no priority adjustment is made from depth. +See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. .. note:: From e0f48c486e4e658e80a4fea22f0fee58c3f01a0c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 27 Jan 2016 13:04:08 +0100 Subject: [PATCH 37/42] Add link to CoC mardown file on Github --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8a7d2c71d..3e050bb1e 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ Contributing ============ Please note that this project is released with a Contributor Code of Conduct -(see CODE_OF_CONDUCT.md). +(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@scrapinghub.com. From 7ca9ae19765d2c49c0e838ebbfc1596d0fbcd7d9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jan 2016 17:54:28 +0500 Subject: [PATCH 38/42] DOC typo fix --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 2e92961a9..82e674cee 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -447,7 +447,7 @@ Response objects The body of this Response. Keep in mind that Response.body is always a bytes object. If you want the unicode version use - :attr:`TextResponse.txt` (only available in :class:`TextResponse` + :attr:`TextResponse.text` (only available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use From f1d971a5c0cdfe0f4fe5619146cd6818324fc98e Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 14:34:46 -0200 Subject: [PATCH 39/42] fix PythonItemExporter for non-string types --- scrapy/exporters.py | 8 ++++---- tests/test_exporters.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 145468dbe..c7c78d054 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -273,10 +273,10 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_dict(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if self.binary: - return to_bytes(value, encoding=self.encoding) - else: - return to_unicode(value, encoding=self.encoding) + encode_func = to_bytes if self.binary else to_unicode + if isinstance(value, (six.text_type, bytes)): + return encode_func(value, encoding=self.encoding) + return value def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1633e1039..662f8ec5c 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -134,6 +134,19 @@ class PythonItemExporterTest(BaseItemExporterTest): expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) + def test_other_python_types_item(self): + from datetime import datetime + now = datetime.now() + item = { + 'boolean': False, + 'number': 22, + 'time': now, + 'float': 3.14, + } + ie = self._get_exporter() + exported = ie.export_item(item) + self.assertEqual(exported, item) + class PprintItemExporterTest(BaseItemExporterTest): From c55ff110a34d39be27bbd3d03fbf52caa271b4c9 Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 15:43:17 -0200 Subject: [PATCH 40/42] Fix CSV exporter for non string Python types. --- scrapy/exporters.py | 2 +- tests/test_exporters.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c7c78d054..55d74332b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter): try: yield to_native_str(s) except TypeError: - yield to_native_str(repr(s)) + yield to_native_str(str(s)) def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 662f8ec5c..97c09a495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -271,6 +271,21 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"[4, 8]",John\r\n', ) + def test_other_python_types_item(self): + from datetime import datetime + now = datetime(2015, 1, 1, 1, 1, 1) + item = { + 'boolean': False, + 'number': 22, + 'time': now, + 'float': 3.14, + } + self.assertExportResult( + item=item, + include_headers_line=False, + expected='22,False,3.14,2015-01-01 01:01:01\r\n' + ) + class XmlItemExporterTest(BaseItemExporterTest): From 27758f60ada4791c044bfe8bc86d267aa930c744 Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 16:28:01 -0200 Subject: [PATCH 41/42] Changes fallback for CSVItemExporter, avoiding to call to_native_str(str()). --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 55d74332b..35f50838b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter): try: yield to_native_str(s) except TypeError: - yield to_native_str(str(s)) + yield s def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: From 3e080c3c52720535519ba1be7dee472258b19647 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Jan 2016 00:59:27 +0500 Subject: [PATCH 42/42] call .text from .body_as_unicode() and not the other way around --- scrapy/http/response/text.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 9c667ab7e..afa430329 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -59,7 +59,12 @@ class TextResponse(Response): def body_as_unicode(self): """Return body as unicode""" - # check for self.encoding before _cached_ubody just in + return self.text + + @property + def text(self): + """ Body as unicode """ + # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: @@ -67,11 +72,6 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody - @property - def text(self): - """ Body as unicode """ - return self.body_as_unicode() - def urljoin(self, url): """Join this Response's url with a possible relative url to form an absolute interpretation of the latter."""