mirror of https://github.com/scrapy/scrapy.git
Feed post-processing plugin support (#5190)
This commit is contained in:
D R Siddhartha
GitHub
parent
3f635eb683
commit
43ea21e830
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.6-v7.3.1
|
||||
PYPY_VERSION: 3.6-v7.3.3
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.6.12
|
||||
|
|
|
|||
|
|
@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, as
|
|||
soon as a file reaches the maximum item count, that file is delivered to the
|
||||
feed URI, allowing item delivery to start way before the end of the crawl.
|
||||
|
||||
|
||||
.. _item-filter:
|
||||
|
||||
Item filtering
|
||||
|
|
@ -312,6 +313,63 @@ ItemFilter
|
|||
:members:
|
||||
|
||||
|
||||
.. _post-processing:
|
||||
|
||||
Post-Processing
|
||||
===============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Scrapy provides an option to activate plugins to post-process feeds before they are exported
|
||||
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
|
||||
These plugins can be activated through the ``postprocessing`` option of a feed.
|
||||
The option must be passed a list of post-processing plugins in the order you want
|
||||
the feed to be processed. These plugins can be declared either as an import string
|
||||
or with the imported class of the plugin. Parameters to plugins can be passed
|
||||
through the feed options. See :ref:`feed options <feed-options>` for examples.
|
||||
|
||||
.. _builtin-plugins:
|
||||
|
||||
Built-in Plugins
|
||||
----------------
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
|
||||
|
||||
.. _custom-plugins:
|
||||
|
||||
Custom Plugins
|
||||
--------------
|
||||
|
||||
Each plugin is a class that must implement the following methods:
|
||||
|
||||
.. method:: __init__(self, file, feed_options)
|
||||
|
||||
Initialize the plugin.
|
||||
|
||||
:param file: file-like object having at least the `write`, `tell` and `close` methods implemented
|
||||
|
||||
:param feed_options: feed-specific :ref:`options <feed-options>`
|
||||
:type feed_options: :class:`dict`
|
||||
|
||||
.. method:: write(self, data)
|
||||
|
||||
Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
|
||||
It must return number of bytes written.
|
||||
|
||||
.. method:: close(self)
|
||||
|
||||
Close the target file object.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
|
|
@ -368,10 +426,12 @@ For instance::
|
|||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path('items.csv'): {
|
||||
pathlib.Path('items.csv.gz'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 5,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
|
||||
|
||||
- ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
|
||||
|
||||
The plugins will be used in the order of the list passed.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from zope.interface import implementer, Interface
|
|||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.conf import feed_complete_default_values_from_settings
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
|
|
@ -396,6 +397,9 @@ class FeedExporter:
|
|||
"""
|
||||
storage = self._get_storage(uri, feed_options)
|
||||
file = storage.open(spider)
|
||||
if "postprocessing" in feed_options:
|
||||
file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)
|
||||
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed_options['format'],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,154 @@
|
|||
"""
|
||||
Extension for processing data before they are exported to feeds.
|
||||
"""
|
||||
from bz2 import BZ2File
|
||||
from gzip import GzipFile
|
||||
from io import IOBase
|
||||
from lzma import LZMAFile
|
||||
from typing import Any, BinaryIO, Dict, List
|
||||
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
class GzipPlugin:
|
||||
"""
|
||||
Compresses received data using `gzip <https://en.wikipedia.org/wiki/Gzip>`_.
|
||||
|
||||
Accepted ``feed_options`` parameters:
|
||||
|
||||
- `gzip_compresslevel`
|
||||
- `gzip_mtime`
|
||||
- `gzip_filename`
|
||||
|
||||
See :py:class:`gzip.GzipFile` for more info about parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
compress_level = self.feed_options.get("gzip_compresslevel", 9)
|
||||
mtime = self.feed_options.get("gzip_mtime")
|
||||
filename = self.feed_options.get("gzip_filename")
|
||||
self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,
|
||||
mtime=mtime, filename=filename)
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
return self.gzipfile.write(data)
|
||||
|
||||
def close(self) -> None:
|
||||
self.gzipfile.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class Bz2Plugin:
|
||||
"""
|
||||
Compresses received data using `bz2 <https://en.wikipedia.org/wiki/Bzip2>`_.
|
||||
|
||||
Accepted ``feed_options`` parameters:
|
||||
|
||||
- `bz2_compresslevel`
|
||||
|
||||
See :py:class:`bz2.BZ2File` for more info about parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
compress_level = self.feed_options.get("bz2_compresslevel", 9)
|
||||
self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
return self.bz2file.write(data)
|
||||
|
||||
def close(self) -> None:
|
||||
self.bz2file.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class LZMAPlugin:
|
||||
"""
|
||||
Compresses received data using `lzma <https://en.wikipedia.org/wiki/Lempel–Ziv–Markov_chain_algorithm>`_.
|
||||
|
||||
Accepted ``feed_options`` parameters:
|
||||
|
||||
- `lzma_format`
|
||||
- `lzma_check`
|
||||
- `lzma_preset`
|
||||
- `lzma_filters`
|
||||
|
||||
.. note::
|
||||
``lzma_filters`` cannot be used in pypy version 7.3.1 and older.
|
||||
|
||||
See :py:class:`lzma.LZMAFile` for more info about parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
|
||||
format = self.feed_options.get("lzma_format")
|
||||
check = self.feed_options.get("lzma_check", -1)
|
||||
preset = self.feed_options.get("lzma_preset")
|
||||
filters = self.feed_options.get("lzma_filters")
|
||||
self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,
|
||||
check=check, preset=preset, filters=filters)
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
return self.lzmafile.write(data)
|
||||
|
||||
def close(self) -> None:
|
||||
self.lzmafile.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager
|
||||
# instance as a file like writable object. This could be needed by some exporters
|
||||
# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper.
|
||||
class PostProcessingManager(IOBase):
|
||||
"""
|
||||
This will manage and use declared plugins to process data in a
|
||||
pipeline-ish way.
|
||||
:param plugins: all the declared plugins for the feed
|
||||
:type plugins: list
|
||||
:param file: final target file where the processed data will be written
|
||||
:type file: file like object
|
||||
"""
|
||||
|
||||
def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.plugins = self._load_plugins(plugins)
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
self.head_plugin = self._get_head_plugin()
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
"""
|
||||
Uses all the declared plugins to process data first, then writes
|
||||
the processed data to target file.
|
||||
:param data: data passed to be written to target file
|
||||
:type data: bytes
|
||||
:return: returns number of bytes written
|
||||
:rtype: int
|
||||
"""
|
||||
return self.head_plugin.write(data)
|
||||
|
||||
def tell(self) -> int:
|
||||
return self.file.tell()
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the target file along with all the plugins.
|
||||
"""
|
||||
self.head_plugin.close()
|
||||
|
||||
def writable(self) -> bool:
|
||||
return True
|
||||
|
||||
def _load_plugins(self, plugins: List[Any]) -> List[Any]:
|
||||
plugins = [load_object(plugin) for plugin in plugins]
|
||||
return plugins
|
||||
|
||||
def _get_head_plugin(self) -> Any:
|
||||
prev = self.file
|
||||
for plugin in self.plugins[::-1]:
|
||||
prev = plugin(prev, self.feed_options)
|
||||
return prev
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
import bz2
|
||||
import csv
|
||||
import gzip
|
||||
import json
|
||||
import lzma
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
|
|
@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase):
|
|||
self.assertEqual(row['expected'], data[feed_options['format']])
|
||||
|
||||
|
||||
class FeedPostProcessedExportsTest(FeedExportTestBase):
|
||||
__test__ = True
|
||||
|
||||
items = [{'foo': 'bar'}]
|
||||
expected = b'foo\r\nbar\r\n'
|
||||
|
||||
class MyPlugin1:
|
||||
def __init__(self, file, feed_options):
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
self.char = self.feed_options.get('plugin1_char', b'')
|
||||
|
||||
def write(self, data):
|
||||
written_count = self.file.write(data)
|
||||
written_count += self.file.write(self.char)
|
||||
return written_count
|
||||
|
||||
def close(self):
|
||||
self.file.close()
|
||||
|
||||
def _named_tempfile(self, name):
|
||||
return os.path.join(self.temp_dir, name)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_export(self, spider_cls, settings):
|
||||
""" Run spider with specified settings; return exported data with filename. """
|
||||
|
||||
FEEDS = settings.get('FEEDS') or {}
|
||||
settings['FEEDS'] = {
|
||||
printf_escape(path_to_url(file_path)): feed_options
|
||||
for file_path, feed_options in FEEDS.items()
|
||||
}
|
||||
|
||||
content = {}
|
||||
try:
|
||||
with MockServer() as s:
|
||||
runner = CrawlerRunner(Settings(settings))
|
||||
spider_cls.start_urls = [s.url('/')]
|
||||
yield runner.crawl(spider_cls)
|
||||
|
||||
for file_path, feed_options in FEEDS.items():
|
||||
if not os.path.exists(str(file_path)):
|
||||
continue
|
||||
|
||||
with open(str(file_path), 'rb') as f:
|
||||
content[str(file_path)] = f.read()
|
||||
|
||||
finally:
|
||||
for file_path in FEEDS.keys():
|
||||
if not os.path.exists(str(file_path)):
|
||||
continue
|
||||
|
||||
os.remove(str(file_path))
|
||||
|
||||
return content
|
||||
|
||||
def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''):
|
||||
data_stream = BytesIO()
|
||||
gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime,
|
||||
compresslevel=compresslevel, mode="wb")
|
||||
gzipf.write(data)
|
||||
gzipf.close()
|
||||
data_stream.seek(0)
|
||||
return data_stream.read()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_gzip_plugin(self):
|
||||
|
||||
filename = self._named_tempfile('gzip_file')
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
filename: {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
try:
|
||||
gzip.decompress(data[filename])
|
||||
except OSError:
|
||||
self.fail("Received invalid gzip data.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_gzip_plugin_compresslevel(self):
|
||||
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0),
|
||||
self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('compresslevel_0'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 0,
|
||||
'gzip_mtime': 0,
|
||||
'gzip_filename': "",
|
||||
},
|
||||
self._named_tempfile('compresslevel_9'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 9,
|
||||
'gzip_mtime': 0,
|
||||
'gzip_filename': "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = gzip.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_gzip_plugin_mtime(self):
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123),
|
||||
self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('mtime_123'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_mtime': 123,
|
||||
'gzip_filename': "",
|
||||
},
|
||||
self._named_tempfile('mtime_123456789'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_mtime': 123456789,
|
||||
'gzip_filename': "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = gzip.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_gzip_plugin_filename(self):
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"),
|
||||
self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('filename_FILE1'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_mtime': 0,
|
||||
'gzip_filename': "FILE1",
|
||||
},
|
||||
self._named_tempfile('filename_FILE2'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_mtime': 0,
|
||||
'gzip_filename': "FILE2",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = gzip.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_lzma_plugin(self):
|
||||
|
||||
filename = self._named_tempfile('lzma_file')
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
filename: {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
try:
|
||||
lzma.decompress(data[filename])
|
||||
except lzma.LZMAError:
|
||||
self.fail("Received invalid lzma data.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_lzma_plugin_format(self):
|
||||
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ),
|
||||
self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('format_FORMAT_XZ'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_format': lzma.FORMAT_XZ,
|
||||
},
|
||||
self._named_tempfile('format_FORMAT_ALONE'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_format': lzma.FORMAT_ALONE,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = lzma.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_lzma_plugin_check(self):
|
||||
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE),
|
||||
self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('check_CHECK_NONE'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_check': lzma.CHECK_NONE,
|
||||
},
|
||||
self._named_tempfile('check_CHECK_CRC256'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_check': lzma.CHECK_SHA256,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = lzma.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_lzma_plugin_preset(self):
|
||||
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0),
|
||||
self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('preset_PRESET_0'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_preset': 0,
|
||||
},
|
||||
self._named_tempfile('preset_PRESET_9'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_preset': 9,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = lzma.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_lzma_plugin_filters(self):
|
||||
import sys
|
||||
if "PyPy" in sys.version:
|
||||
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
|
||||
raise unittest.SkipTest("lzma filters doesn't work in PyPy")
|
||||
|
||||
filters = [{'id': lzma.FILTER_LZMA2}]
|
||||
compressed = lzma.compress(self.expected, filters=filters)
|
||||
filename = self._named_tempfile('filters')
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
filename: {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'lzma_filters': filters,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
self.assertEqual(compressed, data[filename])
|
||||
result = lzma.decompress(data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_bz2_plugin(self):
|
||||
|
||||
filename = self._named_tempfile('bz2_file')
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
filename: {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
try:
|
||||
bz2.decompress(data[filename])
|
||||
except OSError:
|
||||
self.fail("Received invalid bz2 data.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_bz2_plugin_compresslevel(self):
|
||||
|
||||
filename_to_compressed = {
|
||||
self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1),
|
||||
self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('compresslevel_1'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
|
||||
'bz2_compresslevel': 1,
|
||||
},
|
||||
self._named_tempfile('compresslevel_9'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
|
||||
'bz2_compresslevel': 9,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, compressed in filename_to_compressed.items():
|
||||
result = bz2.decompress(data[filename])
|
||||
self.assertEqual(compressed, data[filename])
|
||||
self.assertEqual(self.expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_custom_plugin(self):
|
||||
filename = self._named_tempfile('csv_file')
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
filename: {
|
||||
'format': 'csv',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
self.assertEqual(self.expected, data[filename])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_custom_plugin_with_parameter(self):
|
||||
|
||||
expected = b'foo\r\n\nbar\r\n\n'
|
||||
filename = self._named_tempfile('newline')
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
filename: {
|
||||
'format': 'csv',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
'plugin1_char': b'\n'
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
self.assertEqual(expected, data[filename])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_custom_plugin_with_compression(self):
|
||||
|
||||
expected = b'foo\r\n\nbar\r\n\n'
|
||||
|
||||
filename_to_decompressor = {
|
||||
self._named_tempfile('bz2'): bz2.decompress,
|
||||
self._named_tempfile('lzma'): lzma.decompress,
|
||||
self._named_tempfile('gzip'): gzip.decompress,
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('bz2'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'],
|
||||
'plugin1_char': b'\n',
|
||||
},
|
||||
self._named_tempfile('lzma'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'],
|
||||
'plugin1_char': b'\n',
|
||||
},
|
||||
self._named_tempfile('gzip'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'plugin1_char': b'\n',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, decompressor in filename_to_decompressor.items():
|
||||
result = decompressor(data[filename])
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_exports_compatibility_with_postproc(self):
|
||||
import marshal
|
||||
import pickle
|
||||
filename_to_expected = {
|
||||
self._named_tempfile('csv'): b'foo\r\nbar\r\n',
|
||||
self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]',
|
||||
self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n',
|
||||
self._named_tempfile('xml'): b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items>\n<item><foo>bar</foo></item>\n</items>',
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._named_tempfile('csv'): {
|
||||
'format': 'csv',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
# empty plugin to activate postprocessing.PostProcessingManager
|
||||
},
|
||||
self._named_tempfile('json'): {
|
||||
'format': 'json',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
},
|
||||
self._named_tempfile('jsonlines'): {
|
||||
'format': 'jsonlines',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
},
|
||||
self._named_tempfile('xml'): {
|
||||
'format': 'xml',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
},
|
||||
self._named_tempfile('marshal'): {
|
||||
'format': 'marshal',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
},
|
||||
self._named_tempfile('pickle'): {
|
||||
'format': 'pickle',
|
||||
'postprocessing': [self.MyPlugin1],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(self.items, settings)
|
||||
|
||||
for filename, result in data.items():
|
||||
if 'pickle' in filename:
|
||||
expected, result = self.items[0], pickle.loads(result)
|
||||
elif 'marshal' in filename:
|
||||
expected, result = self.items[0], marshal.loads(result)
|
||||
else:
|
||||
expected = filename_to_expected[filename]
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
|
||||
class BatchDeliveriesTest(FeedExportTestBase):
|
||||
__test__ = True
|
||||
_file_mark = '_%(batch_time)s_#%(batch_id)02d_'
|
||||
|
|
|
|||
Loading…
Reference in New Issue