mirror of https://github.com/scrapy/scrapy.git
Remove deprecated code moved to itemloaders
This commit is contained in:
parent
85c57778b5
commit
aa0a425826
|
|
@ -1,21 +0,0 @@
|
|||
"""Common functions used in Item Loaders code"""
|
||||
|
||||
import warnings
|
||||
|
||||
from itemloaders import common
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
def wrap_loader_context(function, context):
|
||||
"""Wrap functions that receive loader_context to contain the context
|
||||
"pre-loaded" and expose a interface that receives only one argument
|
||||
"""
|
||||
warnings.warn(
|
||||
"scrapy.loader.common.wrap_loader_context has moved to a new library."
|
||||
"Please update your reference to itemloaders.common.wrap_loader_context",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
return common.wrap_loader_context(function, context)
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
"""
|
||||
This module provides some commonly used processors for Item Loaders.
|
||||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from itemloaders import processors
|
||||
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
|
||||
|
||||
Compose = create_deprecated_class("Compose", processors.Compose)
|
||||
|
||||
TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
|
||||
|
||||
Identity = create_deprecated_class("Identity", processors.Identity)
|
||||
|
||||
SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
|
||||
|
||||
Join = create_deprecated_class("Join", processors.Join)
|
||||
|
|
@ -21,17 +21,12 @@ from typing import (
|
|||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Pattern,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from w3lib.html import replace_entities
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Spider
|
||||
|
|
@ -108,39 +103,6 @@ def walk_modules(path: str) -> List[ModuleType]:
|
|||
return mods
|
||||
|
||||
|
||||
def extract_regex(
|
||||
regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
|
||||
) -> List[str]:
|
||||
"""Extract a list of unicode strings from the given text/encoding using the following policies:
|
||||
|
||||
* if the regex contains a named group called "extract" that will be returned
|
||||
* if the regex contains multiple numbered groups, all those will be returned (flattened)
|
||||
* if the regex doesn't contain any group the entire regex matching is returned
|
||||
"""
|
||||
warnings.warn(
|
||||
"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if isinstance(regex, str):
|
||||
regex = re.compile(regex, re.UNICODE)
|
||||
|
||||
try:
|
||||
# named group
|
||||
strings = [regex.search(text).group("extract")] # type: ignore[union-attr]
|
||||
except Exception:
|
||||
# full regex or numbered groups
|
||||
strings = regex.findall(text)
|
||||
strings = flatten(strings)
|
||||
|
||||
if isinstance(text, str):
|
||||
return [replace_entities(s, keep=["lt", "amp"]) for s in strings]
|
||||
return [
|
||||
replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings
|
||||
]
|
||||
|
||||
|
||||
def md5sum(file: IO) -> str:
|
||||
"""Calculate the md5 checksum of a file-like object without reading its
|
||||
whole content in memory.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ Once we remove the references from scrapy, we can remove these tests.
|
|||
"""
|
||||
|
||||
import unittest
|
||||
import warnings
|
||||
from functools import partial
|
||||
|
||||
from itemloaders.processors import (
|
||||
|
|
@ -18,9 +17,6 @@ from itemloaders.processors import (
|
|||
|
||||
from scrapy.item import Field, Item
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.misc import extract_regex
|
||||
|
||||
|
||||
# test items
|
||||
|
|
@ -722,24 +718,5 @@ class FunctionProcessorTestCase(unittest.TestCase):
|
|||
self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
|
||||
|
||||
|
||||
class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
|
||||
def test_deprecated_wrap_loader_context(self):
|
||||
def function(*args):
|
||||
return None
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
wrap_loader_context(function, context={})
|
||||
|
||||
assert len(w) == 1
|
||||
assert issubclass(w[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
def test_deprecated_extract_regex(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
extract_regex(r"\w+", "this is a test")
|
||||
|
||||
assert len(w) == 1
|
||||
assert issubclass(w[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue