diff --git a/docs/contributing.rst b/docs/contributing.rst index f49bc536e..ad9a3805a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -150,9 +150,9 @@ To run all tests go to the root directory of Scrapy source code and run: ``tox`` -To run a specific test (say ``tests/test_contrib_loader.py``) use: +To run a specific test (say ``tests/test_loader.py``) use: - ``tox -- tests/test_contrib_loader.py`` + ``tox -- tests/test_loader.py`` Writing tests @@ -166,11 +166,11 @@ Scrapy uses unit-tests, which are located in the `tests/`_ directory. Their module name typically resembles the full path of the module they're testing. For example, the item loaders code is in:: - scrapy.contrib.loader + scrapy.loader And their unit-tests are in:: - tests/test_contrib_loader.py + tests/test_loader.py .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 603d28f78..a5efa3fb9 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -4,7 +4,7 @@ Item Loaders ============ -.. module:: scrapy.contrib.loader +.. module:: scrapy.loader :synopsis: Item Loader class Item Loaders provide a convenient mechanism for populating scraped :ref:`Items @@ -39,7 +39,7 @@ Here is a typical Item Loader usage in a :ref:`Spider `, using the :ref:`Product item ` declared in the :ref:`Items chapter `:: - from scrapy.contrib.loader import ItemLoader + from scrapy.loader import ItemLoader from myproject.items import Product def parse(self, response): @@ -150,8 +150,8 @@ Declaring Item Loaders Item Loaders are declared like Items, by using a class definition syntax. Here is an example:: - from scrapy.contrib.loader import ItemLoader - from scrapy.contrib.loader.processor import TakeFirst, MapCompose, Join + from scrapy.loader import ItemLoader + from scrapy.loader.processor import TakeFirst, MapCompose, Join class ProductLoader(ItemLoader): @@ -182,7 +182,7 @@ output processors to use: in the :ref:`Item Field ` metadata. Here is an example:: import scrapy - from scrapy.contrib.loader.processor import Join, MapCompose, TakeFirst + from scrapy.loader.processor import Join, MapCompose, TakeFirst from w3lib.html import remove_tags def filter_price(value): @@ -201,7 +201,7 @@ metadata. Here is an example:: :: - >>> from scrapy.contrib.loader import ItemLoader + >>> from scrapy.loader import ItemLoader >>> il = ItemLoader(item=Product()) >>> il.add_value('name', [u'Welcome to my', u'website']) >>> il.add_value('price', [u'€', u'1000']) @@ -309,7 +309,7 @@ ItemLoader objects Examples:: - >>> from scrapy.contrib.loader.processor import TakeFirst + >>> from scrapy.loader.processor import TakeFirst >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') 'FOO` @@ -513,7 +513,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``):: - from scrapy.contrib.loader.processor import MapCompose + from scrapy.loader.processor import MapCompose from myproject.ItemLoaders import ProductLoader def strip_dashes(x): @@ -526,7 +526,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - from scrapy.contrib.loader.processor import MapCompose + from scrapy.loader.processor import MapCompose from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata @@ -551,7 +551,7 @@ needs. Available built-in processors ============================= -.. module:: scrapy.contrib.loader.processor +.. module:: scrapy.loader.processor :synopsis: A collection of processors to use with Item Loaders Even though you can use any callable function as input and output processors, @@ -570,7 +570,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import Identity + >>> from scrapy.loader.processor import Identity >>> proc = Identity() >>> proc(['one', 'two', 'three']) ['one', 'two', 'three'] @@ -583,7 +583,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import TakeFirst + >>> from scrapy.loader.processor import TakeFirst >>> proc = TakeFirst() >>> proc(['', 'one', 'two', 'three']) 'one' @@ -598,7 +598,7 @@ Here is a list of all built-in processors: Examples:: - >>> from scrapy.contrib.loader.processor import Join + >>> from scrapy.loader.processor import Join >>> proc = Join() >>> proc(['one', 'two', 'three']) u'one two three' @@ -619,7 +619,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import Compose + >>> from scrapy.loader.processor import Compose >>> proc = Compose(lambda v: v[0], str.upper) >>> proc(['hello', 'world']) 'HELLO' @@ -666,7 +666,7 @@ Here is a list of all built-in processors: >>> def filter_world(x): ... return None if x == 'world' else x ... - >>> from scrapy.contrib.loader.processor import MapCompose + >>> from scrapy.loader.processor import MapCompose >>> proc = MapCompose(filter_world, unicode.upper) >>> proc([u'hello', u'world', u'this', u'is', u'scrapy']) [u'HELLO, u'THIS', u'IS', u'SCRAPY'] @@ -683,7 +683,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import SelectJmes, Compose, MapCompose + >>> from scrapy.loader.processor import SelectJmes, Compose, MapCompose >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries >>> proc({'foo': 'bar'}) 'bar' diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/loader/__init__.py similarity index 100% rename from scrapy/contrib/loader/__init__.py rename to scrapy/loader/__init__.py diff --git a/scrapy/contrib/loader/common.py b/scrapy/loader/common.py similarity index 100% rename from scrapy/contrib/loader/common.py rename to scrapy/loader/common.py diff --git a/scrapy/contrib/loader/processor.py b/scrapy/loader/processor.py similarity index 100% rename from scrapy/contrib/loader/processor.py rename to scrapy/loader/processor.py diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 3f1da0751..69df2f7a2 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -6,7 +6,7 @@ tests/test_commands.py tests/test_command_version.py tests/test_exporters.py tests/test_linkextractors.py -tests/test_contrib_loader.py +tests/test_loader.py tests/test_crawl.py tests/test_crawler.py tests/test_downloader_handlers.py diff --git a/tests/test_contrib_loader.py b/tests/test_loader.py similarity index 99% rename from tests/test_contrib_loader.py rename to tests/test_loader.py index fd65e8200..c51092ad0 100644 --- a/tests/test_contrib_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,8 @@ import unittest from functools import partial -from scrapy.contrib.loader import ItemLoader -from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \ +from scrapy.loader import ItemLoader +from scrapy.loader.processor import Join, Identity, TakeFirst, \ Compose, MapCompose, SelectJmes from scrapy.item import Item, Field from scrapy.selector import Selector diff --git a/tests/test_squeue.py b/tests/test_squeue.py index 83ffcc4b7..16014a897 100644 --- a/tests/test_squeue.py +++ b/tests/test_squeue.py @@ -2,7 +2,7 @@ from queuelib.tests import test_queue as t from scrapy.squeue import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue from scrapy.item import Item, Field from scrapy.http import Request -from scrapy.contrib.loader import ItemLoader +from scrapy.loader import ItemLoader class TestItem(Item): name = Field()