From 18d303b5f1815ce8d0a41ba04b56cfbe25c9616e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 19 Apr 2011 01:33:52 -0300 Subject: [PATCH] ported internal scrapy.utils imports to w3lib --- scrapy/commands/crawl.py | 3 +-- scrapy/commands/deploy.py | 3 ++- scrapy/commands/fetch.py | 3 ++- scrapy/commands/parse.py | 2 +- scrapy/contrib/downloadermiddleware/httpauth.py | 2 +- scrapy/contrib/downloadermiddleware/httpcache.py | 3 ++- scrapy/contrib/downloadermiddleware/redirect.py | 3 ++- scrapy/contrib/feedexport.py | 4 ++-- scrapy/contrib/linkextractors/htmlparser.py | 3 ++- scrapy/contrib/linkextractors/image.py | 4 ++-- scrapy/contrib/linkextractors/lxmlparser.py | 2 +- scrapy/contrib/linkextractors/regex.py | 4 ++-- scrapy/contrib/linkextractors/sgml.py | 4 +++- scrapy/contrib_exp/crawlspider/reqext.py | 3 ++- scrapy/core/downloader/handlers/file.py | 2 +- scrapy/http/headers.py | 2 +- scrapy/http/request/__init__.py | 3 ++- scrapy/shell.py | 2 +- scrapy/tests/test_contrib_feedexport.py | 4 ++-- scrapy/tests/test_downloader_handlers.py | 2 +- scrapy/utils/misc.py | 2 +- scrapy/utils/request.py | 3 ++- 22 files changed, 36 insertions(+), 27 deletions(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index d27ba7f9b..c2e787a71 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,10 +1,9 @@ -import sys +from w3lib.url import is_url from scrapy import log from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.http import Request -from scrapy.utils.url import is_url from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py index c22b9c710..46c67902f 100644 --- a/scrapy/commands/deploy.py +++ b/scrapy/commands/deploy.py @@ -11,10 +11,11 @@ import netrc from urlparse import urlparse, urljoin from subprocess import Popen, PIPE, check_call +from w3lib.form import encode_multipart + from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.py26 import json -from scrapy.utils.multipart import encode_multipart from scrapy.utils.http import basic_auth_header from scrapy.utils.conf import get_config, closest_scrapy_cfg diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 4d1abe14e..444de6e41 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,10 +1,11 @@ import pprint +from w3lib.url import is_url + from scrapy import log from scrapy.command import ScrapyCommand from scrapy.http import Request from scrapy.spider import BaseSpider -from scrapy.utils.url import is_url from scrapy.exceptions import UsageError class Command(ScrapyCommand): diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a39f12afa..8ab3ee76f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,9 +1,9 @@ +from w3lib.url import is_url from scrapy.command import ScrapyCommand from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, create_spider_for_request -from scrapy.utils.url import is_url from scrapy.exceptions import UsageError from scrapy import log diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py index 23f27ea34..b956838f1 100644 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ b/scrapy/contrib/downloadermiddleware/httpauth.py @@ -4,7 +4,7 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -from scrapy.utils.http import basic_auth_header +from w3lib.http import basic_auth_header from scrapy.utils.python import WeakKeyCache diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index e27db5d80..079189358 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -5,13 +5,14 @@ from os.path import join, exists from time import time import cPickle as pickle +from w3lib.http import headers_dict_to_raw, headers_raw_to_dict + from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.http import Headers from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.core.downloader.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint -from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.project import data_path diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index cbeb41050..204f021ca 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,6 +1,7 @@ +from w3lib.url import urljoin_rfc + from scrapy import log from scrapy.http import HtmlResponse -from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest from scrapy.conf import settings diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index 8ab028c9e..4a04b36de 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -12,14 +12,14 @@ from ftplib import FTP from shutil import copyfileobj from zope.interface import Interface, implements - from twisted.internet import defer, threads +from w3lib.url import file_uri_to_path + from scrapy import log, signals from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object -from scrapy.utils.url import file_uri_to_path from scrapy.conf import settings diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index fb3fd661b..0f979d3c1 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -4,9 +4,10 @@ HTMLParser-based link extractor from HTMLParser import HTMLParser +from w3lib.url import safe_url_string, urljoin_rfc + from scrapy.link import Link from scrapy.utils.python import unique as unique_list -from scrapy.utils.url import safe_url_string, urljoin_rfc class HtmlParserLinkExtractor(HTMLParser): diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 1f29a6797..cac87ec1d 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -3,9 +3,9 @@ This module implements the HtmlImageLinkExtractor for extracting image links only. """ - +from w3lib.url import urljoin_rfc from scrapy.link import Link -from scrapy.utils.url import canonicalize_url, urljoin_rfc +from scrapy.utils.url import canonicalize_url from scrapy.utils.python import unicode_to_str, flatten from scrapy.selector.libxml2sel import XPathSelectorList, HtmlXPathSelector diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py index 1bf35501d..21b06cae8 100644 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ b/scrapy/contrib/linkextractors/lxmlparser.py @@ -7,10 +7,10 @@ because it collides with the lxml library module. from lxml import etree import lxml.html +from w3lib.url import safe_url_string, urljoin_rfc from scrapy.link import Link from scrapy.utils.python import unique as unique_list, str_to_unicode -from scrapy.utils.url import safe_url_string, urljoin_rfc class LxmlLinkExtractor(object): def __init__(self, tag="a", attr="href", process=None, unique=False): diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 1de044df3..9c349ad3a 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -1,7 +1,7 @@ import re -from scrapy.utils.url import urljoin_rfc -from scrapy.utils.markup import remove_tags, remove_entities, replace_escape_chars +from w3lib.url import urljoin_rfc +from w3lib.html import remove_tags, remove_entities, replace_escape_chars from scrapy.link import Link from .sgml import SgmlLinkExtractor diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 8fbcb35e7..fad6a8218 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -4,11 +4,13 @@ SGMLParser-based Link extractors import re +from w3lib.url import safe_url_string, urljoin_rfc + from scrapy.selector import HtmlXPathSelector from scrapy.link import Link from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode -from scrapy.utils.url import safe_url_string, urljoin_rfc, canonicalize_url, url_is_from_any_domain +from scrapy.utils.url import canonicalize_url, url_is_from_any_domain from scrapy.utils.response import get_base_url class BaseSgmlLinkExtractor(FixedSGMLParser): diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py index 98210fa11..eb6e32f2e 100644 --- a/scrapy/contrib_exp/crawlspider/reqext.py +++ b/scrapy/contrib_exp/crawlspider/reqext.py @@ -1,9 +1,10 @@ """Request Extractors""" +from w3lib.url import safe_url_string, urljoin_rfc + from scrapy.http import Request from scrapy.selector import HtmlXPathSelector from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import FixedSGMLParser, str_to_unicode -from scrapy.utils.url import safe_url_string, urljoin_rfc from itertools import ifilter diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index f53642f21..d01c37c9c 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,5 +1,5 @@ +from w3lib.url import file_uri_to_path from scrapy.core.downloader.responsetypes import responsetypes -from scrapy.utils.url import file_uri_to_path from scrapy.utils.decorator import defers class FileDownloadHandler(object): diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 0ab2258e4..fb612a55f 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,5 +1,5 @@ +from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict -from scrapy.utils.http import headers_dict_to_raw class Headers(CaselessDict): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index f791aded5..9f2161674 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -7,8 +7,9 @@ See documentation in docs/topics/request-response.rst import copy +from w3lib.url import safe_url_string + from scrapy.http.headers import Headers -from scrapy.utils.url import safe_url_string from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated from scrapy.http.common import deprecated_setter diff --git a/scrapy/shell.py b/scrapy/shell.py index 15da950b7..eaa850b05 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,6 +7,7 @@ See documentation in docs/topics/shell.rst import signal from twisted.internet import reactor, threads +from w3lib.url import any_to_uri from scrapy.item import BaseItem from scrapy.spider import BaseSpider @@ -14,7 +15,6 @@ from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector from scrapy.utils.spider import create_spider_for_request from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser -from scrapy.utils.url import any_to_uri from scrapy.utils.console import start_python_console from scrapy.settings import Settings from scrapy.http import Request, Response, HtmlResponse, XmlResponse diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index fa3977533..5e07b3f14 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -1,13 +1,13 @@ import os, urlparse +from cStringIO import StringIO from zope.interface.verify import verifyObject from twisted.trial import unittest from twisted.internet import defer -from cStringIO import StringIO +from w3lib.url import path_to_file_uri from scrapy.spider import BaseSpider from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage -from scrapy.utils.url import path_to_file_uri from scrapy.utils.test import assert_aws_environ class FeedStorageTest(unittest.TestCase): diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 446947255..31d024802 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -8,6 +8,7 @@ from twisted.web import server, static, util, resource from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource +from w3lib.url import path_to_file_uri from scrapy.core.downloader.webclient import PartialDownloadError from scrapy.core.downloader.handlers.file import FileDownloadHandler @@ -15,7 +16,6 @@ from scrapy.core.downloader.handlers.http import HttpDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spider import BaseSpider from scrapy.http import Request -from scrapy.utils.url import path_to_file_uri from scrapy import optional_features diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index e61b7aeb4..344d01f62 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -4,8 +4,8 @@ import re import hashlib from pkgutil import iter_modules +from w3lib.html import remove_entities from scrapy.utils.python import flatten -from scrapy.utils.markup import remove_entities def arg_to_iter(arg): """Convert an argument to an iterable. The argument can be a None, single diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 3e3c04742..c3ce51647 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -8,9 +8,10 @@ import weakref from base64 import urlsafe_b64encode from urlparse import urlunparse +from w3lib.http import basic_auth_header + from scrapy.utils.url import canonicalize_url from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.http import basic_auth_header _fingerprint_cache = weakref.WeakKeyDictionary()