diff --git a/sep/sep-002.rst b/sep/sep-002.rst index 2e8a28340..c467cb402 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError(f"Expected iterable, got {type(value).__name__}") + raise TypeError("Expected iterable, got %s" % type(value).__name__) def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b9f5e556f..05b0eb99c 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print(f"{len(scraped_items)} items scraped") + print "%d items scraped" % len(scraped_items) # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 4e3340521..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given ' - f'{rule.matcher!r} in {rule!r}') + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) # prepare callback if callable(rule.callback): @@ -553,7 +553,8 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError(f'Invalid callback {callback!r} can not be resolved') + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index d0169b81e..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = f"{url.scheme}://{netloc}/robots.txt" + robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = fr'^(.*\.)?({"|".join(domains)})$' + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed