From d5bd44a5b9fa28a44c44396d1f87c3697a5e2d42 Mon Sep 17 00:00:00 2001 From: muriloviana Date: Tue, 18 Oct 2016 17:29:16 -0200 Subject: [PATCH 1/5] add middleware to template project --- scrapy/commands/startproject.py | 1 + .../project/module/middleware.py.tmpl | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 scrapy/templates/project/module/middleware.py.tmpl diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e3989baaf..1001b8682 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -17,6 +17,7 @@ TEMPLATES_TO_RENDER = ( ('${project_name}', 'settings.py.tmpl'), ('${project_name}', 'items.py.tmpl'), ('${project_name}', 'pipelines.py.tmpl'), + ('${project_name}', 'middleware.py.tmpl'), ) IGNORE = ignore_patterns('*.pyc', '.svn') diff --git a/scrapy/templates/project/module/middleware.py.tmpl b/scrapy/templates/project/module/middleware.py.tmpl new file mode 100644 index 000000000..0ae9d7160 --- /dev/null +++ b/scrapy/templates/project/module/middleware.py.tmpl @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# http://doc.scrapy.org/en/latest/topics/spider-middleware.html + + +class ${ProjectName}SpiderMiddleware(object): + # This method is called for each response that goes through the + # spider middleware and into the spider, for processing. + + def process_spider_input(response, spider): + # Called for each response that goes to spider + pass + + def process_spider_output(response, result, spider): + # Called for each response with the spider results + pass + + def process_spider_exception(response, exception, spider): + # Called whenever a spider raises an exception + pass From 34f2014c55ed25615066f7017ddd36df871095bd Mon Sep 17 00:00:00 2001 From: muriloviana Date: Wed, 19 Oct 2016 00:37:31 -0200 Subject: [PATCH 2/5] change settings middleware name and updating middleware template --- scrapy/commands/startproject.py | 2 +- .../project/module/middleware.py.tmpl | 23 ----------- .../project/module/middlewares.py.tmpl | 39 +++++++++++++++++++ .../templates/project/module/settings.py.tmpl | 2 +- 4 files changed, 41 insertions(+), 25 deletions(-) delete mode 100644 scrapy/templates/project/module/middleware.py.tmpl create mode 100644 scrapy/templates/project/module/middlewares.py.tmpl diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1001b8682..594106632 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -17,7 +17,7 @@ TEMPLATES_TO_RENDER = ( ('${project_name}', 'settings.py.tmpl'), ('${project_name}', 'items.py.tmpl'), ('${project_name}', 'pipelines.py.tmpl'), - ('${project_name}', 'middleware.py.tmpl'), + ('${project_name}', 'middlewares.py.tmpl'), ) IGNORE = ignore_patterns('*.pyc', '.svn') diff --git a/scrapy/templates/project/module/middleware.py.tmpl b/scrapy/templates/project/module/middleware.py.tmpl deleted file mode 100644 index 0ae9d7160..000000000 --- a/scrapy/templates/project/module/middleware.py.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- - -# Define here the models for your spider middleware -# -# See documentation in: -# http://doc.scrapy.org/en/latest/topics/spider-middleware.html - - -class ${ProjectName}SpiderMiddleware(object): - # This method is called for each response that goes through the - # spider middleware and into the spider, for processing. - - def process_spider_input(response, spider): - # Called for each response that goes to spider - pass - - def process_spider_output(response, result, spider): - # Called for each response with the spider results - pass - - def process_spider_exception(response, exception, spider): - # Called whenever a spider raises an exception - pass diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl new file mode 100644 index 000000000..91d05baa6 --- /dev/null +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# http://doc.scrapy.org/en/latest/topics/spider-middleware.html + + +class ${ProjectName}SpiderMiddleware(object): + + def process_spider_input(response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + pass + + def process_spider_output(response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + pass + + def process_spider_exception(response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Response, dict + # or Item objects. + pass + + def process_start_requests(start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + pass diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index f13e85871..72f25ebef 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -47,7 +47,7 @@ ROBOTSTXT_OBEY = True # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomSpiderMiddleware': 543, +# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares From 38e292a13207443c9f966fb7168e37fb69965a7e Mon Sep 17 00:00:00 2001 From: muriloviana Date: Wed, 19 Oct 2016 12:47:23 -0200 Subject: [PATCH 3/5] add from_crawler method to template and turn the returns methods explicit --- .../project/module/middlewares.py.tmpl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 91d05baa6..4f7c8d191 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -5,22 +5,34 @@ # See documentation in: # http://doc.scrapy.org/en/latest/topics/spider-middleware.html +from scrapy import signals + class ${ProjectName}SpiderMiddleware(object): + # If any of methods bellow is not defined, in practice, the middleware + # manager only add the methods that the middleware has defined. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls(crawler.stats) + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s def process_spider_input(response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. - pass + return None def process_spider_output(response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. # Must return an iterable of Request, dict or Item objects. - pass + for i in result: + yield i def process_spider_exception(response, exception, spider): # Called when a spider or process_spider_input() method @@ -36,4 +48,5 @@ class ${ProjectName}SpiderMiddleware(object): # that it doesn’t have a response associated. # Must return only requests (not items). - pass + for r in start_requests: + yield r From 32fd69281066f16db7aab8741da4c08051884049 Mon Sep 17 00:00:00 2001 From: muriloviana Date: Wed, 19 Oct 2016 13:50:09 -0200 Subject: [PATCH 4/5] define method spider_opened and update class instructions --- scrapy/templates/project/module/middlewares.py.tmpl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 4f7c8d191..8b9827527 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -9,8 +9,9 @@ from scrapy import signals class ${ProjectName}SpiderMiddleware(object): - # If any of methods bellow is not defined, in practice, the middleware - # manager only add the methods that the middleware has defined. + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. @classmethod def from_crawler(cls, crawler): @@ -50,3 +51,6 @@ class ${ProjectName}SpiderMiddleware(object): # Must return only requests (not items). for r in start_requests: yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) From 09c401bf8e1381f2e9109b799fb2889b8c07a651 Mon Sep 17 00:00:00 2001 From: muriloviana Date: Wed, 19 Oct 2016 14:09:01 -0200 Subject: [PATCH 5/5] use crawler only to register signals in from_crawler() --- scrapy/templates/project/module/middlewares.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8b9827527..42318fec2 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -16,7 +16,7 @@ class ${ProjectName}SpiderMiddleware(object): @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. - s = cls(crawler.stats) + s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s