import logging

from testfixtures import LogCapture
from twisted.trial.unittest import TestCase
from twisted.internet import defer

from scrapy.spiders import Spider
from scrapy.item import Item, Field
from scrapy.http import Request
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer


class TestItem(Item):
    value = Field()


class LocalhostSpider(Spider):
    start_urls = ['http://localhost:8998']  # tests.mockserver.MockServer


# ================================================================================
# exceptions from a spider's parse method
class BaseExceptionFromParseMethodSpider(LocalhostSpider):
    custom_settings = {
        'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540}
    }


class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider):
    """ return value is NOT a generator """
    name = 'not_a_generator'

    def parse(self, response):
        raise AssertionError


class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider):
    """ return value is a generator; the exception is raised
    before the items are yielded: no items should be scraped """
    name = 'generator_error_before_items'

    def parse(self, response):
        raise ValueError
        for i in range(3):
            yield {'value': i}


class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider):
    """ return value is a generator; the exception is raised
    after the items are yielded: 3 items should be scraped """
    name = 'generator_error_after_items'

    def parse(self, response):
        for i in range(3):
            yield {'value': i}
        raise FloatingPointError


class CatchExceptionMiddleware(object):
    def process_spider_exception(self, response, exception, spider):
        """ catch an exception and log it """
        logging.warn('{} exception caught'.format(exception.__class__.__name__))
        return None


# ================================================================================
# exception from a previous middleware's process_spider_input method
# process_spider_input is not expected to return an iterable, so there are no
# separate tests for generator/non-generator implementations
class FromPreviousMiddlewareInputSpider(LocalhostSpider):
    name = 'not_a_generator_from_previous_middleware_input'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
            'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545,
            # spider side
        }
    }

    def parse(self, response):
        return None


class RaiseExceptionOnInputMiddleware(object):
    def process_spider_input(self, response, spider):
        raise LookupError


# ================================================================================
# exception from a previous middleware's process_spider_output method (not a generator)
class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider):
    name = 'not_a_generator_from_previous_middleware_output'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
            'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545,
            # spider side
        }
    }

    def parse(self, response):
        return [{'value': i} for i in range(3)]


class RaiseExceptionOnOutputNotAGeneratorMiddleware(object):
    def process_spider_output(self, response, result, spider):
        raise UnicodeError


# ================================================================================
# exception from a previous middleware's process_spider_output method (generator)
class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider):
    name = 'generator_from_previous_middleware_output'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
            'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545,
            # spider side
        }
    }

    def parse(self, response):
        return [{'value': i} for i in range(10, 13)]


class RaiseExceptionOnOutputGeneratorMiddleware(object):
    def process_spider_output(self, response, result, spider):
        for r in result:
            yield r
        raise NameError


# ================================================================================
# do something useful from the exception handler
class DoSomethingSpider(LocalhostSpider):
    name = 'do_something'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.DoSomethingMiddleware': 540,
            'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,
            # spider side
        }
    }

    def parse(self, response):
        yield {'value': response.url}
        raise ImportError


class DoSomethingMiddleware(object):
    def process_spider_exception(self, response, exception, spider):
        return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')]


# ================================================================================
# don't catch _InvalidOutput from scrapy's spider middleware manager
class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider):
    name = 'invalid_return_value_from_previous_middleware_input'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540,
            'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,
            # spider side
        }
    }

    def parse(self, response):
        return None


class InvalidReturnValueInputMiddleware(object):
    def process_spider_input(self, response, spider):
        return 1.0  # <type 'float'>, not None


class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider):
    name = 'invalid_return_value_from_previous_middleware_output'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,
            'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545,
            # spider side
        }
    }

    def parse(self, response):
        return None


class InvalidReturnValueOutputMiddleware(object):
    def process_spider_output(self, response, result, spider):
        return 1  # <type 'int'>, not an iterable


# ================================================================================
# make sure only non already called process_spider_output methods
# are called if process_spider_exception returns an iterable
class ExecutionChainSpider(LocalhostSpider):
    name = 'execution_chain'
    custom_settings = {
        'SPIDER_MIDDLEWARES': {
            # engine side
            'tests.test_spidermiddleware.ThirdMiddleware': 540,
            'tests.test_spidermiddleware.SecondMiddleware': 541,
            'tests.test_spidermiddleware.FirstMiddleware': 542
            # spider side
        },
    }

    def parse(self, response):
        return None


class FirstMiddleware(object):
    def process_spider_output(self, response, result, spider):
        for r in result:
            if isinstance(r, dict):
                r['handled_by_first_middleware'] = True
            yield r

    def process_spider_exception(self, response, exception, spider):
        # log exception, handle control to the next middleware's process_spider_exception
        logging.warn('{} exception caught'.format(exception.__class__.__name__))
        return None


class SecondMiddleware(object):
    def process_spider_output(self, response, result, spider):
        for r in result:
            if isinstance(r, dict):
                r['handled_by_second_middleware'] = True
            yield r
        raise MemoryError


class ThirdMiddleware(object):
    def process_spider_output(self, response, result, spider):
        for r in result:
            if isinstance(r, dict):
                r['handled_by_third_middleware'] = True
            yield r

    def process_spider_exception(self, response, exception, spider):
        # handle control to the next middleware's process_spider_output
        return [{'item': i} for i in range(3)]


class TestSpiderMiddleware(TestCase):

    @classmethod
    def setUpClass(cls):
        cls.mockserver = MockServer()
        cls.mockserver.__enter__()

    @classmethod
    def tearDownClass(cls):
        cls.mockserver.__exit__(None, None, None)

    @defer.inlineCallbacks
    def crawl_log(self, spider):
        crawler = get_crawler(spider)
        with LogCapture() as log:
            yield crawler.crawl()
        raise defer.returnValue(log)

    @defer.inlineCallbacks
    def test_process_spider_exception_from_parse_method_non_generator(self):
        # non-generator return value
        log = yield self.crawl_log(NotAGeneratorSpider)
        self.assertIn("AssertionError exception caught", str(log))
        self.assertIn("spider_exceptions/AssertionError", str(log))

    @defer.inlineCallbacks
    def test_process_spider_exception_from_parse_method_generator_no_items(self):
        # generator return value, no items before the error
        log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider)
        self.assertIn("ValueError exception caught", str(log))
        self.assertIn("spider_exceptions/ValueError", str(log))

    @defer.inlineCallbacks
    def test_process_spider_exception_from_parse_method_generator_with_items(self):
        # generator return value, 3 items before the error
        log = yield self.crawl_log(GeneratorErrorAfterItemsSpider)
        self.assertIn("'item_scraped_count': 3", str(log))
        self.assertIn("FloatingPointError exception caught", str(log))
        self.assertIn("spider_exceptions/FloatingPointError", str(log))

    @defer.inlineCallbacks
    def test_process_spider_exception_from_previous_middleware_input(self):
        log = yield self.crawl_log(FromPreviousMiddlewareInputSpider)
        self.assertIn("LookupError exception caught", str(log))

    @defer.inlineCallbacks
    def test_process_spider_exception_from_previous_middleware_output(self):
        # non-generator output value
        log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider)
        self.assertNotIn("UnicodeError exception caught", str(log))
        # generator output value
        log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider)
        self.assertIn("'item_scraped_count': 3", str(log))
        self.assertIn("NameError exception caught", str(log))

    @defer.inlineCallbacks
    def test_process_spider_exception_do_something(self):
        log = yield self.crawl_log(DoSomethingSpider)
        self.assertIn("ImportError exception caught", str(log))
        self.assertIn("{'value': 10}", str(log))
        self.assertIn("{'value': 'asdf'}", str(log))
        self.assertIn("{'value': 'http://localhost:8998'}", str(log))
        self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log))

    @defer.inlineCallbacks
    def test_process_spider_exception_invalid_return_value_previous_middleware(self):
        """ don't catch _InvalidOutput from middleware """
        # on middleware's input
        log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider)
        self.assertNotIn("_InvalidOutput exception caught", str(log1))
        self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1))
        # on middleware's output
        log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider)
        self.assertNotIn("_InvalidOutput exception caught", str(log2))
        self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2))

    @defer.inlineCallbacks
    def test_process_spider_exception_execution_chain(self):
        # on middleware's input
        log = yield self.crawl_log(ExecutionChainSpider)
        self.assertNotIn("handled_by_first_middleware", str(log))
        self.assertNotIn("handled_by_second_middleware", str(log))
        self.assertIn("MemoryError exception caught", str(log))
        self.assertIn("handled_by_third_middleware", str(log))