From d9437fd3d97430d39ea408bca7c200ec042e1ab7 Mon Sep 17 00:00:00 2001 From: rootavish Date: Wed, 6 Apr 2016 08:47:06 +0530 Subject: [PATCH] Modifying existing gzip read failure recovery mechanism to patch read for broken archives --- scrapy/utils/gz.py | 2 +- .../compressed/unexpected-eof-output.txt | 1 + tests/sample_data/compressed/unexpected-eof.gz | Bin 0 -> 5134 bytes tests/test_utils_gz.py | 10 ++++++++++ 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/sample_data/compressed/unexpected-eof-output.txt create mode 100644 tests/sample_data/compressed/unexpected-eof.gz diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index afc7ed128..73c2eb73b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -43,7 +43,7 @@ def gunzip(data): # contains the whole page content if output or getattr(f, 'extrabuf', None): try: - output += f.extrabuf + output += f.extrabuf[-f.extrasize:] finally: break else: diff --git a/tests/sample_data/compressed/unexpected-eof-output.txt b/tests/sample_data/compressed/unexpected-eof-output.txt new file mode 100644 index 000000000..3b201255f --- /dev/null +++ b/tests/sample_data/compressed/unexpected-eof-output.txt @@ -0,0 +1 @@ +document.write('¡¡¡¡¡°ÎҽкéС±¦£¬ÊÇÒ»¸ö·Ç³£¸ß¶Ë·Ç³£Å£±ÆµÄÖ°Òµ¶þÊÀ×棬ȴÔÚÒ»¸öÔºڷç¸ßÉìÊÖ²»¼ûÎåÖ¸µÄÒ¹ÍíÎÒĪÃûÆäÃîµÄ¾Í±»ÀÏ°Ö¸øÒ»½Åõßµ½ÁËÕâô¸öµØ·½¡­¡­¡±

¡¡¡¡¡°ÔÚ´©Ô½µÄÄÇÒ»¿Ì£¬ÎÒÏëÆðÁËÔÚѧУѧµÄÓïÊýÍâÎﻯÉú¡ª¡ªÎÒ¶¼²»»á¡­¡­¡±

¡¡¡¡¡­¡­

¡¡¡¡ÌìÔ˴󽣬³àÔƹú»Ê¹¬Ö®ÖУ¬»ÊµÛºéÎÄÇåµÄÊé·¿ÐþÇå¸óÄÚ¡£

¡¡¡¡¡°±ÝÏ£¬¡±Ò»Ãû»¤ÎÀ´Ò´ÒÀ´±¨£º¡°Íâ³öÔÆÓÎÊ®¶þÄêµÄ¹úʦ»ØÀ´ÁË£¬ÏÖÔÚÕýÔÚÐþÑôµîÄڵȺò±ÝÏ¡£¡±

¡¡¡¡¡°Å¶£¿¹úʦ»ØÀ´ÁË£¿¡±Ìýµ½»¤ÎÀµÄ»°£¬»ÊµÛºéÎÄÇåÏÈÊÇһ㶣¬Ëæºó·ÅÏÂÊÖÖеÄ×àÕÛ£¬ÆðÉíЦµÀ£º¡°Í¨±¨Ò»Éù£¬Ëµë޾͵½¡£¡±

¡¡¡¡ºéÎÄÇ壬ËÄÊ®°ËË꣬³àÔƹúµÚÊ®°ËÈλʵۣ¬ÉúµÄ·½Ãæ´ó¶ú£¬ÏàòÌÃÌã¬ÓÈÆä°®ÃñÈç×Ó£¬ÉîÊÜ°ÙÐÕ°®´÷£¬ËãµÃÉÏÊÇÖιúÓз½µÄÒ»´úÃ÷¾ý¡£

¡¡¡¡¡°ÊÇ¡£¡±»¤ÎÀ½ÓÁÕâ¾Í×¼±¸À뿪£¬²»¹ý×ßÖ®ºóºöÈ»¶ÙÁËÏ£¬ËæºóµÀ£º¡°¶ÔÁË£¬±ÝÏ£¬³ýÁ˹úʦ´óÈËÒÔ¼°ËûµÄÅ®¶ùÔƲÊóïС½ãÖ®Í⣬»¹ÓÐÃûÄêÇáÈË¡£¾ÝÊôÏ°µÖдò̽£¬´ËÈ˾Ý˵ÊÇÀ×Òô¸ó¶þ³¤ÀÏÒ¶·ÉºèÖ®×ÓÒ¶½úÄþ¡£¡±

¡¡¡¡¡°À×Òô¸ó¶þ³¤ÀÏÖ®×Ó£¿¡±ºéÎÄÇåÖåÁËÖåü£¬ËæºóÇáÇá»ÓÁË»ÓÊÖ£º¡°ÄãÏÈÏÂÈ¥°É¡£¡±

¡¡¡¡¡°ÊÇ¡£¡±

¡¡¡¡»¤ÎÀÏÂÈ¥ºó£¬ºéÎÄÇåÏëÁËÏ룬²»Ñ¡ÁúÅÛ£¬·´¶øÌØÒâ´©ÉÏÒ»Éí±ã×°£¬Ö®ºóÃþÁËÃþÏ°ÍÉϵĺúÐ룬à«à«µÀ£º¡°¹úʦÔÆÓÎÊ®¶þÄ꣬Èç½ñ»ØÀ´£¬Äª²»ÊÇÒòΪÄǸö»éÔ¼£¿¶÷£¬°´ÕÕÄê¼ÍÀ´Ë㣬ÏëÀ´Ó¦¸ÃÊÇÁË¡£Ö»²»¹ý£¬Õâ¸öÀ×Òô¸ó¶þ³¤ÀϵĶù×Ó£¬ÓÖÀ´×öʲô£¿¡±

¡¡¡¡¡­¡­

¡¡¡¡ÐþÑôÒéÊ´óµîÖ®ÄÚ¡£

¡¡¡¡¡°¹þ¹þ£¬Ê®¶þÄê²»¼û£¬¹úʦ·ç²ÉÓÈʤÍùÎô£¬Ïë±ØÎäѧÓÖÓнø¾³£¬¿Éϲ¿ÉºØ°¡¡£¡±»ÊµÛºéÎÄÇåÒ»½øÒéÊ´óµî£¬µÚÒ»ÑÛ¿´µ½µîÄÚÒ»ÃûÇàÅÛÄÐ×Ó£¬Á¢Ê±±ã¼±ÐÐÁ½²½£¬Á½ÈËË«ÊÖÏàÎÕ£¬¹þ¹þ´óЦ£¬ºéÎÄÇåµÀ£º¡°ÕâÒ»±ðÊ®¶þÄ꣬¿ÉÕæÊÇÏëÉ·ÎÒÒ²£¡¡±

¡¡¡¡ÄÇÖÐÄêÄÐ×Ó¿´Äê¼Í´óÔ¼ÎåÊ®À´Ë꣬ÊÖÖÐÒ»°Ñ¶ìëÓðÉÈ£¬Í·ÉÏÁ½³ßÇà½í£¬ò¢ÏÂÈý縳¤Ð룬ÉúµÄÏÉ·çµÀ¹Ç£¬¹ËÅÎÖ®¼ä¼«ÓÐÆøÊÆ¡£

¡¡¡¡ÄܽкéÎÄÇåÕâλ»ÊµÛÈç´Ë¶Ô´ý£¬´ËÈËÕýÊdzàÔƹú»¤¹úÎäÉñ£¬±»×ðΪ¹úʦµÄÆßÐÇÎäÕߣ¬ÔÆÊ¥Ðù£¡

¡¡¡¡¡°±ÝÏ¿ÍÆø¡£¡±ÉíΪ»¤¹úÎäÉñ£¬ÔÆÊ¥ÐùÓë»ÊµÛºéÎÄÇåÖ®¼äµÄ¹ØϵһÏò¼«ÎªÁ¼ºÃ£¬ÏòÀ´Æ½Æðƽ×ø£¬Ò²²»ÒâÍ⣬΢ЦµÀ£º¡°´Ë·¬ÔÆijÔÆÓιéÀ´£¬ÊµÊÇΪÁËСŮ²ÊóïµÄ»éʶøÀ´¡£¡±

¡¡¡¡¹ûÈ»£¡

¡¡¡¡ÌýÁËÔÆÊ¥ÐùµÄ»°£¬ºéÎÄÇåµ±¼´Ïò֮ǰһֱվÔÚÔÆÊ¥Ðù²»Ô¶´¦µÄÒ»ÃûÉÙÅ®¿´È¥¡£

¡¡¡¡ÄÇÉÙÅ®´©×ÅÒ»Éíµ­ÂÌÉ«µÄ³¤È¹£¬´óÔ¼¶þ°ËÄ껪£¬Ò»Ë«ËÆË®µÄíø×Ó£¬Áé¹âÉÁ¶¯¡£·ôÈçÄýÖ¬£¬Ñ©°×ÖÐ͸×ŷۺ죬ËƺõÄÜÅ¡³öË®À´¡£ÓñÊÖʮָÏËÏË£¬ÈáÈôÎ޹ǣ¬ÇáÇáµÄÎÕ×Åһ֧ͨÌå±ÌÂ̵ÄÓñóһͷ¼°ÑüÇàË¿Ëæ·çÎ趯£¬ÏÉ×Ó°ãÉñ²ÊÍÑËס£

¡¡¡¡¡°ÔƲÊó¼û¹ý±ÝÏ¡£¡±ÑÛ¼ûºéÎÄÇåÍûÀ´£¬ÉÙÅ®µ±¼´ÐÐÀñ¡£

¡¡¡¡¾øÃÀµÄÈË£¬¾øºÃµÄ³öÉí¡£

¡¡¡¡ÕýÊÇ»¤¹úÎäÉñÔÆÊ¥ÐùΨһµÄÕÆÉÏÃ÷Ö飬ÔƲÊó

¡¡¡¡¡°°¥Ñ½£¬ÕâÊ®¶þÄêû¼û£¬²Êóï¾ÓÈ»ÒѾ­³öÂäµÄÈç´ËÃÀÀö£¡¡±ºéÎÄÇå֮ǰ¾Í×¢Òâµ½ÁËÕâ¾øÃÀµÄÉÙÅ®£¬´ËʱÌýËýÈ·ÈÏ£¬Ô½·¢ÐÀϲ£¬Ð¦µÀ£º¡°´ó¼Ò¶¼²»ÊÇÍâÈË£¬½ÐÎÒÊåÊå°É£¬ÕâÑùÏÔµÃÇ×½üЩ¡£¡±

¡¡¡¡ÔƲÊóïÇáÇáµÄ¡°¶÷¡±ÁËÒ»Éù£¬ËãÊÇ´ðÓ¦¡£

¡¡¡¡ÔÆÊ¥ÐùÁ½È˺®êÑÍê±Ï£¬ºéÎÄÇå×îºó¿´ÏòÄÇÃû»¤ÎÀ¿ÚÖеÄÄêÇáÈË£¬Î¢Ð¦×ÅÎʵÀ£º¡°ÕâλÉÙÏÀÊÇ¡­¡­¡±

¡¡¡¡ÌýµÃºéÎÄÇåÎÊÆð£¬ÄêÇáÈ˸Ͻô±§È­ÐÐÀñ£º¡°Áù´óÕýͳ֮һ£¬À×Òô¸ó×ù϶þ³¤ÀÏÖ®×Ó£¬Ò¶½úÄþ¼û¹ý±ÝÏ¡£¡±ÕâÒ¶½úÄþÉúµÄÒDZíÌÃÌ㬽£Ã¼ÐÇÄ¿£¬Ò»Éí°×ÒÂÈçÑ©£¬¶ËµÄÒ»¸±ºÃÂôÏà¡£Ö»²»¹ýËûËäÈ»ÊÇÔÚÏòºéÎÄÇåÐÐÀñ£¬¿ÉÊÇÑÛÉñÈ´²»Ê±ÍµÍµ¿´ÏòÔƲÊóÐÐΪ¹îÒì¡£

¡¡¡¡¡°¶÷£¬Ó¢ÐÛ³öÉÙÄ꣬²»´í£¬²»´í¡£¡±Ò¶½úÄþµÄÑÛÉñ±»ºéÎÄÇå¿´ÔÚÑÛÀ˲Ϣ֮¼äºéÎÄÇå±ãÃ÷°×¹ýÀ´£¬Ëûüͷ΢΢ÖåÁËÖ壬²»¹ýºÜ¿ìÊæÕ¹¿ªÀ´£¬ÇáÇáµãÁ˵ãÍ·£¬Í¬Ê±ÐÄÖÐÈ´°µµÀ£º¡°²ÊóïÓ×Äê±ãÒòΪÌ츳¼«¼Ñ£¬ËÍÍùÁù´óÕýͳ֮һµÄÀ×Òô¸óÐÞÁ¶£¬Èç½ñËý»ØÀ´£¬È´¸úÀ´ÁËÕâôһ¸öÈË£¬¿´Ñù×Ó½ñÌìµÄÊÂÇ飬ÅÂÊÇûÄÇô¼òµ¥¡£¡±

¡¡¡¡¡°ºÇºÇ£¬±ÝÏ£¬¡±Èç½ñÈËÒѼû¹ý£¬ÔÆÊ¥ÐùÓðÉÈÇáÒ¡£¬Ð¡ÉùÌáÐѵÀ£º¡°°´ÕÕµ±ÄêµÄ»éÔ¼£¬²»Öª¾Å»Ê×Ó£¬¿É»¹°²ºÃ£¿¡±

¡¡¡¡³àÔƹú£¬Ã¿Èλʵۣ¬Óëµ±´ú»¤¹úÎäÉñ£¬±ØÐëÒª½á³É»éÔ¼£¬ÓÃÒÔÀ­½üË«·½Ö®¼äµÄÓѺùØϵ¡£

¡¡¡¡Èç¹û»¤¹úÎäÉñÑ¡µÄÊÇÄУ¬ÄǻʵÛÕâ±ß£¬±ãÑ¡Ò»¹«Ö÷ϼޣ»¶øÈôÊÇ»¤¹úÎäÉñÕâ±ßÊÇÅ®£¬ÄǻʵÛÕâ±ß£¬ÔòҪѡһÃû»Ê×ÓÓ­È¢´ËÅ®¹ýÃÅ¡£

¡¡¡¡±¾Èλ¤¹úÎäÉñÔÆÊ¥ÐùÏ¥ÏÂÖ»ÓÐһŮ£¬¹Ê¶ø£¬ºéÎÄÇå±ãÐèҪѡ³öÒ»Ãû»Ê×ÓÀ´ÓëÖ®³É»é¡£¶øÕâÃû±»Ñ¡¶¨µÄ»Ê×Ó£¬ÕýÊÇÓëÔƲÊóïÄêÁäÏ൱£¬×îÊܺéÎÄÇåϲ°®ÐÄÌ۵ľŻÊ×Ó£¬ºéС±¦£¡

¡¡¡¡Ö®ËùÒԽкéС±¦£¬ÊÇÒòΪ¾Å»Ê×Ó³öÉúʱÉí×Ó¹ÇÏà¶Ôµ¥±¡£¬ÓÖÊÇ×îСµÄ±¦±´¸í´ñ£¬ËùÒÔÆðÃûС±¦£¬Òâ˼ÊÇÏ£ÍûËûÏñ´«¼Ò±¦Ò»Ñù¼áͦ¡ª¡ªËµÆðÀ´£¬Õâ¸öÃû×Ö»¹Êǵ±ÄêÔÆÊ¥ÐùÇ××ÔÈ¡µÄ¡£

¡¡¡¡¡°°¦£¬ÎÒ¼ÒС±¦Õ⺢×Ó£¬¡±ÌáÆð×Ô¼ºÕâ¸ö×îÌÛ°®È´Ò²ÊÇ×î½ÐÈËÍ·Ì۵ĵھÅ×Ó£¬ºéÎÄÇåÒ²ÊÇÓÐЩÓôÃÆ£º¡°ËûÒ»Ììµ¹ÊÇͦ¿ªÐĵģ¬¾ÍÊÇÓÐЩ²»ÎñÕýÒµ£¬½ÐÈËÍ·ÌÛ¡£¡±

¡¡¡¡¡°Å¶£¿²»ÎñÕýÒµ£¿¡±ÌýÁËÕâ»°£¬ÔÆÊ¥Ðù¶ÙʱÖåüµÀ£º¡°Äª·Çµ±ÄêÎÒÁô¸øËûµÄÄDz¿¡¶Á÷ÔÆӥצ¹¦¡·£¬ËûÒ»Ö±¶¼Ã»ÓÐÐÞÁ¶£¿¡±

¡¡¡¡ºéС±¦ÌåÖÊÈõ£¬ÔÆÊ¥ÐùÊÇÖªµÀµÄ£¬Ëû³öÓÎÇ°ÌØÒâÁô¸øºéС±¦Ò»²¿¹¦·¨Ãؼ®½ÐËûÐÞÁ¶£¬ÒÔ±ã½ÐËûÇ¿Éí½¡Ì壬¿ÉÊÇÌýºéÎÄÇåµÄ»°£¬Õ⹦·¨£¬ËûûÁ·£¿

¡¡¡¡¡°°¦£¬Ò»ÑÔÄѾ¡°¡£¬¡±ºéÎÄÇ峤̾¿ÚÆø£¬ÎÞÄεÄÒ¡ÁËÒ¡Í·£¬µÀ£º¡°¹úʦ»¹ÊÇÓëÎÒÒ»µÀ£¬ÏÈÈ¥¿´¿´ÔÙ˵°É¡£¡±

¡¡¡¡ÕâÊÇÔõô»ØÊ£¬¿´Ñù×ÓÊÂÇéºÃÏñ²»´ó¶ÔÍ·¡£

¡¡¡¡ÔÆÊ¥ÐùÓëÅ®¶ùÔƲÊóï¶ÔÊÓÒ»ÑÛ£¬Ò»ÆðµãÍ·£º¡°ºÃ¡£¡±Õâ¾Í³öÃÅ£¬Ç°Íù¾Å»Ê×ÓËùÔÚµÄÐþÏö¸ó¡£Ò»Â·ÉÏ£¬Ò¶½úÄþ½ô¸úÔƲÊóïÉí±ß£¬ºéÎÄÇå¿´ÔÚÑÛÖУ¬ÐÄÏÂÒѾ­ÁËÈ»¡£

¡¡¡¡½á¹û¸ÕÒ»½øÐþÏö¸óµÄ´óÃÅ£¬¾Í¿´µ½ÎÞÊýµÄÆÍÈËÄÃ×ÅÌúDzù×Óһ··É±¼£¬Óë´ËͬʱһÕó¶ñ³ôζ´«À´£¬ºéÎÄÇåµÈËÄÈËŤͷ¿´È¥£¬¾ÓÈ»¿´µ½¼¸ÃûÆÍÈËÍƹýÀ´Ò»Á¾·à³µ£¡

¡¡¡¡Ò»ÏòÏÉ·çµÀ¹ÇÆÄÓеãƮƮºõÓ𻯵ÇÏÉζµÀµÄÔÆÊ¥ÐùÖ±½Ó¾Í¿´´ôÁËÑÛ£º¡°Ê²Ã´Çé¿ö£¿¡±

¡¡¡¡ÕýËùνʳö·´³£±ØÓÐÑý£¬Ò»½øÞͿ´µ½Õâô¸öÇé¿ö£¬ËÄÈË·´µ¹²»×ż±ÁË£¬ºéÎÄÇåÖÆÖ¹ÁËÓûÒªÇë°²µÄÆÍÈË£¬×öÁ˸öàäÉùµÄÊÖÊÆ£¬Ëæºó¸úÔÚºóÃ棬¿´¿´Õâµ½µ×ÊÇÔÚ¸ãʲôÐþ»ú¡£

¡¡¡¡Ò»Â·Ç°ÐУ¬ºÜ¿ì£¬ËÄÈ˾ÍÌýµ½Ò»°ÑÂÔ΢ÓÐЩÄêÇáµÄÉùÒôÏìÆ𡪡ª

¡¡¡¡¡°ÚÀ£¬ÊÖ½ÅÒ»¶¨Òª×Ðϸµã£¬¶¼Ìýµ½ÁËÂð£¿Õâ¿ÉÊÇ×î¹Ø¼üµÄÒ»²½£¬²»ÄÜÓаëµã²î´í£¡¡±ÄÇÉùÒôÌýÆðÀ´´óÔ¼¶þÊ®À´Ë꣬ËäÈ»²»ËãµÍ³ÁÐÛºñ£¬¿ÉÊÇÌýÔÚ¶úÖе¹Ò²Í¦Êæ·þ£¬ºéÎÄÇåЦµÀ£º¡°Ð¡±¦Õ⺢×Ó£¬²»ÖªÔÚ¸ãʲô£¬ÎÒÃÇÇÒÏÈ¿´ËûÒ»¿´£¡¡±

¡¡¡¡ÔÆÊ¥ÐùºÍÔƲÊóïÒ²ÊǺÃÆ棬Ҷ½úÄþÔò¹´Á˹´×ì½Ç£¬³¶³öһ˿²»Ð¼µÄЦÒâ¡£ÈýÈ˸úºéÎÄÇåÒ»ÆðÈÌס³ô棬ÕÒÁ˸öÒþ±ÎµÄµØ·½£¬ÍµÍµ¹Û¿´¡£

¡¡¡¡ÕâÀïµØ´¦ÐþÏö¸óµÄºóÔº£¬ÖÜΧÂÌÊ÷´Ð´Ð£¬Ð¡ÇÅÁ÷Ë®£¬·ç¾°¼«¼Ñ¡£¶ø¾Å»Ê×ÓºéС±¦£¬ÔòÕýÖ¸»Ó×ÅÆÍÈËÃÇÔÚµØÏÂÍÚ×Åʲô£¬±ßÉÏ»¹·Å×Ų»ÉÙµÄÌú¹Ü¡¢ÌÕ´ÉƬ¡¢Ä¾Í·Ö®ÀàµÄÆ÷Îï¡£×îÆæÃîµÄÊÇ£¬¾ÍÔÚºéС±¦µÄ²»Ô¶´¦£¬»¹°Ú×ÅÒ»¸öÆæ¹ÖµÄ°×É«ÌÕ´ÉÔì¾ÍµÄ¶«Î÷£¡

¡¡¡¡ÄǸö¶«Î÷ÓÐ×ÅÒ»¸öÍÖÔ²ÐεĿÚ×Ó£¬ÏÂÃæÊǵ××ù£¬ºóÃ滹´ø×ÅÒ»¸ö·½·½ÕýÕýµÄÏä×Ó£¬²»ÖªºÎÓá£

¡¡¡¡²»¹ýºÜ¿ìËÄÈ˾ÍÖªµÀÁË¡£Ö»ÌýºéС±¦ËµµÀ£º¡°¶¼Ð¡ÐÄЩ¹þ£¬Õâ¿ÉÊÇÎÒÐÂÑÐÖƳöÀ´µÄÂíÍ°£¬ºÜ½ð¹óµÄ£¬ÌÕ´É×öµÄ£¬±ðÅö»µÁË£¡¡±

¡¡¡¡ÂíÍ°£¿ÄÇÊÇʲô¶«Î÷£¿

¡¡¡¡Ç¿ÈÌסÐÄÖкÃÆ棬ºéÎÄÇåËÄÈ˼ÌÐø͵¿´¡£

¡¡¡¡¡°¶Ô¶Ô¶Ô£¬¾ÍÕâÑù£¬¼ÌÐøÍÚ£¬¶÷£¬ÔÙÍÚÉîµã£¬´óµã£¬ÒªÖ±ÉÏֱϵģ¡¡±ºéС±¦¼ÌÐøÖ¸»Ó£¬È´²»ÏëÆÍÈËÃÇÕýÍÚ×ÅÄØ£¬ºöȻһÃûÆÍÈË´óÉùµÀ£º¡°¾Å»Ê×Ó£¬ÎÒÍÚ³öÀ´Ò»¸ö¶«Î÷£¡¡±

¡¡¡¡ºéС±¦ºÃÆæµÀ£º¡°Ê²Ã´¶«Î÷£¿¸øÎÒ¿´¿´£¡¡±

¡¡¡¡¡°ÔÚÕâÄØ¡£¡±ÄÇÆÍÈ˸Ͻô°ÑÍÚµ½µÄ¶«Î÷½»µ½ºéС±¦Õâλ¾Å»Ê×ÓµÄÊÖÉÏ¡£¶«Î÷ºÜÔ࣬ÉÏÃæÕ´ÂúÄàÍÁ£¬ºéС±¦Ò²²»½éÒ⣬Éì³öÐä×ÓÖ±½Ó²ÁÁ˸ɾ»£¬Ö®ºóÖÕÓÚ¿´µ½Õâµ½µ×ÊǸöʲô¶«Î÷¡£

¡¡¡¡ÄÇÊÇÒ»¸ö·Ç³£Æ¯ÁÁµÄÊÖïí£¬ÉÏÃæÏâןì³È»ÆÂÌÇàÀ¶×Ϻڰ×Ò»¹²¾Å¸öÑÕÉ«µÄÓñʯ£¬ÄÇÓñʯÉÏ»¹É¢·¢×Åһ˿µ­µ­µÄÎíÆø£¬ÃþÔÚÊÖÀﻹÓÐЩůůµÄ£¬·Ç³£ÉñÆæ¡£

¡¡¡¡¡°ºÃ¶«Î÷£¡¡±ºéС±¦´óϲ£¬Ö±½Ó°ÑÄÇ´®Öé×Ó´÷ÊÖÍóÉÏ£¬Ð¦µÀ£º¡°Ã¿ÈËÉͶþÁ½£¬»ØÍ·×Ô¼ºÈ¥Á죡¡±

¡¡¡¡ÖÚÆÍÈ˸ж÷´÷µÂ£º¡°Ð»¾Å»Ê×Ó£¡¡±È»ºó¼ÌÐø¿ªÍÚ£¡

¡¡¡¡ÕâÒ»ÍÚ¾Í×ã×ãÍÚÁËС°ë¸öʱ³½£¬ÖÕÓÚ´ó¿ÓÍں㬾ŻÊ×ÓÏȽÐÈË°ÑÀïÃæÓôóÔ¼Áù³ß×óÓÒ³¤¶ÌµÄÌÕ´ÉƬµæºÃ£¬È»ºóϹÜÂñÍÁ£¬ÔÙÖ®ºó°Ñ·à³µÍÆÀ´£¬ºéС±¦ÑÚ×űÇ×Ó½ÐÆÍÈË°Ñ·à³µÀïÃæÄǶñ³ôµÄ±ãÄçÖ®ÎïÈ«²¿µ¹ÁËÏÂÈ¥¡ª¡ªÕâһϳ¡Éϸü³ôÁË¡­¡­

¡¡¡¡ÕâЩ»î¸ÉÍ֮꣬ºó¾ÍÊÇÉÏ°å¹Ì¶¨£¬×îºóÓÖ°ÑÂíÍ°¹Ì¶¨ºÃ£¬Ò»Õó΢·ç´µ¹ý£¬×ÜËãÒ»Çи㶨¡£

¡¡¡¡¡°Ð¡±¦£¬ÄãÕâ¹Äµ·Ê²Ã´ÄØ£¿¡±ÑÛ¿´ÕâʱºòÆÍÈËÒѾ­¿ªÊ¼ÔÚÖÜΧ¸ÇС·¿×Ó£¬ºéÎÄÇåËÄÈËÕâ²Å×ß³öÀ´£¬ºéÎÄÇåÖ¸×ÅÄǸöÂíÍ°£¬ÎʵÀ£º¡°ÕⶫÎ÷¸ÉʲôµÄ£¿¡±

¡¡¡¡ÑÛ¼û¸¸»ÊÀ´ÎÊ£¬ºéС±¦ºÙºÙЦÁËЦ£¬Óе㲻ºÃÒâ˼£¬´ðµÀ£º¡°ÕⶫÎ÷ÎÒ¹ÜËû½Ð³éË®ÂíÍ°£¬¿ÉÓÐÓÃÁËÄØ£¬È¥Ã©²ÞµÄʱºòÖ»Òª×øÔÚÕâÉÏÃæ·½±ã£¬µÈ·½±ãÍêÁË£¬Ò»°´Õ⿪¹Ø£¬ÕâºóÃæË®ÏäÀïµÄË®¾ÍÁ÷ÏÂÀ´£¬È»ºó¾Í³å¸É¾»ÁË£¬ºÜʵÓõģ¡¡±

¡¡¡¡¡°Äã¹Äµ·Õâô°ëÌ죬¾ÍÊÇŪÁ˸öé²Þ³öÀ´£¿£¡¡±ºéÎÄÇåËÄÈËÈ«²¿¶¼¾ª´ôÁË£¡

¡¡¡¡ÓÈÆäÊÇÔÆÊ¥ÐùºÍÔƲÊó¸üÊÇÌýµÄÄ¿µÉ¿Ú´ô£¡Ò¶½úÄþÔòÊÇŤ¹ýÉíÈ¥£¬¼ç°ò²»×¡¶¶¶¯¡£

¡¡¡¡Õâ¾Å»Ê×Ó֮ǰºéÎÄÇå¾Í˵Ëû²»ÎñÕýÒµ£¬ÏÖÔÚ¿´À´£¬»¹ÕæÊÇ£¡

¡¡¡¡¡°°¡£¬µ±È»£¬²»¹âÊÇé²Þ£¬¡±ºéС±¦·Ç³£µÃÒ⣬µÀ£º¡°ÕâÊÇÎÒµÄÒ»Ïîз¢Ã÷£¬ÎÒ¹ÜÕâ¸ö½ÐÕÓÆø³Ø£¡¼òµ¥À´Ëµ£¬¾ÍÊǵ±ÕâЩ¡­¡­¶÷¶÷£¬»Æ°×Ö®ÎïÔÚÕâ±ã³ØÀï·¢½ÍÖ®ºó£¬»á²úÉúÒ»ÖÖÆøÌ壬¿ÉÒÔÓÃÀ´µã»ðÔì·¹Óõģ¬·½±ã£¬»·±££¬·Ç³£ºÃÓã¡¡±

¡¡¡¡ÔÆÊ¥ÐùÒѾ­ÓеãÌý²»ÏÂÈ¥ÁË¡ª¡ª²»ºÃºÃÁ·Î䣬¸ãÕâЩÓеÄûµÄµÄÂÒÆß°ËÔãµÄ¶«Î÷£¬ÓÐÓã¿

¡¡¡¡¡°¶÷£¬Ð¡±¦°¡£¬¡±ÔÆÊ¥ÐùÉÏÇ°Ò»²½£º¡°ÎÒµ±³õÁô¸øÄãµÄÄDz¿¡¶Á÷ÔÆӥצ¹¦¡·£¬Äã¿ÉÔøÏ°Á·£¿¡±

¡¡¡¡¡°ÄúÊÇÔƲ®²®£¿¡±¿´µ½ÔÆÊ¥Ðù£¬ºéС±¦¼±Ã¦ÉÏÇ°¼ûÀñ£º¡°ÔƲ®²®£¬ºÃ¾Ãû¼ûÀ²£¡Äú¸øÎÒÁôµÄ¹¦·¨£¬ÎÒÊÇÒ»¶¨ÒªÁ·µÄѽ£¡¡±

¡¡¡¡¡°ÄÇÁ·µÄÈçºÎÁË£¿¸øÎÒʹÀ´¿´¿´£¿¡±ÔÆÊ¥Ðù¼±ÎÊ¡£

¡¡¡¡¡°°¡£¬ÄÇÅÂÊDz»ÐУ¬¡±ºéС±¦ÂúÁ³²»ºÃÒâ˼µÄ»ØµÀ£º¡°Á·ÊÇÁ·ÁË£¬¾ÍÊÇûÁ·³É£¡¡±

¡¡¡¡ÕâʱºòÒ»ÅÔµÄÔƲÊóﶼÓеãÌý²»ÏÂÈ¥ÁË£¬ÖåÁËÖåü£¬Å¤¹ýÉí×Ó¡£Ò¶½úÄþÔò¡°àÍ¡±µÄÒ»ÉùЦÁ˳öÀ´£¬´Õµ½ÔƲÊóï¶ú±ß£¬Ð¡ÉùµÀ£º¡°¿´À´ÄãµÄÕâδ»é·ò£¬¹ûÈ»¸ú´«ËµÖÐÒ»Ñù£¬²»Ö»²»ÎñÕýÒµ£¬»¹ÊǸö·Ï²Ä£¡¡±

¡¡¡¡¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª

¡¡¡¡ÐÂÊéÐÂÆøÏó£¬ÐÂÊéÉÏ´«À²£¬ºé¼ÒС±¦¸ç»ªÀöµÇ³¡£¡

¡¡¡¡ºÀÃ˵ÄÐÖµÜÃÇ£¬ÄãÃÇ¿É»¹ÔÚ·ñ£¿

¡¡¡¡ÐµÄÕ÷³Ì¿ªÆô£¬»¹¿É·ñÅãÎÒÒ»Æð³Ë·çÆÆÀË£¬Åû¾£Õ¶¼¬£¿£¡

¡¡¡¡ÇóÊղأ¬ÇóÍƼö£¬Çó´òÉÍ£¬ÇóÊ®·ÖÆÀ¼ÛƱ£¡ÐÂÊéÇóÒ»ÇÐÖ§³Ö£¡£¡

¡¡¡¡¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª¡ª

¡¡¡¡ÁíÍâÍÆÒ»ÏÂÍê±¾ÀÏÊ顶ÖØÉúÖ®Éñ¼¶°Ü¼Ò×Ó¡·£¬¾ù¶©ÆÆÍò×÷Æ·£¬ÖÊÁ¿±£Ö¤£¬Êé»Ä¿É¿´¡£

¡¡¡¡ÆðµãÖÐÎÄÍøwww.qidian.com»¶Ó­¹ã´óÊéÓѹâÁÙÔĶÁ£¬×îС¢×î¿ì¡¢×î»ðµÄÁ¬ÔØ×÷Æ·¾¡ÔÚÆðµãÔ­´´£¡

¡¡¡¡ÆðµãÖÐÎÄÍø www.qidian.com »¶Ó­¹ã´óÊéÓѹâÁÙÔĶÁ£¬×îС¢×î¿ì¡¢×î»ðµÄÁ¬ÔØ×÷Æ·¾¡ÔÚÆðµãÔ­´´£¡ÊÖ»úÓû§Çëµ½m.qidian.comÔĶÁ¡£'); \ No newline at end of file diff --git a/tests/sample_data/compressed/unexpected-eof.gz b/tests/sample_data/compressed/unexpected-eof.gz new file mode 100644 index 0000000000000000000000000000000000000000..96211e4324b11a3ab16c13a4fb7ea3b443389c49 GIT binary patch literal 5134 zcmV+p6!GgHiwFSORRmW617ug*QrlV*{hXPn`GTqWnVIu24|A!hb85avD3D+jN3yl$ zizMH)ZC$o?mnCA705*hh4PXv2c1%sTg~8M$RUyV(y}MVhUaR%DfBp5hfBf>#U;p&q z|NQ;eU;g+94*!T=qzCoWfRna(!7(GXdS5Zie05)8__NDPJwj74BFdxF%D(b~B_~5X zm2~RE%1L$KzHri#s?ELQv`T1Hb1%J!7L0lC$hac3;Yw=D7t`uAQuXfd`$TWE!N`!S z_hh%CbMN7=^yh#7B~K+8%G-%fD1|hdeO~Y)|GanIX2~#o8}Q*FB&rK1h1Cnd;Ppw3r6GpgLr)e(F*4m{8}m(hMHW z9gPA1x|`oW3ImOT5=>uEm?ZU*E!K0mG6^Y>^Vi{k)9D+n>kzRZcA=6 zUhvaoI9MdZ;4!=-S9D0M`Yp!d__bQQ3g29NsL4tF1<~;uLRE1+Ki|T(j1^vBU+a@f z*Y7xK8W5Tt4EA`TcWqw=Tcov2>Yem1QtbygNzT7CT%xaCnFoWfs8bXfJ6l0QhWOd? z@Iqd&XM|RCCq2QR*9ggBN0}cO({#8|x&C;nbD$N&FZL#G?EIB94h+`;M7< zwOJfgpn;Jp&x{|9#TuB7iMe2XdfdL^g~6!dW|~xOoi_`-P*Q7)Y(Fj5C-r+vRX(;E zLXY+4l_?1Vn>VDqXfs4ko>(kOG?C2-?G}}Xq*rn#-vdq`r$u)J2-f1jnA0=!&|)0l z`0U*I9g(DgeA@>;$joAm&x|-`D~s-`6Xmh|rsYRU#jbx#i&-2g`iwrlYT5chJc3Qj zQB9BAEIl@~JZ1CHJXGU0Q)4H^?Bw7fST9_K2|m$p zkaoO43ndlEb?X(VS-n?J6OC^w&{0ty4CtBWwq!O27M!=_c&lnSW*dM^RcsbEuzJ6a z2%*XYawAS#a(3^O3>oqH|B=M0h)$i3JRw(U@zuc99&3*vLd!H!Qg{JVubhk^Fq`1OHI4JD{@PQJm}euTele7aZ>-l8$b?_l{^WyR*lI)9gHGXx?Vxrv zUKEvh?zwPkrU5;}Ynn15Do+y4J9dhIEIfvT9I0ZcTo^m?wQcc&Te1DNUfwqk^h;j2 z-2hH3PO9ZM*ioEK->#gHR?UR8Sl3I7+L5*s2feA&d&ZG|LP5t&2P02KJggI$DT%zu zSwC1f=93z}sm;72`vxXU&91!yqD^RTKQO(c!9Fh}J!dm`;JH_d*mQeS1lqpz3NHcj zqw)C*{Foc);;v63)xXttyh2;6gOd2SExD)nu6uem<%OtJQlk-$tGBU|F_~2$9xWqh zYp2Ll2gVehX93qO;*f(xLSvO_OI{nH+T2o~bM1pjSX`qVl}ll|MYXo``VN`po+ z%+Hk6M3J{pw~V>{v0H4)*5(VICjLs^&bl~sHx0MeZU%9hR=pq$(A>07Xk%)CKUt7j zV_G)=g?sz*`Aa0$iV-=k9@r6~gP!dhKhjfV$@1P8LW!H9j#84(k+Gb4Jkp3QH3}Q= z%l9215%Sk)weE4QR^&28@Q;mXlrI!{UYIg++yO6T3#(%JaHU?^2wHxL7oO=g(Ux0- zW4a@(L=U{kId+ak8#4(X{Dc^0rvD7K*T|J;o;LyP=m#gY*^||d9aFxJx1M3hOGgc< z7tccXO!8Os%dYB^CJ>OC`9jegqL1eel@CDH^_-Qyc&h!;Z-Gm!?>4X`70N_@ zFCVV@`knOFdXK%rhuaVGWUz3{X)#iHAy?__!SAejmyjFWIs-t`mcK)tiA?D-cYd8W zw7{fIq_=%3W>LpF3AcGcVK^^Uscv-c{DYM*usJ)j0NP~e-{yHPa{_^M%y7@Xv19;V za*0b2h!qcm0L14ix>%zq+EBa^>;c0*qQaI``7-|(Q_T}46hlgCeo+$^({Qf^;aJJv zK>5jkFO7{KVpdxqsoC7n+={p1ZRD1nZyxne%?s!2pw=wlnPrp5ZAQj!1q4}eq=2S@ zC5t5Q45l1s^h_F~arQ5&Wo<_bA-3y2tOHMI4t(&Xt>mW57P;+Oa&?AVA1=bUN@Mq% zatQXR-}EnFAMKbhEZvS{8Y*x_)^5q>tJR@y*3H#A#7P($R-q7k`K-j?Cgx$p0k)j| z&_4{qJb83>BH_lmbbBa2Ei}5SgN$g}YE3MGM!)!W?i0LVG*=5^d`={J-8puH^QfR7 zmhC)XY(z=xz5bK!RP@{e;;?R@4Lmp3I;qW90YVPSH7P9Q)Wi!0BMFR08i!n@)^^e# zen(3;&Q30QV(SmB?_tu8Yy2aYU=ssU8HK2-D0_AV|om7p% zgfU#rQKqX|mBzt03|PKjg@h&OJS}3K9Is5FcXJ6=GvysXxr7hpFFwyHP0!j#{sEDA zA%j)+P|#S=8?)sp|sHT*)8OgVGHV>t20?d2oH+oH=|JmU-+qA=I$}dpa|C zQ#(-JIn#EdlZMZX6SAT;$NzjY_}<}0Ul>h}*0nFi6~^ayuVOC4QufW?5jseJ{F0=qL7yEZgTY%FrFrQbXX^<}^K z>4?U3Xx9t{Zql9Pj+l5M7m@OhwwGUSt(aU4bBJ!7)4Q$cl=jyh)@!_D3t<=IMUeRtZ{QvP&@p`)9^@A>Rr2u`GT-sk~-~ zUK@n{H<1FdNNbW}!K!W5dB&QNhmMmOR#!O_q)m-O3)dRnbMsW1dkwFl|IBZVkCLAR zu?!foSWa4<97h%Jt&pNfv4gAfznRM&G>+HdQ`{Na!cq~1{nj*3kJkuz=h5l2d$Ms$ z_1Y26tca4v8)d)Uf~S7fQ{F_g!u&Zq^e>w{iwbeq(hAk(+2?>nG^tXL*;`7)j4bB` z5MhdAQeaYvhOf?Xy!6FbZ4!WB`puN`gBHuS??xDO!1dTgyod6Jh9U9xl-(y+-Yr;Z z1M*o9CaFW3SALfrmT96&WJu*UmaszmcHbPIqh`u$ZQA|?K?48A8}4bIHZP?)e@&`^ zh@Uh(Ek9a=?3iY+Eg0kqZ-jFx60G!p`dwXv#Y|q2I@2;fIv8FkFLk796}iiQXrlG~ z!H$maXUV`6K_aHO;|5t?^@t%E6sSAiMUC;|heo166U(a(b$3BV>j{b3(l%AaL3YmV z57wmhR<*tLZU>{112BmuGNdYdX;H;vvPQy%G?qq&cPr1&dpFWlJg93yUljWO(d0D# zk3AYbh^dG&;{3k#5K=ExzIL0?HRyG$JnObp{1~q%s*#s{pv0lfOVh-TW@Vqj1)owp%o&oD*bZ5AsVqse z1HOhPKC7IREVxEVi3|Lh9Zp(7GHE3?9jY>0 zxE|g>!XaFFBe`U0Cz(r$ai_HXlFXXHXir=aSyaKOMNXVWGh82$X6gDzC&eAuE8B~z zk8uWe*?Ab_;qkdUMfY%h>;>1;JdS2WKot-=hSXt8!aTA4L0myXAg15zr^Olh+QhjS zi}uPri>z9LVG7atmYvgYuY_evC;<2J7S&YHHQ0=8GR|j=P8s2;%lf!GlY;F4-TP=+ z*!<=@+cG&X`}&nIFg$zYpY`N2(Pw0)42|#h)#dRRGX?VgA0cY-H)<%@Q}zJS(3Jj5 zKW_XTZQXe(X2VJE2J_AIeg9+sBM^i=gFEkHC-b^f$D7?!T)+n7X3b;Q7aNmrWiCDp zC(#0Zwk9@``e6{sFBWl#F^v?+y1`rPY9|$wSrz_Neh?odyJExFaO0tYp)UX#^N?W1 zZOA7xq8VBCc{z_}nB+7sDS|D5c0pvtg_%b)%!k-Q$pBKXdEC2kE^-HyZ9NH}%0-?D zZ{!h?N<}Bq1>V4hiLIU5cgSdQf^0R9(RL$`HEi7en9$nt$zr8AJo`OXcYQQ>QqXhv z>!cCi3_kFtq+tL(c#B?ylP9*o1&R7gpiaCq${e4k?~Q9Ui2h;_fnE)g<`-=wzk~AE z7x{$R92GJ?!BL?l;yvXnwP5Pfh}T0m`V#OiXCdZig-X z2W|khV;&CM#S9d#`YETFTX_vK7@za~@)K%r)8wI8g)aL{Qpl3yg$R z)fo1~cvy!~KhaOp14t~nh_xx;8|6IpkTh(ZuGFGpA4MDme`=$trC6)AH&WI_yq2D#lmga+Gy z8DEBw;`yza?;okZpc}|em{MTQ#QS}&ZtJwk8*LqYhfZvNZT=wj#mO?}f*T+))MKp0 z_U=ZZ6h3c_557|}MO@U-FlyeNp^^BPsOqX&1}S!2h1mh*3%npXptd?Y=t@vZ$#rTg zd(HzWT04bv>!jn{dlaVo93~E=k0cf%sRg71`0@kVhT%0!lnQdznhVq3jk4E?PyzJ2 zKlxR#BJ(uy9+{)birk^@H#csJD@Chva>TN1D6M_QT)LAx|I$f7I3PA!Q`eztq}ro; z1mJYXQsU1P@;MGh&$y03y2rIM?^81@=C8uw2!PwMOgKAY!t4^;BuFp}Ce^6pt5B&W z{DfVj^bv;tb6Mve!~1_qg*rkbQhO_^QYF?hr2ywVqgYOtZdOyA=<2jXQ_d zf}s*A%5)15du0iA5tHSdGMHG(khVY}0RXU`La=iJMnjp=FJM+Gv)BGTRm6y!FU1)8 z%bZNzYejI)vxsW}g~1cpgprguM&PQ|Rxo*g0{{U3|EyU{Pr^VD{wv&!7q~0(Um72Q z1HMAS@4=cF5r{w^U@L8*Y|~;bN?8y&!GY9Qco;yuxFg|}m>3h&WVf4bX20&t&U`Zq zrK>3%Fn!Po3G9sE@O?T4)&9(cxhS!4g#yHe&^BRlU?9aMk=RPDc!227&+KX{Z#d;V z8e7DU;a#^##+L#mPU{_+ju4fHtd@?@GinFwN=`IOAbmf=J=T-PT!q4z}YvMJsZg%Oj_ zeQ@Upnf=-bkr6f&DM{l~l9oAwD(Op9(@VMh$Gdx5{!r*}!{Z5u!|tQNPJr5XZyyAG zPyoQ?Th$z|P6%8jo^3M&!>Qb5Tgrp2atg%Gaz1iWdy`(E6}qgd#Oyji$VKc1$g<-M wVTWE1<@@9EuGJaWKLDV71{Nljf^eoiv)92T=^%`35cz0ree(+d0RR630QaH)cmMzZ literal 0 HcmV?d00001 diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 2b47bf8da..7148185f4 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,6 +1,8 @@ import unittest from os.path import join +from w3lib.encoding import html_to_unicode + from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, Headers from tests import tests_datadir @@ -66,3 +68,11 @@ class GunzipTest(unittest.TestCase): hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) + + def test_gunzip_illegal_eof(self): + with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: + text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1] + with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o: + expected_text = o.read().decode("utf-8") + self.assertEqual(len(text), len(expected_text)) + self.assertEqual(text, expected_text)