HttpUtil: fix gunzipping with partial content; and debug response in url plugin earlier

2 files changed, 17 insertions, 3 deletions

diff --git a/data/rbot/plugins/url.rb b/data/rbot/plugins/url.rb
index 8a05cafa..10cc419a 100644
--- a/data/rbot/plugins/url.rb
+++ b/data/rbot/plugins/url.rb
@@ -52,6 +52,8 @@ class UrlPlugin < Plugin
         case resp
         when Net::HTTPSuccess
 
+          debug resp.to_hash
+
           if resp['content-type'] =~ /^text\/|(?:x|ht)ml/
             # The page is text or HTML, so we can try finding a title and, if
             # requested, the first par.
@@ -76,8 +78,6 @@ class UrlPlugin < Plugin
           # if nothing was found, provide more basic info, as for non-html pages
           end
 
-          debug resp.to_hash.inspect
-
           enc = resp['content-encoding']
 
           extra << ", #{Bold}encoding#{Bold}: #{enc}" if enc
diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index b4219f66..3c949686 100644
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -78,7 +78,21 @@ module ::Net
         return str
       when 'gzip', 'x-gzip'
         debug "gunzipping body"
-        return Zlib::GzipReader.new(StringIO.new(str)).read
+        begin
+          return Zlib::GzipReader.new(StringIO.new(str)).read
+        rescue Zlib::Error => e
+          # If we can't unpack the whole stream (e.g. because we're doing a
+          # partial read
+          debug "full gunzipping failed (#{e}), trying to recover as much as possible"
+          ret = ""
+          begin
+            Zlib::GzipReader.new(StringIO.new(str)).each_byte { |byte|
+              ret << byte
+            }
+          rescue
+          end
+          return ret
+        end
       else
         raise "Unhandled content encoding #{method}"
       end