zlib: support concatenated gzip files

Reviewed-By: Fedor Indutny <fedor@indutny.com>
PR-URL: https://github.com/joyent/node/pull/6442

lib/zlib.js

@@ -580,7 +580,7 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) {
       self._buffer = new Buffer(self._chunkSize);
     }
 
-    if (availOutAfter === 0) {
+    if (availOutAfter === 0 || availInAfter > 0) {
       // Not actually done.  Need to reprocess.
       // Also, update the availInBefore to the availInAfter value,
       // so that if we have to hit it a third (fourth, etc.) time,
@@ -588,6 +588,13 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) {
       inOff += (availInBefore - availInAfter);
       availInBefore = availInAfter;
 
+      if (availOutAfter !== 0) {
+        // There is still some data available for reading.
+        // This is usually a concatenated stream, so, reset and restart.
+        self.reset();
+        self._offset = 0;
+      }
+
       if (!async)
         return true;
 

src/node_zlib.cc

@@ -63,6 +63,11 @@ enum node_zlib_mode {
   UNZIP
 };
 
+enum node_zlib_error {
+  NO_ERROR,
+  FAILED,
+  WRITE_PENDING
+};
 
 void InitZlib(v8::Handle<v8::Object> target);
 
@@ -203,7 +208,7 @@ class ZCtx : public AsyncWrap {
     if (!async) {
       // sync version
       Process(work_req);
-      if (CheckError(ctx))
+      if (CheckError(ctx) == NO_ERROR)
         AfterSync(ctx, args);
       return;
     }
@@ -287,7 +292,7 @@ class ZCtx : public AsyncWrap {
   }
 
 
-  static bool CheckError(ZCtx* ctx) {
+  static node_zlib_error CheckError(ZCtx* ctx) {
     // Acceptable error states depend on the type of zlib stream.
     switch (ctx->err_) {
     case Z_OK:
@@ -300,14 +305,18 @@ class ZCtx : public AsyncWrap {
         ZCtx::Error(ctx, "Missing dictionary");
       else
         ZCtx::Error(ctx, "Bad dictionary");
-      return false;
+      return FAILED;
     default:
       // something else.
-      ZCtx::Error(ctx, "Zlib error");
+      if (ctx->strm_.total_out == 0) {
-      return false;
+        ZCtx::Error(ctx, "Zlib error");
+        return FAILED;
+      } else {
+        return WRITE_PENDING;
+      }
     }
 
-    return true;
+    return NO_ERROR;
   }
 
 
@@ -321,7 +330,8 @@ class ZCtx : public AsyncWrap {
     HandleScope handle_scope(env->isolate());
     Context::Scope context_scope(env->context());
 
-    if (!CheckError(ctx))
+    node_zlib_error error = CheckError(ctx);
+    if (error == FAILED)
       return;
 
     Local<Integer> avail_out = Integer::New(env->isolate(),
@@ -335,6 +345,11 @@ class ZCtx : public AsyncWrap {
     Local<Value> args[2] = { avail_in, avail_out };
     ctx->MakeCallback(env->callback_string(), ARRAY_SIZE(args), args);
 
+    if (error == WRITE_PENDING) {
+      ZCtx::Error(ctx, "Zlib error");
+      return;
+    }
+
     ctx->Unref();
     if (ctx->pending_close_)
       ctx->Close();
@@ -539,10 +554,12 @@ class ZCtx : public AsyncWrap {
     switch (ctx->mode_) {
       case DEFLATE:
       case DEFLATERAW:
+      case GZIP:
         ctx->err_ = deflateReset(&ctx->strm_);
         break;
       case INFLATE:
       case INFLATERAW:
+      case GUNZIP:
         ctx->err_ = inflateReset(&ctx->strm_);
         break;
       default:

test/simple/test-zlib-from-multiple-gzip-with-garbage.js

@@ -0,0 +1,83 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// test unzipping a file that was created by concatenating multiple gzip
+// streams.
+
+var common = require('../common');
+var assert = require('assert');
+var zlib = require('zlib');
+
+var util = require('util');
+
+var gzipBuffer = new Buffer(128);
+var gzipOffset = 0;
+
+var stream1 = '123\n';
+var stream2 = '456\n';
+var stream3 = '789\n';
+
+function gzipAppend(data) {
+  data.copy(gzipBuffer, gzipOffset);
+  gzipOffset += data.length;
+}
+
+function writeGzipStream(text, cb) {
+  var gzip = zlib.createGzip();
+  gzip.on('data', gzipAppend);
+  gzip.write(text, function() {
+    gzip.flush(function() {
+      gzip.end(function() {
+        cb();
+      });
+    });
+  });
+}
+
+function writeGarbageStream(text, cb) {
+  gzipAppend(new Buffer(text));
+  cb();
+}
+
+writeGzipStream(stream1, function() {
+  writeGzipStream(stream2, function() {
+    writeGarbageStream(stream3, function() {
+      var gunzip = zlib.createGunzip();
+      var gunzippedData = new Buffer(2 * 1024);
+      var gunzippedOffset = 0;
+      gunzip.on('data', function (data) {
+        data.copy(gunzippedData, gunzippedOffset);
+        gunzippedOffset += data.length;
+      });
+      gunzip.on('error', function() {
+        assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset),
+                     stream1 + stream2);
+      });
+      gunzip.on('end', function() {
+        assert.fail('end event not expected');
+      });
+
+      gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
+        gunzip.end();
+      });
+    });
+  });
+});

test/simple/test-zlib-from-multiple-gzip.js

@@ -0,0 +1,74 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// test unzipping a file that was created by concatenating multiple gzip
+// streams.
+
+var common = require('../common');
+var assert = require('assert');
+var zlib = require('zlib');
+
+var util = require('util');
+
+var gzipBuffer = new Buffer(128);
+var gzipOffset = 0;
+
+var stream1 = '123\n';
+var stream2 = '456\n';
+var stream3 = '789\n';
+
+function gzipAppend(data) {
+  data.copy(gzipBuffer, gzipOffset);
+  gzipOffset += data.length;
+}
+
+function writeGzipStream(text, cb) {
+  var gzip = zlib.createGzip();
+  gzip.on('data', gzipAppend);
+  gzip.write(text, function() {
+    gzip.flush(function() {
+      gzip.end(function() {
+        cb();
+      });
+    });
+  });
+}
+
+writeGzipStream(stream1, function() {
+  writeGzipStream(stream2, function() {
+    writeGzipStream(stream3, function() {
+      var gunzip = zlib.createGunzip();
+      var gunzippedData = new Buffer(2 * 1024);
+      var gunzippedOffset = 0;
+      gunzip.on('data', function (data) {
+        data.copy(gunzippedData, gunzippedOffset);
+        gunzippedOffset += data.length;
+      });
+      gunzip.on('end', function() {
+        assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), stream1 + stream2 + stream3);
+      });
+
+      gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
+        gunzip.end();
+      });
+    });
+  });
+});

test/simple/test-zlib-from-multiple-huge-gzip.js

@@ -0,0 +1,93 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// test unzipping a file that was created by concatenating multiple gzip
+// streams.
+
+var common = require('../common');
+var assert = require('assert');
+var zlib = require('zlib');
+
+var util = require('util');
+
+var HUGE = 64 * 1024;
+
+var originalBuffer = new Buffer(3 * HUGE);
+var originalOffset = 0;
+
+var gzipBuffer = new Buffer(3 * HUGE);
+var gzipOffset = 0;
+
+function getRandomLetter() {
+  return (Math.random() * (122 - 97)) + 97;
+}
+
+function generateHugeStream() {
+  var buffer = new Buffer(HUGE);
+  for (var i = 0; i < HUGE; i++)
+    buffer.writeUInt8(getRandomLetter(), i);
+
+  buffer.copy(originalBuffer, originalOffset);
+  originalOffset += HUGE;
+
+  return buffer;
+}
+
+function gzipAppend(data) {
+  data.copy(gzipBuffer, gzipOffset);
+  gzipOffset += data.length;
+}
+
+function writeGzipStream(text, cb) {
+  var gzip = zlib.createGzip();
+  gzip.on('data', gzipAppend);
+  gzip.write(text, function() {
+    gzip.flush(function() {
+      gzip.end(function() {
+        cb();
+      });
+    });
+  });
+}
+
+writeGzipStream(generateHugeStream(), function() {
+  writeGzipStream(generateHugeStream(), function() {
+    writeGzipStream(generateHugeStream(), function() {
+      var gunzip = zlib.createGunzip();
+      var gunzippedData = new Buffer(3 * HUGE);
+      var gunzippedOffset = 0;
+      gunzip.on('data', function (data) {
+        data.copy(gunzippedData, gunzippedOffset);
+        gunzippedOffset += data.length;
+      });
+      gunzip.on('end', function() {
+        var gunzippedStr = gunzippedData.toString('utf8', 0, gunzippedOffset);
+        var originalStr  = originalBuffer.toString('utf8', 0, 3 * HUGE);
+
+        assert.equal(gunzippedStr, originalStr);
+      });
+
+      gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
+        gunzip.end();
+      });
+    });
+  });
+});