Browse Source

zlib: Make the finish flush flag configurable

Up to now, `Z_FINISH` was always the flushing flag that was used
for the last chunk of input data. This patch makes this choice
configurable so that advanced users can perform e.g. decompression of
partial data using `Z_SYNC_FLUSH`, if that suits their needs.

Add tests to make sure that an error is thrown upon encountering
invalid `flush` or `finishFlush` flags.

Fixes: https://github.com/nodejs/node/issues/5761
PR-URL: https://github.com/nodejs/node/pull/6069
Reviewed-By: James M Snell <jasnell@gmail.com>
process-exit-stdio-flushing
Anna Henningsen 9 years ago
parent
commit
978166796e
No known key found for this signature in database GPG Key ID: D8B9F5AEAE84E4CF
  1. 26
      doc/api/zlib.markdown
  2. 28
      lib/zlib.js
  3. 28
      test/parallel/test-zlib-flush-flags.js
  4. 17
      test/parallel/test-zlib-truncated.js

26
doc/api/zlib.markdown

@ -109,6 +109,31 @@ http.createServer((request, response) => {
}).listen(1337); }).listen(1337);
``` ```
By default, the zlib methods with throw an error when decompressing
truncated data. However, if it is known that the data is incomplete, or
the desire is to inspect only the beginning of a compressed file, it is
possible to suppress the default error handling by changing the flushing
method that is used to compressed the last chunk of input data:
```js
// This is a truncated version of the buffer from the above examples
const buffer = new Buffer('eJzT0yMA', 'base64');
zlib.unzip(buffer, { finishFlush: zlib.Z_SYNC_FLUSH }, (err, buffer) => {
if (!err) {
console.log(buffer.toString());
} else {
// handle error
}
});
```
This will not change the behavior in other error-throwing situations, e.g.
when the input data has an invalid format. Using this method, it will not be
possible to determine whether the input ended prematurely or lacks the
integrity checks, making it necessary to manually check that the
decompressed result is valid.
## Memory Usage Tuning ## Memory Usage Tuning
<!--type=misc--> <!--type=misc-->
@ -231,6 +256,7 @@ Note that some options are only relevant when compressing, and are
ignored by the decompression classes. ignored by the decompression classes.
* flush (default: `zlib.Z_NO_FLUSH`) * flush (default: `zlib.Z_NO_FLUSH`)
* finishFlush (default: `zlib.Z_FINISH`)
* chunkSize (default: 16*1024) * chunkSize (default: 16*1024)
* windowBits * windowBits
* level (compression only) * level (compression only)

28
lib/zlib.js

@ -234,7 +234,7 @@ function zlibBufferSync(engine, buffer) {
if (!(buffer instanceof Buffer)) if (!(buffer instanceof Buffer))
throw new TypeError('Not a string or buffer'); throw new TypeError('Not a string or buffer');
var flushFlag = binding.Z_FINISH; var flushFlag = engine._finishFlushFlag;
return engine._processChunk(buffer, flushFlag); return engine._processChunk(buffer, flushFlag);
} }
@ -282,6 +282,14 @@ function Unzip(opts) {
Zlib.call(this, opts, binding.UNZIP); Zlib.call(this, opts, binding.UNZIP);
} }
function isValidFlushFlag(flag) {
return flag === binding.Z_NO_FLUSH ||
flag === binding.Z_PARTIAL_FLUSH ||
flag === binding.Z_SYNC_FLUSH ||
flag === binding.Z_FULL_FLUSH ||
flag === binding.Z_FINISH ||
flag === binding.Z_BLOCK;
}
// the Zlib class they all inherit from // the Zlib class they all inherit from
// This thing manages the queue of requests, and returns // This thing manages the queue of requests, and returns
@ -294,17 +302,16 @@ function Zlib(opts, mode) {
Transform.call(this, opts); Transform.call(this, opts);
if (opts.flush) { if (opts.flush && !isValidFlushFlag(opts.flush)) {
if (opts.flush !== binding.Z_NO_FLUSH &&
opts.flush !== binding.Z_PARTIAL_FLUSH &&
opts.flush !== binding.Z_SYNC_FLUSH &&
opts.flush !== binding.Z_FULL_FLUSH &&
opts.flush !== binding.Z_FINISH &&
opts.flush !== binding.Z_BLOCK) {
throw new Error('Invalid flush flag: ' + opts.flush); throw new Error('Invalid flush flag: ' + opts.flush);
} }
if (opts.finishFlush && !isValidFlushFlag(opts.finishFlush)) {
throw new Error('Invalid flush flag: ' + opts.finishFlush);
} }
this._flushFlag = opts.flush || binding.Z_NO_FLUSH; this._flushFlag = opts.flush || binding.Z_NO_FLUSH;
this._finishFlushFlag = typeof opts.finishFlush !== 'undefined' ?
opts.finishFlush : binding.Z_FINISH;
if (opts.chunkSize) { if (opts.chunkSize) {
if (opts.chunkSize < exports.Z_MIN_CHUNK || if (opts.chunkSize < exports.Z_MIN_CHUNK ||
@ -486,12 +493,13 @@ Zlib.prototype._transform = function(chunk, encoding, cb) {
if (this._closed) if (this._closed)
return cb(new Error('zlib binding closed')); return cb(new Error('zlib binding closed'));
// If it's the last chunk, or a final flush, we use the Z_FINISH flush flag. // If it's the last chunk, or a final flush, we use the Z_FINISH flush flag
// (or whatever flag was provided using opts.finishFlush).
// If it's explicitly flushing at some other time, then we use // If it's explicitly flushing at some other time, then we use
// Z_FULL_FLUSH. Otherwise, use Z_NO_FLUSH for maximum compression // Z_FULL_FLUSH. Otherwise, use Z_NO_FLUSH for maximum compression
// goodness. // goodness.
if (last) if (last)
flushFlag = binding.Z_FINISH; flushFlag = this._finishFlushFlag;
else { else {
flushFlag = this._flushFlag; flushFlag = this._flushFlag;
// once we've flushed the last of the queue, stop flushing and // once we've flushed the last of the queue, stop flushing and

28
test/parallel/test-zlib-flush-flags.js

@ -0,0 +1,28 @@
'use strict';
require('../common');
const assert = require('assert');
const zlib = require('zlib');
assert.doesNotThrow(() => {
zlib.createGzip({ flush: zlib.Z_SYNC_FLUSH });
});
assert.throws(() => {
zlib.createGzip({ flush: 'foobar' });
}, /Invalid flush flag: foobar/);
assert.throws(() => {
zlib.createGzip({ flush: 10000 });
}, /Invalid flush flag: 10000/);
assert.doesNotThrow(() => {
zlib.createGzip({ finishFlush: zlib.Z_SYNC_FLUSH });
});
assert.throws(() => {
zlib.createGzip({ finishFlush: 'foobar' });
}, /Invalid flush flag: foobar/);
assert.throws(() => {
zlib.createGzip({ finishFlush: 10000 });
}, /Invalid flush flag: 10000/);

17
test/parallel/test-zlib-truncated.js

@ -45,5 +45,22 @@ const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing el'
zlib[methods.decomp](truncated, function(err, result) { zlib[methods.decomp](truncated, function(err, result) {
assert(/unexpected end of file/.test(err.message)); assert(/unexpected end of file/.test(err.message));
}); });
const syncFlushOpt = { finishFlush: zlib.Z_SYNC_FLUSH };
// sync truncated input test, finishFlush = Z_SYNC_FLUSH
assert.doesNotThrow(function() {
const result = zlib[methods.decompSync](truncated, syncFlushOpt)
.toString();
assert.equal(result, inputString.substr(0, result.length));
});
// async truncated input test, finishFlush = Z_SYNC_FLUSH
zlib[methods.decomp](truncated, syncFlushOpt, function(err, decompressed) {
assert.ifError(err);
const result = decompressed.toString();
assert.equal(result, inputString.substr(0, result.length));
});
}); });
}); });

Loading…
Cancel
Save