diff --git a/benchmark/string_decoder/string-decoder-create.js b/benchmark/string_decoder/string-decoder-create.js new file mode 100644 index 0000000000..17c0f6750d --- /dev/null +++ b/benchmark/string_decoder/string-decoder-create.js @@ -0,0 +1,22 @@ +'use strict'; +const common = require('../common.js'); +const StringDecoder = require('string_decoder').StringDecoder; + +const bench = common.createBenchmark(main, { + encoding: [ + 'ascii', 'utf8', 'utf-8', 'base64', 'ucs2', 'UTF-8', 'AscII', 'UTF-16LE' + ], + n: [25e6] +}); + +function main(conf) { + const encoding = conf.encoding; + const n = conf.n | 0; + + bench.start(); + for (var i = 0; i < n; ++i) { + const sd = new StringDecoder(encoding); + !!sd.encoding; + } + bench.end(n); +} diff --git a/benchmark/string_decoder/string-decoder.js b/benchmark/string_decoder/string-decoder.js index 705ef84103..7a86fb6ef5 100644 --- a/benchmark/string_decoder/string-decoder.js +++ b/benchmark/string_decoder/string-decoder.js @@ -1,51 +1,79 @@ 'use strict'; -var common = require('../common.js'); -var StringDecoder = require('string_decoder').StringDecoder; +const common = require('../common.js'); +const StringDecoder = require('string_decoder').StringDecoder; -var bench = common.createBenchmark(main, { - encoding: ['ascii', 'utf8', 'base64-utf8', 'base64-ascii'], - inlen: [32, 128, 1024], +const bench = common.createBenchmark(main, { + encoding: ['ascii', 'utf8', 'base64-utf8', 'base64-ascii', 'utf16le'], + inlen: [32, 128, 1024, 4096], chunk: [16, 64, 256, 1024], - n: [25e4] + n: [25e5] }); -var UTF_ALPHA = 'Blåbærsyltetøy'; -var ASC_ALPHA = 'Blueberry jam'; +const UTF8_ALPHA = 'Blåbærsyltetøy'; +const ASC_ALPHA = 'Blueberry jam'; +const UTF16_BUF = Buffer.from('Blåbærsyltetøy', 'utf16le'); function main(conf) { - var encoding = conf.encoding; - var inLen = conf.inlen | 0; - var chunkLen = conf.chunk | 0; - var n = conf.n | 0; + const encoding = conf.encoding; + const inLen = conf.inlen | 0; + const chunkLen = conf.chunk | 0; + const n = conf.n | 0; var alpha; - var chunks = []; + var buf; + const chunks = []; var str = ''; - var isBase64 = (encoding === 'base64-ascii' || encoding === 'base64-utf8'); + const isBase64 = (encoding === 'base64-ascii' || encoding === 'base64-utf8'); var i; if (encoding === 'ascii' || encoding === 'base64-ascii') alpha = ASC_ALPHA; else if (encoding === 'utf8' || encoding === 'base64-utf8') - alpha = UTF_ALPHA; - else + alpha = UTF8_ALPHA; + else if (encoding === 'utf16le') { + buf = UTF16_BUF; + str = Buffer.alloc(0); + } else throw new Error('Bad encoding'); - var sd = new StringDecoder(isBase64 ? 'base64' : encoding); + const sd = new StringDecoder(isBase64 ? 'base64' : encoding); for (i = 0; i < inLen; ++i) { if (i > 0 && (i % chunkLen) === 0 && !isBase64) { - chunks.push(Buffer.from(str, encoding)); - str = ''; + if (alpha) { + chunks.push(Buffer.from(str, encoding)); + str = ''; + } else { + chunks.push(str); + str = Buffer.alloc(0); + } + } + if (alpha) + str += alpha[i % alpha.length]; + else { + var start = i; + var end = i + 2; + if (i % 2 !== 0) { + ++start; + ++end; + } + str = Buffer.concat([ + str, + buf.slice(start % buf.length, end % buf.length) + ]); } - str += alpha[i % alpha.length]; } - if (str.length > 0 && !isBase64) + + if (!alpha) { + if (str.length > 0) + chunks.push(str); + } else if (str.length > 0 && !isBase64) chunks.push(Buffer.from(str, encoding)); + if (isBase64) { str = Buffer.from(str, 'utf8').toString('base64'); while (str.length > 0) { - var len = Math.min(chunkLen, str.length); + const len = Math.min(chunkLen, str.length); chunks.push(Buffer.from(str.substring(0, len), 'utf8')); str = str.substring(len); } diff --git a/lib/string_decoder.js b/lib/string_decoder.js index 4047489e6c..aaadfd8934 100644 --- a/lib/string_decoder.js +++ b/lib/string_decoder.js @@ -2,224 +2,234 @@ const Buffer = require('buffer').Buffer; -function assertEncoding(encoding) { - // Do not cache `Buffer.isEncoding`, some modules monkey-patch it to support - // additional encodings - if (encoding && !Buffer.isEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); +// Do not cache `Buffer.isEncoding` when checking encoding names as some +// modules monkey-patch it to support additional encodings +function normalizeEncoding(enc) { + if (!enc) return 'utf8'; + var low; + for (;;) { + switch (enc) { + case 'utf8': + case 'utf-8': + return 'utf8'; + case 'ucs2': + case 'utf16le': + case 'ucs-2': + case 'utf-16le': + return 'utf16le'; + case 'base64': + case 'ascii': + case 'binary': + case 'hex': + return enc; + default: + if (low) { + if (!Buffer.isEncoding(enc)) + throw new Error('Unknown encoding: ' + enc); + return enc; + } + low = true; + enc = ('' + enc).toLowerCase(); + } } } // StringDecoder provides an interface for efficiently splitting a series of // buffers into a series of JS strings without breaking apart multi-byte -// characters. CESU-8 is handled as part of the UTF-8 encoding. -// -// @TODO Handling all encodings inside a single object makes it very difficult -// to reason about this code, so it should be split up in the future. -// @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code -// points as used by CESU-8. -const StringDecoder = exports.StringDecoder = function(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); +// characters. +exports.StringDecoder = StringDecoder; +function StringDecoder(encoding) { + this.encoding = normalizeEncoding(encoding); + var nb; switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; + this.text = utf16Text; + this.end = utf16End; + // fall through + case 'utf8': + nb = 4; break; case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; + this.text = base64Text; + this.end = base64End; + nb = 3; break; default: - this.write = passThroughWrite; + this.write = simpleWrite; + this.end = simpleEnd; return; } + this.lastNeed = 0; + this.lastTotal = 0; + this.lastChar = Buffer.allocUnsafe(nb); +} - // Enough space to store all bytes of a single character. UTF-8 needs 4 - // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). - this.charBuffer = Buffer.allocUnsafe(6); - // Number of bytes received for the current incomplete multi-byte character. - this.charReceived = 0; - // Number of bytes expected for the current incomplete multi-byte character. - this.charLength = 0; -}; - - -// write decodes the given buffer and returns it as JS string that is -// guaranteed to not contain any partial multi-byte characters. Any partial -// character found at the end of the buffer is buffered up, and will be -// returned when calling write again with the remaining bytes. -// -// Note: Converting a Buffer containing an orphan surrogate to a String -// currently works, but converting a String to a Buffer (via `Buffer.from()`, -// or Buffer#write) will replace incomplete surrogates with the unicode -// replacement character. See https://codereview.chromium.org/121173009/ . -StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - var buflen = buffer.length; - var charBuffer = this.charBuffer; - var charLength = this.charLength; - var charReceived = this.charReceived; - var surrogateSize = this.surrogateSize; - var encoding = this.encoding; - var charCode; - // if our last write ended with an incomplete multibyte character - while (charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var diff = charLength - charReceived; - var available = (buflen >= diff) ? diff : buflen; - - // add the new bytes to the char buffer - buffer.copy(charBuffer, charReceived, 0, available); - charReceived += available; - - if (charReceived < charLength) { - // still not enough chars in this buffer? wait for more ... - - this.charLength = charLength; - this.charReceived = charReceived; - +StringDecoder.prototype.write = function(buf) { + if (buf.length === 0) + return ''; + var r; + var i; + if (this.lastNeed) { + r = this.fillLast(buf); + if (r === undefined) return ''; - } - - // remove bytes belonging to the current character from the buffer - buffer = buffer.slice(available, buflen); - buflen = buffer.length; - - // get the character that was split - charStr = charBuffer.toString(encoding, 0, charLength); + i = this.lastNeed; + this.lastNeed = 0; + } else { + i = 0; + } + if (i < buf.length) + return (r ? r + this.text(buf, i) : this.text(buf, i)); + return r || ''; +}; - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - charLength += surrogateSize; - charStr = ''; - continue; - } - charReceived = charLength = 0; +StringDecoder.prototype.end = utf8End; - // if there are no more bytes in this buffer, just emit our char - if (buflen === 0) { - this.charLength = charLength; - this.charReceived = charReceived; +// Returns only complete characters in a Buffer +StringDecoder.prototype.text = utf8Text; - return charStr; - } +// Attempts to complete a partial character using bytes from a Buffer +StringDecoder.prototype.fillLast = function(buf) { + if (this.lastNeed <= buf.length) { + buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, this.lastNeed); + return this.lastChar.toString(this.encoding, 0, this.lastTotal); } + buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, buf.length); + this.lastNeed -= buf.length; +}; - // determine and set charLength / charReceived - if (this.detectIncompleteChar(buffer)) - charLength = this.charLength; - charReceived = this.charReceived; +// Checks the type of a UTF-8 byte, whether it's ASCII, a leading byte, or a +// continuation byte. +function utf8CheckByte(byte) { + if (byte <= 0x7F) + return 0; + else if (byte >> 5 === 0x06) + return 2; + else if (byte >> 4 === 0x0E) + return 3; + else if (byte >> 3 === 0x1E) + return 4; + return -1; +} - var end = buflen; - if (charLength) { - // buffer the incomplete character bytes we got - buffer.copy(charBuffer, 0, buflen - charReceived, end); - end -= charReceived; +// Checks at most the last 3 bytes of a Buffer for an incomplete UTF-8 +// character, returning the total number of bytes needed to complete the partial +// character (if applicable). +function utf8CheckIncomplete(self, buf, i) { + var j = buf.length - 1; + if (j < i) + return 0; + var nb = utf8CheckByte(buf[j--]); + if (nb >= 0) { + if (nb > 0) + self.lastNeed = nb + 1 - (buf.length - j); + return nb; } - - this.charLength = charLength; - charStr += buffer.toString(encoding, 0, end); - - end = charStr.length - 1; - charCode = charStr.charCodeAt(end); - // CESU-8: lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - charLength += surrogateSize; - charReceived += surrogateSize; - charBuffer.copy(charBuffer, surrogateSize, 0, surrogateSize); - buffer.copy(charBuffer, 0, 0, surrogateSize); - - this.charLength = charLength; - this.charReceived = charReceived; - - return charStr.substring(0, end); + if (j < i) + return 0; + nb = utf8CheckByte(buf[j--]); + if (nb >= 0) { + if (nb > 0) + self.lastNeed = nb + 1 - (buf.length - j); + return nb; } + if (j < i) + return 0; + nb = utf8CheckByte(buf[j--]); + if (nb >= 0) { + if (nb > 0) + self.lastNeed = nb + 1 - (buf.length - j); + return nb; + } + return 0; +} - // or just emit the charStr - return charStr; -}; - -// detectIncompleteChar determines if there is an incomplete UTF-8 character at -// the end of the given buffer. If so, it sets this.charLength to the byte -// length that character, and sets this.charReceived to the number of bytes -// that are available for this character. -StringDecoder.prototype.detectIncompleteChar = function(buffer) { - var buflen = buffer.length; - // determine how many bytes we have to check at the end of this buffer - var i = (buflen >= 3) ? 3 : buflen; - var newlen = false; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buflen - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i === 1 && c >> 5 === 0x06) { - this.charLength = 2; - newlen = true; - break; - } +// Returns all complete UTF-8 characters in a Buffer. If the Buffer ended on a +// partial character, the character's bytes are buffered until the required +// number of bytes are available. +function utf8Text(buf, i) { + const total = utf8CheckIncomplete(this, buf, i); + if (!this.lastNeed) + return buf.toString('utf8', i); + this.lastTotal = total; + const end = buf.length - (total - this.lastNeed); + buf.copy(this.lastChar, 0, end); + return buf.toString('utf8', i, end); +} - // 1110XXXX - if (i <= 2 && c >> 4 === 0x0E) { - this.charLength = 3; - newlen = true; - break; - } +// For UTF-8, a replacement character for each buffered byte of a (partial) +// character needs to be added to the output. +function utf8End(buf) { + const r = (buf && buf.length ? this.write(buf) : ''); + if (this.lastNeed) + return r + '\ufffd'.repeat(this.lastTotal - this.lastNeed); + return r; +} - // 11110XXX - if (i <= 3 && c >> 3 === 0x1E) { - this.charLength = 4; - newlen = true; - break; +// UTF-16LE typically needs two bytes per character, but even if we have an even +// number of bytes available, we need to check if we end on a leading/high +// surrogate. In that case, we need to wait for the next two bytes in order to +// decode the last character properly. +function utf16Text(buf, i) { + if ((buf.length - i) % 2 === 0) { + const r = buf.toString('utf16le', i); + if (r) { + const c = r.charCodeAt(r.length - 1); + if (c >= 0xD800 && c <= 0xDBFF) { + this.lastNeed = 2; + this.lastTotal = 4; + this.lastChar[0] = buf[buf.length - 2]; + this.lastChar[1] = buf[buf.length - 1]; + return r.slice(0, -1); + } } + return r; } + this.lastNeed = 1; + this.lastTotal = 2; + this.lastChar[0] = buf[buf.length - 1]; + return buf.toString('utf16le', i, buf.length - 1); +} - this.charReceived = i; - - return newlen; -}; +// For UTF-16LE we do not explicitly append special replacement characters if we +// end on a partial character, we simply let v8 handle that. +function utf16End(buf) { + const r = (buf && buf.length ? this.write(buf) : ''); + if (this.lastNeed) { + const end = this.lastTotal - this.lastNeed; + return r + this.lastChar.toString('utf16le', 0, end); + } + return r; +} -StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - var charReceived = this.charReceived; - if (charReceived) { - var cr = charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.toString(enc, 0, cr); +function base64Text(buf, i) { + const n = (buf.length - i) % 3; + if (n === 0) + return buf.toString('base64', i); + this.lastNeed = 3 - n; + this.lastTotal = 3; + if (n === 1) { + this.lastChar[0] = buf[buf.length - 1]; + } else { + this.lastChar[0] = buf[buf.length - 2]; + this.lastChar[1] = buf[buf.length - 1]; } + return buf.toString('base64', i, buf.length - n); +} - return res; -}; -function passThroughWrite(buffer) { - return buffer.toString(this.encoding); +function base64End(buf) { + const r = (buf && buf.length ? this.write(buf) : ''); + if (this.lastNeed) + return r + this.lastChar.toString('base64', 0, 3 - this.lastNeed); + return r; } -function utf16DetectIncompleteChar(buffer) { - var charReceived = this.charReceived = buffer.length % 2; - this.charLength = charReceived ? 2 : 0; - return true; +// Pass bytes on through for single-byte encodings (e.g. ascii, binary, hex) +function simpleWrite(buf) { + return buf.toString(this.encoding); } -function base64DetectIncompleteChar(buffer) { - var charReceived = this.charReceived = buffer.length % 3; - this.charLength = charReceived ? 3 : 0; - return true; +function simpleEnd(buf) { + return (buf && buf.length ? this.write(buf) : ''); } diff --git a/test/parallel/test-string-decoder-end.js b/test/parallel/test-string-decoder-end.js index e13ddcb167..33b47a034a 100644 --- a/test/parallel/test-string-decoder-end.js +++ b/test/parallel/test-string-decoder-end.js @@ -51,6 +51,8 @@ function testBuf(encoding, buf) { var res3 = buf.toString(encoding); console.log('expect=%j', res3); + console.log('res1=%j', res1); + console.log('res2=%j', res2); assert.equal(res1, res3, 'one byte at a time should match toString'); assert.equal(res2, res3, 'all bytes at once should match toString'); } diff --git a/test/parallel/test-string-decoder.js b/test/parallel/test-string-decoder.js index 4f7cbd7fad..14933c46fc 100644 --- a/test/parallel/test-string-decoder.js +++ b/test/parallel/test-string-decoder.js @@ -1,8 +1,13 @@ 'use strict'; require('../common'); var assert = require('assert'); +var inspect = require('util').inspect; var StringDecoder = require('string_decoder').StringDecoder; +// Test default encoding +var decoder = new StringDecoder(); +assert.strictEqual(decoder.encoding, 'utf8'); + process.stdout.write('scanning '); // UTF-8 @@ -27,10 +32,49 @@ test( test('ucs2', Buffer.from('ababc', 'ucs2'), 'ababc'); // UTF-16LE -test('ucs2', Buffer.from('3DD84DDC', 'hex'), '\ud83d\udc4d'); // thumbs up +test('utf16le', Buffer.from('3DD84DDC', 'hex'), '\ud83d\udc4d'); // thumbs up console.log(' crayon!'); +// Additional UTF-8 tests +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('E1', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('E18B', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ufffd\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('\ufffd')), '\ufffd'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('\ufffd\ufffd\ufffd')), + '\ufffd\ufffd\ufffd'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(Buffer.from('efbfbde2', 'hex')), '\ufffd'); +assert.strictEqual(decoder.end(), '\ufffd'); + + +// Additional UTF-16LE surrogate pair tests +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('4D', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('DC', 'hex')), '\ud83d\udc4d'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ud83d'); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(Buffer.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.write(Buffer.from('4D', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ud83d'); + // test verifies that StringDecoder will correctly decode the given input // buffer with the given encoding to the expected output. It will attempt all // possible ways to write() the input buffer, see writeSequences(). The @@ -54,9 +98,9 @@ function test(encoding, input, expected, singleSequence) { var message = 'Expected "' + unicodeEscape(expected) + '", ' + 'but got "' + unicodeEscape(output) + '"\n' + + 'input: ' + input.toString('hex').match(/.{2}/g) + '\n' + 'Write sequence: ' + JSON.stringify(sequence) + '\n' + - 'Decoder charBuffer: 0x' + decoder.charBuffer.toString('hex') + '\n' + - 'Full Decoder State: ' + JSON.stringify(decoder, null, 2); + 'Full Decoder State: ' + inspect(decoder); assert.fail(output, expected, message); } });