string_decoder: Add 'end' method, do base64 properly

doc/api/string_decoder.markdown

@@ -19,6 +19,10 @@ additional support for utf8.
 
 Accepts a single argument, `encoding` which defaults to `utf8`.
 
-### StringDecoder.write(buffer)
+### decoder.write(buffer)
 
 Returns a decoded string.
+
+### decoder.end()
+
+Returns any trailing bytes that were left in the buffer.

lib/fs.js

@@ -1402,6 +1402,11 @@ ReadStream.prototype._read = function() {
     }
 
     if (bytesRead === 0) {
+      if (this._decoder) {
+        var ret = this._decoder.end();
+        if (ret)
+          this.emit('data', ret);
+      }
       self.emit('end');
       self.destroy();
       return;

lib/http.js

@@ -363,6 +363,11 @@ IncomingMessage.prototype._emitData = function(d) {
 
 IncomingMessage.prototype._emitEnd = function() {
   if (!this._endEmitted) {
+    if (this._decoder) {
+      var ret = this._decoder.end();
+      if (ret)
+        this.emit('data', ret);
+    }
     this.emit('end');
   }
 
@@ -1859,6 +1864,11 @@ Client.prototype.request = function(method, path, headers) {
   // but it will get removed when we remove this legacy interface.
   c.on('socket', function(s) {
     s.on('end', function() {
+      if (self._decoder) {
+        var ret = self._decoder.end();
+        if (ret)
+          self.emit('data', ret);
+      }
       self.emit('end');
     });
   });

lib/net.js

@@ -414,6 +414,11 @@ function onread(buffer, offset, length) {
     if (!self.writable) self._destroy();
 
     if (!self.allowHalfOpen) self.end();
+    if (self._decoder) {
+      var ret = self._decoder.end();
+      if (ret)
+        self.emit('data', ret);
+    }
     if (self._events && self._events['end']) self.emit('end');
     if (self.onend) self.onend();
   } else {

lib/string_decoder.js

@@ -32,6 +32,11 @@ var StringDecoder = exports.StringDecoder = function(encoding) {
       this.surrogateSize = 2;
       this.detectIncompleteChar = utf16DetectIncompleteChar;
       break;
+    case 'base64':
+      // Base-64 stores 3 bytes in 4 chars, and pads the remainder.
+      this.surrogateSize = 3;
+      this.detectIncompleteChar = base64DetectIncompleteChar;
+      break;
     default:
       this.write = passThroughWrite;
       return;
@@ -145,6 +150,21 @@ StringDecoder.prototype.detectIncompleteChar = function(buffer) {
   return i;
 };
 
+StringDecoder.prototype.end = function(buffer) {
+  var res = '';
+  if (buffer && buffer.length)
+    res = this.write(buffer);
+
+  if (this.charReceived) {
+    var cr = this.charReceived;
+    var buf = this.charBuffer;
+    var enc = this.encoding;
+    res += buf.slice(0, cr).toString(enc);
+  }
+
+  return res;
+};
+
 function passThroughWrite(buffer) {
   return buffer.toString(this.encoding);
 }
@@ -154,3 +174,9 @@ function utf16DetectIncompleteChar(buffer) {
   this.charLength = incomplete ? 2 : 0;
   return incomplete;
 }
+
+function base64DetectIncompleteChar(buffer) {
+  var incomplete = this.charReceived = buffer.length % 3;
+  this.charLength = incomplete ? 3 : 0;
+  return incomplete;
+}

lib/tls.js

@@ -967,6 +967,11 @@ SecurePair.prototype.destroy = function() {
     self.cleartext.writable = self.cleartext.readable = false;
 
     process.nextTick(function() {
+      if (self.cleartext._decoder) {
+        var ret = self.cleartext._decoder.end();
+        if (ret)
+          self.cleartext.emit('data', ret);
+      }
       self.cleartext.emit('end');
       self.encrypted.emit('close');
       self.cleartext.emit('close');

test/simple/test-string-decoder-end.js

@@ -0,0 +1,75 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// verify that the string decoder works getting 1 byte at a time,
+// the whole buffer at once, and that both match the .toString(enc)
+// result of the entire buffer.
+
+var assert = require('assert');
+var SD = require('string_decoder').StringDecoder;
+var encodings = ['base64', 'hex', 'utf8', 'utf16le', 'ucs2'];
+
+var bufs = [ '☃💩', 'asdf' ].map(function(b) {
+  return new Buffer(b);
+});
+
+// also test just arbitrary bytes from 0-15.
+for (var i = 1; i <= 16; i++) {
+  var bytes = new Array(i).join('.').split('.').map(function(_, j) {
+    return j + 0x78;
+  });
+  bufs.push(new Buffer(bytes));
+}
+
+encodings.forEach(testEncoding);
+
+console.log('ok');
+
+function testEncoding(encoding) {
+  bufs.forEach(function(buf) {
+    testBuf(encoding, buf);
+  });
+}
+
+function testBuf(encoding, buf) {
+  console.error('# %s', encoding, buf);
+
+  // write one byte at a time.
+  var s = new SD(encoding);
+  var res1 = '';
+  for (var i = 0; i < buf.length; i++) {
+    res1 += s.write(buf.slice(i, i + 1));
+  }
+  res1 += s.end();
+
+  // write the whole buffer at once.
+  var res2 = '';
+  var s = new SD(encoding);
+  res2 += s.write(buf);
+  res2 += s.end();
+
+  // .toString() on the buffer
+  var res3 = buf.toString(encoding);
+
+  console.log('expect=%j', res3);
+  assert.equal(res1, res3, 'one byte at a time should match toString');
+  assert.equal(res2, res3, 'all bytes at once should match toString');
+}