// Copyright Joyent, Inc. and other Node contributors. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to permit // persons to whom the Software is furnished to do so, subject to the // following conditions: // // The above copyright notice and this permission notice shall be included // in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN // NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. module.exports = Readable; Readable.ReadableState = ReadableState; var EE = require('events').EventEmitter; var Stream = require('stream'); var util = require('util'); var StringDecoder; util.inherits(Readable, Stream); function ReadableState(options, stream) { options = options || {}; // the point at which it stops calling _read() to fill the buffer // Note: 0 is a valid value, means "don't call _read preemptively ever" var hwm = options.highWaterMark; this.highWaterMark = (hwm || hwm === 0) ? hwm : 16 * 1024; // cast to ints. this.highWaterMark = ~~this.highWaterMark; this.buffer = []; this.length = 0; this.pipes = null; this.pipesCount = 0; this.flowing = false; this.ended = false; this.endEmitted = false; this.reading = false; // In streams that never have any data, and do push(null) right away, // the consumer can miss the 'end' event if they do some I/O before // consuming the stream. So, we don't emit('end') until some reading // happens. this.calledRead = false; // a flag to be able to tell if the onwrite cb is called immediately, // or on a later tick. We set this to true at first, becuase any // actions that shouldn't happen until "later" should generally also // not happen before the first write call. this.sync = true; // whenever we return null, then we set a flag to say // that we're awaiting a 'readable' event emission. this.needReadable = false; this.emittedReadable = false; // object stream flag. Used to make read(n) ignore n and to // make all the buffer merging and length checks go away this.objectMode = !!options.objectMode; // when piping, we only care about 'readable' events that happen // after read()ing all the bytes and not getting any pushback. this.ranOut = false; // the number of writers that are awaiting a drain event in .pipe()s this.awaitDrain = 0; // if true, a maybeReadMore has been scheduled this.readingMore = false; this.decoder = null; if (options.encoding) { if (!StringDecoder) StringDecoder = require('string_decoder').StringDecoder; this.decoder = new StringDecoder(options.encoding); } } function Readable(options) { if (!(this instanceof Readable)) return new Readable(options); this._readableState = new ReadableState(options, this); // legacy this.readable = true; Stream.call(this); } // Manually shove something into the read() buffer. // This returns true if the highWaterMark has not been hit yet, // similar to how Writable.write() returns true if you should // write() some more. Readable.prototype.push = function(chunk) { var state = this._readableState; if (typeof chunk === 'string' && !state.objectMode) chunk = new Buffer(chunk, arguments[1]); return readableAddChunk(this, state, chunk, false); }; Readable.prototype.unshift = function(chunk) { var state = this._readableState; if (typeof chunk === 'string' && !state.objectMode) chunk = new Buffer(chunk, arguments[1]); return readableAddChunk(this, state, chunk, true); }; function readableAddChunk(stream, state, chunk, addToFront) { state.reading = false; var er = chunkInvalid(state, chunk); if (er) { stream.emit('error', er); } else if (chunk === null || chunk === undefined) { onEofChunk(stream, state); } else if (state.objectMode || chunk && chunk.length > 0) { if (state.decoder) chunk = state.decoder.write(chunk); // update the buffer info. state.length += state.objectMode ? 1 : chunk.length; if (addToFront) state.buffer.unshift(chunk); else state.buffer.push(chunk); if (state.needReadable) emitReadable(stream); maybeReadMore(stream, state); } return needMoreData(state); } // if it's past the high water mark, we can push in some more. // Also, if we have no data yet, we can stand some // more bytes. This is to work around cases where hwm=0, // such as the repl. Also, if the push() triggered a // readable event, and the user called read(largeNumber) such that // needReadable was set, then we ought to push more, so that another // 'readable' event will be triggered. function needMoreData(state) { return !state.ended && (state.needReadable || state.length < state.highWaterMark || state.length === 0); } // backwards compatibility. Readable.prototype.setEncoding = function(enc) { if (!StringDecoder) StringDecoder = require('string_decoder').StringDecoder; this._readableState.decoder = new StringDecoder(enc); }; // Don't raise the hwm > 128MB var MAX_HWM = 0x800000; function roundUpToNextPowerOf2(n) { if (n >= MAX_HWM) { n = MAX_HWM; } else { // Get the next highest power of 2 n--; for (var p = 1; p < 32; p <<= 1) n |= n >> p; n++; } return n; } function howMuchToRead(n, state) { if (state.length === 0 && state.ended) return 0; if (state.objectMode) return n === 0 ? 0 : 1; if (isNaN(n) || n === null) { // only flow one buffer at a time if (state.flowing && state.buffer.length) return state.buffer[0].length; else return state.length; } if (n <= 0) return 0; // If we're asking for more than the target buffer level, // then raise the water mark. Bump up to the next highest // power of 2, to prevent increasing it excessively in tiny // amounts. if (n > state.highWaterMark) state.highWaterMark = roundUpToNextPowerOf2(n); // don't have that much. return null, unless we've ended. if (n > state.length) { if (!state.ended) { state.needReadable = true; return 0; } else return state.length; } return n; } // you can override either this method, or the async _read(n) below. Readable.prototype.read = function(n) { var state = this._readableState; state.calledRead = true; var nOrig = n; if (typeof n !== 'number' || n > 0) state.emittedReadable = false; // if we're doing read(0) to trigger a readable event, but we // already have a bunch of data in the buffer, then just trigger // the 'readable' event and move on. if (n === 0 && state.needReadable && state.length >= state.highWaterMark) { emitReadable(this); return null; } n = howMuchToRead(n, state); // if we've ended, and we're now clear, then finish it up. if (n === 0 && state.ended) { if (state.length === 0) endReadable(this); return null; } // All the actual chunk generation logic needs to be // *below* the call to _read. The reason is that in certain // synthetic stream cases, such as passthrough streams, _read // may be a completely synchronous operation which may change // the state of the read buffer, providing enough data when // before there was *not* enough. // // So, the steps are: // 1. Figure out what the state of things will be after we do // a read from the buffer. // // 2. If that resulting state will trigger a _read, then call _read. // Note that this may be asynchronous, or synchronous. Yes, it is // deeply ugly to write APIs this way, but that still doesn't mean // that the Readable class should behave improperly, as streams are // designed to be sync/async agnostic. // Take note if the _read call is sync or async (ie, if the read call // has returned yet), so that we know whether or not it's safe to emit // 'readable' etc. // // 3. Actually pull the requested chunks out of the buffer and return. // if we need a readable event, then we need to do some reading. var doRead = state.needReadable; // if we currently have less than the highWaterMark, then also read some if (state.length - n <= state.highWaterMark) doRead = true; // however, if we've ended, then there's no point, and if we're already // reading, then it's unnecessary. if (state.ended || state.reading) doRead = false; if (doRead) { state.reading = true; state.sync = true; // if the length is currently zero, then we *need* a readable event. if (state.length === 0) state.needReadable = true; // call internal read method this._read(state.highWaterMark); state.sync = false; } // If _read called its callback synchronously, then `reading` // will be false, and we need to re-evaluate how much data we // can return to the user. if (doRead && !state.reading) n = howMuchToRead(nOrig, state); var ret; if (n > 0) ret = fromList(n, state); else ret = null; if (ret === null) { state.needReadable = true; n = 0; } state.length -= n; // If we have nothing in the buffer, then we want to know // as soon as we *do* get something into the buffer. if (state.length === 0 && !state.ended) state.needReadable = true; // If we happened to read() exactly the remaining amount in the // buffer, and the EOF has been seen at this point, then make sure // that we emit 'end' on the very next tick. if (state.ended && !state.endEmitted && state.length === 0) endReadable(this); return ret; }; function chunkInvalid(state, chunk) { var er = null; if (!Buffer.isBuffer(chunk) && 'string' !== typeof chunk && chunk !== null && chunk !== undefined && !state.objectMode && !er) { er = new TypeError('Invalid non-string/buffer chunk'); } return er; } function onEofChunk(stream, state) { state.ended = true; if (state.decoder) { var chunk = state.decoder.end(); if (chunk && chunk.length) { state.buffer.push(chunk); state.length += state.objectMode ? 1 : chunk.length; } } // if we've ended and we have some data left, then emit // 'readable' now to make sure it gets picked up. if (state.length > 0) emitReadable(stream); else endReadable(stream); } // Don't emit readable right away in sync mode, because this can trigger // another read() call => stack overflow. This way, it might trigger // a nextTick recursion warning, but that's not so bad. function emitReadable(stream) { var state = stream._readableState; state.needReadable = false; if (state.emittedReadable) return; state.emittedReadable = true; if (state.sync) process.nextTick(function() { emitReadable_(stream); }); else emitReadable_(stream); } function emitReadable_(stream) { var state = stream._readableState; stream.emit('readable'); } // at this point, the user has presumably seen the 'readable' event, // and called read() to consume some data. that may have triggered // in turn another _read(n) call, in which case reading = true if // it's in progress. // However, if we're not ended, or reading, and the length < hwm, // then go ahead and try to read some more preemptively. function maybeReadMore(stream, state) { if (!state.readingMore) { state.readingMore = true; process.nextTick(function() { maybeReadMore_(stream, state); }); } } function maybeReadMore_(stream, state) { var len = state.length; while (!state.reading && !state.flowing && !state.ended && state.length < state.highWaterMark) { stream.read(0); if (len === state.length) // didn't get any data, stop spinning. break; else len = state.length; } state.readingMore = false; } // abstract method. to be overridden in specific implementation classes. // call cb(er, data) where data is <= n in length. // for virtual (non-string, non-buffer) streams, "length" is somewhat // arbitrary, and perhaps not very meaningful. Readable.prototype._read = function(n) { this.emit('error', new Error('not implemented')); }; Readable.prototype.pipe = function(dest, pipeOpts) { var src = this; var state = this._readableState; switch (state.pipesCount) { case 0: state.pipes = dest; break; case 1: state.pipes = [state.pipes, dest]; break; default: state.pipes.push(dest); break; } state.pipesCount += 1; var doEnd = (!pipeOpts || pipeOpts.end !== false) && dest !== process.stdout && dest !== process.stderr; var endFn = doEnd ? onend : cleanup; if (state.endEmitted) process.nextTick(endFn); else src.once('end', endFn); dest.on('unpipe', onunpipe); function onunpipe(readable) { if (readable !== src) return; cleanup(); } function onend() { dest.end(); } // when the dest drains, it reduces the awaitDrain counter // on the source. This would be more elegant with a .once() // handler in flow(), but adding and removing repeatedly is // too slow. var ondrain = pipeOnDrain(src); dest.on('drain', ondrain); function cleanup() { // cleanup event handlers once the pipe is broken dest.removeListener('close', onclose); dest.removeListener('finish', onfinish); dest.removeListener('drain', ondrain); dest.removeListener('error', onerror); dest.removeListener('unpipe', onunpipe); src.removeListener('end', onend); src.removeListener('end', cleanup); // if the reader is waiting for a drain event from this // specific writer, then it would cause it to never start // flowing again. // So, if this is awaiting a drain, then we just call it now. // If we don't know, then assume that we are waiting for one. if (!dest._writableState || dest._writableState.needDrain) ondrain(); } // if the dest has an error, then stop piping into it. // however, don't suppress the throwing behavior for this. function onerror(er) { unpipe(); if (EE.listenerCount(dest, 'error') === 0) dest.emit('error', er); } dest.once('error', onerror); // Both close and finish should trigger unpipe, but only once. function onclose() { dest.removeListener('finish', onfinish); unpipe(); } dest.once('close', onclose); function onfinish() { dest.removeListener('close', onclose); unpipe(); } dest.once('finish', onfinish); function unpipe() { src.unpipe(dest); } // tell the dest that it's being piped to dest.emit('pipe', src); // start the flow if it hasn't been started already. if (!state.flowing) { // the handler that waits for readable events after all // the data gets sucked out in flow. // This would be easier to follow with a .once() handler // in flow(), but that is too slow. this.on('readable', pipeOnReadable); state.flowing = true; process.nextTick(function() { flow(src); }); } return dest; }; function pipeOnDrain(src) { return function() { var dest = this; var state = src._readableState; state.awaitDrain--; if (state.awaitDrain === 0) flow(src); }; } function flow(src) { var state = src._readableState; var chunk; state.awaitDrain = 0; function write(dest, i, list) { var written = dest.write(chunk); if (false === written) { state.awaitDrain++; } } while (state.pipesCount && null !== (chunk = src.read())) { if (state.pipesCount === 1) write(state.pipes, 0, null); else state.pipes.forEach(write); src.emit('data', chunk); // if anyone needs a drain, then we have to wait for that. if (state.awaitDrain > 0) return; } // if every destination was unpiped, either before entering this // function, or in the while loop, then stop flowing. // // NB: This is a pretty rare edge case. if (state.pipesCount === 0) { state.flowing = false; // if there were data event listeners added, then switch to old mode. if (EE.listenerCount(src, 'data') > 0) emitDataEvents(src); return; } // at this point, no one needed a drain, so we just ran out of data // on the next readable event, start it over again. state.ranOut = true; } function pipeOnReadable() { if (this._readableState.ranOut) { this._readableState.ranOut = false; flow(this); } } Readable.prototype.unpipe = function(dest) { var state = this._readableState; // if we're not piping anywhere, then do nothing. if (state.pipesCount === 0) return this; // just one destination. most common case. if (state.pipesCount === 1) { // passed in one, but it's not the right one. if (dest && dest !== state.pipes) return this; if (!dest) dest = state.pipes; // got a match. state.pipes = null; state.pipesCount = 0; this.removeListener('readable', pipeOnReadable); state.flowing = false; if (dest) dest.emit('unpipe', this); return this; } // slow case. multiple pipe destinations. if (!dest) { // remove all. var dests = state.pipes; var len = state.pipesCount; state.pipes = null; state.pipesCount = 0; this.removeListener('readable', pipeOnReadable); state.flowing = false; for (var i = 0; i < len; i++) dests[i].emit('unpipe', this); return this; } // try to find the right one. var i = state.pipes.indexOf(dest); if (i === -1) return this; state.pipes.splice(i, 1); state.pipesCount -= 1; if (state.pipesCount === 1) state.pipes = state.pipes[0]; dest.emit('unpipe', this); return this; }; // set up data events if they are asked for // Ensure readable listeners eventually get something Readable.prototype.on = function(ev, fn) { var res = Stream.prototype.on.call(this, ev, fn); if (ev === 'data' && !this._readableState.flowing) emitDataEvents(this); if (ev === 'readable' && !this._readableState.reading) this.read(0); return res; }; Readable.prototype.addListener = Readable.prototype.on; // pause() and resume() are remnants of the legacy readable stream API // If the user uses them, then switch into old mode. Readable.prototype.resume = function() { emitDataEvents(this); this.read(0); this.emit('resume'); }; Readable.prototype.pause = function() { emitDataEvents(this, true); this.emit('pause'); }; function emitDataEvents(stream, startPaused) { var state = stream._readableState; if (state.flowing) { // https://github.com/isaacs/readable-stream/issues/16 throw new Error('Cannot switch to old mode now.'); } var paused = startPaused || false; var readable = false; // convert to an old-style stream. stream.readable = true; stream.pipe = Stream.prototype.pipe; stream.on = stream.addListener = Stream.prototype.on; stream.on('readable', function() { readable = true; var c; while (!paused && (null !== (c = stream.read()))) stream.emit('data', c); if (c === null) { readable = false; stream._readableState.needReadable = true; } }); stream.pause = function() { paused = true; this.emit('pause'); }; stream.resume = function() { paused = false; if (readable) process.nextTick(function() { stream.emit('readable'); }); else this.read(0); this.emit('resume'); }; // now make it start, just in case it hadn't already. stream.emit('readable'); } // wrap an old-style stream as the async data source. // This is *not* part of the readable stream interface. // It is an ugly unfortunate mess of history. Readable.prototype.wrap = function(stream) { var state = this._readableState; var paused = false; var self = this; stream.on('end', function() { state.ended = true; if (state.decoder) { var chunk = state.decoder.end(); if (chunk && chunk.length) self.push(chunk); } self.push(null); }); stream.on('data', function(chunk) { if (state.decoder) chunk = state.decoder.write(chunk); if (!chunk || !chunk.length) return; var ret = self.push(chunk); if (!ret) { paused = true; stream.pause(); } }); // proxy all the other methods. // important when wrapping filters and duplexes. for (var i in stream) { if (typeof stream[i] === 'function' && typeof this[i] === 'undefined') { this[i] = function(method) { return function() { return stream[method].apply(stream, arguments); }}(i); } } // proxy certain important events. var events = ['error', 'close', 'destroy', 'pause', 'resume']; events.forEach(function(ev) { stream.on(ev, self.emit.bind(self, ev)); }); // when we try to consume some more bytes, simply unpause the // underlying stream. self._read = function(n) { if (paused) { stream.resume(); paused = false; } }; }; // exposed for testing purposes only. Readable._fromList = fromList; // Pluck off n bytes from an array of buffers. // Length is the combined lengths of all the buffers in the list. function fromList(n, state) { var list = state.buffer; var length = state.length; var stringMode = !!state.decoder; var objectMode = !!state.objectMode; var ret; // nothing in the list, definitely empty. if (list.length === 0) return null; if (length === 0) ret = null; else if (objectMode) ret = list.shift(); else if (!n || n >= length) { // read it all, truncate the array. if (stringMode) ret = list.join(''); else ret = Buffer.concat(list, length); list.length = 0; } else { // read just some of it. if (n < list[0].length) { // just take a part of the first list item. // slice is the same for buffers and strings. var buf = list[0]; ret = buf.slice(0, n); list[0] = buf.slice(n); } else if (n === list[0].length) { // first list is a perfect match ret = list.shift(); } else { // complex case. // we have enough to cover it, but it spans past the first buffer. if (stringMode) ret = ''; else ret = new Buffer(n); var c = 0; for (var i = 0, l = list.length; i < l && c < n; i++) { var buf = list[0]; var cpy = Math.min(n - c, buf.length); if (stringMode) ret += buf.slice(0, cpy); else buf.copy(ret, c, 0, cpy); if (cpy < buf.length) list[0] = buf.slice(cpy); else list.shift(); c += cpy; } } } return ret; } function endReadable(stream) { var state = stream._readableState; // If we get here before consuming all the bytes, then that is a // bug in node. Should never happen. if (state.length > 0) throw new Error('endReadable called on non-empty stream'); if (!state.endEmitted && state.calledRead) { state.ended = true; state.endEmitted = true; process.nextTick(function() { stream.readable = false; stream.emit('end'); }); } }