diff --git a/node.gyp b/node.gyp index 8ee17a70f9..cbd9dd61ac 100644 --- a/node.gyp +++ b/node.gyp @@ -105,6 +105,7 @@ 'src/node_zlib.cc', 'src/pipe_wrap.cc', 'src/signal_wrap.cc', + 'src/smalloc.cc', 'src/string_bytes.cc', 'src/stream_wrap.cc', 'src/slab_allocator.cc', @@ -134,6 +135,7 @@ 'src/node_wrap.h', 'src/pipe_wrap.h', 'src/queue.h', + 'src/smalloc.h', 'src/tty_wrap.h', 'src/tcp_wrap.h', 'src/udp_wrap.h', diff --git a/src/node_extensions.h b/src/node_extensions.h index 8e1c7021c7..240dc07d4f 100644 --- a/src/node_extensions.h +++ b/src/node_extensions.h @@ -29,6 +29,7 @@ NODE_EXT_LIST_ITEM(node_evals) NODE_EXT_LIST_ITEM(node_fs) NODE_EXT_LIST_ITEM(node_http_parser) NODE_EXT_LIST_ITEM(node_os) +NODE_EXT_LIST_ITEM(node_smalloc) NODE_EXT_LIST_ITEM(node_zlib) // libuv rewrite diff --git a/src/smalloc.cc b/src/smalloc.cc new file mode 100644 index 0000000000..1e54ec72f6 --- /dev/null +++ b/src/smalloc.cc @@ -0,0 +1,305 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + + +#include "node.h" +#include "smalloc.h" + +#include "v8.h" +#include "v8-profiler.h" + +#include +#include + +#define ALLOC_ID (0xA10C) + +namespace node { + +namespace smalloc { + +using v8::Arguments; +using v8::FunctionTemplate; +using v8::Handle; +using v8::HandleScope; +using v8::HeapProfiler; +using v8::Isolate; +using v8::Local; +using v8::Object; +using v8::Persistent; +using v8::RetainedObjectInfo; +using v8::String; +using v8::Uint32; +using v8::Value; +using v8::kExternalUnsignedByteArray; + + +struct CallbackInfo { + void* hint; + FreeCallback cb; +}; + +typedef v8::WeakReferenceCallbacks::Revivable Callback; +typedef v8::WeakReferenceCallbacks::Revivable CallbackFree; + +Callback target_cb; +CallbackFree target_free_cb; + +void TargetCallback(Isolate* isolate, Persistent* target, char* arg); +void TargetFreeCallback(Isolate* isolate, + Persistent* target, + void* arg); + + +// for internal use: copyOnto(source, source_start, dest, dest_start, length) +Handle CopyOnto(const Arguments& args) { + HandleScope scope(node_isolate); + + Local source = args[0]->ToObject(); + Local dest = args[2]->ToObject(); + size_t source_start = args[1]->Uint32Value(); + size_t dest_start = args[3]->Uint32Value(); + size_t length = args[4]->Uint32Value(); + size_t source_length = source->GetIndexedPropertiesExternalArrayDataLength(); + size_t dest_length = dest->GetIndexedPropertiesExternalArrayDataLength(); + char* source_data = static_cast( + source->GetIndexedPropertiesExternalArrayData()); + char* dest_data = static_cast( + dest->GetIndexedPropertiesExternalArrayData()); + + assert(source_data != NULL); + assert(dest_data != NULL); + // necessary to check in case (source|dest)_start _and_ length overflow + assert(length <= source_length); + assert(length <= dest_length); + assert(source_start <= source_length); + assert(dest_start <= dest_length); + // now we can guarantee these will catch oob access and *_start overflow + assert(source_start + length <= source_length); + assert(dest_start + length <= dest_length); + + memmove(dest_data + dest_start, source_data + source_start, length); + + return Undefined(node_isolate); +} + + +// for internal use: dest._data = sliceOnto(source, dest, start, end); +Handle SliceOnto(const Arguments& args) { + HandleScope scope(node_isolate); + + Local source = args[0]->ToObject(); + Local dest = args[1]->ToObject(); + char* source_data = static_cast(source-> + GetIndexedPropertiesExternalArrayData()); + size_t source_len = source->GetIndexedPropertiesExternalArrayDataLength(); + size_t start = args[2]->Uint32Value(); + size_t end = args[3]->Uint32Value(); + + assert(!dest->HasIndexedPropertiesInExternalArrayData()); + assert(source_data != NULL); + assert(end <= source_len); + assert(start <= end); + + dest->SetIndexedPropertiesToExternalArrayData(source_data + start, + kExternalUnsignedByteArray, + end - start); + + return scope.Close(source); +} + + +// for internal use: alloc(obj, n); +Handle Alloc(const Arguments& args) { + HandleScope scope(node_isolate); + + Local obj = args[0]->ToObject(); + size_t length = args[1]->Uint32Value(); + + Alloc(obj, length); + + return scope.Close(obj); +} + + +void Alloc(Handle obj, size_t length) { + assert(length <= kMaxLength); + + char* data = new char[length]; + Alloc(obj, data, length); +} + + +void Alloc(Handle obj, char* data, size_t length) { + assert(data != NULL); + + Persistent p_obj(node_isolate, obj); + + node_isolate->AdjustAmountOfExternalAllocatedMemory(length); + p_obj.MakeWeak(node_isolate, data, target_cb); + p_obj.MarkIndependent(node_isolate); + p_obj.SetWrapperClassId(node_isolate, ALLOC_ID); + p_obj->SetIndexedPropertiesToExternalArrayData(data, + kExternalUnsignedByteArray, + length); +} + + +void TargetCallback(Isolate* isolate, Persistent* target, char* data) { + int len = (*target)->GetIndexedPropertiesExternalArrayDataLength(); + if (data != NULL && len > 0) { + isolate->AdjustAmountOfExternalAllocatedMemory(-len); + delete[] data; + } + (*target).Dispose(); + (*target).Clear(); +} + + +void Alloc(Handle obj, size_t length, FreeCallback fn, void* hint) { + assert(length <= kMaxLength); + + char* data = new char[length]; + Alloc(obj, data, length, fn, hint); +} + + +void Alloc(Handle obj, + char* data, + size_t length, + FreeCallback fn, + void* hint) { + assert(data != NULL); + + CallbackInfo* cb_info = new CallbackInfo; + cb_info->cb = fn; + cb_info->hint = hint; + Persistent p_obj(node_isolate, obj); + + node_isolate->AdjustAmountOfExternalAllocatedMemory(length + + sizeof(*cb_info)); + p_obj.MakeWeak(node_isolate, static_cast(cb_info), target_free_cb); + p_obj.MarkIndependent(node_isolate); + p_obj.SetWrapperClassId(node_isolate, ALLOC_ID); + p_obj->SetIndexedPropertiesToExternalArrayData(data, + kExternalUnsignedByteArray, + length); +} + + +// TODO(trevnorris): running AllocDispose will cause data == NULL, which is +// then passed to cb_info->cb, which the user will need to check for. +void TargetFreeCallback(Isolate* isolate, + Persistent* target, + void* arg) { + Local obj = **target; + int len = obj->GetIndexedPropertiesExternalArrayDataLength(); + char* data = static_cast(obj->GetIndexedPropertiesExternalArrayData()); + CallbackInfo* cb_info = static_cast(arg); + isolate->AdjustAmountOfExternalAllocatedMemory(-(len + sizeof(*cb_info))); + (*target).Dispose(); + (*target).Clear(); + cb_info->cb(data, cb_info->hint); + delete cb_info; +} + + +class RetainedAllocInfo: public RetainedObjectInfo { +public: + RetainedAllocInfo(Handle wrapper); + virtual void Dispose(); + virtual bool IsEquivalent(RetainedObjectInfo* other); + virtual intptr_t GetHash(); + virtual const char* GetLabel(); + virtual intptr_t GetSizeInBytes(); + +private: + static const char label_[]; + char* data_; + int length_; +}; + + +const char RetainedAllocInfo::label_[] = "smalloc"; + + +RetainedAllocInfo::RetainedAllocInfo(Handle wrapper) { + Local obj = wrapper->ToObject(); + length_ = obj->GetIndexedPropertiesExternalArrayDataLength(); + data_ = static_cast(obj->GetIndexedPropertiesExternalArrayData()); +} + + +void RetainedAllocInfo::Dispose() { + delete this; +} + + +bool RetainedAllocInfo::IsEquivalent(RetainedObjectInfo* other) { + return label_ == other->GetLabel() && + data_ == static_cast(other)->data_; +} + + +intptr_t RetainedAllocInfo::GetHash() { + return reinterpret_cast(data_); +} + + +const char* RetainedAllocInfo::GetLabel() { + return label_; +} + + +intptr_t RetainedAllocInfo::GetSizeInBytes() { + return length_; +} + + +RetainedObjectInfo* WrapperInfo(uint16_t class_id, Handle wrapper) { + return new RetainedAllocInfo(wrapper); +} + + +void Initialize(Handle exports) { + exports->Set(String::New("copyOnto"), + FunctionTemplate::New(CopyOnto)->GetFunction()); + exports->Set(String::New("sliceOnto"), + FunctionTemplate::New(SliceOnto)->GetFunction()); + + exports->Set(String::New("alloc"), + FunctionTemplate::New(Alloc)->GetFunction()); + + exports->Set(String::New("kMaxLength"), + Uint32::New(kMaxLength, node_isolate)); + + target_cb = TargetCallback; + target_free_cb = TargetFreeCallback; + + HeapProfiler* heap_profiler = node_isolate->GetHeapProfiler(); + heap_profiler->SetWrapperClassInfoProvider(ALLOC_ID, WrapperInfo); +} + + +} // namespace smalloc + +} // namespace node + +NODE_MODULE(node_smalloc, node::smalloc::Initialize) diff --git a/src/smalloc.h b/src/smalloc.h new file mode 100644 index 0000000000..88a9530af6 --- /dev/null +++ b/src/smalloc.h @@ -0,0 +1,63 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +#ifndef NODE_SMALLOC_H_ +#define NODE_SMALLOC_H_ + +#include "v8.h" + +namespace node { + +/** + * Simple memory allocator. + * + * Utilities for external memory allocation management. Is an abstraction for + * v8's external array data handling to simplify and centralize how this is + * managed. + */ +namespace smalloc { + +// mirrors deps/v8/src/objects.h +static const unsigned int kMaxLength = 0x3fffffff; + +typedef void (*FreeCallback)(char* data, void* hint); + +/** + * Allocate external memory and set to passed object. If data is passed then + * will use that instead of allocating new. + */ +void Alloc(v8::Handle obj, size_t length); +void Alloc(v8::Handle obj, char* data, size_t length); +void Alloc(v8::Handle obj, + size_t length, + FreeCallback fn, + void* hint); +void Alloc(v8::Handle obj, + char* data, + size_t length, + FreeCallback fn, + void* hint); + +} // namespace smalloc + +} // namespace node + +#endif // NODE_SMALLOC_H_ diff --git a/test/simple/test-smalloc-alloc-segfault.js b/test/simple/test-smalloc-alloc-segfault.js new file mode 100644 index 0000000000..6d484857fa --- /dev/null +++ b/test/simple/test-smalloc-alloc-segfault.js @@ -0,0 +1,56 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var common = require('../common'); +var assert = require('assert'); + +var spawn = require('child_process').spawn; +var alloc = process.binding('smalloc').alloc; + + +// child +if (process.argv[2] === 'child') { + + // test that many allocations won't cause early ones to free prematurely + // note: if memory frees prematurely then reading out the values will cause + // it to seg fault + var arr = []; + for (var i = 0; i < 1e4; i++) { + arr.push(alloc({}, 1)); + alloc({}, 1024); + if (i % 10 === 0) gc(); + } + var sum = 0; + for (var i = 0; i < 1e4; i++) { + sum += arr[i][0]; + arr[i][0] = sum; + } + +} else { + // test case + var child = spawn(process.execPath, + ['--expose_gc', __filename, 'child']); + + child.on('exit', function(code, signal) { + assert.equal(code, 0, signal); + console.log('alloc didn\'t segfault'); + }); +} diff --git a/test/simple/test-smalloc-sliceonto-segfault.js b/test/simple/test-smalloc-sliceonto-segfault.js new file mode 100644 index 0000000000..62a438c6e3 --- /dev/null +++ b/test/simple/test-smalloc-sliceonto-segfault.js @@ -0,0 +1,59 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var common = require('../common'); +var assert = require('assert'); + +var spawn = require('child_process').spawn; +var alloc = process.binding('smalloc').alloc; +var sliceOnto = process.binding('smalloc').sliceOnto; + + +// child +if (process.argv[2] === 'child') { + + // test that many slices won't cause early ones to free prematurely + // note: if this fails then it will cause v8 to seg fault + var arr = []; + for (var i = 0; i < 1e4; i++) { + var a = alloc({}, 256); + var b = {}; + b._data = sliceOnto(a, b, 0, 1); + if (i % 2 === 0) + arr.push(b); + if (i % 10 === 0) gc(); + } + var sum = 0; + for (var i = 0; i < 5e3; i++) { + sum += arr[i][0]; + arr[i][0] = sum; + } + +} else { + // test case + var child = spawn(process.execPath, + ['--expose_gc', __filename, 'child']); + + child.on('exit', function(code, signal) { + assert.equal(code, 0, signal); + console.log('sliceOnto didn\'t segfault'); + }); +} diff --git a/test/simple/test-smalloc.js b/test/simple/test-smalloc.js new file mode 100644 index 0000000000..14450afa44 --- /dev/null +++ b/test/simple/test-smalloc.js @@ -0,0 +1,89 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var common = require('../common'); +var assert = require('assert'); + +var smalloc = process.binding('smalloc'); +var alloc = smalloc.alloc; +var copyOnto = smalloc.copyOnto; +var sliceOnto = smalloc.sliceOnto; + + +// verify initial allocation + +var b = alloc({}, 5); +assert.ok(typeof b === 'object'); +for (var i = 0; i < 5; i++) + assert.ok(b[i] !== undefined); + + +var b = {}; +var c = alloc(b, 5); +assert.equal(b, c); +assert.deepEqual(b, c); + + +var b = alloc({}, 5); +var c = {}; +c._data = sliceOnto(b, c, 0, 5); +assert.ok(typeof c._data === 'object'); +assert.equal(b, c._data); +assert.deepEqual(b, c._data); + + +// verify writes + +var b = alloc({}, 5); +for (var i = 0; i < 5; i++) + b[i] = i; +for (var i = 0; i < 5; i++) + assert.equal(b[i], i); + + +var b = alloc({}, 6); +var c0 = {}; +var c1 = {}; +c0._data = sliceOnto(b, c0, 0, 3); +c1._data = sliceOnto(b, c1, 3, 6); +for (var i = 0; i < 3; i++) { + c0[i] = i; + c1[i] = i + 3; +} +for (var i = 0; i < 3; i++) + assert.equal(b[i], i); +for (var i = 3; i < 6; i++) + assert.equal(b[i], i); + + +var a = alloc({}, 6); +var b = alloc({}, 6); +var c = alloc({}, 12); +for (var i = 0; i < 6; i++) { + a[i] = i; + b[i] = i * 2; +} +copyOnto(a, 0, c, 0, 6); +copyOnto(b, 0, c, 6, 6); +for (var i = 0; i < 6; i++) { + assert.equal(c[i], i); + assert.equal(c[i + 6], i * 2); +}