lukechilds
/
node
mirror of https://github.com/lukechilds/node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

buffer: add buffer.transcode 

Add buffer.transcode(source, from, to) method. Primarily uses ICU
to transcode a buffer's content from one of Node.js' supported
encodings to another.

Originally part of a proposal to add a new unicode module. Decided
to refactor the approach towrds individual PRs without a new module.

Refs: https://github.com/nodejs/node/pull/8075
PR-URL: https://github.com/nodejs/node/pull/9038
Reviewed-By: Anna Henningsen <anna@addaleax.net>
v7.x
James M Snell 9 years ago
committed by Evan Lucas
parent
a5c62cb4f2
commit
0939edd4ed
9 changed files with 437 additions and 39 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 27
      doc/api/buffer.md
  2. 4
      lib/buffer.js
  3. 30
      lib/internal/buffer.js
  4. 1
      node.gyp
  5. 55
      src/node_buffer.cc
  6. 280
      src/node_i18n.cc
  7. 27
      src/util.h
  8. 48
      test/parallel/test-icu-transcode.js
  9. 4
      tools/icu/icu-generic.gyp

27
doc/api/buffer.md
View File

@ -2302,6 +2302,33 @@ added: v3.0.0
On 32-bit architectures, this value is `(2^30)-1` (~1GB).
On 64-bit architectures, this value is `(2^31)-1` (~2GB).
## buffer.transcode(source, fromEnc, toEnc)
<!-- YAML
added: REPLACEME
-->
* `source` {Buffer} A `Buffer` instance
* `fromEnc` {String} The current encoding
* `toEnc` {String} To target encoding
Re-encodes the given `Buffer` instance from one character encoding to another.
Returns a new `Buffer` instance.
Throws if the `fromEnc` or `toEnc` specify invalid character encodings or if
conversion from `fromEnc` to `toEnc` is not permitted.
The transcoding process will use substitution characters if a given byte
sequence cannot be adequately represented in the target encoding. For instance:
```js
const newBuf = buffer.transcode(Buffer.from('€'), 'utf8', 'ascii');
console.log(newBuf.toString('ascii'));
// prints '?'
```
Because the Euro (`€`) sign is not representable in US-ASCII, it is replaced
with `?` in the transcoded `Buffer`.
## Class: SlowBuffer
<!-- YAML
deprecated: v6.0.0

4
lib/buffer.js
View File

@ -1360,3 +1360,7 @@ Buffer.prototype.swap64 = function swap64() {
};
Buffer.prototype.toLocaleString = Buffer.prototype.toString;
// Put this at the end because internal/buffer has a circular
// dependency on Buffer.
exports.transcode = require('internal/buffer').transcode;

30
lib/internal/buffer.js
View File

@ -0,0 +1,30 @@
'use strict';
if (!process.binding('config').hasIntl) {
return;
}
const normalizeEncoding = require('internal/util').normalizeEncoding;
const Buffer = require('buffer').Buffer;
const icu = process.binding('icu');
// Transcodes the Buffer from one encoding to another, returning a new
// Buffer instance.
exports.transcode = function transcode(source, fromEncoding, toEncoding) {
if (!Buffer.isBuffer(source))
throw new TypeError('"source" argument must be a Buffer');
if (source.length === 0) return Buffer.alloc(0);
fromEncoding = normalizeEncoding(fromEncoding) || fromEncoding;
toEncoding = normalizeEncoding(toEncoding) || toEncoding;
const result = icu.transcode(source, fromEncoding, toEncoding);
if (Buffer.isBuffer(result))
return result;
const code = icu.icuErrName(result);
const err = new Error(`Unable to transcode Buffer [${code}]`);
err.code = code;
err.errno = result;
throw err;
};

1
node.gyp
View File

@ -74,6 +74,7 @@
'lib/v8.js',
'lib/vm.js',
'lib/zlib.js',
'lib/internal/buffer.js',
'lib/internal/child_process.js',
'lib/internal/cluster.js',
'lib/internal/freelist.js',

55
src/node_buffer.cc
View File

@ -22,23 +22,6 @@
if (!(r)) return env->ThrowRangeError("out of range index"); \
} while (0)
#define THROW_AND_RETURN_UNLESS_BUFFER(env, obj) \
do { \
if (!HasInstance(obj)) \
return env->ThrowTypeError("argument should be a Buffer"); \
} while (0)
#define SPREAD_ARG(val, name) \
CHECK((val)->IsUint8Array()); \
Local<Uint8Array> name = (val).As<Uint8Array>(); \
ArrayBuffer::Contents name##_c = name->Buffer()->GetContents(); \
const size_t name##_offset = name->ByteOffset(); \
const size_t name##_length = name->ByteLength(); \
char* const name##_data = \
static_cast<char*>(name##_c.Data()) + name##_offset; \
if (name##_length > 0) \
CHECK_NE(name##_data, nullptr);
#define SLICE_START_END(start_arg, end_arg, end_max) \
size_t start; \
size_t end; \
@ -448,7 +431,7 @@ void StringSlice(const FunctionCallbackInfo<Value>& args) {
Isolate* isolate = env->isolate();
THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
SPREAD_ARG(args.This(), ts_obj);
SPREAD_BUFFER_ARG(args.This(), ts_obj);
if (ts_obj_length == 0)
return args.GetReturnValue().SetEmptyString();
@ -465,7 +448,7 @@ void StringSlice<UCS2>(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
SPREAD_ARG(args.This(), ts_obj);
SPREAD_BUFFER_ARG(args.This(), ts_obj);
if (ts_obj_length == 0)
return args.GetReturnValue().SetEmptyString();
@ -543,8 +526,8 @@ void Copy(const FunctionCallbackInfo<Value> &args) {
THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
Local<Object> target_obj = args[0].As<Object>();
SPREAD_ARG(args.This(), ts_obj);
SPREAD_ARG(target_obj, target);
SPREAD_BUFFER_ARG(args.This(), ts_obj);
SPREAD_BUFFER_ARG(target_obj, target);
size_t target_start;
size_t source_start;
@ -577,7 +560,7 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
size_t start = args[2]->Uint32Value();
size_t end = args[3]->Uint32Value();
@ -590,7 +573,7 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
// First check if Buffer has been passed.
if (Buffer::HasInstance(args[1])) {
SPREAD_ARG(args[1], fill_obj);
SPREAD_BUFFER_ARG(args[1], fill_obj);
str_length = fill_obj_length;
memcpy(ts_obj_data + start, fill_obj_data, MIN(str_length, fill_length));
goto start_fill;
@ -669,7 +652,7 @@ void StringWrite(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
SPREAD_ARG(args.This(), ts_obj);
SPREAD_BUFFER_ARG(args.This(), ts_obj);
if (!args[0]->IsString())
return env->ThrowTypeError("Argument must be a string");
@ -747,7 +730,7 @@ static inline void Swizzle(char* start, unsigned int len) {
template <typename T, enum Endianness endianness>
void ReadFloatGeneric(const FunctionCallbackInfo<Value>& args) {
THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
uint32_t offset = args[1]->Uint32Value();
CHECK_LE(offset + sizeof(T), ts_obj_length);
@ -881,8 +864,8 @@ void CompareOffset(const FunctionCallbackInfo<Value> &args) {
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_ARG(args[1], target);
SPREAD_BUFFER_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[1], target);
size_t target_start;
size_t source_start;
@ -921,8 +904,8 @@ void Compare(const FunctionCallbackInfo<Value> &args) {
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]);
SPREAD_ARG(args[0], obj_a);
SPREAD_ARG(args[1], obj_b);
SPREAD_BUFFER_ARG(args[0], obj_a);
SPREAD_BUFFER_ARG(args[1], obj_b);
size_t cmp_length = MIN(obj_a_length, obj_b_length);
@ -977,7 +960,7 @@ void IndexOfString(const FunctionCallbackInfo<Value>& args) {
UTF8);
THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
Local<String> needle = args[1].As<String>();
int64_t offset_i64 = args[2]->IntegerValue();
@ -1084,8 +1067,8 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[0]);
THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[1]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_ARG(args[1], buf);
SPREAD_BUFFER_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[1], buf);
int64_t offset_i64 = args[2]->IntegerValue();
bool is_forward = args[4]->IsTrue();
@ -1143,7 +1126,7 @@ void IndexOfNumber(const FunctionCallbackInfo<Value>& args) {
ASSERT(args[3]->IsBoolean());
THROW_AND_RETURN_UNLESS_BUFFER(Environment::GetCurrent(args), args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
uint32_t needle = args[1]->Uint32Value();
int64_t offset_i64 = args[2]->IntegerValue();
@ -1171,7 +1154,7 @@ void IndexOfNumber(const FunctionCallbackInfo<Value>& args) {
void Swap16(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
SwapBytes16(ts_obj_data, ts_obj_length);
args.GetReturnValue().Set(args[0]);
}
@ -1180,7 +1163,7 @@ void Swap16(const FunctionCallbackInfo<Value>& args) {
void Swap32(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
SwapBytes32(ts_obj_data, ts_obj_length);
args.GetReturnValue().Set(args[0]);
}
@ -1189,7 +1172,7 @@ void Swap32(const FunctionCallbackInfo<Value>& args) {
void Swap64(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
SPREAD_ARG(args[0], ts_obj);
SPREAD_BUFFER_ARG(args[0], ts_obj);
SwapBytes64(ts_obj_data, ts_obj_length);
args.GetReturnValue().Set(args[0]);
}

280
src/node_i18n.cc
View File

@ -24,6 +24,7 @@
#if defined(NODE_HAVE_I18N_SUPPORT)
#include "node.h"
#include "node_buffer.h"
#include "env.h"
#include "env-inl.h"
#include "util.h"
@ -34,6 +35,10 @@
#include <unicode/uchar.h>
#include <unicode/udata.h>
#include <unicode/uidna.h>
#include <unicode/utypes.h>
#include <unicode/ucnv.h>
#include <unicode/utf8.h>
#include <unicode/utf16.h>
#ifdef NODE_HAVE_SMALL_ICU
/* if this is defined, we have a 'secondary' entry point.
@ -54,7 +59,9 @@ namespace node {
using v8::Context;
using v8::FunctionCallbackInfo;
using v8::Isolate;
using v8::Local;
using v8::MaybeLocal;
using v8::Object;
using v8::String;
using v8::Value;
@ -63,6 +70,275 @@ bool flag_icu_data_dir = false;
namespace i18n {
const size_t kStorageSize = 1024;
// TODO(jasnell): This could potentially become a member of MaybeStackBuffer
// at some point in the future. Care would need to be taken with the
// MaybeStackBuffer<UChar> variant below.
MaybeLocal<Object> AsBuffer(Isolate* isolate,
MaybeStackBuffer<char>* buf,
size_t len) {
if (buf->IsAllocated()) {
MaybeLocal<Object> ret = Buffer::New(isolate, buf->out(), len);
if (!ret.IsEmpty()) buf->Release();
return ret;
}
return Buffer::Copy(isolate, buf->out(), len);
}
MaybeLocal<Object> AsBuffer(Isolate* isolate,
MaybeStackBuffer<UChar>* buf,
size_t len) {
char* dst = reinterpret_cast<char*>(**buf);
MaybeLocal<Object> ret;
if (buf->IsAllocated()) {
ret = Buffer::New(isolate, dst, len);
if (!ret.IsEmpty()) buf->Release();
} else {
ret = Buffer::Copy(isolate, dst, len);
}
if (!ret.IsEmpty() && IsBigEndian()) {
SPREAD_BUFFER_ARG(ret.ToLocalChecked(), buf);
SwapBytes16(buf_data, buf_length);
}
return ret;
}
struct Converter {
explicit Converter(const char* name, const char* sub = NULL)
: conv(nullptr) {
UErrorCode status = U_ZERO_ERROR;
conv = ucnv_open(name, &status);
CHECK(U_SUCCESS(status));
if (sub != NULL) {
ucnv_setSubstChars(conv, sub, strlen(sub), &status);
}
}
~Converter() {
ucnv_close(conv);
}
UConverter* conv;
};
// One-Shot Converters
void CopySourceBuffer(MaybeStackBuffer<UChar>* dest,
const char* data,
const size_t length,
const size_t length_in_chars) {
dest->AllocateSufficientStorage(length_in_chars);
char* dst = reinterpret_cast<char*>(**dest);
memcpy(dst, data, length);
if (IsBigEndian()) {
SwapBytes16(dst, length);
}
}
typedef MaybeLocal<Object> (*TranscodeFunc)(Isolate* isolate,
const char* fromEncoding,
const char* toEncoding,
const char* source,
const size_t source_length,
UErrorCode* status);
MaybeLocal<Object> Transcode(Isolate* isolate,
const char* fromEncoding,
const char* toEncoding,
const char* source,
const size_t source_length,
UErrorCode* status) {
*status = U_ZERO_ERROR;
MaybeLocal<Object> ret;
MaybeStackBuffer<char> result;
Converter to(toEncoding, "?");
Converter from(fromEncoding);
const uint32_t limit = source_length * ucnv_getMaxCharSize(to.conv);
result.AllocateSufficientStorage(limit);
char* target = *result;
ucnv_convertEx(to.conv, from.conv, &target, target + limit,
&source, source + source_length, nullptr, nullptr,
nullptr, nullptr, true, true, status);
if (U_SUCCESS(*status))
ret = AsBuffer(isolate, &result, target - &result[0]);
return ret;
}
MaybeLocal<Object> TranscodeToUcs2(Isolate* isolate,
const char* fromEncoding,
const char* toEncoding,
const char* source,
const size_t source_length,
UErrorCode* status) {
*status = U_ZERO_ERROR;
MaybeLocal<Object> ret;
MaybeStackBuffer<UChar> destbuf(source_length);
Converter from(fromEncoding);
const size_t length_in_chars = source_length * sizeof(*destbuf);
ucnv_toUChars(from.conv, *destbuf, length_in_chars,
source, source_length, status);
if (U_SUCCESS(*status))
ret = AsBuffer(isolate, &destbuf, length_in_chars);
return ret;
}
MaybeLocal<Object> TranscodeFromUcs2(Isolate* isolate,
const char* fromEncoding,
const char* toEncoding,
const char* source,
const size_t source_length,
UErrorCode* status) {
*status = U_ZERO_ERROR;
MaybeStackBuffer<UChar> sourcebuf;
MaybeLocal<Object> ret;
Converter to(toEncoding, "?");
const size_t length_in_chars = source_length / sizeof(UChar);
CopySourceBuffer(&sourcebuf, source, source_length, length_in_chars);
MaybeStackBuffer<char> destbuf(length_in_chars);
const uint32_t len = ucnv_fromUChars(to.conv, *destbuf, length_in_chars,
*sourcebuf, length_in_chars, status);
if (U_SUCCESS(*status))
ret = AsBuffer(isolate, &destbuf, len);
return ret;
}
MaybeLocal<Object> TranscodeUcs2FromUtf8(Isolate* isolate,
const char* fromEncoding,
const char* toEncoding,
const char* source,
const size_t source_length,
UErrorCode* status) {
*status = U_ZERO_ERROR;
MaybeStackBuffer<UChar, kStorageSize> destbuf;
int32_t result_length;
u_strFromUTF8(*destbuf, kStorageSize, &result_length,
source, source_length, status);
MaybeLocal<Object> ret;
if (U_SUCCESS(*status)) {
ret = AsBuffer(isolate, &destbuf, result_length * sizeof(**destbuf));
} else if (*status == U_BUFFER_OVERFLOW_ERROR) {
*status = U_ZERO_ERROR;
destbuf.AllocateSufficientStorage(result_length);
u_strFromUTF8(*destbuf, result_length, &result_length,
source, source_length, status);
if (U_SUCCESS(*status))
ret = AsBuffer(isolate, &destbuf, result_length * sizeof(**destbuf));
}
return ret;
}
MaybeLocal<Object> TranscodeUtf8FromUcs2(Isolate* isolate,
const char* fromEncoding,
const char* toEncoding,
const char* source,
const size_t source_length,
UErrorCode* status) {
*status = U_ZERO_ERROR;
MaybeLocal<Object> ret;
const size_t length_in_chars = source_length / sizeof(UChar);
int32_t result_length;
MaybeStackBuffer<UChar> sourcebuf;
MaybeStackBuffer<char, kStorageSize> destbuf;
CopySourceBuffer(&sourcebuf, source, source_length, length_in_chars);
u_strToUTF8(*destbuf, kStorageSize, &result_length,
*sourcebuf, length_in_chars, status);
if (U_SUCCESS(*status)) {
ret = AsBuffer(isolate, &destbuf, result_length);
} else if (*status == U_BUFFER_OVERFLOW_ERROR) {
*status = U_ZERO_ERROR;
destbuf.AllocateSufficientStorage(result_length);
u_strToUTF8(*destbuf, result_length, &result_length, *sourcebuf,
length_in_chars, status);
if (U_SUCCESS(*status)) {
ret = Buffer::New(isolate, *destbuf, result_length);
destbuf.Release();
}
}
return ret;
}
const char* EncodingName(const enum encoding encoding) {
switch (encoding) {
case ASCII: return "us-ascii";
case LATIN1: return "iso8859-1";
case UCS2: return "utf16le";
case UTF8: return "utf-8";
default: return NULL;
}
}
bool SupportedEncoding(const enum encoding encoding) {
switch (encoding) {
case ASCII:
case LATIN1:
case UCS2:
case UTF8: return true;
default: return false;
}
}
void Transcode(const FunctionCallbackInfo<Value>&args) {
Environment* env = Environment::GetCurrent(args);
Isolate* isolate = env->isolate();
UErrorCode status = U_ZERO_ERROR;
MaybeLocal<Object> result;
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
SPREAD_BUFFER_ARG(args[0], ts_obj);
const enum encoding fromEncoding = ParseEncoding(isolate, args[1], BUFFER);
const enum encoding toEncoding = ParseEncoding(isolate, args[2], BUFFER);
if (SupportedEncoding(fromEncoding) && SupportedEncoding(toEncoding)) {
TranscodeFunc tfn = &Transcode;
switch (fromEncoding) {
case ASCII:
case LATIN1:
if (toEncoding == UCS2)
tfn = &TranscodeToUcs2;
break;
case UTF8:
if (toEncoding == UCS2)
tfn = &TranscodeUcs2FromUtf8;
break;
case UCS2:
switch (toEncoding) {
case UCS2:
tfn = &Transcode;
break;
case UTF8:
tfn = &TranscodeUtf8FromUcs2;
break;
default:
tfn = TranscodeFromUcs2;
}
break;
default:
// This should not happen because of the SupportedEncoding checks
ABORT();
}
result = tfn(isolate, EncodingName(fromEncoding), EncodingName(toEncoding),
ts_obj_data, ts_obj_length, &status);
} else {
status = U_ILLEGAL_ARGUMENT_ERROR;
}
if (result.IsEmpty())
return args.GetReturnValue().Set(status);
return args.GetReturnValue().Set(result.ToLocalChecked());
}
static void ICUErrorName(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
UErrorCode status = static_cast<UErrorCode>(args[0]->Int32Value());
args.GetReturnValue().Set(
String::NewFromUtf8(env->isolate(),
u_errorName(status),
v8::NewStringType::kNormal).ToLocalChecked());
}
bool InitializeICUDirectory(const char* icu_data_path) {
if (icu_data_path != nullptr) {
flag_icu_data_dir = true;
@ -282,6 +558,10 @@ void Init(Local<Object> target,
env->SetMethod(target, "toUnicode", ToUnicode);
env->SetMethod(target, "toASCII", ToASCII);
env->SetMethod(target, "getStringWidth", GetStringWidth);
// One-shot converters
env->SetMethod(target, "icuErrName", ICUErrorName);
env->SetMethod(target, "transcode", Transcode);
}
} // namespace i18n

27
src/util.h
View File

@ -343,6 +343,15 @@ class MaybeStackBuffer {
buf_ = nullptr;
}
bool IsAllocated() {
return buf_ != buf_st_;
}
void Release() {
buf_ = buf_st_;
length_ = 0;
}
MaybeStackBuffer() : length_(0), buf_(buf_st_) {
// Default to a zero-length, null-terminated buffer.
buf_[0] = T();
@ -378,6 +387,24 @@ class BufferValue : public MaybeStackBuffer<char> {
explicit BufferValue(v8::Isolate* isolate, v8::Local<v8::Value> value);
};
#define THROW_AND_RETURN_UNLESS_BUFFER(env, obj) \
do { \
if (!Buffer::HasInstance(obj)) \
return env->ThrowTypeError("argument should be a Buffer"); \
} while (0)
#define SPREAD_BUFFER_ARG(val, name) \
CHECK((val)->IsUint8Array()); \
Local<v8::Uint8Array> name = (val).As<v8::Uint8Array>(); \
v8::ArrayBuffer::Contents name##_c = name->Buffer()->GetContents(); \
const size_t name##_offset = name->ByteOffset(); \
const size_t name##_length = name->ByteLength(); \
char* const name##_data = \
static_cast<char*>(name##_c.Data()) + name##_offset; \
if (name##_length > 0) \
CHECK_NE(name##_data, nullptr);
} // namespace node
#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS

48
test/parallel/test-icu-transcode.js
View File

@ -0,0 +1,48 @@
'use strict';
require('../common');
const buffer = require('buffer');
const assert = require('assert');
const orig = Buffer.from('tést €', 'utf8');
// Test Transcoding
const tests = {
'latin1': [0x74, 0xe9, 0x73, 0x74, 0x20, 0x3f],
'ascii': [0x74, 0x3f, 0x73, 0x74, 0x20, 0x3f],
'ucs2': [0x74, 0x00, 0xe9, 0x00, 0x73,
0x00, 0x74, 0x00, 0x20, 0x00,
0xac, 0x20]
};
for (const test in tests) {
const dest = buffer.transcode(orig, 'utf8', test);
assert.strictEqual(dest.length, tests[test].length);
for (var n = 0; n < tests[test].length; n++)
assert.strictEqual(dest[n], tests[test][n]);
}
{
const dest = buffer.transcode(Buffer.from(tests.ucs2), 'ucs2', 'utf8');
assert.strictEqual(dest.toString(), orig.toString());
}
{
const utf8 = Buffer.from('€'.repeat(4000), 'utf8');
const ucs2 = Buffer.from('€'.repeat(4000), 'ucs2');
const utf8_to_ucs2 = buffer.transcode(utf8, 'utf8', 'ucs2');
const ucs2_to_utf8 = buffer.transcode(ucs2, 'ucs2', 'utf8');
assert.deepStrictEqual(utf8, ucs2_to_utf8);
assert.deepStrictEqual(ucs2, utf8_to_ucs2);
assert.strictEqual(ucs2_to_utf8.toString('utf8'),
utf8_to_ucs2.toString('ucs2'));
}
assert.throws(
() => buffer.transcode(Buffer.from('a'), 'b', 'utf8'),
/Unable to transcode Buffer \[U_ILLEGAL_ARGUMENT_ERROR\]/
);
assert.throws(
() => buffer.transcode(Buffer.from('a'), 'uf8', 'b'),
/Unable to transcode Buffer \[U_ILLEGAL_ARGUMENT_ERROR\]/
);

4
tools/icu/icu-generic.gyp
View File

@ -20,9 +20,7 @@
'type': 'none',
'toolsets': [ 'target' ],
'direct_dependent_settings': {
'defines': [
'UCONFIG_NO_CONVERSION=1',
]
'defines': []
},
},
{

Write Preview
Loading…
Cancel
Save