mirror of https://github.com/lukechilds/node.git
Browse Source
Provide an (initially experimental) implementation of the WHATWG Encoding Standard API (`TextDecoder` and `TextEncoder`). The is the same API implemented on the browser side. By default, with small-icu, only the UTF-8, UTF-16le and UTF-16be decoders are supported. With full-icu enabled, every encoding other than iso-8859-16 is supported. This provides a basic test, but does not include the full web platform tests. Note: many of the web platform tests for this would fail by default because we ship with small-icu by default. A process warning will be emitted on first use to indicate that the API is still experimental. No runtime flag is required to use the feature. Refs: https://encoding.spec.whatwg.org/ PR-URL: https://github.com/nodejs/node/pull/13644 Reviewed-By: Timothy Gu <timothygu99@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com>v6
James M Snell
8 years ago
12 changed files with 1189 additions and 16 deletions
@ -0,0 +1,458 @@ |
|||
'use strict'; |
|||
|
|||
// An implementation of the WHATWG Encoding Standard
|
|||
// https://encoding.spec.whatwg.org
|
|||
|
|||
const errors = require('internal/errors'); |
|||
const kHandle = Symbol('handle'); |
|||
const kFlags = Symbol('flags'); |
|||
const kEncoding = Symbol('encoding'); |
|||
const kDecoder = Symbol('decoder'); |
|||
const kEncoder = Symbol('encoder'); |
|||
|
|||
let warned = false; |
|||
const experimental = |
|||
'The WHATWG Encoding Standard implementation is an experimental API. It ' + |
|||
'should not yet be used in production applications.'; |
|||
|
|||
const { |
|||
getConstructorOf, |
|||
customInspectSymbol: inspect |
|||
} = require('internal/util'); |
|||
|
|||
const { |
|||
isArrayBuffer |
|||
} = process.binding('util'); |
|||
|
|||
const { |
|||
encodeUtf8String |
|||
} = process.binding('buffer'); |
|||
|
|||
const { |
|||
decode: _decode, |
|||
getConverter, |
|||
hasConverter |
|||
} = process.binding('icu'); |
|||
|
|||
const CONVERTER_FLAGS_FLUSH = 0x1; |
|||
const CONVERTER_FLAGS_FATAL = 0x2; |
|||
const CONVERTER_FLAGS_IGNORE_BOM = 0x4; |
|||
|
|||
const empty = new Uint8Array(0); |
|||
|
|||
const encodings = new Map([ |
|||
['unicode-1-1-utf-8', 'utf-8'], |
|||
['utf8', 'utf-8'], |
|||
['utf-8', 'utf-8'], |
|||
['866', 'ibm866'], |
|||
['cp866', 'ibm866'], |
|||
['csibm866', 'ibm866'], |
|||
['ibm866', 'ibm866'], |
|||
['csisolatin2', 'iso-8859-2'], |
|||
['iso-8859-2', 'iso-8859-2'], |
|||
['iso-ir-101', 'iso-8859-2'], |
|||
['iso8859-2', 'iso-8859-2'], |
|||
['iso88592', 'iso-8859-2'], |
|||
['iso_8859-2', 'iso-8859-2'], |
|||
['iso_8859-2:1987', 'iso-8859-2'], |
|||
['l2', 'iso-8859-2'], |
|||
['latin2', 'iso-8859-2'], |
|||
['csisolatin3', 'iso-8859-3'], |
|||
['iso-8859-3', 'iso-8859-3'], |
|||
['iso-ir-109', 'iso-8859-3'], |
|||
['iso8859-3', 'iso-8859-3'], |
|||
['iso88593', 'iso-8859-3'], |
|||
['iso_8859-3', 'iso-8859-3'], |
|||
['iso_8859-3:1988', 'iso-8859-3'], |
|||
['l3', 'iso-8859-3'], |
|||
['latin3', 'iso-8859-3'], |
|||
['csisolatin4', 'iso-8859-4'], |
|||
['iso-8859-4', 'iso-8859-4'], |
|||
['iso-ir-110', 'iso-8859-4'], |
|||
['iso8859-4', 'iso-8859-4'], |
|||
['iso88594', 'iso-8859-4'], |
|||
['iso_8859-4', 'iso-8859-4'], |
|||
['iso_8859-4:1988', 'iso-8859-4'], |
|||
['l4', 'iso-8859-4'], |
|||
['latin4', 'iso-8859-4'], |
|||
['csisolatincyrillic', 'iso-8859-5'], |
|||
['cyrillic', 'iso-8859-5'], |
|||
['iso-8859-5', 'iso-8859-5'], |
|||
['iso-ir-144', 'iso-8859-5'], |
|||
['iso8859-5', 'iso-8859-5'], |
|||
['iso88595', 'iso-8859-5'], |
|||
['iso_8859-5', 'iso-8859-5'], |
|||
['iso_8859-5:1988', 'iso-8859-5'], |
|||
['arabic', 'iso-8859-6'], |
|||
['asmo-708', 'iso-8859-6'], |
|||
['csiso88596e', 'iso-8859-6'], |
|||
['csiso88596i', 'iso-8859-6'], |
|||
['csisolatinarabic', 'iso-8859-6'], |
|||
['ecma-114', 'iso-8859-6'], |
|||
['iso-8859-6', 'iso-8859-6'], |
|||
['iso-8859-6-e', 'iso-8859-6'], |
|||
['iso-8859-6-i', 'iso-8859-6'], |
|||
['iso-ir-127', 'iso-8859-6'], |
|||
['iso8859-6', 'iso-8859-6'], |
|||
['iso88596', 'iso-8859-6'], |
|||
['iso_8859-6', 'iso-8859-6'], |
|||
['iso_8859-6:1987', 'iso-8859-6'], |
|||
['csisolatingreek', 'iso-8859-7'], |
|||
['ecma-118', 'iso-8859-7'], |
|||
['elot_928', 'iso-8859-7'], |
|||
['greek', 'iso-8859-7'], |
|||
['greek8', 'iso-8859-7'], |
|||
['iso-8859-7', 'iso-8859-7'], |
|||
['iso-ir-126', 'iso-8859-7'], |
|||
['iso8859-7', 'iso-8859-7'], |
|||
['iso88597', 'iso-8859-7'], |
|||
['iso_8859-7', 'iso-8859-7'], |
|||
['iso_8859-7:1987', 'iso-8859-7'], |
|||
['sun_eu_greek', 'iso-8859-7'], |
|||
['csiso88598e', 'iso-8859-8'], |
|||
['csisolatinhebrew', 'iso-8859-8'], |
|||
['hebrew', 'iso-8859-8'], |
|||
['iso-8859-8', 'iso-8859-8'], |
|||
['iso-8859-8-e', 'iso-8859-8'], |
|||
['iso-ir-138', 'iso-8859-8'], |
|||
['iso8859-8', 'iso-8859-8'], |
|||
['iso88598', 'iso-8859-8'], |
|||
['iso_8859-8', 'iso-8859-8'], |
|||
['iso_8859-8:1988', 'iso-8859-8'], |
|||
['visual', 'iso-8859-8'], |
|||
['csiso88598i', 'iso-8859-8-i'], |
|||
['iso-8859-8-i', 'iso-8859-8-i'], |
|||
['logical', 'iso-8859-8-i'], |
|||
['csisolatin6', 'iso-8859-10'], |
|||
['iso-8859-10', 'iso-8859-10'], |
|||
['iso-ir-157', 'iso-8859-10'], |
|||
['iso8859-10', 'iso-8859-10'], |
|||
['iso885910', 'iso-8859-10'], |
|||
['l6', 'iso-8859-10'], |
|||
['latin6', 'iso-8859-10'], |
|||
['iso-8859-13', 'iso-8859-13'], |
|||
['iso8859-13', 'iso-8859-13'], |
|||
['iso885913', 'iso-8859-13'], |
|||
['iso-8859-14', 'iso-8859-14'], |
|||
['iso8859-14', 'iso-8859-14'], |
|||
['iso885914', 'iso-8859-14'], |
|||
['csisolatin9', 'iso-8859-15'], |
|||
['iso-8859-15', 'iso-8859-15'], |
|||
['iso8859-15', 'iso-8859-15'], |
|||
['iso885915', 'iso-8859-15'], |
|||
['iso_8859-15', 'iso-8859-15'], |
|||
['l9', 'iso-8859-15'], |
|||
['cskoi8r', 'koi8-r'], |
|||
['koi', 'koi8-r'], |
|||
['koi8', 'koi8-r'], |
|||
['koi8-r', 'koi8-r'], |
|||
['koi8_r', 'koi8-r'], |
|||
['koi8-ru', 'koi8-u'], |
|||
['koi8-u', 'koi8-u'], |
|||
['csmacintosh', 'macintosh'], |
|||
['mac', 'macintosh'], |
|||
['macintosh', 'macintosh'], |
|||
['x-mac-roman', 'macintosh'], |
|||
['dos-874', 'windows-874'], |
|||
['iso-8859-11', 'windows-874'], |
|||
['iso8859-11', 'windows-874'], |
|||
['iso885911', 'windows-874'], |
|||
['tis-620', 'windows-874'], |
|||
['windows-874', 'windows-874'], |
|||
['cp1250', 'windows-1250'], |
|||
['windows-1250', 'windows-1250'], |
|||
['x-cp1250', 'windows-1250'], |
|||
['cp1251', 'windows-1251'], |
|||
['windows-1251', 'windows-1251'], |
|||
['x-cp1251', 'windows-1251'], |
|||
['ansi_x3.4-1968', 'windows-1252'], |
|||
['ascii', 'windows-1252'], |
|||
['cp1252', 'windows-1252'], |
|||
['cp819', 'windows-1252'], |
|||
['csisolatin1', 'windows-1252'], |
|||
['ibm819', 'windows-1252'], |
|||
['iso-8859-1', 'windows-1252'], |
|||
['iso-ir-100', 'windows-1252'], |
|||
['iso8859-1', 'windows-1252'], |
|||
['iso88591', 'windows-1252'], |
|||
['iso_8859-1', 'windows-1252'], |
|||
['iso_8859-1:1987', 'windows-1252'], |
|||
['l1', 'windows-1252'], |
|||
['latin1', 'windows-1252'], |
|||
['us-ascii', 'windows-1252'], |
|||
['windows-1252', 'windows-1252'], |
|||
['x-cp1252', 'windows-1252'], |
|||
['cp1253', 'windows-1253'], |
|||
['windows-1253', 'windows-1253'], |
|||
['x-cp1253', 'windows-1253'], |
|||
['cp1254', 'windows-1254'], |
|||
['csisolatin5', 'windows-1254'], |
|||
['iso-8859-9', 'windows-1254'], |
|||
['iso-ir-148', 'windows-1254'], |
|||
['iso8859-9', 'windows-1254'], |
|||
['iso88599', 'windows-1254'], |
|||
['iso_8859-9', 'windows-1254'], |
|||
['iso_8859-9:1989', 'windows-1254'], |
|||
['l5', 'windows-1254'], |
|||
['latin5', 'windows-1254'], |
|||
['windows-1254', 'windows-1254'], |
|||
['x-cp1254', 'windows-1254'], |
|||
['cp1255', 'windows-1255'], |
|||
['windows-1255', 'windows-1255'], |
|||
['x-cp1255', 'windows-1255'], |
|||
['cp1256', 'windows-1256'], |
|||
['windows-1256', 'windows-1256'], |
|||
['x-cp1256', 'windows-1256'], |
|||
['cp1257', 'windows-1257'], |
|||
['windows-1257', 'windows-1257'], |
|||
['x-cp1257', 'windows-1257'], |
|||
['cp1258', 'windows-1258'], |
|||
['windows-1258', 'windows-1258'], |
|||
['x-cp1258', 'windows-1258'], |
|||
['x-mac-cyrillic', 'x-mac-cyrillic'], |
|||
['x-mac-ukrainian', 'x-mac-cyrillic'], |
|||
['chinese', 'gbk'], |
|||
['csgb2312', 'gbk'], |
|||
['csiso58gb231280', 'gbk'], |
|||
['gb2312', 'gbk'], |
|||
['gb_2312', 'gbk'], |
|||
['gb_2312-80', 'gbk'], |
|||
['gbk', 'gbk'], |
|||
['iso-ir-58', 'gbk'], |
|||
['x-gbk', 'gbk'], |
|||
['gb18030', 'gb18030'], |
|||
['big5', 'big5'], |
|||
['big5-hkscs', 'big5'], |
|||
['cn-big5', 'big5'], |
|||
['csbig5', 'big5'], |
|||
['x-x-big5', 'big5'], |
|||
['cseucpkdfmtjapanese', 'euc-jp'], |
|||
['euc-jp', 'euc-jp'], |
|||
['x-euc-jp', 'euc-jp'], |
|||
['csiso2022jp', 'iso-2022-jp'], |
|||
['iso-2022-jp', 'iso-2022-jp'], |
|||
['csshiftjis', 'shift_jis'], |
|||
['ms932', 'shift_jis'], |
|||
['ms_kanji', 'shift_jis'], |
|||
['shift-jis', 'shift_jis'], |
|||
['shift_jis', 'shift_jis'], |
|||
['sjis', 'shift_jis'], |
|||
['windows-31j', 'shift_jis'], |
|||
['x-sjis', 'shift_jis'], |
|||
['cseuckr', 'euc-kr'], |
|||
['csksc56011987', 'euc-kr'], |
|||
['euc-kr', 'euc-kr'], |
|||
['iso-ir-149', 'euc-kr'], |
|||
['korean', 'euc-kr'], |
|||
['ks_c_5601-1987', 'euc-kr'], |
|||
['ks_c_5601-1989', 'euc-kr'], |
|||
['ksc5601', 'euc-kr'], |
|||
['ksc_5601', 'euc-kr'], |
|||
['windows-949', 'euc-kr'], |
|||
['utf-16be', 'utf-16be'], |
|||
['utf-16le', 'utf-16le'], |
|||
['utf-16', 'utf-16le'] |
|||
]); |
|||
|
|||
// Unfortunately, String.prototype.trim also removes non-ascii whitespace,
|
|||
// so we have to do this manually
|
|||
function trimAsciiWhitespace(label) { |
|||
var s = 0; |
|||
var e = label.length; |
|||
while (s < e && ( |
|||
label[s] === '\u0009' || |
|||
label[s] === '\u000a' || |
|||
label[s] === '\u000c' || |
|||
label[s] === '\u000d' || |
|||
label[s] === '\u0020')) { |
|||
s++; |
|||
} |
|||
while (e > s && ( |
|||
label[e - 1] === '\u0009' || |
|||
label[e - 1] === '\u000a' || |
|||
label[e - 1] === '\u000c' || |
|||
label[e - 1] === '\u000d' || |
|||
label[e - 1] === '\u0020')) { |
|||
e--; |
|||
} |
|||
return label.slice(s, e); |
|||
} |
|||
|
|||
function getEncodingFromLabel(label) { |
|||
const enc = encodings.get(label); |
|||
if (enc !== undefined) return enc; |
|||
return encodings.get(trimAsciiWhitespace(label.toLowerCase())); |
|||
} |
|||
|
|||
function hasTextDecoder(encoding = 'utf-8') { |
|||
if (typeof encoding !== 'string') |
|||
throw new errors.Error('ERR_INVALID_ARG_TYPE', 'encoding', 'string'); |
|||
return hasConverter(getEncodingFromLabel(encoding)); |
|||
} |
|||
|
|||
var Buffer; |
|||
function lazyBuffer() { |
|||
if (Buffer === undefined) |
|||
Buffer = require('buffer').Buffer; |
|||
return Buffer; |
|||
} |
|||
|
|||
class TextDecoder { |
|||
constructor(encoding = 'utf-8', options = {}) { |
|||
if (!warned) { |
|||
warned = true; |
|||
process.emitWarning(experimental, 'ExperimentalWarning'); |
|||
} |
|||
|
|||
encoding = `${encoding}`; |
|||
if (typeof options !== 'object') |
|||
throw new errors.Error('ERR_INVALID_ARG_TYPE', 'options', 'object'); |
|||
|
|||
const enc = getEncodingFromLabel(encoding); |
|||
if (enc === undefined) |
|||
throw new errors.RangeError('ERR_ENCODING_NOT_SUPPORTED', encoding); |
|||
|
|||
var flags = 0; |
|||
if (options !== null) { |
|||
flags |= options.fatal ? CONVERTER_FLAGS_FATAL : 0; |
|||
flags |= options.ignoreBOM ? CONVERTER_FLAGS_IGNORE_BOM : 0; |
|||
} |
|||
|
|||
const handle = getConverter(enc, flags); |
|||
if (handle === undefined) |
|||
throw new errors.Error('ERR_ENCODING_NOT_SUPPORTED', encoding); |
|||
|
|||
this[kHandle] = handle; |
|||
this[kFlags] = flags; |
|||
this[kEncoding] = enc; |
|||
} |
|||
|
|||
get encoding() { |
|||
if (this == null || this[kDecoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextDecoder'); |
|||
return this[kEncoding]; |
|||
} |
|||
|
|||
get fatal() { |
|||
if (this == null || this[kDecoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextDecoder'); |
|||
return (this[kFlags] & CONVERTER_FLAGS_FATAL) === CONVERTER_FLAGS_FATAL; |
|||
} |
|||
|
|||
get ignoreBOM() { |
|||
if (this == null || this[kDecoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextDecoder'); |
|||
return (this[kFlags] & CONVERTER_FLAGS_IGNORE_BOM) === |
|||
CONVERTER_FLAGS_IGNORE_BOM; |
|||
} |
|||
|
|||
decode(input = empty, options = {}) { |
|||
if (this == null || this[kDecoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextDecoder'); |
|||
if (isArrayBuffer(input)) { |
|||
input = lazyBuffer().from(input); |
|||
} else if (!ArrayBuffer.isView(input)) { |
|||
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'input', |
|||
['ArrayBuffer', 'ArrayBufferView']); |
|||
} |
|||
if (typeof options !== 'object') { |
|||
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'options', 'object'); |
|||
} |
|||
|
|||
var flags = 0; |
|||
if (options !== null) |
|||
flags |= options.stream ? 0 : CONVERTER_FLAGS_FLUSH; |
|||
|
|||
const ret = _decode(this[kHandle], input, flags); |
|||
if (typeof ret === 'number') { |
|||
const err = new errors.TypeError('ERR_ENCODING_INVALID_ENCODED_DATA', |
|||
this.encoding); |
|||
err.errno = ret; |
|||
throw err; |
|||
} |
|||
return ret.toString('ucs2'); |
|||
} |
|||
|
|||
[inspect](depth, opts) { |
|||
if (this == null || this[kDecoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextDecoder'); |
|||
if (typeof depth === 'number' && depth < 0) |
|||
return opts.stylize('[Object]', 'special'); |
|||
var ctor = getConstructorOf(this); |
|||
var obj = Object.create({ |
|||
constructor: ctor === null ? TextDecoder : ctor |
|||
}); |
|||
obj.encoding = this.encoding; |
|||
obj.fatal = this.fatal; |
|||
obj.ignoreBOM = this.ignoreBOM; |
|||
if (opts.showHidden) { |
|||
obj[kFlags] = this[kFlags]; |
|||
obj[kHandle] = this[kHandle]; |
|||
} |
|||
// Lazy to avoid circular dependency
|
|||
return require('util').inspect(obj, opts); |
|||
} |
|||
} |
|||
|
|||
class TextEncoder { |
|||
constructor() { |
|||
if (!warned) { |
|||
warned = true; |
|||
process.emitWarning(experimental, 'ExperimentalWarning'); |
|||
} |
|||
} |
|||
|
|||
get encoding() { |
|||
if (this == null || this[kEncoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextEncoder'); |
|||
return 'utf-8'; |
|||
} |
|||
|
|||
encode(input = '') { |
|||
if (this == null || this[kEncoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextEncoder'); |
|||
return encodeUtf8String(`${input}`); |
|||
} |
|||
|
|||
[inspect](depth, opts) { |
|||
if (this == null || this[kEncoder] !== true) |
|||
throw new errors.TypeError('ERR_INVALID_THIS', 'TextEncoder'); |
|||
if (typeof depth === 'number' && depth < 0) |
|||
return opts.stylize('[Object]', 'special'); |
|||
var ctor = getConstructorOf(this); |
|||
var obj = Object.create({ |
|||
constructor: ctor === null ? TextEncoder : ctor |
|||
}); |
|||
obj.encoding = this.encoding; |
|||
// Lazy to avoid circular dependency
|
|||
return require('util').inspect(obj, opts); |
|||
} |
|||
} |
|||
|
|||
Object.defineProperties( |
|||
TextDecoder.prototype, { |
|||
[kDecoder]: { enumerable: false, value: true, configurable: false }, |
|||
'decode': { enumerable: true }, |
|||
'encoding': { enumerable: true }, |
|||
'fatal': { enumerable: true }, |
|||
'ignoreBOM': { enumerable: true }, |
|||
[Symbol.toStringTag]: { |
|||
configurable: true, |
|||
value: 'TextDecoder' |
|||
} }); |
|||
Object.defineProperties( |
|||
TextEncoder.prototype, { |
|||
[kEncoder]: { enumerable: false, value: true, configurable: false }, |
|||
'encode': { enumerable: true }, |
|||
'encoding': { enumerable: true }, |
|||
[Symbol.toStringTag]: { |
|||
configurable: true, |
|||
value: 'TextEncoder' |
|||
} }); |
|||
|
|||
module.exports = { |
|||
getEncodingFromLabel, |
|||
hasTextDecoder, |
|||
TextDecoder, |
|||
TextEncoder |
|||
}; |
|||
@ -0,0 +1,385 @@ |
|||
// Flags: --expose-internals
|
|||
'use strict'; |
|||
|
|||
const common = require('../common'); |
|||
const assert = require('assert'); |
|||
const { TextEncoder, TextDecoder } = require('util'); |
|||
const { customInspectSymbol: inspect } = require('internal/util'); |
|||
const { getEncodingFromLabel } = require('internal/encoding'); |
|||
|
|||
const encoded = Buffer.from([0xef, 0xbb, 0xbf, 0x74, 0x65, |
|||
0x73, 0x74, 0xe2, 0x82, 0xac]); |
|||
|
|||
if (!common.hasIntl) { |
|||
common.skip('WHATWG Encoding tests because ICU is not present.'); |
|||
} |
|||
|
|||
// Make Sure TextDecoder and TextEncoder exist
|
|||
assert(TextDecoder); |
|||
assert(TextEncoder); |
|||
|
|||
// Test TextEncoder
|
|||
const enc = new TextEncoder(); |
|||
assert(enc); |
|||
const buf = enc.encode('\ufefftest€'); |
|||
|
|||
assert.strictEqual(Buffer.compare(buf, encoded), 0); |
|||
|
|||
|
|||
// Test TextDecoder, UTF-8, fatal: false, ignoreBOM: false
|
|||
{ |
|||
['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => { |
|||
const dec = new TextDecoder(i); |
|||
const res = dec.decode(buf); |
|||
assert.strictEqual(res, 'test€'); |
|||
}); |
|||
|
|||
['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => { |
|||
const dec = new TextDecoder(i); |
|||
let res = ''; |
|||
res += dec.decode(buf.slice(0, 8), { stream: true }); |
|||
res += dec.decode(buf.slice(8)); |
|||
assert.strictEqual(res, 'test€'); |
|||
}); |
|||
} |
|||
|
|||
// Test TextDecoder, UTF-8, fatal: false, ignoreBOM: true
|
|||
{ |
|||
['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => { |
|||
const dec = new TextDecoder(i, { ignoreBOM: true }); |
|||
const res = dec.decode(buf); |
|||
assert.strictEqual(res, '\ufefftest€'); |
|||
}); |
|||
|
|||
['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => { |
|||
const dec = new TextDecoder(i, { ignoreBOM: true }); |
|||
let res = ''; |
|||
res += dec.decode(buf.slice(0, 8), { stream: true }); |
|||
res += dec.decode(buf.slice(8)); |
|||
assert.strictEqual(res, '\ufefftest€'); |
|||
}); |
|||
} |
|||
|
|||
// Test TextDecoder, UTF-8, fatal: true, ignoreBOM: false
|
|||
{ |
|||
['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => { |
|||
const dec = new TextDecoder(i, { fatal: true }); |
|||
assert.throws(() => dec.decode(buf.slice(0, 8)), |
|||
common.expectsError({ |
|||
code: 'ERR_ENCODING_INVALID_ENCODED_DATA', |
|||
type: TypeError, |
|||
message: |
|||
/^The encoded data was not valid for encoding utf-8$/ |
|||
})); |
|||
}); |
|||
|
|||
['unicode-1-1-utf-8', 'utf8', 'utf-8'].forEach((i) => { |
|||
const dec = new TextDecoder(i, { fatal: true }); |
|||
assert.doesNotThrow(() => dec.decode(buf.slice(0, 8), { stream: true })); |
|||
assert.doesNotThrow(() => dec.decode(buf.slice(8))); |
|||
}); |
|||
} |
|||
|
|||
// Test TextDecoder, UTF-16le
|
|||
{ |
|||
const dec = new TextDecoder('utf-16le'); |
|||
const res = dec.decode(Buffer.from('test€', 'utf-16le')); |
|||
assert.strictEqual(res, 'test€'); |
|||
} |
|||
|
|||
// Test TextDecoder, UTF-16be
|
|||
{ |
|||
const dec = new TextDecoder('utf-16be'); |
|||
const res = dec.decode(Buffer.from([0x00, 0x74, 0x00, 0x65, 0x00, |
|||
0x73, 0x00, 0x74, 0x20, 0xac])); |
|||
assert.strictEqual(res, 'test€'); |
|||
} |
|||
|
|||
{ |
|||
const fn = TextDecoder.prototype[inspect]; |
|||
fn.call(new TextDecoder(), Infinity, {}); |
|||
|
|||
[{}, [], true, 1, '', new TextEncoder()].forEach((i) => { |
|||
assert.throws(() => fn.call(i, Infinity, {}), |
|||
common.expectsError({ |
|||
code: 'ERR_INVALID_THIS', |
|||
message: 'Value of "this" must be of type TextDecoder' |
|||
})); |
|||
}); |
|||
} |
|||
|
|||
{ |
|||
const fn = TextEncoder.prototype[inspect]; |
|||
fn.call(new TextEncoder(), Infinity, {}); |
|||
|
|||
[{}, [], true, 1, '', new TextDecoder()].forEach((i) => { |
|||
assert.throws(() => fn.call(i, Infinity, {}), |
|||
common.expectsError({ |
|||
code: 'ERR_INVALID_THIS', |
|||
message: 'Value of "this" must be of type TextEncoder' |
|||
})); |
|||
}); |
|||
} |
|||
|
|||
// Test Encoding Mappings
|
|||
{ |
|||
|
|||
const mappings = { |
|||
'utf-8': [ |
|||
'unicode-1-1-utf-8', |
|||
'utf8' |
|||
], |
|||
'utf-16be': [], |
|||
'utf-16le': [ |
|||
'utf-16' |
|||
], |
|||
'ibm866': [ |
|||
'866', |
|||
'cp866', |
|||
'csibm866' |
|||
], |
|||
'iso-8859-2': [ |
|||
'csisolatin2', |
|||
'iso-ir-101', |
|||
'iso8859-2', |
|||
'iso88592', |
|||
'iso_8859-2', |
|||
'iso_8859-2:1987', |
|||
'l2', |
|||
'latin2' |
|||
], |
|||
'iso-8859-3': [ |
|||
'csisolatin3', |
|||
'iso-ir-109', |
|||
'iso8859-3', |
|||
'iso88593', |
|||
'iso_8859-3', |
|||
'iso_8859-3:1988', |
|||
'l3', |
|||
'latin3' |
|||
], |
|||
'iso-8859-4': [ |
|||
'csisolatin4', |
|||
'iso-ir-110', |
|||
'iso8859-4', |
|||
'iso88594', |
|||
'iso_8859-4', |
|||
'iso_8859-4:1988', |
|||
'l4', |
|||
'latin4' |
|||
], |
|||
'iso-8859-5': [ |
|||
'csisolatincyrillic', |
|||
'cyrillic', |
|||
'iso-ir-144', |
|||
'iso8859-5', |
|||
'iso88595', |
|||
'iso_8859-5', |
|||
'iso_8859-5:1988' |
|||
], |
|||
'iso-8859-6': [ |
|||
'arabic', |
|||
'asmo-708', |
|||
'csiso88596e', |
|||
'csiso88596i', |
|||
'csisolatinarabic', |
|||
'ecma-114', |
|||
'iso-8859-6-e', |
|||
'iso-8859-6-i', |
|||
'iso-ir-127', |
|||
'iso8859-6', |
|||
'iso88596', |
|||
'iso_8859-6', |
|||
'iso_8859-6:1987' |
|||
], |
|||
'iso-8859-7': [ |
|||
'csisolatingreek', |
|||
'ecma-118', |
|||
'elot_928', |
|||
'greek', |
|||
'greek8', |
|||
'iso-ir-126', |
|||
'iso8859-7', |
|||
'iso88597', |
|||
'iso_8859-7', |
|||
'iso_8859-7:1987', |
|||
'sun_eu_greek' |
|||
], |
|||
'iso-8859-8': [ |
|||
'csiso88598e', |
|||
'csisolatinhebrew', |
|||
'hebrew', |
|||
'iso-8859-8-e', |
|||
'iso-ir-138', |
|||
'iso8859-8', |
|||
'iso88598', |
|||
'iso_8859-8', |
|||
'iso_8859-8:1988', |
|||
'visual' |
|||
], |
|||
'iso-8859-8-i': [ |
|||
'csiso88598i', |
|||
'logical' |
|||
], |
|||
'iso-8859-10': [ |
|||
'csisolatin6', |
|||
'iso-ir-157', |
|||
'iso8859-10', |
|||
'iso885910', |
|||
'l6', |
|||
'latin6' |
|||
], |
|||
'iso-8859-13': [ |
|||
'iso8859-13', |
|||
'iso885913' |
|||
], |
|||
'iso-8859-14': [ |
|||
'iso8859-14', |
|||
'iso885914' |
|||
], |
|||
'iso-8859-15': [ |
|||
'csisolatin9', |
|||
'iso8859-15', |
|||
'iso885915', |
|||
'iso_8859-15', |
|||
'l9' |
|||
], |
|||
'koi8-r': [ |
|||
'cskoi8r', |
|||
'koi', |
|||
'koi8', |
|||
'koi8_r' |
|||
], |
|||
'koi8-u': [ |
|||
'koi8-ru' |
|||
], |
|||
'macintosh': [ |
|||
'csmacintosh', |
|||
'mac', |
|||
'x-mac-roman' |
|||
], |
|||
'windows-874': [ |
|||
'dos-874', |
|||
'iso-8859-11', |
|||
'iso8859-11', |
|||
'iso885911', |
|||
'tis-620' |
|||
], |
|||
'windows-1250': [ |
|||
'cp1250', |
|||
'x-cp1250' |
|||
], |
|||
'windows-1251': [ |
|||
'cp1251', |
|||
'x-cp1251' |
|||
], |
|||
'windows-1252': [ |
|||
'ansi_x3.4-1968', |
|||
'ascii', |
|||
'cp1252', |
|||
'cp819', |
|||
'csisolatin1', |
|||
'ibm819', |
|||
'iso-8859-1', |
|||
'iso-ir-100', |
|||
'iso8859-1', |
|||
'iso88591', |
|||
'iso_8859-1', |
|||
'iso_8859-1:1987', |
|||
'l1', |
|||
'latin1', |
|||
'us-ascii', |
|||
'x-cp1252' |
|||
], |
|||
'windows-1253': [ |
|||
'cp1253', |
|||
'x-cp1253' |
|||
], |
|||
'windows-1254': [ |
|||
'cp1254', |
|||
'csisolatin5', |
|||
'iso-8859-9', |
|||
'iso-ir-148', |
|||
'iso8859-9', |
|||
'iso88599', |
|||
'iso_8859-9', |
|||
'iso_8859-9:1989', |
|||
'l5', |
|||
'latin5', |
|||
'x-cp1254' |
|||
], |
|||
'windows-1255': [ |
|||
'cp1255', |
|||
'x-cp1255' |
|||
], |
|||
'windows-1256': [ |
|||
'cp1256', |
|||
'x-cp1256' |
|||
], |
|||
'windows-1257': [ |
|||
'cp1257', |
|||
'x-cp1257' |
|||
], |
|||
'windows-1258': [ |
|||
'cp1258', |
|||
'x-cp1258' |
|||
], |
|||
'x-mac-cyrillic': [ |
|||
'x-mac-ukrainian' |
|||
], |
|||
'gbk': [ |
|||
'chinese', |
|||
'csgb2312', |
|||
'csiso58gb231280', |
|||
'gb2312', |
|||
'gb_2312', |
|||
'gb_2312-80', |
|||
'iso-ir-58', |
|||
'x-gbk' |
|||
], |
|||
'gb18030': [ ], |
|||
'big5': [ |
|||
'big5-hkscs', |
|||
'cn-big5', |
|||
'csbig5', |
|||
'x-x-big5' |
|||
], |
|||
'euc-jp': [ |
|||
'cseucpkdfmtjapanese', |
|||
'x-euc-jp' |
|||
], |
|||
'iso-2022-jp': [ |
|||
'csiso2022jp' |
|||
], |
|||
'shift_jis': [ |
|||
'csshiftjis', |
|||
'ms932', |
|||
'ms_kanji', |
|||
'shift-jis', |
|||
'sjis', |
|||
'windows-31j', |
|||
'x-sjis' |
|||
], |
|||
'euc-kr': [ |
|||
' euc-kr \t', |
|||
'EUC-kr \n', |
|||
'cseuckr', |
|||
'csksc56011987', |
|||
'iso-ir-149', |
|||
'korean', |
|||
'ks_c_5601-1987', |
|||
'ks_c_5601-1989', |
|||
'ksc5601', |
|||
'ksc_5601', |
|||
'windows-949' |
|||
] |
|||
}; |
|||
Object.entries(mappings).forEach((i) => { |
|||
const enc = i[0]; |
|||
const labels = i[1]; |
|||
assert.strictEqual(getEncodingFromLabel(enc), enc); |
|||
labels.forEach((l) => assert.strictEqual(getEncodingFromLabel(l), enc)); |
|||
}); |
|||
|
|||
assert.strictEqual(getEncodingFromLabel('made-up'), undefined); |
|||
} |
|||
Loading…
Reference in new issue