lukechilds
/
node
mirror of https://github.com/lukechilds/node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

src: support UTF-8 in compiled-in JS source files 

Detect it when source files in lib/ are not ASCII.  Decode them as UTF-8
and store them as UTF-16 in the binary so they can be used as external
string resources without non-ASCII characters getting mangled.

Fixes: https://github.com/nodejs/node/issues/10673
PR-URL: https://github.com/nodejs/node/pull/11129
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: James M Snell <jasnell@gmail.com>
v7.x
Ben Noordhuis 8 years ago
committed by Italo A. Casas
parent
5350f04e42
commit
3380cd5fdb
No known key found for this signature in database GPG Key ID: 23EFEFE93C4CFFFE
3 changed files with 81 additions and 95 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      node.gyp
  2. 51
      src/node_javascript.cc
  3. 117
      tools/js2c.py

8
node.gyp
View File

@ -145,7 +145,7 @@
'src',
'tools/msvs/genfiles',
'deps/uv/src/ares',
'<(SHARED_INTERMEDIATE_DIR)', # for node_natives.h
'<(SHARED_INTERMEDIATE_DIR)',
],
'sources': [
@ -166,7 +166,6 @@
'src/node_debug_options.cc',
'src/node_file.cc',
'src/node_http_parser.cc',
'src/node_javascript.cc',
'src/node_main.cc',
'src/node_os.cc',
'src/node_revert.cc',
@ -234,11 +233,11 @@
'deps/http_parser/http_parser.h',
'deps/v8/include/v8.h',
'deps/v8/include/v8-debug.h',
'<(SHARED_INTERMEDIATE_DIR)/node_natives.h',
# javascript files to make for an even more pleasant IDE experience
'<@(library_files)',
# node.gyp is added to the project by default.
'common.gypi',
'<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
],
'defines': [
@ -711,12 +710,13 @@
'actions': [
{
'action_name': 'node_js2c',
'process_outputs_as_sources': 1,
'inputs': [
'<@(library_files)',
'./config.gypi',
],
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/node_natives.h',
'<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
],
'conditions': [
[ 'node_use_dtrace=="false" and node_use_etw=="false"', {

51
src/node_javascript.cc
View File

@ -1,51 +0,0 @@
#include "node.h"
#include "node_natives.h"
#include "v8.h"
#include "env.h"
#include "env-inl.h"
namespace node {
using v8::Local;
using v8::NewStringType;
using v8::Object;
using v8::String;
// id##_data is defined in node_natives.h.
#define V(id) \
static struct : public String::ExternalOneByteStringResource { \
const char* data() const override { \
return reinterpret_cast<const char*>(id##_data); \
} \
size_t length() const override { return sizeof(id##_data); } \
void Dispose() override { /* Default calls `delete this`. */ } \
} id##_external_data;
NODE_NATIVES_MAP(V)
#undef V
Local<String> MainSource(Environment* env) {
auto maybe_string =
String::NewExternalOneByte(
env->isolate(),
&internal_bootstrap_node_external_data);
return maybe_string.ToLocalChecked();
}
void DefineJavaScript(Environment* env, Local<Object> target) {
auto context = env->context();
#define V(id) \
do { \
auto key = \
String::NewFromOneByte( \
env->isolate(), id##_name, NewStringType::kNormal, \
sizeof(id##_name)).ToLocalChecked(); \
auto value = \
String::NewExternalOneByte( \
env->isolate(), &id##_external_data).ToLocalChecked(); \
CHECK(target->Set(context, key, value).FromJust()); \
} while (0);
NODE_NATIVES_MAP(V)
#undef V
}
} // namespace node

117
tools/js2c.py
View File

@ -37,13 +37,16 @@ import sys
import string
def ToCString(contents):
step = 20
slices = (contents[i:i+step] for i in xrange(0, len(contents), step))
slices = map(lambda s: ','.join(str(ord(c)) for c in s), slices)
def ToCArray(elements, step=10):
slices = (elements[i:i+step] for i in xrange(0, len(elements), step))
slices = map(lambda s: ','.join(str(x) for x in s), slices)
return ',\n'.join(slices)
def ToCString(contents):
return ToCArray(map(ord, contents), step=20)
def ReadFile(filename):
file = open(filename, "rt")
try:
@ -161,34 +164,72 @@ def ReadMacros(lines):
return (constants, macros)
HEADER_TEMPLATE = """\
#ifndef NODE_NATIVES_H_
#define NODE_NATIVES_H_
TEMPLATE = """
#include "node.h"
#include "node_javascript.h"
#include "v8.h"
#include "env.h"
#include "env-inl.h"
#include <stdint.h>
namespace node {{
#define NODE_NATIVES_MAP(V) \\
{node_natives_map}
{definitions}
v8::Local<v8::String> MainSource(Environment* env) {{
return internal_bootstrap_node_value.ToStringChecked(env->isolate());
}}
void DefineJavaScript(Environment* env, v8::Local<v8::Object> target) {{
{initializers}
}}
namespace node {{
{sources}
}} // namespace node
"""
#endif // NODE_NATIVES_H_
ONE_BYTE_STRING = """
static const uint8_t raw_{var}[] = {{ {data} }};
static struct : public v8::String::ExternalOneByteStringResource {{
const char* data() const override {{
return reinterpret_cast<const char*>(raw_{var});
}}
size_t length() const override {{ return arraysize(raw_{var}); }}
void Dispose() override {{ /* Default calls `delete this`. */ }}
v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{
return v8::String::NewExternalOneByte(isolate, this).ToLocalChecked();
}}
}} {var};
"""
TWO_BYTE_STRING = """
static const uint16_t raw_{var}[] = {{ {data} }};
static struct : public v8::String::ExternalStringResource {{
const uint16_t* data() const override {{ return raw_{var}; }}
size_t length() const override {{ return arraysize(raw_{var}); }}
void Dispose() override {{ /* Default calls `delete this`. */ }}
v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{
return v8::String::NewExternalTwoByte(isolate, this).ToLocalChecked();
}}
}} {var};
"""
NODE_NATIVES_MAP = """\
V({escaped_id}) \\
INITIALIZER = """\
CHECK(target->Set(env->context(),
{key}.ToStringChecked(env->isolate()),
{value}.ToStringChecked(env->isolate())).FromJust());
"""
SOURCES = """\
static const uint8_t {escaped_id}_name[] = {{
{name}}};
static const uint8_t {escaped_id}_data[] = {{
{data}}};
"""
def Render(var, data):
# Treat non-ASCII as UTF-8 and convert it to UTF-16.
if any(ord(c) > 127 for c in data):
template = TWO_BYTE_STRING
data = map(ord, data.decode('utf-8').encode('utf-16be'))
data = [data[i] * 256 + data[i+1] for i in xrange(0, len(data), 2)]
data = ToCArray(data)
else:
template = ONE_BYTE_STRING
data = ToCString(data)
return template.format(var=var, data=data)
def JS2C(source, target):
@ -207,36 +248,32 @@ def JS2C(source, target):
(consts, macros) = ReadMacros(macro_lines)
# Build source code lines
node_natives_map = []
sources = []
definitions = []
initializers = []
for s in modules:
lines = ReadFile(str(s))
for name in modules:
lines = ReadFile(str(name))
lines = ExpandConstants(lines, consts)
lines = ExpandMacros(lines, macros)
data = ToCString(lines)
# On Windows, "./foo.bar" in the .gyp file is passed as "foo.bar"
# so don't assume there is always a slash in the file path.
if '/' in s or '\\' in s:
id = '/'.join(re.split('/|\\\\', s)[1:])
else:
id = s
if '.' in id:
id = id.split('.', 1)[0]
if '/' in name or '\\' in name:
name = '/'.join(re.split('/|\\\\', name)[1:])
name = ToCString(id)
escaped_id = id.replace('-', '_').replace('/', '_')
node_natives_map.append(NODE_NATIVES_MAP.format(**locals()))
sources.append(SOURCES.format(**locals()))
name = name.split('.', 1)[0]
var = name.replace('-', '_').replace('/', '_')
key = '%s_key' % var
value = '%s_value' % var
node_natives_map = ''.join(node_natives_map)
sources = ''.join(sources)
definitions.append(Render(key, name))
definitions.append(Render(value, lines))
initializers.append(INITIALIZER.format(key=key, value=value))
# Emit result
output = open(str(target[0]), "w")
output.write(HEADER_TEMPLATE.format(**locals()))
output.write(TEMPLATE.format(definitions=''.join(definitions),
initializers=''.join(initializers)))
output.close()
def main():

Write Preview
Loading…
Cancel
Save