From 03a8637f7934b12516d36ea3c5c9fb2443bdfb8d Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 8 Apr 2016 19:03:24 -0700 Subject: [PATCH] tools: Check in tools for shrinking ICU size, change default to small-icu * Change configure default to "small-icu" (Intl on, English only) * add "--without-intl" and "vcbuild without-intl" options, equivalent to --with-intl=none * update BUILDING.md with above changes * Checks in tools that generate the deps/icu-small source directory from ICU source * Tools and process for updating ICU documented in tools/icu/README.md Fixes: https://github.com/nodejs/node/issues/3476 PR-URL: https://github.com/nodejs/node/pull/6088 Reviewed-By: James M Snell --- BUILDING.md | 38 +++---- configure | 58 +++++++++-- tools/icu/README.md | 78 +++++++++++++++ tools/icu/icu-generic.gyp | 194 +++++++++++++++++++++++------------- tools/icu/shrink-icu-src.py | 126 +++++++++++++++++++++++ vcbuild.bat | 4 +- 6 files changed, 388 insertions(+), 110 deletions(-) create mode 100644 tools/icu/shrink-icu-src.py diff --git a/BUILDING.md b/BUILDING.md index 0d55a2047f..da52875e7b 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -117,29 +117,14 @@ $ make ### `Intl` (ECMA-402) support: -[Intl](https://github.com/nodejs/node/wiki/Intl) support is not -enabled by default. +[Intl](https://github.com/nodejs/node/wiki/Intl) support is +enabled by default, with English data only. +#### Default: `small-icu` (English only) support -#### "small" (English only) support - -This option will build with "small" (English only) support, but -the full `Intl` (ECMA-402) APIs. With `--download=all` it will -download the ICU library as needed. - -##### Unix / OS X: - -```text -$ ./configure --with-intl=small-icu --download=all -``` - -##### Windows: - -```text -> vcbuild small-icu download-all -``` - -The `small-icu` mode builds with English-only data. You can add full +By default, only English data is included, but +the full `Intl` (ECMA-402) APIs. It does not need to download +any dependencies to function. You can add full data at runtime. *Note:* more docs are on @@ -148,7 +133,8 @@ data at runtime. #### Build with full ICU support (all locales supported by ICU): With the `--download=all`, this may download ICU if you don't have an -ICU in `deps/icu`. +ICU in `deps/icu`. (The embedded `small-icu` included in the default +Node.js source does not include all locales.) ##### Unix / OS X: @@ -164,19 +150,19 @@ $ ./configure --with-intl=full-icu --download=all #### Building without Intl support -The `Intl` object will not be available. This is the default at -present, so this option is not normally needed. +The `Intl` object will not be available, nor some other APIs such as +`String.normalize`. ##### Unix / OS X: ```text -$ ./configure --with-intl=none +$ ./configure --without-intl ``` ##### Windows: ```text -> vcbuild intl-none +> vcbuild without-intl ``` #### Use existing installed ICU (Unix / OS X only): diff --git a/configure b/configure index 03a889d04a..7da8efcff3 100755 --- a/configure +++ b/configure @@ -303,20 +303,28 @@ parser.add_option('--with-etw', intl_optgroup.add_option('--with-intl', action='store', dest='with_intl', - default='none', + default='small-icu', choices=valid_intl_modes, help='Intl mode (valid choices: {0}) [default: %default]'.format( ', '.join(valid_intl_modes))) +intl_optgroup.add_option('--without-intl', + action='store_const', + dest='with_intl', + const='none', + help='Disable Intl, same as --with-intl=none') + intl_optgroup.add_option('--with-icu-path', action='store', dest='with_icu_path', help='Path to icu.gyp (ICU i18n, Chromium version only.)') +icu_default_locales='root,en' + intl_optgroup.add_option('--with-icu-locales', action='store', dest='with_icu_locales', - default='root,en', + default=icu_default_locales, help='Comma-separated list of locales for "small-icu". "root" is assumed. ' '[default: %default]') @@ -880,7 +888,7 @@ do_not_edit = '# Do not edit. Generated by the configure script.\n' def glob_to_var(dir_base, dir_sub, patch_dir): list = [] - dir_all = os.path.join(dir_base, dir_sub) + dir_all = '%s/%s' % (dir_base, dir_sub) files = os.walk(dir_all) for ent in files: (path, dirs, files) = ent @@ -968,6 +976,7 @@ def configure_intl(o): locs = set(options.with_icu_locales.split(',')) locs.add('root') # must have root o['variables']['icu_locales'] = string.join(locs,',') + # We will check a bit later if we can use the canned deps/icu-small elif with_intl == 'full-icu': # full ICU o['variables']['v8_enable_i18n_support'] = 1 @@ -994,15 +1003,42 @@ def configure_intl(o): # this is just the 'deps' dir. Used for unpacking. icu_parent_path = os.path.join(root_dir, 'deps') - # The full path to the ICU source directory. - icu_full_path = os.path.join(icu_parent_path, 'icu') + # The full path to the ICU source directory. Should not include './'. + icu_full_path = 'deps/icu' # icu-tmp is used to download and unpack the ICU tarball. icu_tmp_path = os.path.join(icu_parent_path, 'icu-tmp') + # canned ICU. see tools/icu/README.md to update. + canned_icu_dir = 'deps/icu-small' + + # We can use 'deps/icu-small' - pre-canned ICU *iff* + # - with_intl == small-icu (the default!) + # - with_icu_locales == 'root,en' (the default!) + # - deps/icu-small exists! + # - with_icu_source is unset (i.e. no other ICU was specified) + # (Note that this is the *DEFAULT CASE*.) + # + # This is *roughly* equivalent to + # $ configure --with-intl=small-icu --with-icu-source=deps/icu-small + # .. Except that we avoid copying icu-small over to deps/icu. + # In this default case, deps/icu is ignored, although make clean will + # still harmlessly remove deps/icu. + + # are we using default locales? + using_default_locales = ( options.with_icu_locales == icu_default_locales ) + + # make sure the canned ICU really exists + canned_icu_available = os.path.isdir(canned_icu_dir) + + if (o['variables']['icu_small'] == b(True)) and using_default_locales and (not with_icu_source) and canned_icu_available: + # OK- we can use the canned ICU. + icu_config['variables']['icu_small_canned'] = 1 + icu_full_path = canned_icu_dir + # --with-icu-source processing - # first, check that they didn't pass --with-icu-source=deps/icu - if with_icu_source and os.path.abspath(icu_full_path) == os.path.abspath(with_icu_source): + # now, check that they didn't pass --with-icu-source=deps/icu + elif with_icu_source and os.path.abspath(icu_full_path) == os.path.abspath(with_icu_source): print 'Ignoring redundant --with-icu-source=%s' % (with_icu_source) with_icu_source = None # if with_icu_source is still set, try to use it. @@ -1043,7 +1079,7 @@ def configure_intl(o): # ICU mode. (icu-generic.gyp) o['variables']['icu_gyp_path'] = 'tools/icu/icu-generic.gyp' - # ICU source dir relative to root + # ICU source dir relative to tools/icu (for .gyp file) o['variables']['icu_path'] = icu_full_path if not os.path.isdir(icu_full_path): print '* ECMA-402 (Intl) support didn\'t find ICU in %s..' % (icu_full_path) @@ -1083,14 +1119,14 @@ def configure_intl(o): 'source/data/in', icu_data_file_l) # relative to dep.. - icu_data_in = os.path.join('../../deps/icu/source/data/in', icu_data_file_l) + icu_data_in = os.path.join('..','..', icu_full_path, 'source/data/in', icu_data_file_l) if not os.path.isfile(icu_data_path) and icu_endianness != 'l': # use host endianness icu_data_path = os.path.join(icu_full_path, 'source/data/in', icu_data_file) # relative to dep.. - icu_data_in = os.path.join('icu/source/data/in', + icu_data_in = os.path.join('..', icu_full_path, 'source/data/in', icu_data_file) # this is the input '.dat' file to use .. icudt*.dat # may be little-endian if from a icu-project.org tarball @@ -1117,7 +1153,7 @@ def configure_intl(o): # with a list of the src files to use for i in icu_src: var = 'icu_src_%s' % i - path = '../../deps/icu/source/%s' % icu_src[i] + path = '../../%s/source/%s' % (icu_full_path, icu_src[i]) icu_config['variables'][var] = glob_to_var('tools/icu', path, 'patches/%s/source/%s' % (icu_ver_major, icu_src[i]) ) # write updated icu_config.gypi with a bunch of paths write(icu_config_name, do_not_edit + diff --git a/tools/icu/README.md b/tools/icu/README.md index 40d5287d9f..6c08bf1598 100644 --- a/tools/icu/README.md +++ b/tools/icu/README.md @@ -1,6 +1,84 @@ Notes about the icu directory. === +How to upgrade ICU +--- + +- Make sure your node workspace is clean (clean `git status`) should be sufficient. +- Configure Node with the specific [ICU version](http://icu-project.org/download) you want to upgrade to, for example: + +``` +./configure \ + --with-intl=small-icu \ + --with-icu-source=http://download.icu-project.org/files/icu4c/56.1/icu4c-56_1-src.zip +make +``` + +(the equivalent `vcbuild.bat` commands should work also.) + +- (note- may need to make changes in `icu-generic.gyp` or `tools/icu/patches` for +version specific stuff) + +- Verify the node build works + +``` +make test-ci +``` + +Also running +```js + new Intl.DateTimeFormat('es',{month:'long'}).format(new Date(9E8)); +``` + +…Should return `January` not `enero`. +(TODO here: improve [testing](https://github.com/nodejs/Intl/issues/16)) + + +- Now, copy `deps/icu` over to `deps/icu-small` + +``` +python tools/icu/shrink-icu-src.py +``` + +- Now, do a clean rebuild of node to test: + +(TODO: fix this when these options become default) + +``` +./configure --with-intl=small-icu --with-icu-source=deps/icu-small +make +``` + +- Test this newly default-generated Node.js +```js +process.versions.icu; +new Intl.DateTimeFormat('es',{month:'long'}).format(new Date(9E8)); +``` + +(should return your updated ICU version number, and also `January` again.) + +- You are ready to check in the updated `deps/small-icu`. +This is a big commit, so make this a separate commit from other changes. + +- Now, fix the default URL for the `full-icu` build in `/configure`, in +the `configure_intl()` function. It should match the ICU URL used in the +first step. When this is done, the following should build with full ICU. + +``` +# clean up +rm -rf out deps/icu deps/icu4c* +./configure --with-intl=full-icu --download=all +make +make test-ci +``` + +- commit the change to `configure`. + +----- + +Notes about these tools +--- + The files in this directory were written for the node.js effort. It's the intent of their author (Steven R. Loomis / srl295) to merge them upstream into ICU, pending much discussion within the ICU-PMC. diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index 222a9e9566..a61b294141 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -8,8 +8,8 @@ { 'variables': { 'icu_src_derb': [ - '../../deps/icu/source/tools/genrb/derb.c', - '../../deps/icu/source/tools/genrb/derb.cpp' + '<(icu_path)/source/tools/genrb/derb.c', + '<(icu_path)/source/tools/genrb/derb.cpp' ], }, 'includes': [ '../../icu_config.gypi' ], @@ -121,49 +121,82 @@ 'sources': [ '<@(icu_src_i18n)' ], + ## if your compiler can dead-strip, these exclusions will + ## make ZERO difference to binary size. + ## Made ICU-specific for future-proofing. 'conditions': [ [ 'icu_ver_major == 55', { 'sources!': [ - ## Strip out the following for ICU 55 only. - ## add more conditions in the future? - ## if your compiler can dead-strip, this will - ## make ZERO difference to binary size. - ## Made ICU-specific for future-proofing. + # alphabetic index + '<(icu_path)/source/i18n/alphaindex.cpp', + # BOCSU + # misc + '<(icu_path)/source/i18n/regexcmp.cpp', + '<(icu_path)/source/i18n/regexcmp.h', + '<(icu_path)/source/i18n/regexcst.h', + '<(icu_path)/source/i18n/regeximp.cpp', + '<(icu_path)/source/i18n/regeximp.h', + '<(icu_path)/source/i18n/regexst.cpp', + '<(icu_path)/source/i18n/regexst.h', + '<(icu_path)/source/i18n/regextxt.cpp', + '<(icu_path)/source/i18n/regextxt.h', + '<(icu_path)/source/i18n/region.cpp', + '<(icu_path)/source/i18n/region_impl.h', + '<(icu_path)/source/i18n/reldatefmt.cpp', + '<(icu_path)/source/i18n/reldatefmt.h' + '<(icu_path)/source/i18n/scientificformathelper.cpp', + '<(icu_path)/source/i18n/tmunit.cpp', + '<(icu_path)/source/i18n/tmutamt.cpp', + '<(icu_path)/source/i18n/tmutfmt.cpp', + '<(icu_path)/source/i18n/uregex.cpp', + '<(icu_path)/source/i18n/uregexc.cpp', + '<(icu_path)/source/i18n/uregion.cpp', + '<(icu_path)/source/i18n/uspoof.cpp', + '<(icu_path)/source/i18n/uspoof_build.cpp', + '<(icu_path)/source/i18n/uspoof_conf.cpp', + '<(icu_path)/source/i18n/uspoof_conf.h', + '<(icu_path)/source/i18n/uspoof_impl.cpp', + '<(icu_path)/source/i18n/uspoof_impl.h', + '<(icu_path)/source/i18n/uspoof_wsconf.cpp', + '<(icu_path)/source/i18n/uspoof_wsconf.h', + ]}], + [ 'icu_ver_major == 57', { 'sources!': [ # alphabetic index - '../../deps/icu/source/i18n/alphaindex.cpp', + '<(icu_path)/source/i18n/alphaindex.cpp', # BOCSU # misc - '../../deps/icu/source/i18n/regexcmp.cpp', - '../../deps/icu/source/i18n/regexcmp.h', - '../../deps/icu/source/i18n/regexcst.h', - '../../deps/icu/source/i18n/regeximp.cpp', - '../../deps/icu/source/i18n/regeximp.h', - '../../deps/icu/source/i18n/regexst.cpp', - '../../deps/icu/source/i18n/regexst.h', - '../../deps/icu/source/i18n/regextxt.cpp', - '../../deps/icu/source/i18n/regextxt.h', - '../../deps/icu/source/i18n/region.cpp', - '../../deps/icu/source/i18n/region_impl.h', - '../../deps/icu/source/i18n/reldatefmt.cpp', - '../../deps/icu/source/i18n/reldatefmt.h' - '../../deps/icu/source/i18n/scientificformathelper.cpp', - '../../deps/icu/source/i18n/tmunit.cpp', - '../../deps/icu/source/i18n/tmutamt.cpp', - '../../deps/icu/source/i18n/tmutfmt.cpp', - '../../deps/icu/source/i18n/uregex.cpp', - '../../deps/icu/source/i18n/uregexc.cpp', - '../../deps/icu/source/i18n/uregion.cpp', - '../../deps/icu/source/i18n/uspoof.cpp', - '../../deps/icu/source/i18n/uspoof_build.cpp', - '../../deps/icu/source/i18n/uspoof_conf.cpp', - '../../deps/icu/source/i18n/uspoof_conf.h', - '../../deps/icu/source/i18n/uspoof_impl.cpp', - '../../deps/icu/source/i18n/uspoof_impl.h', - '../../deps/icu/source/i18n/uspoof_wsconf.cpp', - '../../deps/icu/source/i18n/uspoof_wsconf.h', - ]}]], + '<(icu_path)/source/i18n/regexcmp.cpp', + '<(icu_path)/source/i18n/regexcmp.h', + '<(icu_path)/source/i18n/regexcst.h', + '<(icu_path)/source/i18n/regeximp.cpp', + '<(icu_path)/source/i18n/regeximp.h', + '<(icu_path)/source/i18n/regexst.cpp', + '<(icu_path)/source/i18n/regexst.h', + '<(icu_path)/source/i18n/regextxt.cpp', + '<(icu_path)/source/i18n/regextxt.h', + '<(icu_path)/source/i18n/region.cpp', + '<(icu_path)/source/i18n/region_impl.h', + '<(icu_path)/source/i18n/reldatefmt.cpp', + '<(icu_path)/source/i18n/reldatefmt.h' + '<(icu_path)/source/i18n/scientificformathelper.cpp', + '<(icu_path)/source/i18n/tmunit.cpp', + '<(icu_path)/source/i18n/tmutamt.cpp', + '<(icu_path)/source/i18n/tmutfmt.cpp', + '<(icu_path)/source/i18n/uregex.cpp', + '<(icu_path)/source/i18n/uregexc.cpp', + '<(icu_path)/source/i18n/uregion.cpp', + '<(icu_path)/source/i18n/uspoof.cpp', + '<(icu_path)/source/i18n/uspoof_build.cpp', + '<(icu_path)/source/i18n/uspoof_conf.cpp', + '<(icu_path)/source/i18n/uspoof_conf.h', + '<(icu_path)/source/i18n/uspoof_impl.cpp', + '<(icu_path)/source/i18n/uspoof_impl.h', + '<(icu_path)/source/i18n/uspoof_wsconf.cpp', + '<(icu_path)/source/i18n/uspoof_wsconf.h', + ]}], + ], 'include_dirs': [ - '../../deps/icu/source/i18n', + '<(icu_path)/source/i18n', ], 'defines': [ 'U_I18N_IMPLEMENTATION=1', @@ -171,7 +204,7 @@ 'dependencies': [ 'icuucx', 'icu_implementation', 'icu_uconfig', 'icu_uconfig_target' ], 'direct_dependent_settings': { 'include_dirs': [ - '../../deps/icu/source/i18n', + '<(icu_path)/source/i18n', ], }, 'export_dependent_settings': [ 'icuucx', 'icu_uconfig_target' ], @@ -253,7 +286,7 @@ 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.c' ], 'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ], 'include_dirs': [ - '../../deps/icu/source/common', + '<(icu_path)/source/common', ], 'actions': [ { @@ -333,7 +366,7 @@ 'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.c' ], # for umachine.h 'include_dirs': [ - '../../deps/icu/source/common', + '<(icu_path)/source/common', ], }]], # end icu_small == true }]], # end OS != win @@ -349,7 +382,7 @@ '<@(icu_src_stubdata)' ], 'include_dirs': [ - '../../deps/icu/source/common', + '<(icu_path)/source/common', ], }, # this target is for v8 consumption. @@ -379,34 +412,51 @@ 'sources': [ '<@(icu_src_common)', ], - 'conditions': [ - [ 'icu_ver_major == 55', { 'sources!': [ - ## Strip out the following for ICU 55 only. - ## add more conditions in the future? ## if your compiler can dead-strip, this will ## make ZERO difference to binary size. ## Made ICU-specific for future-proofing. + 'conditions': [ + [ 'icu_ver_major == 55', { 'sources!': [ + + # bidi- not needed (yet!) + '<(icu_path)/source/common/ubidi.c', + '<(icu_path)/source/common/ubidiimp.h', + '<(icu_path)/source/common/ubidiln.c', + '<(icu_path)/source/common/ubidiwrt.c', + #'<(icu_path)/source/common/ubidi_props.c', + #'<(icu_path)/source/common/ubidi_props.h', + #'<(icu_path)/source/common/ubidi_props_data.h', + # and the callers + '<(icu_path)/source/common/ushape.cpp', + '<(icu_path)/source/common/usprep.cpp', + '<(icu_path)/source/common/uts46.cpp', + '<(icu_path)/source/common/uidna.cpp', + ]}], + [ 'icu_ver_major == 57', { 'sources!': [ + # work around http://bugs.icu-project.org/trac/ticket/12451 + # (benign afterwards) + '<(icu_path)/source/common/cstr.cpp', # bidi- not needed (yet!) - '../../deps/icu/source/common/ubidi.c', - '../../deps/icu/source/common/ubidiimp.h', - '../../deps/icu/source/common/ubidiln.c', - '../../deps/icu/source/common/ubidiwrt.c', - #'../../deps/icu/source/common/ubidi_props.c', - #'../../deps/icu/source/common/ubidi_props.h', - #'../../deps/icu/source/common/ubidi_props_data.h', + '<(icu_path)/source/common/ubidi.c', + '<(icu_path)/source/common/ubidiimp.h', + '<(icu_path)/source/common/ubidiln.c', + '<(icu_path)/source/common/ubidiwrt.c', + #'<(icu_path)/source/common/ubidi_props.c', + #'<(icu_path)/source/common/ubidi_props.h', + #'<(icu_path)/source/common/ubidi_props_data.h', # and the callers - '../../deps/icu/source/common/ushape.cpp', - '../../deps/icu/source/common/usprep.cpp', - '../../deps/icu/source/common/uts46.cpp', - '../../deps/icu/source/common/uidna.cpp', + '<(icu_path)/source/common/ushape.cpp', + '<(icu_path)/source/common/usprep.cpp', + '<(icu_path)/source/common/uts46.cpp', + '<(icu_path)/source/common/uidna.cpp', ]}], [ 'OS == "solaris"', { 'defines': [ '_XOPEN_SOURCE_EXTENDED=0', ]}], ], 'include_dirs': [ - '../../deps/icu/source/common', + '<(icu_path)/source/common', ], 'defines': [ 'U_COMMON_IMPLEMENTATION=1', @@ -415,7 +465,7 @@ 'export_dependent_settings': [ 'icu_uconfig', 'icu_uconfig_target' ], 'direct_dependent_settings': { 'include_dirs': [ - '../../deps/icu/source/common', + '<(icu_path)/source/common', ], 'conditions': [ [ 'OS=="win"', { @@ -426,7 +476,7 @@ ], }, }, - # tools library + # tools library. This builds all of ICU together. { 'target_name': 'icutools', 'type': '<(library)', @@ -440,16 +490,16 @@ '<@(icu_src_stubdata)', ], 'sources!': [ - '../../deps/icu/source/tools/toolutil/udbgutil.cpp', - '../../deps/icu/source/tools/toolutil/udbgutil.h', - '../../deps/icu/source/tools/toolutil/dbgutil.cpp', - '../../deps/icu/source/tools/toolutil/dbgutil.h', + '<(icu_path)/source/tools/toolutil/udbgutil.cpp', + '<(icu_path)/source/tools/toolutil/udbgutil.h', + '<(icu_path)/source/tools/toolutil/dbgutil.cpp', + '<(icu_path)/source/tools/toolutil/dbgutil.h', ], 'include_dirs': [ - '../../deps/icu/source/common', - '../../deps/icu/source/i18n', - '../../deps/icu/source/io', - '../../deps/icu/source/tools/toolutil', + '<(icu_path)/source/common', + '<(icu_path)/source/i18n', + '<(icu_path)/source/io', + '<(icu_path)/source/tools/toolutil', ], 'defines': [ 'U_COMMON_IMPLEMENTATION=1', @@ -466,10 +516,10 @@ ], 'direct_dependent_settings': { 'include_dirs': [ - '../../deps/icu/source/common', - '../../deps/icu/source/i18n', - '../../deps/icu/source/io', - '../../deps/icu/source/tools/toolutil', + '<(icu_path)/source/common', + '<(icu_path)/source/i18n', + '<(icu_path)/source/io', + '<(icu_path)/source/tools/toolutil', ], 'conditions': [ [ 'OS=="win"', { diff --git a/tools/icu/shrink-icu-src.py b/tools/icu/shrink-icu-src.py new file mode 100644 index 0000000000..1760b4ff47 --- /dev/null +++ b/tools/icu/shrink-icu-src.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +import optparse +import os +import pprint +import re +import shlex +import subprocess +import sys +import shutil +import string + +parser = optparse.OptionParser() + +parser.add_option('--icu-small', + action='store', + dest='icusmall', + default='deps/icu-small', + help='path to target ICU directory to shrink. Will be deleted.') + +parser.add_option('--icu-src', + action='store', + dest='icusrc', + default='deps/icu', + help='path to source ICU directory.') + +parser.add_option('--icutmp', + action='store', + dest='icutmp', + default='out/Release/gen/icutmp', + help='path to icutmp dir.') + + +(options, args) = parser.parse_args() + +if os.path.isdir(options.icusmall): + print 'Deleting existing icusmall %s' % (options.icusmall) + shutil.rmtree(options.icusmall) + +if not os.path.isdir(options.icusrc): + print 'Missing source ICU dir --icusrc=%' % (options.icusrc) + sys.exit(1) + + + +ignore_regex = re.compile('^.*\.(vcxproj|filters|nrm|icu|dat|xml|txt|ac|guess|m4|in|sub|py|mak)$') + +def icu_ignore(dir, files): + subdir = dir[len(options.icusrc)+1::] + ign = [] + if len(subdir) == 0: + # remove all files at root level + ign = ign + files + # except... + ign.remove('source') + ign.remove('license.html') + ign.remove('LICENSE') + elif subdir == 'source': + ign = ign + ['layout','samples','test','extra','config','layoutex','allinone'] + ign = ign + ['runConfigureICU','install-sh','mkinstalldirs','configure'] + elif subdir == 'source/tools': + ign = ign + ['tzcode','ctestfw','gensprep','gennorm2','gendict','icuswap', + 'genbrk','gencfu','gencolusb','genren','memcheck','makeconv','gencnval','icuinfo','gentest'] + elif subdir == 'source/data': + ign = ign + ['unidata','curr','zone','unit','lang','region','misc','sprep'] + # else: + # print '!%s! [%s]' % (subdir, files) + ign = ign + ['.DS_Store', 'Makefile', 'Makefile.in'] + + for file in files: + if ignore_regex.match(file): + ign = ign + [file] + + # print '>%s< [%s]' % (subdir, ign) + return ign + +# copied from configure +def icu_info(icu_full_path): + uvernum_h = os.path.join(icu_full_path, 'source/common/unicode/uvernum.h') + if not os.path.isfile(uvernum_h): + print ' Error: could not load %s - is ICU installed?' % uvernum_h + sys.exit(1) + icu_ver_major = None + matchVerExp = r'^\s*#define\s+U_ICU_VERSION_SHORT\s+"([^"]*)".*' + match_version = re.compile(matchVerExp) + for line in open(uvernum_h).readlines(): + m = match_version.match(line) + if m: + icu_ver_major = m.group(1) + if not icu_ver_major: + print ' Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h + sys.exit(1) + icu_endianness = sys.byteorder[0]; # TODO(srl295): EBCDIC should be 'e' + return (icu_ver_major, icu_endianness) + +(icu_ver_major, icu_endianness) = icu_info(options.icusrc) +print "icudt%s%s" % (icu_ver_major, icu_endianness) + +src_datafile = os.path.join(options.icutmp, "icusmdt%s.dat" % (icu_ver_major)) +dst_datafile = os.path.join(options.icusmall, "source","data","in", "icudt%s%s.dat" % (icu_ver_major, icu_endianness)) + +if not os.path.isfile(src_datafile): + print "Could not find source datafile %s - did you build small-icu node?" % src_datafile + sys.exit(1) +else: + print "will use small datafile %s" % (src_datafile) +print '%s --> %s' % (options.icusrc, options.icusmall) +shutil.copytree(options.icusrc, options.icusmall, ignore=icu_ignore) +print '%s --> %s' % (src_datafile, dst_datafile) + +# OK, now copy the data file +shutil.copy(src_datafile, dst_datafile) + +# Now, print a short notice +readme_name = os.path.join(options.icusmall, "README-SMALL-ICU.txt" ) + +fi = open(readme_name, 'wb') +print >>fi, "Small ICU sources - auto generated by shrink-icu-src.py" +print >>fi, "" +print >>fi, "This directory contains the ICU subset used by --with-intl=small-icu (the default)" +print >>fi, "It is a strict subset of ICU %s source files with the following exception(s):" % (icu_ver_major) +print >>fi, "* %s : Reduced-size data file" % (dst_datafile) +print >>fi, "" +print >>fi, "" +print >>fi, "To rebuild this directory, see ../../tools/icu/README.md" +print >>fi, "" +fi.close() diff --git a/vcbuild.bat b/vcbuild.bat index b51fa8eda5..cceb815a05 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -73,6 +73,7 @@ if /i "%1"=="upload" set upload=1&goto arg-ok if /i "%1"=="small-icu" set i18n_arg=%1&goto arg-ok if /i "%1"=="full-icu" set i18n_arg=%1&goto arg-ok if /i "%1"=="intl-none" set i18n_arg=%1&goto arg-ok +if /i "%1"=="without-intl" set i18n_arg=%1&goto arg-ok if /i "%1"=="download-all" set download_arg="--download=all"&goto arg-ok if /i "%1"=="ignore-flaky" set test_args=%test_args% --flaky-tests=dontcare&goto arg-ok if /i "%1"=="enable-vtune" set enable_vtune_arg=1&goto arg-ok @@ -108,6 +109,7 @@ if defined enable_vtune_arg set configure_flags=%configure_flags% --enable-vtune if "%i18n_arg%"=="full-icu" set configure_flags=%configure_flags% --with-intl=full-icu if "%i18n_arg%"=="small-icu" set configure_flags=%configure_flags% --with-intl=small-icu if "%i18n_arg%"=="intl-none" set configure_flags=%configure_flags% --with-intl=none +if "%i18n_arg%"=="without-intl" set configure_flags=%configure_flags% --without-intl if defined config_flags set configure_flags=%configure_flags% %config_flags% @@ -309,7 +311,7 @@ echo Failed to create vc project files. goto exit :help -echo vcbuild.bat [debug/release] [msi] [test-all/test-uv/test-internet/test-pummel/test-simple/test-message] [clean] [noprojgen] [small-icu/full-icu/intl-none] [nobuild] [nosign] [x86/x64] [vc2013/vc2015] [download-all] [enable-vtune] +echo vcbuild.bat [debug/release] [msi] [test-all/test-uv/test-internet/test-pummel/test-simple/test-message] [clean] [noprojgen] [small-icu/full-icu/without-intl] [nobuild] [nosign] [x86/x64] [vc2013/vc2015] [download-all] [enable-vtune] echo Examples: echo vcbuild.bat : builds release build echo vcbuild.bat debug : builds debug build