@ -2,6 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import copy
import hashlib
import json
@ -13,6 +14,7 @@ import subprocess
import sys
import gyp
import gyp . common
from gyp . common import OrderedSet
import gyp . msvs_emulation
import gyp . MSVSUtil as MSVSUtil
import gyp . xcode_emulation
@ -60,17 +62,7 @@ generator_additional_path_sections = []
generator_extra_sources_for_rules = [ ]
generator_filelist_paths = None
# TODO: figure out how to not build extra host objects in the non-cross-compile
# case when this is enabled, and enable unconditionally.
generator_supports_multiple_toolsets = (
os . environ . get ( ' GYP_CROSSCOMPILE ' ) or
os . environ . get ( ' AR_host ' ) or
os . environ . get ( ' CC_host ' ) or
os . environ . get ( ' CXX_host ' ) or
os . environ . get ( ' AR_target ' ) or
os . environ . get ( ' CC_target ' ) or
os . environ . get ( ' CXX_target ' ) )
generator_supports_multiple_toolsets = gyp . common . CrossCompileRequested ( )
def StripPrefix ( arg , prefix ) :
if arg . startswith ( prefix ) :
@ -106,7 +98,7 @@ def AddArch(output, arch):
return ' %s . %s %s ' % ( output , arch , extension )
class Target :
class Target ( object ) :
""" Target represents the paths used within a single gyp target.
Conceptually , building a single target A is a series of steps :
@ -210,8 +202,8 @@ class Target:
# an output file; the result can be namespaced such that it is unique
# to the input file name as well as the output target name.
class NinjaWriter :
def __init__ ( self , qualified_target , target_outputs , base_dir , build_dir ,
class NinjaWriter ( object ) :
def __init__ ( self , hash_for_rules , target_outputs , base_dir , build_dir ,
output_file , toplevel_build , output_file_name , flavor ,
toplevel_dir = None ) :
"""
@ -221,7 +213,7 @@ class NinjaWriter:
toplevel_dir : path to the toplevel directory
"""
self . qualified_target = qualified_target
self . hash_for_rules = hash_for_rules
self . target_outputs = target_outputs
self . base_dir = base_dir
self . build_dir = build_dir
@ -338,12 +330,15 @@ class NinjaWriter:
obj + = ' . ' + self . toolset
path_dir , path_basename = os . path . split ( path )
assert not os . path . isabs ( path_dir ) , (
" ' %s ' can not be absolute path (see crbug.com/462153). " % path_dir )
if qualified :
path_basename = self . name + ' . ' + path_basename
return os . path . normpath ( os . path . join ( obj , self . base_dir , path_dir ,
path_basename ) )
def WriteCollapsedDependencies ( self , name , targets ) :
def WriteCollapsedDependencies ( self , name , targets , order_only = None ) :
""" Given a list of targets, return a path for a single file
representing the result of building all the targets or None .
@ -351,10 +346,11 @@ class NinjaWriter:
assert targets == filter ( None , targets ) , targets
if len ( targets ) == 0 :
assert not order_only
return None
if len ( targets ) > 1 :
if len ( targets ) > 1 or order_only :
stamp = self . GypPathToUniqueOutput ( name + ' .stamp ' )
targets = self . ninja . build ( stamp , ' stamp ' , targets )
targets = self . ninja . build ( stamp , ' stamp ' , targets , order_only = order_only )
self . ninja . newline ( )
return targets [ 0 ]
@ -391,6 +387,9 @@ class NinjaWriter:
self . ninja . variable ( ' arch ' , self . win_env [ arch ] )
self . ninja . variable ( ' cc ' , ' $cl_ ' + arch )
self . ninja . variable ( ' cxx ' , ' $cl_ ' + arch )
self . ninja . variable ( ' cc_host ' , ' $cl_ ' + arch )
self . ninja . variable ( ' cxx_host ' , ' $cl_ ' + arch )
self . ninja . variable ( ' asm ' , ' $ml_ ' + arch )
if self . flavor == ' mac ' :
self . archs = self . xcode_settings . GetActiveArchs ( config_name )
@ -472,6 +471,8 @@ class NinjaWriter:
else :
print " Warning: Actions/rules writing object files don ' t work with " \
" multiarch targets, dropping. (target %s ) " % spec [ ' target_name ' ]
elif self . flavor == ' mac ' and len ( self . archs ) > 1 :
link_deps = collections . defaultdict ( list )
if self . flavor == ' win ' and self . target . type == ' static_library ' :
@ -523,7 +524,7 @@ class NinjaWriter:
def WriteWinIdlFiles ( self , spec , prebuild ) :
""" Writes rules to match MSVS ' s implicit idl handling. """
assert self . flavor == ' win '
if self . msvs_settings . HasExplicitIdlRules ( spec ) :
if self . msvs_settings . HasExplicitIdlRulesOrActions ( spec ) :
return [ ]
outputs = [ ]
for source in filter ( lambda x : x . endswith ( ' .idl ' ) , spec [ ' sources ' ] ) :
@ -557,9 +558,10 @@ class NinjaWriter:
stamp = self . WriteCollapsedDependencies ( ' actions_rules_copies ' , outputs )
if self . is_mac_bundle :
self . WriteMacBundleResources (
xcassets = self . WriteMacBundleResources (
extra_mac_bundle_resources + mac_bundle_resources , mac_bundle_depends )
self . WriteMacInfoPlist ( mac_bundle_depends )
partial_info_plist = self . WriteMacXCassets ( xcassets , mac_bundle_depends )
self . WriteMacInfoPlist ( partial_info_plist , mac_bundle_depends )
return stamp
@ -580,23 +582,24 @@ class NinjaWriter:
def WriteActions ( self , actions , extra_sources , prebuild ,
extra_mac_bundle_resources ) :
# Actions cd into the base directory.
env = self . GetSortedXcodeEnv ( )
if self . flavor == ' win ' :
env = self . msvs_settings . GetVSMacroEnv (
' $!PRODUCT_DIR ' , config = self . config_name )
env = self . GetToolchainEnv ( )
all_outputs = [ ]
for action in actions :
# First write out a rule for the action.
name = ' %s _ %s ' % ( action [ ' action_name ' ] ,
hashlib . md5 ( self . qualified_target ) . hexdigest ( ) )
name = ' %s _ %s ' % ( action [ ' action_name ' ] , self . hash_for_rules )
description = self . GenerateDescription ( ' ACTION ' ,
action . get ( ' message ' , None ) ,
name )
is_cygwin = ( self . msvs_settings . IsRuleRunUnderCygwin ( action )
if self . flavor == ' win ' else False )
args = action [ ' action ' ]
depfile = action . get ( ' depfile ' , None )
if depfile :
depfile = self . ExpandSpecial ( depfile , self . base_to_build )
pool = ' console ' if int ( action . get ( ' ninja_use_console ' , 0 ) ) else None
rule_name , _ = self . WriteNewNinjaRule ( name , args , description ,
is_cygwin , env = env )
is_cygwin , env , pool ,
depfile = depfile )
inputs = [ self . GypPathToNinja ( i , env ) for i in action [ ' inputs ' ] ]
if int ( action . get ( ' process_outputs_as_sources ' , False ) ) :
@ -616,15 +619,16 @@ class NinjaWriter:
def WriteRules ( self , rules , extra_sources , prebuild ,
mac_bundle_resources , extra_mac_bundle_resources ) :
env = self . GetSortedXcode Env ( )
env = self . GetToolchain Env ( )
all_outputs = [ ]
for rule in rules :
# First write out a rule for the rule action.
name = ' %s _ %s ' % ( rule [ ' rule_name ' ] ,
hashlib . md5 ( self . qualified_target ) . hexdigest ( ) )
# Skip a rule with no action and no inputs.
if ' action ' not in rule and not rule . get ( ' rule_sources ' , [ ] ) :
continue
# First write out a rule for the rule action.
name = ' %s _ %s ' % ( rule [ ' rule_name ' ] , self . hash_for_rules )
args = rule [ ' action ' ]
description = self . GenerateDescription (
' RULE ' ,
@ -632,8 +636,9 @@ class NinjaWriter:
( ' %s ' + generator_default_variables [ ' RULE_INPUT_PATH ' ] ) % name )
is_cygwin = ( self . msvs_settings . IsRuleRunUnderCygwin ( rule )
if self . flavor == ' win ' else False )
pool = ' console ' if int ( rule . get ( ' ninja_use_console ' , 0 ) ) else None
rule_name , args = self . WriteNewNinjaRule (
name , args , description , is_cygwin , env = env )
name , args , description , is_cygwin , env , pool )
# TODO: if the command references the outputs directly, we should
# simplify it to just use $out.
@ -645,16 +650,31 @@ class NinjaWriter:
needed_variables = set ( [ ' source ' ] )
for argument in args :
for var in special_locals :
if ( ' $ { %s } ' % var ) in argument :
if ' $ { %s } ' % var in argument :
needed_variables . add ( var )
def cygwin_munge ( path ) :
# pylint: disable=cell-var-from-loop
if is_cygwin :
return path . replace ( ' \\ ' , ' / ' )
return path
inputs = [ self . GypPathToNinja ( i , env ) for i in rule . get ( ' inputs ' , [ ] ) ]
# If there are n source files matching the rule, and m additional rule
# inputs, then adding 'inputs' to each build edge written below will
# write m * n inputs. Collapsing reduces this to m + n.
sources = rule . get ( ' rule_sources ' , [ ] )
num_inputs = len ( inputs )
if prebuild :
num_inputs + = 1
if num_inputs > 2 and len ( sources ) > 2 :
inputs = [ self . WriteCollapsedDependencies (
rule [ ' rule_name ' ] , inputs , order_only = prebuild ) ]
prebuild = [ ]
# For each source file, write an edge that generates all the outputs.
for source in rule . get ( ' rule_sources ' , [ ] ) :
for source in sources :
source = os . path . normpath ( source )
dirname , basename = os . path . split ( source )
root , ext = os . path . splitext ( basename )
@ -663,9 +683,6 @@ class NinjaWriter:
outputs = [ self . ExpandRuleVariables ( o , root , dirname ,
source , ext , basename )
for o in rule [ ' outputs ' ] ]
inputs = [ self . ExpandRuleVariables ( i , root , dirname ,
source , ext , basename )
for i in rule . get ( ' inputs ' , [ ] ) ]
if int ( rule . get ( ' process_outputs_as_sources ' , False ) ) :
extra_sources + = outputs
@ -703,10 +720,11 @@ class NinjaWriter:
else :
assert var == None , repr ( var )
inputs = [ self . GypPathToNinja ( i , env ) for i in inputs ]
outputs = [ self . GypPathToNinja ( o , env ) for o in outputs ]
extra_bindings . append ( ( ' unique_name ' ,
hashlib . md5 ( outputs [ 0 ] ) . hexdigest ( ) ) )
if self . flavor == ' win ' :
# WriteNewNinjaRule uses unique_name for creating an rsp file on win.
extra_bindings . append ( ( ' unique_name ' ,
hashlib . md5 ( outputs [ 0 ] ) . hexdigest ( ) ) )
self . ninja . build ( outputs , rule_name , self . GypPathToNinja ( source ) ,
implicit = inputs ,
order_only = prebuild ,
@ -718,7 +736,7 @@ class NinjaWriter:
def WriteCopies ( self , copies , prebuild , mac_bundle_depends ) :
outputs = [ ]
env = self . GetSortedXcode Env ( )
env = self . GetToolchain Env ( )
for copy in copies :
for path in copy [ ' files ' ] :
# Normalize the path so trailing slashes don't confuse us.
@ -742,15 +760,68 @@ class NinjaWriter:
def WriteMacBundleResources ( self , resources , bundle_depends ) :
""" Writes ninja edges for ' mac_bundle_resources ' . """
xcassets = [ ]
for output , res in gyp . xcode_emulation . GetMacBundleResources (
generator_default_variables [ ' PRODUCT_DIR ' ] ,
self . xcode_settings , map ( self . GypPathToNinja , resources ) ) :
output = self . ExpandSpecial ( output )
self . ninja . build ( output , ' mac_tool ' , res ,
variables = [ ( ' mactool_cmd ' , ' copy-bundle-resource ' ) ] )
bundle_depends . append ( output )
if os . path . splitext ( output ) [ - 1 ] != ' .xcassets ' :
isBinary = self . xcode_settings . IsBinaryOutputFormat ( self . config_name )
self . ninja . build ( output , ' mac_tool ' , res ,
variables = [ ( ' mactool_cmd ' , ' copy-bundle-resource ' ) , \
( ' binary ' , isBinary ) ] )
bundle_depends . append ( output )
else :
xcassets . append ( res )
return xcassets
def WriteMacXCassets ( self , xcassets , bundle_depends ) :
""" Writes ninja edges for ' mac_bundle_resources ' .xcassets files.
This add an invocation of ' actool ' via the ' mac_tool.py ' helper script .
It assumes that the assets catalogs define at least one imageset and
thus an Assets . car file will be generated in the application resources
directory . If this is not the case , then the build will probably be done
at each invocation of ninja . """
if not xcassets :
return
extra_arguments = { }
settings_to_arg = {
' XCASSETS_APP_ICON ' : ' app-icon ' ,
' XCASSETS_LAUNCH_IMAGE ' : ' launch-image ' ,
}
settings = self . xcode_settings . xcode_settings [ self . config_name ]
for settings_key , arg_name in settings_to_arg . iteritems ( ) :
value = settings . get ( settings_key )
if value :
extra_arguments [ arg_name ] = value
partial_info_plist = None
if extra_arguments :
partial_info_plist = self . GypPathToUniqueOutput (
' assetcatalog_generated_info.plist ' )
extra_arguments [ ' output-partial-info-plist ' ] = partial_info_plist
def WriteMacInfoPlist ( self , bundle_depends ) :
outputs = [ ]
outputs . append (
os . path . join (
self . xcode_settings . GetBundleResourceFolder ( ) ,
' Assets.car ' ) )
if partial_info_plist :
outputs . append ( partial_info_plist )
keys = QuoteShellArgument ( json . dumps ( extra_arguments ) , self . flavor )
extra_env = self . xcode_settings . GetPerTargetSettings ( )
env = self . GetSortedXcodeEnv ( additional_settings = extra_env )
env = self . ComputeExportEnvString ( env )
bundle_depends . extend ( self . ninja . build (
outputs , ' compile_xcassets ' , xcassets ,
variables = [ ( ' env ' , env ) , ( ' keys ' , keys ) ] ) )
return partial_info_plist
def WriteMacInfoPlist ( self , partial_info_plist , bundle_depends ) :
""" Write build rules for bundle Info.plist files. """
info_plist , out , defines , extra_env = gyp . xcode_emulation . GetMacInfoPlist (
generator_default_variables [ ' PRODUCT_DIR ' ] ,
@ -770,10 +841,18 @@ class NinjaWriter:
env = self . GetSortedXcodeEnv ( additional_settings = extra_env )
env = self . ComputeExportEnvString ( env )
if partial_info_plist :
intermediate_plist = self . GypPathToUniqueOutput ( ' merged_info.plist ' )
info_plist = self . ninja . build (
intermediate_plist , ' merge_infoplist ' ,
[ partial_info_plist , info_plist ] )
keys = self . xcode_settings . GetExtraPlistItems ( self . config_name )
keys = QuoteShellArgument ( json . dumps ( keys ) , self . flavor )
isBinary = self . xcode_settings . IsBinaryOutputFormat ( self . config_name )
self . ninja . build ( out , ' copy_infoplist ' , info_plist ,
variables = [ ( ' env ' , env ) , ( ' keys ' , keys ) ] )
variables = [ ( ' env ' , env ) , ( ' keys ' , keys ) ,
( ' binary ' , isBinary ) ] )
bundle_depends . append ( out )
def WriteSources ( self , ninja_file , config_name , config , sources , predepends ,
@ -785,6 +864,8 @@ class NinjaWriter:
self . ninja . variable ( ' cxx ' , ' $cxx_host ' )
self . ninja . variable ( ' ld ' , ' $ld_host ' )
self . ninja . variable ( ' ldxx ' , ' $ldxx_host ' )
self . ninja . variable ( ' nm ' , ' $nm_host ' )
self . ninja . variable ( ' readelf ' , ' $readelf_host ' )
if self . flavor != ' mac ' or len ( self . archs ) == 1 :
return self . WriteSourcesForArch (
@ -810,6 +891,7 @@ class NinjaWriter:
cflags_objcc = [ ' $cflags_cc ' ] + \
self . xcode_settings . GetCflagsObjCC ( config_name )
elif self . flavor == ' win ' :
asmflags = self . msvs_settings . GetAsmflags ( config_name )
cflags = self . msvs_settings . GetCflags ( config_name )
cflags_c = self . msvs_settings . GetCflagsC ( config_name )
cflags_cc = self . msvs_settings . GetCflagsCC ( config_name )
@ -844,22 +926,31 @@ class NinjaWriter:
self . WriteVariableList ( ninja_file , ' defines ' ,
[ Define ( d , self . flavor ) for d in defines ] )
if self . flavor == ' win ' :
self . WriteVariableList ( ninja_file , ' asmflags ' ,
map ( self . ExpandSpecial , asmflags ) )
self . WriteVariableList ( ninja_file , ' rcflags ' ,
[ QuoteShellArgument ( self . ExpandSpecial ( f ) , self . flavor )
for f in self . msvs_settings . GetRcflags ( config_name ,
self . GypPathToNinja ) ] )
include_dirs = config . get ( ' include_dirs ' , [ ] )
env = self . GetSortedXcodeEnv ( )
env = self . GetToolchainEnv ( )
if self . flavor == ' win ' :
env = self . msvs_settings . GetVSMacroEnv ( ' $!PRODUCT_DIR ' ,
config = config_name )
include_dirs = self . msvs_settings . AdjustIncludeDirs ( include_dirs ,
config_name )
self . WriteVariableList ( ninja_file , ' includes ' ,
[ QuoteShellArgument ( ' -I ' + self . GypPathToNinja ( i , env ) , self . flavor )
for i in include_dirs ] )
if self . flavor == ' win ' :
midl_include_dirs = config . get ( ' midl_include_dirs ' , [ ] )
midl_include_dirs = self . msvs_settings . AdjustMidlIncludeDirs (
midl_include_dirs , config_name )
self . WriteVariableList ( ninja_file , ' midl_includes ' ,
[ QuoteShellArgument ( ' -I ' + self . GypPathToNinja ( i , env ) , self . flavor )
for i in midl_include_dirs ] )
pch_commands = precompiled_header . GetPchBuildCommands ( arch )
if self . flavor == ' mac ' :
# Most targets use no precompiled headers, so only write these if needed.
@ -868,6 +959,8 @@ class NinjaWriter:
include = precompiled_header . GetInclude ( ext , arch )
if include : ninja_file . variable ( var , include )
arflags = config . get ( ' arflags ' , [ ] )
self . WriteVariableList ( ninja_file , ' cflags ' ,
map ( self . ExpandSpecial , cflags ) )
self . WriteVariableList ( ninja_file , ' cflags_c ' ,
@ -879,6 +972,8 @@ class NinjaWriter:
map ( self . ExpandSpecial , cflags_objc ) )
self . WriteVariableList ( ninja_file , ' cflags_objcc ' ,
map ( self . ExpandSpecial , cflags_objcc ) )
self . WriteVariableList ( ninja_file , ' arflags ' ,
map ( self . ExpandSpecial , arflags ) )
ninja_file . newline ( )
outputs = [ ]
has_rc_source = False
@ -894,9 +989,7 @@ class NinjaWriter:
elif ext == ' s ' and self . flavor != ' win ' : # Doesn't generate .o.d files.
command = ' cc_s '
elif ( self . flavor == ' win ' and ext == ' asm ' and
self . msvs_settings . GetArch ( config_name ) == ' x86 ' and
not self . msvs_settings . HasExplicitAsmRules ( spec ) ) :
# Asm files only get auto assembled for x86 (not x64).
command = ' asm '
# Add the _asm suffix as msvs is capable of handling .cc and
# .asm files of the same name without collision.
@ -968,9 +1061,19 @@ class NinjaWriter:
arch = arch )
for arch in self . archs ]
extra_bindings = [ ]
build_output = output
if not self . is_mac_bundle :
self . AppendPostbuildVariable ( extra_bindings , spec , output , output )
self . ninja . build ( output , ' lipo ' , inputs , variables = extra_bindings )
# TODO(yyanagisawa): more work needed to fix:
# https://code.google.com/p/gyp/issues/detail?id=411
if ( spec [ ' type ' ] in ( ' shared_library ' , ' loadable_module ' ) and
not self . is_mac_bundle ) :
extra_bindings . append ( ( ' lib ' , output ) )
self . ninja . build ( [ output , output + ' .TOC ' ] , ' solipo ' , inputs ,
variables = extra_bindings )
else :
self . ninja . build ( build_output , ' lipo ' , inputs , variables = extra_bindings )
return output
def WriteLinkForArch ( self , ninja_file , spec , config_name , config ,
@ -1063,10 +1166,10 @@ class NinjaWriter:
rpath = ' lib/ '
if self . toolset != ' target ' :
rpath + = self . toolset
ldflags . append ( ' -Wl,-rpath= \ $$ORIGIN/ %s ' % rpath )
ldflags . append ( r ' -Wl,-rpath= \ $$ORIGIN/ %s ' % rpath )
ldflags . append ( ' -Wl,-rpath-link= %s ' % rpath )
self . WriteVariableList ( ninja_file , ' ldflags ' ,
gyp . common . uniquer ( map ( self . ExpandSpecial , ldflags ) ) )
map ( self . ExpandSpecial , ldflags ) )
library_dirs = config . get ( ' library_dirs ' , [ ] )
if self . flavor == ' win ' :
@ -1095,9 +1198,27 @@ class NinjaWriter:
extra_bindings . append ( ( ' soname ' , os . path . split ( output ) [ 1 ] ) )
extra_bindings . append ( ( ' lib ' ,
gyp . common . EncodePOSIXShellArgument ( output ) ) )
if self . flavor != ' win ' :
link_file_list = output
if self . is_mac_bundle :
# 'Dependency Framework.framework/Versions/A/Dependency Framework' ->
# 'Dependency Framework.framework.rsp'
link_file_list = self . xcode_settings . GetWrapperName ( )
if arch :
link_file_list + = ' . ' + arch
link_file_list + = ' .rsp '
# If an rspfile contains spaces, ninja surrounds the filename with
# quotes around it and then passes it to open(), creating a file with
# quotes in its name (and when looking for the rsp file, the name
# makes it through bash which strips the quotes) :-/
link_file_list = link_file_list . replace ( ' ' , ' _ ' )
extra_bindings . append (
( ' link_file_list ' ,
gyp . common . EncodePOSIXShellArgument ( link_file_list ) ) )
if self . flavor == ' win ' :
extra_bindings . append ( ( ' binary ' , output ) )
if ' /NOENTRY ' not in ldflags :
if ( ' /NOENTRY ' not in ldflags and
not self . msvs_settings . GetNoImportLibrary ( config_name ) ) :
self . target . import_lib = output + ' .lib '
extra_bindings . append ( ( ' implibflag ' ,
' /IMPLIB: %s ' % self . target . import_lib ) )
@ -1196,6 +1317,19 @@ class NinjaWriter:
self . target . bundle = output
return output
def GetToolchainEnv ( self , additional_settings = None ) :
""" Returns the variables toolchain would set for build steps. """
env = self . GetSortedXcodeEnv ( additional_settings = additional_settings )
if self . flavor == ' win ' :
env = self . GetMsvsToolchainEnv (
additional_settings = additional_settings )
return env
def GetMsvsToolchainEnv ( self , additional_settings = None ) :
""" Returns the variables Visual Studio would set for build steps. """
return self . msvs_settings . GetVSMacroEnv ( ' $!PRODUCT_DIR ' ,
config = self . config_name )
def GetSortedXcodeEnv ( self , additional_settings = None ) :
""" Returns the variables Xcode would set for build steps. """
assert self . abs_build_dir
@ -1377,7 +1511,8 @@ class NinjaWriter:
values = [ ]
ninja_file . variable ( var , ' ' . join ( values ) )
def WriteNewNinjaRule ( self , name , args , description , is_cygwin , env ) :
def WriteNewNinjaRule ( self , name , args , description , is_cygwin , env , pool ,
depfile = None ) :
""" Write out a new ninja " rule " statement for a given command.
Returns the name of the new rule , and a copy of | args | with variables
@ -1435,7 +1570,8 @@ class NinjaWriter:
# GYP rules/actions express being no-ops by not touching their outputs.
# Avoid executing downstream dependencies in this case by specifying
# restat=1 to ninja.
self . ninja . rule ( rule_name , command , description , restat = True ,
self . ninja . rule ( rule_name , command , description , depfile = depfile ,
restat = True , pool = pool ,
rspfile = rspfile , rspfile_content = rspfile_content )
self . ninja . newline ( )
@ -1466,12 +1602,13 @@ def CalculateVariables(default_variables, params):
generator_extra_sources_for_rules = getattr ( xcode_generator ,
' generator_extra_sources_for_rules ' , [ ] )
elif flavor == ' win ' :
exts = gyp . MSVSUtil . TARGET_TYPE_EXT
default_variables . setdefault ( ' OS ' , ' win ' )
default_variables [ ' EXECUTABLE_SUFFIX ' ] = ' .exe '
default_variables [ ' EXECUTABLE_SUFFIX ' ] = ' . ' + exts [ ' executabl e ' ]
default_variables [ ' STATIC_LIB_PREFIX ' ] = ' '
default_variables [ ' STATIC_LIB_SUFFIX ' ] = ' .lib '
default_variables [ ' STATIC_LIB_SUFFIX ' ] = ' . ' + exts [ ' static_ library ' ]
default_variables [ ' SHARED_LIB_PREFIX ' ] = ' '
default_variables [ ' SHARED_LIB_SUFFIX ' ] = ' .dll '
default_variables [ ' SHARED_LIB_SUFFIX ' ] = ' . ' + exts [ ' shared_library ' ]
# Copy additional generator configuration data from VS, which is shared
# by the Windows Ninja generator.
@ -1535,6 +1672,10 @@ def CommandWithWrapper(cmd, wrappers, prog):
def GetDefaultConcurrentLinks ( ) :
""" Returns a best-guess for a number of concurrent links. """
pool_size = int ( os . getenv ( ' GYP_LINK_CONCURRENCY ' , 0 ) )
if pool_size :
return pool_size
if sys . platform in ( ' win32 ' , ' cygwin ' ) :
import ctypes
@ -1557,19 +1698,17 @@ def GetDefaultConcurrentLinks():
mem_limit = max ( 1 , stat . ullTotalPhys / ( 4 * ( 2 * * 30 ) ) ) # total / 4GB
hard_cap = max ( 1 , int ( os . getenv ( ' GYP_LINK_CONCURRENCY_MAX ' , 2 * * 32 ) ) )
# return min(mem_limit, hard_cap)
# TODO(scottmg): Temporary speculative fix for OOM on builders
# See http://crbug.com/333000.
return 2
return min ( mem_limit , hard_cap )
elif sys . platform . startswith ( ' linux ' ) :
with open ( " /proc/meminfo " ) as meminfo :
memtotal_re = re . compile ( r ' ^MemTotal: \ s*( \ d*) \ s*kB ' )
for line in meminfo :
match = memtotal_re . match ( line )
if not match :
continue
# Allow 8Gb per link on Linux because Gold is quite memory hungry
return max ( 1 , int ( match . group ( 1 ) ) / ( 8 * ( 2 * * 20 ) ) )
if os . path . exists ( " /proc/meminfo " ) :
with open ( " /proc/meminfo " ) as meminfo :
memtotal_re = re . compile ( r ' ^MemTotal: \ s*( \ d*) \ s*kB ' )
for line in meminfo :
match = memtotal_re . match ( line )
if not match :
continue
# Allow 8Gb per link on Linux because Gold is quite memory hungry
return max ( 1 , int ( match . group ( 1 ) ) / ( 8 * ( 2 * * 20 ) ) )
return 1
elif sys . platform == ' darwin ' :
try :
@ -1666,14 +1805,15 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
# 'CC_host'/'CXX_host' enviroment variable, cc_host/cxx_host should be set
# to cc/cxx.
if flavor == ' win ' :
# Overridden by local arch choice in the use_deps case.
# Chromium's ffmpeg c99conv.py currently looks for a 'cc =' line in
# build.ninja so needs something valid here. http://crbug.com/233985
cc = ' cl.exe '
cxx = ' cl.exe '
ar = ' lib.exe '
# cc and cxx must be set to the correct architecture by overriding with one
# of cl_x86 or cl_x64 below.
cc = ' UNSET '
cxx = ' UNSET '
ld = ' link.exe '
ld_host = ' $ld '
else :
ar = ' ar '
cc = ' cc '
cxx = ' c++ '
ld = ' $cc '
@ -1681,10 +1821,16 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
ld_host = ' $cc_host '
ldxx_host = ' $cxx_host '
ar_host = ' ar '
cc_host = None
cxx_host = None
cc_host_global_setting = None
cxx_host_global_setting = None
clang_cl = None
nm = ' nm '
nm_host = ' nm '
readelf = ' readelf '
readelf_host = ' readelf '
build_file , _ , _ = gyp . common . ParseQualifiedTarget ( target_list [ 0 ] )
make_global_settings = data [ build_file ] . get ( ' make_global_settings ' , [ ] )
@ -1692,8 +1838,14 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
options . toplevel_dir )
wrappers = { }
for key , value in make_global_settings :
if key == ' AR ' :
ar = os . path . join ( build_to_root , value )
if key == ' AR.host ' :
ar_host = os . path . join ( build_to_root , value )
if key == ' CC ' :
cc = os . path . join ( build_to_root , value )
if cc . endswith ( ' clang-cl ' ) :
clang_cl = cc
if key == ' CXX ' :
cxx = os . path . join ( build_to_root , value )
if key == ' CC.host ' :
@ -1702,6 +1854,18 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
if key == ' CXX.host ' :
cxx_host = os . path . join ( build_to_root , value )
cxx_host_global_setting = value
if key == ' LD ' :
ld = os . path . join ( build_to_root , value )
if key == ' LD.host ' :
ld_host = os . path . join ( build_to_root , value )
if key == ' NM ' :
nm = os . path . join ( build_to_root , value )
if key == ' NM.host ' :
nm_host = os . path . join ( build_to_root , value )
if key == ' READELF ' :
readelf = os . path . join ( build_to_root , value )
if key == ' READELF.host ' :
readelf_host = os . path . join ( build_to_root , value )
if key . endswith ( ' _wrapper ' ) :
wrappers [ key [ : - len ( ' _wrapper ' ) ] ] = os . path . join ( build_to_root , value )
@ -1713,12 +1877,25 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
wrappers [ key_prefix ] = os . path . join ( build_to_root , value )
if flavor == ' win ' :
configs = [ target_dicts [ qualified_target ] [ ' configurations ' ] [ config_name ]
for qualified_target in target_list ]
shared_system_includes = None
if not generator_flags . get ( ' ninja_use_custom_environment_files ' , 0 ) :
shared_system_includes = \
gyp . msvs_emulation . ExtractSharedMSVSSystemIncludes (
configs , generator_flags )
cl_paths = gyp . msvs_emulation . GenerateEnvironmentFiles (
toplevel_build , generator_flags , OpenOutput )
toplevel_build , generator_flags , shared_system_includes , OpenOutput )
for arch , path in cl_paths . iteritems ( ) :
master_ninja . variable (
' cl_ ' + arch , CommandWithWrapper ( ' CC ' , wrappers ,
QuoteShellArgument ( path , flavor ) ) )
if clang_cl :
# If we have selected clang-cl, use that instead.
path = clang_cl
command = CommandWithWrapper ( ' CC ' , wrappers ,
QuoteShellArgument ( path , ' win ' ) )
if clang_cl :
# Use clang-cl to cross-compile for x86 or x86_64.
command + = ( ' -m32 ' if arch == ' x86 ' else ' -m64 ' )
master_ninja . variable ( ' cl_ ' + arch , command )
cc = GetEnvironFallback ( [ ' CC_target ' , ' CC ' ] , cc )
master_ninja . variable ( ' cc ' , CommandWithWrapper ( ' CC ' , wrappers , cc ) )
@ -1728,14 +1905,22 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
if flavor == ' win ' :
master_ninja . variable ( ' ld ' , ld )
master_ninja . variable ( ' idl ' , ' midl.exe ' )
master_ninja . variable ( ' ar ' , ' lib.exe ' )
master_ninja . variable ( ' ar ' , ar )
master_ninja . variable ( ' rc ' , ' rc.exe ' )
master_ninja . variable ( ' asm ' , ' ml.exe ' )
master_ninja . variable ( ' ml_x86 ' , ' ml.exe ' )
master_ninja . variable ( ' ml_x64 ' , ' ml64.exe ' )
master_ninja . variable ( ' mt ' , ' mt.exe ' )
else :
master_ninja . variable ( ' ld ' , CommandWithWrapper ( ' LINK ' , wrappers , ld ) )
master_ninja . variable ( ' ldxx ' , CommandWithWrapper ( ' LINK ' , wrappers , ldxx ) )
master_ninja . variable ( ' ar ' , GetEnvironFallback ( [ ' AR_target ' , ' AR ' ] , ' ar ' ) )
master_ninja . variable ( ' ar ' , GetEnvironFallback ( [ ' AR_target ' , ' AR ' ] , ar ) )
if flavor != ' mac ' :
# Mac does not use readelf/nm for .TOC generation, so avoiding polluting
# the master ninja with extra unused variables.
master_ninja . variable (
' nm ' , GetEnvironFallback ( [ ' NM_target ' , ' NM ' ] , nm ) )
master_ninja . variable (
' readelf ' , GetEnvironFallback ( [ ' READELF_target ' , ' READELF ' ] , readelf ) )
if generator_supports_multiple_toolsets :
if not cc_host :
@ -1743,7 +1928,10 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
if not cxx_host :
cxx_host = cxx
master_ninja . variable ( ' ar_host ' , GetEnvironFallback ( [ ' AR_host ' ] , ' ar ' ) )
master_ninja . variable ( ' ar_host ' , GetEnvironFallback ( [ ' AR_host ' ] , ar_host ) )
master_ninja . variable ( ' nm_host ' , GetEnvironFallback ( [ ' NM_host ' ] , nm_host ) )
master_ninja . variable ( ' readelf_host ' ,
GetEnvironFallback ( [ ' READELF_host ' ] , readelf_host ) )
cc_host = GetEnvironFallback ( [ ' CC_host ' ] , cc_host )
cxx_host = GetEnvironFallback ( [ ' CXX_host ' ] , cxx_host )
@ -1826,7 +2014,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
description = ' IDL $in ' ,
command = ( ' %s gyp-win-tool midl-wrapper $arch $outdir '
' $tlb $h $dlldata $iid $proxy $in '
' $idlflags ' % sys . executable ) )
' $midl_includes $ idlflags ' % sys . executable ) )
master_ninja . rule (
' rc ' ,
description = ' RC $in ' ,
@ -1836,20 +2024,20 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
sys . executable ) )
master_ninja . rule (
' asm ' ,
description = ' ASM $in ' ,
description = ' ASM $out ' ,
command = ( ' %s gyp-win-tool asm-wrapper '
' $arch $asm $defines $includes /c /Fo $out $in ' %
' $arch $asm $defines $includes $asmflags /c /Fo $out $in ' %
sys . executable ) )
if flavor != ' mac ' and flavor != ' win ' :
master_ninja . rule (
' alink ' ,
description = ' AR $out ' ,
command = ' rm -f $out && $ar rcs $out $in ' )
command = ' rm -f $out && $ar rcs $arflags $ out $in ' )
master_ninja . rule (
' alink_thin ' ,
description = ' AR $out ' ,
command = ' rm -f $out && $ar rcsT $out $in ' )
command = ' rm -f $out && $ar rcsT $arflags $ out $in ' )
# This allows targets that only need to depend on $lib's API to declare an
# order-only dependency on $lib.TOC and avoid relinking such downstream
@ -1857,38 +2045,39 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
# The resulting string leaves an uninterpolated %{suffix} which
# is used in the final substitution below.
mtime_preserving_solink_base = (
' if [ ! -e $lib -o ! -e $ { lib} '
' %(solink)s && %(extract_toc)s > $ { lib} '
' %(solink)s && %(extract_toc)s > $ { lib} '
' if ! cmp -s $ { lib} { lib} { lib} { lib} '
' if [ ! -e $lib -o ! -e $lib.TOC ]; then '
' %(solink)s && %(extract_toc)s > $lib.TOC; else '
' %(solink)s && %(extract_toc)s > $lib.tmp && '
' if ! cmp -s $lib.tmp $lib.TOC; then mv $lib.tmp $lib.TOC ; '
' fi; fi '
% { ' solink ' :
' $ld -shared $ldflags -o $lib -Wl,-soname=$soname %(suffix)s ' ,
' extract_toc ' :
( ' { readelf -d $ { lib} '
' nm -gD -f p $ { lib} \' \' ; } ' ) } )
( ' { $ readelf -d $lib | grep SONAME ; '
' $ nm -gD -f p $lib | cut -f1-2 -d\' \' ; } ' ) } )
master_ninja . rule (
' solink ' ,
description = ' SOLINK $lib ' ,
restat = True ,
command = ( mtime_preserving_solink_base % {
' suffix ' : ' -Wl,--whole-archive $in $solibs -Wl,--no-whole-archive '
' $libs ' } ) ,
command = mtime_preserving_solink_base % { ' suffix ' : ' @$link_file_list ' } ,
rspfile = ' $link_file_list ' ,
rspfile_content =
' -Wl,--whole-archive $in $solibs -Wl,--no-whole-archive $libs ' ,
pool = ' link_pool ' )
master_ninja . rule (
' solink_module ' ,
description = ' SOLINK(module) $lib ' ,
restat = True ,
command = ( mtime_preserving_solink_base % {
' suffix ' : ' -Wl,--start-group $in $solibs -Wl,--end-group '
' $libs ' } ) ,
command = mtime_preserving_solink_base % { ' suffix ' : ' @$link_file_list ' } ,
rspfile = ' $link_file_list ' ,
rspfile_content = ' -Wl,--start-group $in -Wl,--end-group $solibs $libs ' ,
pool = ' link_pool ' )
master_ninja . rule (
' link ' ,
description = ' LINK $out ' ,
command = ( ' $ld $ldflags -o $out '
' -Wl,--start-group $in $solibs -Wl,--end-group $libs ' ) ,
' -Wl,--start-group $in -Wl,--end-group $solibs $libs ' ) ,
pool = ' link_pool ' )
elif flavor == ' win ' :
master_ninja . rule (
@ -1927,21 +2116,31 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
' lipo ' ,
description = ' LIPO $out, POSTBUILDS ' ,
command = ' rm -f $out && lipo -create $in -output $out$postbuilds ' )
master_ninja . rule (
' solipo ' ,
description = ' SOLIPO $out, POSTBUILDS ' ,
command = (
' rm -f $lib $lib.TOC && lipo -create $in -output $lib$postbuilds && '
' %(extract_toc)s > $lib.TOC '
% { ' extract_toc ' :
' { otool -l $lib | grep LC_ID_DYLIB -A 5; '
' nm -gP $lib | cut -f1-2 -d \' \' | grep -v U$$; true; } ' } ) )
# Record the public interface of $lib in $lib.TOC. See the corresponding
# comment in the posix section above for details.
solink_base = ' $ld %(type)s $ldflags -o $lib %(suffix)s '
mtime_preserving_solink_base = (
' if [ ! -e $lib -o ! -e $ {lib} .TOC ] || '
' if [ ! -e $lib -o ! -e $lib.TOC ] || '
# Always force dependent targets to relink if this library
# reexports something. Handling this correctly would require
# recursive TOC dumping but this is rare in practice, so punt.
' otool -l $lib | grep -q LC_REEXPORT_DYLIB ; then '
' %(solink)s && %(extract_toc)s > $ { lib} '
' %(solink)s && %(extract_toc)s > $lib.TOC; '
' else '
' %(solink)s && %(extract_toc)s > $ { lib} '
' if ! cmp -s $ { lib} { lib} '
' mv $ { lib} { lib} '
' %(solink)s && %(extract_toc)s > $lib.tmp && '
' if ! cmp -s $lib.tmp $lib.TOC; then '
' mv $lib.tmp $lib.TOC ; '
' fi; '
' fi '
% { ' solink ' : solink_base ,
@ -1949,34 +2148,42 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
' { otool -l $lib | grep LC_ID_DYLIB -A 5; '
' nm -gP $lib | cut -f1-2 -d \' \' | grep -v U$$; true; } ' } )
solink_suffix = ' $in $solibs $libs$postbuilds '
solink_suffix = ' @$link_file_list$postbuilds '
master_ninja . rule (
' solink ' ,
description = ' SOLINK $lib, POSTBUILDS ' ,
restat = True ,
command = mtime_preserving_solink_base % { ' suffix ' : solink_suffix ,
' type ' : ' -shared ' } ,
rspfile = ' $link_file_list ' ,
rspfile_content = ' $in $solibs $libs ' ,
pool = ' link_pool ' )
master_ninja . rule (
' solink_notoc ' ,
description = ' SOLINK $lib, POSTBUILDS ' ,
restat = True ,
command = solink_base % { ' suffix ' : solink_suffix , ' type ' : ' -shared ' } ,
rspfile = ' $link_file_list ' ,
rspfile_content = ' $in $solibs $libs ' ,
pool = ' link_pool ' )
solink_module_suffix = ' $in $solibs $libs$postbuilds '
master_ninja . rule (
' solink_module ' ,
description = ' SOLINK(module) $lib, POSTBUILDS ' ,
restat = True ,
command = mtime_preserving_solink_base % { ' suffix ' : solink_module_ suffix ,
command = mtime_preserving_solink_base % { ' suffix ' : solink_suffix ,
' type ' : ' -bundle ' } ,
rspfile = ' $link_file_list ' ,
rspfile_content = ' $in $solibs $libs ' ,
pool = ' link_pool ' )
master_ninja . rule (
' solink_module_notoc ' ,
description = ' SOLINK(module) $lib, POSTBUILDS ' ,
restat = True ,
command = solink_base % { ' suffix ' : solink_module_suffix , ' type ' : ' -bundle ' } ,
command = solink_base % { ' suffix ' : solink_suffix , ' type ' : ' -bundle ' } ,
rspfile = ' $link_file_list ' ,
rspfile_content = ' $in $solibs $libs ' ,
pool = ' link_pool ' )
master_ninja . rule (
@ -1993,11 +2200,19 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
master_ninja . rule (
' copy_infoplist ' ,
description = ' COPY INFOPLIST $in ' ,
command = ' $env ./gyp-mac-tool copy-info-plist $in $out $keys ' )
command = ' $env ./gyp-mac-tool copy-info-plist $in $out $binary $keys ' )
master_ninja . rule (
' merge_infoplist ' ,
description = ' MERGE INFOPLISTS $in ' ,
command = ' $env ./gyp-mac-tool merge-info-plist $out $in ' )
master_ninja . rule (
' compile_xcassets ' ,
description = ' COMPILE XCASSETS $in ' ,
command = ' $env ./gyp-mac-tool compile-xcassets $keys $in ' )
master_ninja . rule (
' mac_tool ' ,
description = ' MACTOOL $mactool_cmd $in ' ,
command = ' $env ./gyp-mac-tool $mactool_cmd $in $out ' )
command = ' $env ./gyp-mac-tool $mactool_cmd $in $out $binary ' )
master_ninja . rule (
' package_framework ' ,
description = ' PACKAGE FRAMEWORK $out, POSTBUILDS ' ,
@ -2037,6 +2252,15 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
# objects.
target_short_names = { }
# short name of targets that were skipped because they didn't contain anything
# interesting.
# NOTE: there may be overlap between this an non_empty_target_names.
empty_target_names = set ( )
# Set of non-empty short target names.
# NOTE: there may be overlap between this an empty_target_names.
non_empty_target_names = set ( )
for qualified_target in target_list :
# qualified_target is like: third_party/icu/icu.gyp:icui18n#target
build_file , name , toolset = \
@ -2053,6 +2277,10 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
build_file = gyp . common . RelativePath ( build_file , options . toplevel_dir )
qualified_target_for_hash = gyp . common . QualifiedTarget ( build_file , name ,
toolset )
hash_for_rules = hashlib . md5 ( qualified_target_for_hash ) . hexdigest ( )
base_path = os . path . dirname ( build_file )
obj = ' obj '
if toolset != ' target ' :
@ -2060,7 +2288,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
output_file = os . path . join ( obj , base_path , name + ' .ninja ' )
ninja_output = StringIO ( )
writer = NinjaWriter ( qualified_target , target_outputs , base_path , build_dir ,
writer = NinjaWriter ( hash_for_rules , target_outputs , base_path , build_dir ,
ninja_output ,
toplevel_build , output_file ,
flavor , toplevel_dir = options . toplevel_dir )
@ -2080,6 +2308,9 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
target_outputs [ qualified_target ] = target
if qualified_target in all_targets :
all_outputs . add ( target . FinalOutput ( ) )
non_empty_target_names . add ( name )
else :
empty_target_names . add ( name )
if target_short_names :
# Write a short name to build this target. This benefits both the
@ -2091,6 +2322,16 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
master_ninja . build ( short_name , ' phony ' , [ x . FinalOutput ( ) for x in
target_short_names [ short_name ] ] )
# Write phony targets for any empty targets that weren't written yet. As
# short names are not necessarily unique only do this for short names that
# haven't already been output for another target.
empty_target_names = empty_target_names - non_empty_target_names
if empty_target_names :
master_ninja . newline ( )
master_ninja . comment ( ' Empty targets (output for completeness). ' )
for name in sorted ( empty_target_names ) :
master_ninja . build ( name , ' phony ' )
if all_outputs :
master_ninja . newline ( )
master_ninja . build ( ' all ' , ' phony ' , list ( all_outputs ) )
João Reis
10 years ago