1 | #!/usr/bin/env python2
|
2 | """
|
3 | build/ninja_main.py - invoked by ./NINJA-config.sh
|
4 |
|
5 | See build/README.md for the code and data layout.
|
6 |
|
7 | """
|
8 | from __future__ import print_function
|
9 |
|
10 | import cStringIO
|
11 | from glob import glob
|
12 | import os
|
13 | import sys
|
14 |
|
15 | from build import ninja_lib
|
16 | from build.ninja_lib import log
|
17 |
|
18 | from asdl import NINJA_subgraph as asdl_subgraph
|
19 | from bin import NINJA_subgraph as bin_subgraph
|
20 | from core import NINJA_subgraph as core_subgraph
|
21 | from cpp import NINJA_subgraph as cpp_subgraph
|
22 | from data_lang import NINJA_subgraph as data_lang_subgraph
|
23 | from frontend import NINJA_subgraph as frontend_subgraph
|
24 | from ysh import NINJA_subgraph as ysh_subgraph
|
25 | from osh import NINJA_subgraph as osh_subgraph
|
26 | from mycpp import NINJA_subgraph as mycpp_subgraph
|
27 | from pea import NINJA_subgraph as pea_subgraph
|
28 | from prebuilt import NINJA_subgraph as prebuilt_subgraph
|
29 | from yaks import NINJA_subgraph as yaks_subgraph
|
30 |
|
31 | from vendor import ninja_syntax
|
32 |
|
33 |
|
34 | # The file Ninja runs by default.
|
35 | BUILD_NINJA = 'build.ninja'
|
36 |
|
37 |
|
38 | def TarballManifest(cc_h_files):
|
39 | names = []
|
40 |
|
41 | # Code we know about
|
42 | names.extend(cc_h_files)
|
43 |
|
44 | names.extend([
|
45 | # Text
|
46 | 'LICENSE.txt',
|
47 | 'README-native.txt',
|
48 | 'INSTALL.txt',
|
49 | 'configure',
|
50 | 'install',
|
51 | 'doc/osh.1',
|
52 |
|
53 | # Build Scripts
|
54 | 'build/common.sh',
|
55 | 'build/native.sh',
|
56 |
|
57 | # These 2 are used by build/ninja-rules-cpp.sh
|
58 | 'build/py2.sh',
|
59 | 'build/dev-shell.sh',
|
60 |
|
61 | 'build/ninja-rules-cpp.sh',
|
62 | 'mycpp/common.sh',
|
63 |
|
64 | # Generated
|
65 | '_build/oils.sh',
|
66 |
|
67 | # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
|
68 | #'_gen/frontend/help_meta.h',
|
69 | '_gen/frontend/match.re2c.h',
|
70 | '_gen/frontend/id_kind.asdl_c.h',
|
71 | '_gen/frontend/types.asdl_c.h',
|
72 | ])
|
73 |
|
74 | # For configure
|
75 | names.extend(glob('build/detect-*.c'))
|
76 |
|
77 | # TODO: crawl headers
|
78 | # We can now use the headers=[] attribute
|
79 | names.extend(glob('mycpp/*.h'))
|
80 | names.extend(glob('cpp/*.h'))
|
81 |
|
82 | # ONLY the headers
|
83 | names.extend(glob('prebuilt/*/*.h'))
|
84 |
|
85 | names.sort() # Pass them to tar sorted
|
86 |
|
87 | # Check for dupes here
|
88 | unique = sorted(set(names))
|
89 | if names != unique:
|
90 | dupes = [n for n in names if names.count(n) > 1]
|
91 | raise AssertionError("Tarball manifest shouldn't have duplicates: %s" % dupes)
|
92 |
|
93 | for name in names:
|
94 | print(name)
|
95 |
|
96 |
|
97 | def ShellFunctions(cc_sources, f, argv0):
|
98 | """
|
99 | Generate a shell script that invokes the same function that build.ninja does
|
100 | """
|
101 | print('''\
|
102 | #!/bin/sh
|
103 | #
|
104 | # _build/oils.sh - generated by %s
|
105 | #
|
106 | # Usage:
|
107 | # _build/oils.sh COMPILER? VARIANT? SKIP_REBUILD?
|
108 | #
|
109 | # COMPILER: 'cxx' for system compiler, or 'clang' [default cxx]
|
110 | # VARIANT: 'dbg' or 'opt' [default dbg]
|
111 | # SKIP_REBUILD: if non-empty, checks if the output exists before building
|
112 |
|
113 | . build/ninja-rules-cpp.sh
|
114 |
|
115 | OILS_PARALLEL_BUILD=${OILS_PARALLEL_BUILD:-1}
|
116 |
|
117 | _compile_one() {
|
118 | local src=$4
|
119 |
|
120 | echo "CXX $src"
|
121 |
|
122 | # Delegate to function in build/ninja-rules-cpp.sh
|
123 | if test "${_do_fork:-}" = 1; then
|
124 | compile_one "$@" & # we will wait later
|
125 | else
|
126 | compile_one "$@"
|
127 | fi
|
128 | }
|
129 |
|
130 | main() {
|
131 | ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
|
132 |
|
133 | local compiler=${1:-cxx} # default is system compiler
|
134 | local variant=${2:-opt} # default is optimized build
|
135 | local skip_rebuild=${3:-} # if the output exists, skip build'
|
136 | ''' % (argv0), file=f)
|
137 |
|
138 | out_dir = '_bin/$compiler-$variant-sh'
|
139 | print(' local out_dir=%s' % out_dir, file=f)
|
140 |
|
141 | print('''\
|
142 | local out=$out_dir/oils-for-unix
|
143 |
|
144 | if test -n "$skip_rebuild" && test -f "$out"; then
|
145 | echo
|
146 | echo "$0: SKIPPING build because $out exists"
|
147 | echo
|
148 | return
|
149 | fi
|
150 |
|
151 | echo
|
152 | echo "$0: Building oils-for-unix: $out"
|
153 | echo "$0: PWD = $PWD"
|
154 | echo
|
155 | ''', file=f)
|
156 |
|
157 | objects = []
|
158 |
|
159 | in_out = []
|
160 | for src in sorted(cc_sources):
|
161 | # e.g. _build/obj/cxx-dbg-sh/posix.o
|
162 | prefix, _ = os.path.splitext(src)
|
163 | obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
|
164 | in_out.append((src, obj))
|
165 |
|
166 | bin_dir = '_bin/$compiler-$variant-sh'
|
167 | obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
|
168 |
|
169 | all_dirs = [bin_dir] + obj_dirs
|
170 | # Double quote
|
171 | all_dirs = ['"%s"' % d for d in all_dirs]
|
172 |
|
173 | print(' mkdir -p \\', file=f)
|
174 | print(' %s' % ' \\\n '.join(all_dirs), file=f)
|
175 | print('', file=f)
|
176 |
|
177 | do_fork = ''
|
178 |
|
179 | for i, (src, obj) in enumerate(in_out):
|
180 | obj_quoted = '"%s"' % obj
|
181 | objects.append(obj_quoted)
|
182 |
|
183 | # Only fork one translation unit that we know to be slow
|
184 | if 'oils_for_unix.mycpp.cc' in src:
|
185 | # There should only be one forked translation unit
|
186 | # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
|
187 | assert do_fork == ''
|
188 | do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
|
189 | else:
|
190 | do_fork = ''
|
191 |
|
192 | if do_fork:
|
193 | print(' # Potentially fork this translation unit with &', file=f)
|
194 | print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork, file=f)
|
195 | print(' %s %s' % (src, obj_quoted), file=f)
|
196 | print('', file=f)
|
197 |
|
198 | print(' # wait for the translation unit before linking', file=f)
|
199 | print(' echo WAIT', file=f)
|
200 | # time -p shows any excess parallelism on 2 cores
|
201 | # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
|
202 | # other translation units combined!
|
203 |
|
204 | # Timing isn't POSIX
|
205 | #print(' time -p wait', file=f)
|
206 | print(' wait', file=f)
|
207 | print('', file=f)
|
208 |
|
209 | print(' echo "LINK $out"', file=f)
|
210 | # note: can't have spaces in filenames
|
211 | print(' link "$compiler" "$variant" "" "$out" \\', file=f)
|
212 | # put each object on its own line, and indent by 4
|
213 | print(' %s' % (' \\\n '.join(objects)), file=f)
|
214 | print('', file=f)
|
215 |
|
216 | # Strip opt binary
|
217 | # TODO: provide a way for the user to get symbols?
|
218 |
|
219 | print('''\
|
220 | local out_name=oils-for-unix
|
221 | if test "$variant" = opt; then
|
222 | strip -o "$out.stripped" "$out"
|
223 |
|
224 | # Symlink to unstripped binary for benchmarking
|
225 | # out_name=$out_name.stripped
|
226 | fi
|
227 |
|
228 | cd $out_dir
|
229 | for symlink in osh ysh; do
|
230 | # like ln -v, which we can't use portably
|
231 | echo " $symlink -> $out_name"
|
232 | ln -s -f $out_name $symlink
|
233 | done
|
234 | }
|
235 |
|
236 | main "$@"
|
237 | ''', file=f)
|
238 |
|
239 |
|
240 | def Preprocessed(n, cc_sources):
|
241 | # See how much input we're feeding to the compiler. Test C++ template
|
242 | # explosion, e.g. <unordered_map>
|
243 | #
|
244 | # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
|
245 | # metrics/source-code.sh
|
246 |
|
247 | pre_matrix = [
|
248 | ('cxx', 'dbg'),
|
249 | ('cxx', 'opt'),
|
250 | ('clang', 'dbg'),
|
251 | ('clang', 'opt'),
|
252 | ]
|
253 | for compiler, variant in pre_matrix:
|
254 | preprocessed = []
|
255 | for src in cc_sources:
|
256 | # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
|
257 | pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
|
258 | preprocessed.append(pre)
|
259 |
|
260 | # Summary file
|
261 | n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
|
262 | 'line_count',
|
263 | preprocessed)
|
264 | n.newline()
|
265 |
|
266 |
|
267 | def InitSteps(n):
|
268 | """Wrappers for build/ninja-rules-*.sh
|
269 |
|
270 | Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
|
271 | """
|
272 |
|
273 | #
|
274 | # Compiling and linking
|
275 | #
|
276 |
|
277 | # Preprocess one translation unit
|
278 | n.rule('preprocess',
|
279 | # compile_one detects the _build/preprocessed path
|
280 | command='build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
|
281 | description='PP $compiler $variant $more_cxx_flags $in $out')
|
282 | n.newline()
|
283 |
|
284 | n.rule('line_count',
|
285 | command='build/ninja-rules-cpp.sh line_count $out $in',
|
286 | description='line_count $out $in')
|
287 | n.newline()
|
288 |
|
289 | # Compile one translation unit
|
290 | n.rule('compile_one',
|
291 | command='build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
|
292 | depfile='$out.d',
|
293 | # no prefix since the compiler is the first arg
|
294 | description='$compiler $variant $more_cxx_flags $in $out')
|
295 | n.newline()
|
296 |
|
297 | # Link objects together
|
298 | n.rule('link',
|
299 | command='build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
|
300 | description='LINK $compiler $variant $more_link_flags $out $in')
|
301 | n.newline()
|
302 |
|
303 | # 1 input and 2 outputs
|
304 | n.rule('strip',
|
305 | command='build/ninja-rules-cpp.sh strip_ $in $out',
|
306 | description='STRIP $in $out')
|
307 | n.newline()
|
308 |
|
309 | # cc_binary can have symliks
|
310 | n.rule('symlink',
|
311 | command='build/ninja-rules-cpp.sh symlink $dir $target $new',
|
312 | description='SYMLINK $dir $target $new')
|
313 | n.newline()
|
314 |
|
315 | #
|
316 | # Code generators
|
317 | #
|
318 |
|
319 | n.rule('write-shwrap',
|
320 | # $in must start with main program
|
321 | command='build/ninja-rules-py.sh write-shwrap $template $out $in',
|
322 | description='make-pystub $out $in')
|
323 | n.newline()
|
324 |
|
325 | n.rule('gen-oils-for-unix',
|
326 | command='build/ninja-rules-py.sh gen-oils-for-unix $main_name $out_prefix $preamble $in',
|
327 | description='gen-oils-for-unix $main_name $out_prefix $preamble $in')
|
328 | n.newline()
|
329 |
|
330 |
|
331 | def main(argv):
|
332 | try:
|
333 | action = argv[1]
|
334 | except IndexError:
|
335 | action = 'ninja'
|
336 |
|
337 | if action == 'ninja':
|
338 | f = open(BUILD_NINJA, 'w')
|
339 | else:
|
340 | f = cStringIO.StringIO() # thrown away
|
341 |
|
342 | n = ninja_syntax.Writer(f)
|
343 | ru = ninja_lib.Rules(n)
|
344 |
|
345 | ru.comment('InitSteps()')
|
346 | InitSteps(n)
|
347 |
|
348 | #
|
349 | # Create the graph.
|
350 | #
|
351 |
|
352 | asdl_subgraph.NinjaGraph(ru)
|
353 | ru.comment('')
|
354 |
|
355 | bin_subgraph.NinjaGraph(ru)
|
356 | ru.comment('')
|
357 |
|
358 | core_subgraph.NinjaGraph(ru)
|
359 | ru.comment('')
|
360 |
|
361 | cpp_subgraph.NinjaGraph(ru)
|
362 | ru.comment('')
|
363 |
|
364 | data_lang_subgraph.NinjaGraph(ru)
|
365 | ru.comment('')
|
366 |
|
367 | frontend_subgraph.NinjaGraph(ru)
|
368 | ru.comment('')
|
369 |
|
370 | mycpp_subgraph.NinjaGraph(ru)
|
371 | ru.comment('')
|
372 |
|
373 | ysh_subgraph.NinjaGraph(ru)
|
374 | ru.comment('')
|
375 |
|
376 | osh_subgraph.NinjaGraph(ru)
|
377 | ru.comment('')
|
378 |
|
379 | pea_subgraph.NinjaGraph(ru)
|
380 | ru.comment('')
|
381 |
|
382 | prebuilt_subgraph.NinjaGraph(ru)
|
383 | ru.comment('')
|
384 |
|
385 | yaks_subgraph.NinjaGraph(ru)
|
386 | ru.comment('')
|
387 |
|
388 |
|
389 | # Materialize all the cc_binary() rules
|
390 | ru.WriteRules()
|
391 |
|
392 | # Collect sources for metrics, tarball, shell script
|
393 | cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
|
394 |
|
395 | if 0:
|
396 | from pprint import pprint
|
397 | pprint(cc_sources)
|
398 |
|
399 | # TODO: could thin these out, not generate for unit tests, etc.
|
400 | Preprocessed(n, cc_sources)
|
401 |
|
402 | ru.WritePhony()
|
403 |
|
404 | n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
|
405 |
|
406 | if action == 'ninja':
|
407 | log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
|
408 | n.num_build_targets())
|
409 |
|
410 | elif action == 'shell':
|
411 | out = '_build/oils.sh'
|
412 | with open(out, 'w') as f:
|
413 | ShellFunctions(cc_sources, f, argv[0])
|
414 | log(' (%s) -> %s', argv[0], out)
|
415 |
|
416 | elif action == 'tarball-manifest':
|
417 | h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
|
418 | TarballManifest(cc_sources + h)
|
419 |
|
420 | else:
|
421 | raise RuntimeError('Invalid action %r' % action)
|
422 |
|
423 |
|
424 | if __name__ == '__main__':
|
425 | try:
|
426 | main(sys.argv)
|
427 | except RuntimeError as e:
|
428 | print('FATAL: %s' % e, file=sys.stderr)
|
429 | sys.exit(1)
|
430 |
|
431 | # vim: sw=2
|