2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
4 # The LLVM Compiler Infrastructure
6 # This file is distributed under the University of Illinois Open Source
7 # License. See LICENSE.TXT for details.
9 ##===----------------------------------------------------------------------===##
11 # This script builds many different flavors of the LLVM ecosystem. It
12 # will build LLVM, Clang and dragonegg as well as run tests on them.
13 # This script is convenient to use to check builds and tests before
14 # committing changes to the upstream repository
16 # A typical source setup uses three trees and looks like this:
34 # In a typical workflow, the "official" tree always contains unchanged
35 # sources from the main LLVM project repositories. The "staging" tree
36 # is where local work is done. A set of changes resides there waiting
37 # to be moved upstream. The "commit" tree is where changes from
38 # "staging" make their way upstream. Individual incremental changes
39 # from "staging" are applied to "commit" and committed upstream after
40 # a successful build and test run. A successful build is one in which
41 # testing results in no more failures than seen in the testing of the
44 # A build may be invoked as such:
46 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official
47 # --build=debug --build=release --build=paranoid
48 # --prefix=/home/greened/install --builddir=/home/greened/build
50 # This will build the LLVM ecosystem, including LLVM, Clangand
51 # dragonegg, putting build results in ~/build and installing tools in
52 # ~/install. llvm-compilers-check creates separate build and install
53 # directories for each source/build flavor. In the above example,
54 # llvmbuild will build debug, release and paranoid (debug+checks)
55 # flavors from each source tree (official, staging and commit) for a
56 # total of nine builds. All builds will be run in parallel.
58 # The user may control parallelism via the --jobs and --threads
59 # switches. --jobs tells llvm-compilers-checl the maximum total
60 # number of builds to activate in parallel. The user may think of it
61 # as equivalent to the GNU make -j switch. --threads tells
62 # llvm-compilers-check how many worker threads to use to accomplish
63 # those builds. If --threads is less than --jobs, --threads workers
64 # will be launched and each one will pick a source/flavor combination
65 # to build. Then llvm-compilers-check will invoke GNU make with -j
66 # (--jobs / --threads) to use up the remaining job capacity. Once a
67 # worker is finished with a build, it will pick another combination
68 # off the list and start building it.
70 ##===----------------------------------------------------------------------===##
82 # TODO: Use shutil.which when it is available (3.2 or later)
83 def find_executable(executable, path=None):
84 """Try to find 'executable' in the directories listed in 'path' (a
85 string listing directories separated by 'os.pathsep'; defaults to
86 os.environ['PATH']). Returns the complete filename or None if not
90 path = os.environ['PATH']
91 paths = path.split(os.pathsep)
94 (base, ext) = os.path.splitext(executable)
95 # executable files on OS/2 can have an arbitrary extension, but
96 # .exe is automatically appended if no dot is present in the name
98 executable = executable + ".exe"
99 elif sys.platform == 'win32':
100 pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
101 (base, ext) = os.path.splitext(executable)
102 if ext.lower() not in pathext:
105 execname = executable + ext
106 if os.path.isfile(execname):
110 f = os.path.join(p, execname)
111 if os.path.isfile(f):
116 def is_executable(fpath):
117 return os.path.exists(fpath) and os.access(fpath, os.X_OK)
119 def add_options(parser):
120 parser.add_option("-v", "--verbose", action="store_true",
122 help=("Output informational messages"
123 " [default: %default]"))
124 parser.add_option("--src", action="append",
125 help=("Top-level source directory [default: %default]"))
126 parser.add_option("--build", action="append",
127 help=("Build types to run [default: %default]"))
128 parser.add_option("--cc", default=find_executable("cc"),
129 help=("The C compiler to use [default: %default]"))
130 parser.add_option("--cxx", default=find_executable("c++"),
131 help=("The C++ compiler to use [default: %default]"))
132 parser.add_option("--threads", default=4, type="int",
133 help=("The number of worker threads to use "
134 "[default: %default]"))
135 parser.add_option("--jobs", "-j", default=8, type="int",
136 help=("The number of simultaneous build jobs "
137 "[default: %default]"))
138 parser.add_option("--prefix",
139 help=("Root install directory [default: %default]"))
140 parser.add_option("--builddir",
141 help=("Root build directory [default: %default]"))
142 parser.add_option("--extra-llvm-config-flags", default="",
143 help=("Extra flags to pass to llvm configure [default: %default]"))
144 parser.add_option("--force-configure", default=False, action="store_true",
145 help=("Force reconfigure of all components"))
146 parser.add_option("--no-dragonegg", default=False, action="store_true",
147 help=("Do not build dragonegg"))
148 parser.add_option("--no-install", default=False, action="store_true",
149 help=("Do not do installs"))
150 parser.add_option("--keep-going", default=False, action="store_true",
151 help=("Keep going after failures"))
152 parser.add_option("--no-flavor-prefix", default=False, action="store_true",
153 help=("Do not append the build flavor to the install path"))
154 parser.add_option("--enable-werror", default=False, action="store_true",
155 help=("Build with -Werror"))
158 def check_options(parser, options, valid_builds):
159 # See if we're building valid flavors.
160 for build in options.build:
161 if (build not in valid_builds):
162 parser.error("'" + build + "' is not a valid build flavor "
165 # See if we can find source directories.
166 for src in options.src:
167 for component in components:
168 component = component.rstrip("2")
169 compsrc = src + "/" + component
170 if (not os.path.isdir(compsrc)):
171 parser.error("'" + compsrc + "' does not exist")
173 # See if we can find the compilers
174 options.cc = find_executable(options.cc)
175 options.cxx = find_executable(options.cxx)
179 # Find a unique short name for the given set of paths. This searches
180 # back through path components until it finds unique component names
181 # among all given paths.
182 def get_path_abbrevs(paths):
183 # Find the number of common starting characters in the last component
185 unique_paths = list(paths)
187 class NotFoundException(Exception): pass
189 # Find a unique component of each path.
190 unique_bases = unique_paths[:]
192 while len(unique_paths) > 0:
193 bases = [os.path.basename(src) for src in unique_paths]
194 components = { c for c in bases }
195 # Account for single entry in paths.
196 if len(components) > 1 or len(components) == len(bases):
197 # We found something unique.
199 if bases.count(c) == 1:
200 index = bases.index(c)
201 unique_bases[index] = c
202 # Remove the corresponding path from the set under
204 unique_paths[index] = None
205 unique_paths = [ p for p in unique_paths if p is not None ]
206 unique_paths = [os.path.dirname(src) for src in unique_paths]
208 if len(unique_paths) > 0:
209 raise NotFoundException()
211 abbrevs = dict(zip(paths, [base for base in unique_bases]))
215 # Given a set of unique names, find a short character sequence that
216 # uniquely identifies them.
217 def get_short_abbrevs(unique_bases):
218 # Find a unique start character for each path base.
219 my_unique_bases = unique_bases[:]
220 unique_char_starts = unique_bases[:]
221 while len(my_unique_bases) > 0:
222 for start, char_tuple in enumerate(zip(*[base
223 for base in my_unique_bases])):
224 chars = { c for c in char_tuple }
225 # Account for single path.
226 if len(chars) > 1 or len(chars) == len(char_tuple):
227 # We found something unique.
229 if char_tuple.count(c) == 1:
230 index = char_tuple.index(c)
231 unique_char_starts[index] = start
232 # Remove the corresponding path from the set under
234 my_unique_bases[index] = None
235 my_unique_bases = [ b for b in my_unique_bases
239 if len(my_unique_bases) > 0:
240 raise NotFoundException()
242 abbrevs = [abbrev[start_index:start_index+3]
243 for abbrev, start_index
244 in zip([base for base in unique_bases],
245 [index for index in unique_char_starts])]
247 abbrevs = dict(zip(unique_bases, abbrevs))
251 class Builder(threading.Thread):
252 class ExecutableNotFound(Exception): pass
253 class FileNotExecutable(Exception): pass
255 def __init__(self, work_queue, jobs,
256 build_abbrev, source_abbrev,
259 self.work_queue = work_queue
262 self.cxx = options.cxx
263 self.build_abbrev = build_abbrev
264 self.source_abbrev = source_abbrev
265 self.build_prefix = options.builddir
266 self.install_prefix = options.prefix
267 self.options = options
268 self.component_abbrev = dict(
274 source, build = self.work_queue.get()
275 self.dobuild(source, build)
277 traceback.print_exc()
279 self.work_queue.task_done()
281 def execute(self, command, execdir, env, component):
282 prefix = self.component_abbrev[component.replace("-", "_")]
284 if not os.path.exists(execdir):
287 execenv = os.environ.copy()
289 for key, value in env.items():
292 self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
293 + " ".join(command));
296 proc = subprocess.Popen(command,
299 stdout=subprocess.PIPE,
300 stderr=subprocess.STDOUT)
302 line = proc.stdout.readline()
304 self.logger.info("[" + prefix + "] "
305 + str(line, "utf-8").rstrip())
306 line = proc.stdout.readline()
308 (stdoutdata, stderrdata) = proc.communicate()
309 retcode = proc.wait()
314 traceback.print_exc()
316 # Get a list of C++ include directories to pass to clang.
317 def get_includes(self):
318 # Assume we're building with g++ for now.
320 command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
322 self.logger.debug(command)
324 proc = subprocess.Popen(command,
325 stdout=subprocess.PIPE,
326 stderr=subprocess.STDOUT)
329 line = proc.stdout.readline()
331 self.logger.debug(line)
332 if re.search("End of search list", str(line)) is not None:
333 self.logger.debug("Stop Gather")
336 includes.append(str(line, "utf-8").strip())
337 if re.search("#include <...> search starts", str(line)) is not None:
338 self.logger.debug("Start Gather")
340 line = proc.stdout.readline()
343 traceback.print_exc()
344 self.logger.debug(includes)
347 def dobuild(self, source, build):
350 ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
352 prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
353 if (not self.options.no_flavor_prefix):
354 self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
356 build_suffix += "/" + self.source_abbrev[source] + "/" + build
358 self.logger = logging.getLogger(prefix)
360 self.logger.debug(self.install_prefix)
362 # Assume we're building with gcc for now.
363 cxxincludes = self.get_includes()
364 cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version
365 cxxroot = os.path.dirname(cxxroot) # Remove the c++
366 cxxroot = os.path.dirname(cxxroot) # Remove the include
368 configure_flags = dict(
369 llvm=dict(debug=["--prefix=" + self.install_prefix,
370 "--enable-assertions",
371 "--disable-optimized",
372 "--with-gcc-toolchain=" + cxxroot],
373 release=["--prefix=" + self.install_prefix,
374 "--enable-optimized",
375 "--with-gcc-toolchain=" + cxxroot],
376 paranoid=["--prefix=" + self.install_prefix,
377 "--enable-assertions",
378 "--enable-expensive-checks",
379 "--disable-optimized",
380 "--with-gcc-toolchain=" + cxxroot]),
381 dragonegg=dict(debug=[],
385 if (self.options.enable_werror):
386 configure_flags["llvm"]["debug"].append("--enable-werror")
387 configure_flags["llvm"]["release"].append("--enable-werror")
388 configure_flags["llvm"]["paranoid"].append("--enable-werror")
390 configure_env = dict(
391 llvm=dict(debug=dict(CC=self.cc,
393 release=dict(CC=self.cc,
395 paranoid=dict(CC=self.cc,
397 dragonegg=dict(debug=dict(CC=self.cc,
399 release=dict(CC=self.cc,
401 paranoid=dict(CC=self.cc,
405 llvm=dict(debug=["-j" + str(self.jobs)],
406 release=["-j" + str(self.jobs)],
407 paranoid=["-j" + str(self.jobs)]),
408 dragonegg=dict(debug=["-j" + str(self.jobs)],
409 release=["-j" + str(self.jobs)],
410 paranoid=["-j" + str(self.jobs)]))
413 llvm=dict(debug=dict(),
416 dragonegg=dict(debug=dict(GCC=self.cc,
417 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
418 release=dict(GCC=self.cc,
419 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
420 paranoid=dict(GCC=self.cc,
421 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
423 make_install_flags = dict(
424 llvm=dict(debug=["install"],
426 paranoid=["install"]),
427 dragonegg=dict(debug=["install"],
429 paranoid=["install"]))
431 make_install_env = dict(
432 llvm=dict(debug=dict(),
435 dragonegg=dict(debug=dict(),
439 make_check_flags = dict(
440 llvm=dict(debug=["check"],
443 dragonegg=dict(debug=["check"],
447 make_check_env = dict(
448 llvm=dict(debug=dict(),
451 dragonegg=dict(debug=dict(),
455 for component in components:
458 if (self.options.no_dragonegg):
459 if (comp == 'dragonegg'):
460 self.logger.info("Skipping " + component + " in "
464 srcdir = source + "/" + comp.rstrip("2")
465 builddir = self.build_prefix + "/" + comp + "/" + build_suffix
466 installdir = self.install_prefix
468 comp_key = comp.replace("-", "_")
470 config_args = configure_flags[comp_key][build][:]
471 config_args.extend(getattr(self.options,
472 "extra_" + comp_key.rstrip("2")
476 self.logger.info("Configuring " + component + " in " + builddir)
477 configrc = self.configure(component, srcdir, builddir,
479 configure_env[comp_key][build])
481 if (configrc == None) :
482 self.logger.info("[None] Failed to configure " + component + " in " + installdir)
484 if (configrc == 0 or self.options.keep_going) :
485 self.logger.info("Building " + component + " in " + builddir)
486 self.logger.info("Build: make " + str(make_flags[comp_key][build]))
487 buildrc = self.make(component, srcdir, builddir,
488 make_flags[comp_key][build],
489 make_env[comp_key][build])
491 if (buildrc == None) :
492 self.logger.info("[None] Failed to build " + component + " in " + installdir)
494 if (buildrc == 0 or self.options.keep_going) :
495 self.logger.info("Testing " + component + " in " + builddir)
496 self.logger.info("Test: make "
497 + str(make_check_flags[comp_key][build]))
498 testrc = self.make(component, srcdir, builddir,
499 make_check_flags[comp_key][build],
500 make_check_env[comp_key][build])
502 if (testrc == None) :
503 self.logger.info("[None] Failed to test " + component + " in " + installdir)
505 if ((testrc == 0 or self.options.keep_going)
506 and not self.options.no_install):
507 self.logger.info("Installing " + component + " in " + installdir)
508 self.make(component, srcdir, builddir,
509 make_install_flags[comp_key][build],
510 make_install_env[comp_key][build])
512 self.logger.info("Failed testing " + component + " in " + installdir)
515 self.logger.info("Failed to build " + component + " in " + installdir)
518 self.logger.info("Failed to configure " + component + " in " + installdir)
520 def configure(self, component, srcdir, builddir, flags, env):
521 prefix = self.component_abbrev[component.replace("-", "_")]
523 self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
526 configure_files = dict(
527 llvm=[(srcdir + "/configure", builddir + "/Makefile")],
528 dragonegg=[(None,None)])
532 for conf, mf in configure_files[component.replace("-", "_")]:
534 # No configure necessary
537 if not os.path.exists(conf):
538 self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf)
541 if os.path.exists(conf) and os.path.exists(mf):
542 confstat = os.stat(conf)
543 makestat = os.stat(mf)
544 if confstat.st_mtime > makestat.st_mtime:
551 if not doconfig and not self.options.force_configure:
554 program = srcdir + "/configure"
555 if not is_executable(program):
556 self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program)
560 args += ["--verbose"]
562 return self.execute(args, builddir, env, component)
564 def make(self, component, srcdir, builddir, flags, env):
565 program = find_executable("make")
567 raise ExecutableNotFound
569 if not is_executable(program):
570 raise FileNotExecutable
574 return self.execute(args, builddir, env, component)
577 build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
578 components = ["llvm", "dragonegg"]
581 parser = optparse.OptionParser(version="%prog 1.0")
583 (options, args) = parser.parse_args()
584 check_options(parser, options, build_abbrev.keys());
587 logging.basicConfig(level=logging.DEBUG,
588 format='%(name)-13s: %(message)s')
590 logging.basicConfig(level=logging.INFO,
591 format='%(name)-13s: %(message)s')
593 source_abbrev = get_path_abbrevs(set(options.src))
595 work_queue = queue.Queue()
597 jobs = options.jobs // options.threads
601 numthreads = options.threads
603 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
604 + str(numthreads) + " threads using " + str(jobs)
607 logging.getLogger().info("CC = " + str(options.cc))
608 logging.getLogger().info("CXX = " + str(options.cxx))
610 for t in range(numthreads):
611 builder = Builder(work_queue, jobs,
612 build_abbrev, source_abbrev,
614 builder.daemon = True
617 for build in set(options.build):
618 for source in set(options.src):
619 work_queue.put((source, build))