utils/llvmbuild

   1 #!/usr/bin/python3
   2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
   3 #
   4 #                     The LLVM Compiler Infrastructure
   5 #
   6 # This file is distributed under the University of Illinois Open Source
   7 # License. See LICENSE.TXT for details.
   8 #
   9 ##===----------------------------------------------------------------------===##
  10 #
  11 # This script builds many different flavors of the LLVM ecosystem.  It
  12 # will build LLVM, Clang, llvm-gcc, and dragonegg as well as run tests
  13 # on them.  This script is convenient to use to check builds and tests
  14 # before committing changes to the upstream repository
  15 #
  16 # A typical source setup uses three trees and looks like this:
  17 #
  18 # official
  19 #   dragonegg
  20 #     trunk
  21 #   gcc
  22 #     trunk
  23 #   llvm
  24 #     trunk
  25 #       tools
  26 #         clang
  27 #     tags
  28 #       RELEASE_28
  29 #         tools
  30 #           clang
  31 #   llvm-gcc
  32 #     trunk
  33 #     tags
  34 #       RELEASE_28
  35 # staging
  36 #   dragonegg
  37 #     trunk
  38 #   gcc
  39 #     trunk
  40 #   llvm
  41 #     trunk
  42 #       tools
  43 #         clang
  44 #     tags
  45 #       RELEASE_28
  46 #         tools
  47 #           clang
  48 #   llvm-gcc
  49 #     trunk
  50 #     tags
  51 #       RELEASE_28
  52 # commit
  53 #   dragonegg
  54 #     trunk
  55 #   gcc
  56 #     trunk
  57 #   llvm
  58 #     trunk
  59 #       tools
  60 #         clang
  61 #     tags
  62 #       RELEASE_28
  63 #         tools
  64 #           clang
  65 #   llvm-gcc
  66 #     trunk
  67 #     tags
  68 #       RELEASE_28
  69 #
  70 # "gcc" above is the upstream FSF gcc and "gcc/trunk" refers to the
  71 # 4.5 branch as discussed in the dragonegg build guide.
  72 #
  73 # In a typical workflow, the "official" tree always contains unchanged
  74 # sources from the main LLVM project repositories.  The "staging" tree
  75 # is where local work is done.  A set of changes resides there waiting
  76 # to be moved upstream.  The "commit" tree is where changes from
  77 # "staging" make their way upstream.  Individual incremental changes
  78 # from "staging" are applied to "commit" and committed upstream after
  79 # a successful build and test run.  A successful build is one in which
  80 # testing results in no more failures than seen in the testing of the
  81 # "official" tree.
  82 #
  83 # A build may be invoked as such:
  84 #
  85 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging
  86 #   --src=~/llvm/official --branch=trunk --branch=tags/RELEASE_28
  87 #   --build=debug --build=release --build=paranoid
  88 #   --prefix=/home/greened/install --builddir=/home/greened/build
  89 #
  90 # This will build the LLVM ecosystem, including LLVM, Clang, llvm-gcc,
  91 # gcc 4.5 and dragonegg, putting build results in ~/build and
  92 # installing tools in ~/install.  llvmbuild creates separate build and
  93 # install directories for each source/branch/build flavor.  In the
  94 # above example, llvmbuild will build debug, release and paranoid
  95 # (debug+checks) flavors of the trunk and RELEASE_28 branches from
  96 # each source tree (official, staging and commit) for a total of
  97 # eighteen builds.  All builds will be run in parallel.
  98 #
  99 # The user may control parallelism via the --jobs and --threads
 100 # switches.  --jobs tells llvmbuild the maximum total number of builds
 101 # to activate in parallel.  The user may think of it as equivalent to
 102 # the GNU make -j switch.  --threads tells llvmbuild how many worker
 103 # threads to use to accomplish those builds.  If --threads is less
 104 # than --jobs, --threads workers will be launched and each one will
 105 # pick a source/branch/flavor combination to build.  Then llvmbuild
 106 # will invoke GNU make with -j (--jobs / --threads) to use up the
 107 # remaining job capacity.  Once a worker is finished with a build, it
 108 # will pick another combination off the list and start building it.
 109 #
 110 ##===----------------------------------------------------------------------===##
 111
 112 import optparse
 113 import os
 114 import sys
 115 import threading
 116 import queue
 117 import logging
 118 import traceback
 119 import subprocess
 120 import re
 121
 122 # TODO: Use shutil.which when it is available (3.2 or later)
 123 def find_executable(executable, path=None):
 124     """Try to find 'executable' in the directories listed in 'path' (a
 125     string listing directories separated by 'os.pathsep'; defaults to
 126     os.environ['PATH']).  Returns the complete filename or None if not
 127     found
 128     """
 129     if path is None:
 130         path = os.environ['PATH']
 131     paths = path.split(os.pathsep)
 132     extlist = ['']
 133     if os.name == 'os2':
 134         (base, ext) = os.path.splitext(executable)
 135         # executable files on OS/2 can have an arbitrary extension, but
 136         # .exe is automatically appended if no dot is present in the name
 137         if not ext:
 138             executable = executable + ".exe"
 139     elif sys.platform == 'win32':
 140         pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
 141         (base, ext) = os.path.splitext(executable)
 142         if ext.lower() not in pathext:
 143             extlist = pathext
 144     for ext in extlist:
 145         execname = executable + ext
 146         if os.path.isfile(execname):
 147             return execname
 148         else:
 149             for p in paths:
 150                 f = os.path.join(p, execname)
 151                 if os.path.isfile(f):
 152                     return f
 153     else:
 154         return None
 155
 156 def is_executable(fpath):
 157     return os.path.exists(fpath) and os.access(fpath, os.X_OK)
 158
 159 def add_options(parser):
 160     parser.add_option("-v", "--verbose", action="store_true",
 161                       default=False,
 162                       help=("Output informational messages"
 163                             " [default: %default]"))
 164     parser.add_option("--src", action="append",
 165                       help=("Top-level source directory [default: %default]"))
 166     parser.add_option("--build", action="append",
 167                       help=("Build types to run [default: %default]"))
 168     parser.add_option("--branch", action="append",
 169                       help=("Source branch to build [default: %default]"))
 170     parser.add_option("--cc", default=find_executable("cc"),
 171                       help=("The C compiler to use [default: %default]"))
 172     parser.add_option("--cxx", default=find_executable("c++"),
 173                       help=("The C++ compiler to use [default: %default]"))
 174     parser.add_option("--threads", default=4, type="int",
 175                       help=("The number of worker threads to use "
 176                             "[default: %default]"))
 177     parser.add_option("--jobs", "-j", default=8, type="int",
 178                       help=("The number of simultaneous build jobs "
 179                             "[default: %default]"))
 180     parser.add_option("--prefix",
 181                       help=("Root install directory [default: %default]"))
 182     parser.add_option("--builddir",
 183                       help=("Root build directory [default: %default]"))
 184     parser.add_option("--extra-llvm-config-flags", default="",
 185                       help=("Extra flags to pass to llvm configure [default: %default]"))
 186     parser.add_option("--extra-llvm-gcc-config-flags", default="",
 187                       help=("Extra flags to pass to llvm-gcc configure [default: %default]"))
 188     parser.add_option("--extra-gcc-config-flags", default="",
 189                       help=("Extra flags to pass to gcc configure [default: %default]"))
 190     parser.add_option("--force-configure", default=False, action="store_true",
 191                       help=("Force reconfigure of all components"))
 192     parser.add_option("--no-gcc", default=False, action="store_true",
 193                       help=("Do not build dragonegg and gcc"))
 194     parser.add_option("--no-install", default=False, action="store_true",
 195                       help=("Do not do installs"))
 196     return
 197
 198 def check_options(parser, options, valid_builds):
 199     # See if we're building valid flavors.
 200     for build in options.build:
 201         if (build not in valid_builds):
 202             parser.error("'" + build + "' is not a valid build flavor "
 203                          + str(valid_builds))
 204
 205     # See if we can find source directories.
 206     for src in options.src:
 207         for component in components:
 208             component = component.rstrip("2")
 209             compsrc = src + "/" + component
 210             if (not os.path.isdir(compsrc)):
 211                 parser.error("'" + compsrc + "' does not exist")
 212                 if (options.branch is not None):
 213                     for branch in options.branch:
 214                         if (not os.path.isdir(os.path.join(compsrc, branch))):
 215                             parser.error("'" + os.path.join(compsrc, branch)
 216                                          + "' does not exist")
 217
 218     # See if we can find the compilers
 219     options.cc = find_executable(options.cc)
 220     options.cxx = find_executable(options.cxx)
 221
 222     return
 223
 224 # Find a unique short name for the given set of paths.  This searches
 225 # back through path components until it finds unique component names
 226 # among all given paths.
 227 def get_path_abbrevs(paths):
 228     # Find the number of common starting characters in the last component
 229     # of the paths.
 230     unique_paths = list(paths)
 231
 232     class NotFoundException(Exception): pass
 233
 234     # Find a unique component of each path.
 235     unique_bases = unique_paths[:]
 236     found = 0
 237     while len(unique_paths) > 0:
 238         bases = [os.path.basename(src) for src in unique_paths]
 239         components = { c for c in bases }
 240         # Account for single entry in paths.
 241         if len(components) > 1 or len(components) == len(bases):
 242             # We found something unique.
 243             for c in components:
 244                 if bases.count(c) == 1:
 245                    index = bases.index(c)
 246                    unique_bases[index] = c
 247                    # Remove the corresponding path from the set under
 248                    # consideration.
 249                    unique_paths[index] = None
 250             unique_paths = [ p for p in unique_paths if p is not None ]
 251         unique_paths = [os.path.dirname(src) for src in unique_paths]
 252
 253     if len(unique_paths) > 0:
 254         raise NotFoundException()
 255
 256     abbrevs = dict(zip(paths, [base for base in unique_bases]))
 257
 258     return abbrevs
 259
 260 # Given a set of unique names, find a short character sequence that
 261 # uniquely identifies them.
 262 def get_short_abbrevs(unique_bases):
 263     # Find a unique start character for each path base.
 264     my_unique_bases = unique_bases[:]
 265     unique_char_starts = unique_bases[:]
 266     while len(my_unique_bases) > 0:
 267         for start, char_tuple in enumerate(zip(*[base
 268                                                  for base in my_unique_bases])):
 269             chars = { c for c in char_tuple }
 270             # Account for single path.
 271             if len(chars) > 1 or len(chars) == len(char_tuple):
 272                 # We found something unique.
 273                 for c in chars:
 274                     if char_tuple.count(c) == 1:
 275                         index = char_tuple.index(c)
 276                         unique_char_starts[index] = start
 277                         # Remove the corresponding path from the set under
 278                         # consideration.
 279                         my_unique_bases[index] = None
 280                 my_unique_bases = [ b for b in my_unique_bases
 281                                     if b is not None ]
 282                 break
 283
 284     if len(my_unique_bases) > 0:
 285         raise NotFoundException()
 286
 287     abbrevs = [abbrev[start_index:start_index+3]
 288                for abbrev, start_index
 289                in zip([base for base in unique_bases],
 290                       [index for index in unique_char_starts])]
 291
 292     abbrevs = dict(zip(unique_bases, abbrevs))
 293
 294     return abbrevs
 295
 296 class Builder(threading.Thread):
 297     class ExecutableNotFound(Exception): pass
 298     class FileNotExecutable(Exception): pass
 299
 300     def __init__(self, work_queue, jobs,
 301                  build_abbrev, source_abbrev, branch_abbrev,
 302                  options):
 303         super().__init__()
 304         self.work_queue = work_queue
 305         self.jobs = jobs
 306         self.cc = options.cc
 307         self.cxx = options.cxx
 308         self.build_abbrev = build_abbrev
 309         self.source_abbrev = source_abbrev
 310         self.branch_abbrev = branch_abbrev
 311         self.build_prefix = options.builddir
 312         self.install_prefix = options.prefix
 313         self.options = options
 314         self.component_abbrev = dict(
 315             llvm="llvm",
 316             llvm_gcc="lgcc",
 317             llvm2="llv2",
 318             gcc="ugcc",
 319             dagonegg="degg")
 320     def run(self):
 321         while True:
 322             try:
 323                 source, branch, build = self.work_queue.get()
 324                 self.dobuild(source, branch, build)
 325             except:
 326                 traceback.print_exc()
 327             finally:
 328                 self.work_queue.task_done()
 329
 330     def execute(self, command, execdir, env, component):
 331         prefix = self.component_abbrev[component.replace("-", "_")]
 332         pwd = os.getcwd()
 333         if not os.path.exists(execdir):
 334             os.makedirs(execdir)
 335
 336         execenv = os.environ.copy()
 337
 338         for key, value in env.items():
 339             execenv[key] = value
 340
 341         self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
 342                           + " ".join(command));
 343
 344         try:
 345             proc = subprocess.Popen(command,
 346                                     cwd=execdir,
 347                                     env=execenv,
 348                                     stdout=subprocess.PIPE,
 349                                     stderr=subprocess.STDOUT)
 350
 351             line = proc.stdout.readline()
 352             while line:
 353                 self.logger.info("[" + prefix + "] "
 354                                  + str(line, "utf-8").rstrip())
 355                 line = proc.stdout.readline()
 356
 357         except:
 358             traceback.print_exc()
 359
 360     # Get a list of C++ include directories to pass to clang.
 361     def get_includes(self):
 362         # Assume we're building with g++ for now.
 363         command = [self.cxx]
 364         command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
 365         includes = []
 366         self.logger.debug(command)
 367         try:
 368             proc = subprocess.Popen(command,
 369                                     stdout=subprocess.PIPE,
 370                                     stderr=subprocess.STDOUT)
 371
 372             gather = False
 373             line = proc.stdout.readline()
 374             while line:
 375                 self.logger.debug(line)
 376                 if re.search("End of search list", str(line)) is not None:
 377                     self.logger.debug("Stop Gather")
 378                     gather = False
 379                 if gather:
 380                     includes.append(str(line, "utf-8").strip())
 381                 if re.search("#include <...> search starts", str(line)) is not None:
 382                     self.logger.debug("Start Gather")
 383                     gather = True
 384                 line = proc.stdout.readline()
 385         except:
 386             traceback.print_exc()
 387         self.logger.debug(includes)
 388         return includes
 389
 390     def dobuild(self, source, branch, build):
 391         build_suffix = ""
 392
 393         ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
 394
 395         if branch is not None:
 396             sbabbrev = get_short_abbrevs([ab for ab in self.branch_abbrev.values()])
 397
 398             prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + sbabbrev[self.branch_abbrev[branch]] + "-" + self.build_abbrev[build] + "]"
 399             self.install_prefix += "/" + self.source_abbrev[source] + "/" + branch + "/" + build
 400             build_suffix += self.source_abbrev[source] + "/" + branch + "/" + build
 401         else:
 402             prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
 403             self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
 404             build_suffix += "/" + self.source_abbrev[source] + "/" + build
 405
 406         self.logger = logging.getLogger(prefix)
 407
 408         self.logger.debug(self.install_prefix)
 409
 410         # Assume we're building with gcc for now.
 411         cxxincludes = self.get_includes()
 412         cxxroot = cxxincludes[0]
 413         cxxarch = os.path.basename(cxxincludes[1])
 414
 415         configure_flags = dict(
 416             llvm=dict(debug=["--prefix=" + self.install_prefix,
 417                              "--with-extra-options=-Werror",
 418                              "--enable-assertions",
 419                              "--disable-optimized",
 420                              "--with-cxx-include-root=" + cxxroot,
 421                              "--with-cxx-include-arch=" + cxxarch],
 422                       release=["--prefix=" + self.install_prefix,
 423                                "--with-extra-options=-Werror",
 424                                "--enable-optimized",
 425                                "--with-cxx-include-root=" + cxxroot,
 426                                "--with-cxx-include-arch=" + cxxarch],
 427                       paranoid=["--prefix=" + self.install_prefix,
 428                                 "--with-extra-options=-Werror",
 429                                 "--enable-assertions",
 430                                 "--enable-expensive-checks",
 431                                 "--disable-optimized",
 432                                 "--with-cxx-include-root=" + cxxroot,
 433                                 "--with-cxx-include-arch=" + cxxarch]),
 434             llvm_gcc=dict(debug=["--prefix=" + self.install_prefix,
 435                                  "--enable-checking",
 436                                  "--program-prefix=llvm-",
 437                                  "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
 438 # Fortran install seems to be broken.
 439 #                                 "--enable-languages=c,c++,fortran"],
 440                                  "--enable-languages=c,c++"],
 441                           release=["--prefix=" + self.install_prefix,
 442                                    "--program-prefix=llvm-",
 443                                    "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
 444 # Fortran install seems to be broken.
 445 #                                   "--enable-languages=c,c++,fortran"],
 446                                    "--enable-languages=c,c++"],
 447                           paranoid=["--prefix=" + self.install_prefix,
 448                                     "--enable-checking",
 449                                     "--program-prefix=llvm-",
 450                                     "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
 451 # Fortran install seems to be broken.
 452 #                                    "--enable-languages=c,c++,fortran"]),
 453                                     "--enable-languages=c,c++"]),
 454             llvm2=dict(debug=["--prefix=" + self.install_prefix,
 455                               "--with-extra-options=-Werror",
 456                               "--enable-assertions",
 457                               "--disable-optimized",
 458                               "--with-llvmgccdir=" + self.install_prefix + "/bin",
 459                               "--with-cxx-include-root=" + cxxroot,
 460                               "--with-cxx-include-arch=" + cxxarch],
 461                        release=["--prefix=" + self.install_prefix,
 462                                 "--with-extra-options=-Werror",
 463                                 "--enable-optimized",
 464                                 "--with-llvmgccdir=" + self.install_prefix + "/bin",
 465                                 "--with-cxx-include-root=" + cxxroot,
 466                                 "--with-cxx-include-arch=" + cxxarch],
 467                        paranoid=["--prefix=" + self.install_prefix,
 468                                  "--with-extra-options=-Werror",
 469                                  "--enable-assertions",
 470                                  "--enable-expensive-checks",
 471                                  "--disable-optimized",
 472                                  "--with-llvmgccdir=" + self.install_prefix + "/bin",
 473                                  "--with-cxx-include-root=" + cxxroot,
 474                                  "--with-cxx-include-arch=" + cxxarch]),
 475             gcc=dict(debug=["--prefix=" + self.install_prefix,
 476                             "--enable-checking"],
 477                      release=["--prefix=" + self.install_prefix],
 478                      paranoid=["--prefix=" + self.install_prefix,
 479                                "--enable-checking"]),
 480             dragonegg=dict(debug=[],
 481                            release=[],
 482                            paranoid=[]))
 483
 484         configure_env = dict(
 485             llvm=dict(debug=dict(CC=self.cc,
 486                                  CXX=self.cxx),
 487                       release=dict(CC=self.cc,
 488                                    CXX=self.cxx),
 489                       paranoid=dict(CC=self.cc,
 490                                     CXX=self.cxx)),
 491             llvm_gcc=dict(debug=dict(CC=self.cc,
 492                                      CXX=self.cxx),
 493                           release=dict(CC=self.cc,
 494                                        CXX=self.cxx),
 495                           paranoid=dict(CC=self.cc,
 496                                         CXX=self.cxx)),
 497             llvm2=dict(debug=dict(CC=self.cc,
 498                                   CXX=self.cxx),
 499                        release=dict(CC=self.cc,
 500                                     CXX=self.cxx),
 501                        paranoid=dict(CC=self.cc,
 502                                      CXX=self.cxx)),
 503             gcc=dict(debug=dict(CC=self.cc,
 504                                 CXX=self.cxx),
 505                      release=dict(CC=self.cc,
 506                                   CXX=self.cxx),
 507                      paranoid=dict(CC=self.cc,
 508                                    CXX=self.cxx)),
 509             dragonegg=dict(debug=dict(CC=self.cc,
 510                                       CXX=self.cxx),
 511                            release=dict(CC=self.cc,
 512                                         CXX=self.cxx),
 513                            paranoid=dict(CC=self.cc,
 514                                          CXX=self.cxx)))
 515
 516         make_flags = dict(
 517             llvm=dict(debug=["-j" + str(self.jobs)],
 518                       release=["-j" + str(self.jobs)],
 519                       paranoid=["-j" + str(self.jobs)]),
 520             llvm_gcc=dict(debug=["-j" + str(self.jobs),
 521                                  "bootstrap"],
 522                           release=["-j" + str(self.jobs),
 523                                    "bootstrap"],
 524                           paranoid=["-j" + str(self.jobs),
 525                                     "bootstrap"]),
 526             llvm2=dict(debug=["-j" + str(self.jobs)],
 527                        release=["-j" + str(self.jobs)],
 528                        paranoid=["-j" + str(self.jobs)]),
 529             gcc=dict(debug=["-j" + str(self.jobs),
 530                             "bootstrap"],
 531                      release=["-j" + str(self.jobs),
 532                               "bootstrap"],
 533                      paranoid=["-j" + str(self.jobs),
 534                                "bootstrap"]),
 535             dragonegg=dict(debug=["-j" + str(self.jobs)],
 536                            release=["-j" + str(self.jobs)],
 537                            paranoid=["-j" + str(self.jobs)]))
 538
 539         make_env = dict(
 540             llvm=dict(debug=dict(),
 541                       release=dict(),
 542                       paranoid=dict()),
 543             llvm_gcc=dict(debug=dict(),
 544                           release=dict(),
 545                           paranoid=dict()),
 546             llvm2=dict(debug=dict(),
 547                        release=dict(),
 548                        paranoid=dict()),
 549             gcc=dict(debug=dict(),
 550                      release=dict(),
 551                      paranoid=dict()),
 552             dragonegg=dict(debug=dict(GCC=self.install_prefix + "/bin/gcc",
 553                                       LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
 554                            release=dict(GCC=self.install_prefix + "/bin/gcc",
 555                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
 556                            paranoid=dict(GCC=self.install_prefix + "/bin/gcc",
 557                                          LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
 558
 559         make_install_flags = dict(
 560             llvm=dict(debug=["install"],
 561                       release=["install"],
 562                       paranoid=["install"]),
 563             llvm_gcc=dict(debug=["install"],
 564                           release=["install"],
 565                           paranoid=["install"]),
 566             llvm2=dict(debug=["install"],
 567                        release=["install"],
 568                        paranoid=["install"]),
 569             gcc=dict(debug=["install"],
 570                      release=["install"],
 571                      paranoid=["install"]),
 572             dragonegg=dict(debug=["install"],
 573                            release=["install"],
 574                            paranoid=["install"]))
 575
 576         make_install_env = dict(
 577             llvm=dict(debug=dict(),
 578                       release=dict(),
 579                       paranoid=dict()),
 580             llvm_gcc=dict(debug=dict(),
 581                           release=dict(),
 582                           paranoid=dict()),
 583             llvm2=dict(debug=dict(),
 584                        release=dict(),
 585                        paranoid=dict()),
 586             gcc=dict(debug=dict(),
 587                      release=dict(),
 588                      paranoid=dict()),
 589             dragonegg=dict(debug=dict(),
 590                            release=dict(),
 591                            paranoid=dict()))
 592
 593         make_check_flags = dict(
 594             llvm=dict(debug=["check"],
 595                       release=["check"],
 596                       paranoid=["check"]),
 597             llvm_gcc=dict(debug=["check"],
 598                           release=["check"],
 599                           paranoid=["check"]),
 600             llvm2=dict(debug=["check"],
 601                        release=["check"],
 602                        paranoid=["check"]),
 603             gcc=dict(debug=["check"],
 604                      release=["check"],
 605                      paranoid=["check"]),
 606             dragonegg=dict(debug=["check"],
 607                            release=["check"],
 608                            paranoid=["check"]))
 609
 610         make_check_env = dict(
 611             llvm=dict(debug=dict(),
 612                       release=dict(),
 613                       paranoid=dict()),
 614             llvm_gcc=dict(debug=dict(),
 615                           release=dict(),
 616                           paranoid=dict()),
 617             llvm2=dict(debug=dict(),
 618                        release=dict(),
 619                        paranoid=dict()),
 620             gcc=dict(debug=dict(),
 621                      release=dict(),
 622                      paranoid=dict()),
 623             dragonegg=dict(debug=dict(),
 624                            release=dict(),
 625                            paranoid=dict()))
 626
 627         for component in components:
 628             comp = component[:]
 629
 630             if (self.options.no_gcc):
 631                 if (comp == 'gcc' or comp == 'dragonegg' or comp == 'llvm2'):
 632                     self.logger.info("Skipping " + component + " in "
 633                                      + builddir)
 634                     continue
 635
 636             srcdir = source + "/" + comp.rstrip("2")
 637             builddir = self.build_prefix + "/" + comp + "/" + build_suffix
 638             installdir = self.install_prefix
 639
 640             if (branch is not None):
 641                 srcdir += "/" + branch
 642
 643             comp_key = comp.replace("-", "_")
 644
 645             config_args = configure_flags[comp_key][build][:]
 646             config_args.extend(getattr(self.options,
 647                                        "extra_" + comp_key.rstrip("2")
 648                                        + "_config_flags").split())
 649
 650             self.logger.info("Configuring " + component + " in " + builddir)
 651             self.configure(component, srcdir, builddir,
 652                            config_args,
 653                            configure_env[comp_key][build])
 654
 655             self.logger.info("Building " + component + " in " + builddir)
 656             self.logger.info("Build: make " + str(make_flags[comp_key][build]))
 657             self.make(component, srcdir, builddir,
 658                       make_flags[comp_key][build],
 659                       make_env[comp_key][build])
 660
 661             if (not self.options.no_install):
 662                 self.logger.info("Installing " + component + " in " + installdir)
 663                 self.make(component, srcdir, builddir,
 664                           make_install_flags[comp_key][build],
 665                           make_install_env[comp_key][build])
 666
 667             self.logger.info("Testing " + component + " in " + builddir)
 668             self.logger.info("Test: make "
 669                              + str(make_check_flags[comp_key][build]))
 670             self.make(component, srcdir, builddir,
 671                       make_check_flags[comp_key][build],
 672                       make_check_env[comp_key][build])
 673
 674
 675     def configure(self, component, srcdir, builddir, flags, env):
 676         self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
 677                           + str(builddir))
 678
 679         configure_files = dict(
 680             llvm=[(srcdir + "/configure", builddir + "/Makefile")],
 681             llvm_gcc=[(srcdir + "/configure", builddir + "/Makefile"),
 682                       (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")],
 683             llvm2=[(srcdir + "/configure", builddir + "/Makefile")],
 684             gcc=[(srcdir + "/configure", builddir + "/Makefile"),
 685                  (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")],
 686             dragonegg=[()])
 687
 688
 689         doconfig = False
 690         for conf, mf in configure_files[component.replace("-", "_")]:
 691             if not os.path.exists(conf):
 692                 return
 693             if os.path.exists(conf) and os.path.exists(mf):
 694                 confstat = os.stat(conf)
 695                 makestat = os.stat(mf)
 696                 if confstat.st_mtime > makestat.st_mtime:
 697                     doconfig = True
 698                     break
 699             else:
 700                 doconfig = True
 701                 break
 702
 703         if not doconfig and not self.options.force_configure:
 704             return
 705
 706         program = srcdir + "/configure"
 707         if not is_executable(program):
 708             return
 709
 710         args = [program]
 711         args += ["--verbose"]
 712         args += flags
 713         self.execute(args, builddir, env, component)
 714
 715     def make(self, component, srcdir, builddir, flags, env):
 716         program = find_executable("make")
 717         if program is None:
 718             raise ExecutableNotFound
 719
 720         if not is_executable(program):
 721             raise FileNotExecutable
 722
 723         args = [program]
 724         args += flags
 725         self.execute(args, builddir, env, component)
 726
 727 # Global constants
 728 build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
 729 #components = ["llvm", "llvm-gcc", "llvm2", "gcc", "dragonegg"]
 730 components = ["llvm", "llvm2", "gcc", "dragonegg"]
 731
 732 # Parse options
 733 parser = optparse.OptionParser(version="%prog 1.0")
 734 add_options(parser)
 735 (options, args) = parser.parse_args()
 736 check_options(parser, options, build_abbrev.keys());
 737
 738 if options.verbose:
 739     logging.basicConfig(level=logging.DEBUG,
 740                         format='%(name)-13s: %(message)s')
 741 else:
 742     logging.basicConfig(level=logging.INFO,
 743                         format='%(name)-13s: %(message)s')
 744
 745 source_abbrev = get_path_abbrevs(set(options.src))
 746
 747 branch_abbrev = None
 748 if options.branch is not None:
 749     branch_abbrev = get_path_abbrevs(set(options.branch))
 750
 751 work_queue = queue.Queue()
 752
 753 jobs = options.jobs // options.threads
 754 if jobs == 0:
 755     jobs = 1
 756
 757 numthreads = options.threads
 758
 759 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
 760                          + str(numthreads) + " threads using " + str(jobs)
 761                          + " make jobs")
 762
 763 for t in range(numthreads):
 764     builder = Builder(work_queue, jobs,
 765                       build_abbrev, source_abbrev, branch_abbrev,
 766                       options)
 767     builder.daemon = True
 768     builder.start()
 769
 770 for build in set(options.build):
 771     for source in set(options.src):
 772         if options.branch is not None:
 773             for branch in set(options.branch):
 774                 work_queue.put((source, branch, build))
 775         else:
 776             work_queue.put((source, None, build))
 777
 778 work_queue.join()