Revert r247080, "LowerBitSets: Extend pass to support functions as bitset
[oota-llvm.git] / utils / llvm-compilers-check
1 #!/usr/bin/python3
2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
3 #
4 #                     The LLVM Compiler Infrastructure
5 #
6 # This file is distributed under the University of Illinois Open Source
7 # License. See LICENSE.TXT for details.
8 #
9 ##===----------------------------------------------------------------------===##
10 #
11 # This script builds many different flavors of the LLVM ecosystem.  It
12 # will build LLVM, Clang and dragonegg as well as run tests on them.
13 # This script is convenient to use to check builds and tests before
14 # committing changes to the upstream repository
15 #
16 # A typical source setup uses three trees and looks like this:
17 #
18 # official
19 #   dragonegg
20 #   llvm
21 #     tools
22 #       clang
23 # staging
24 #   dragonegg
25 #   llvm
26 #     tools
27 #       clang
28 # commit
29 #   dragonegg
30 #   llvm
31 #     tools
32 #       clang
33 #
34 # In a typical workflow, the "official" tree always contains unchanged
35 # sources from the main LLVM project repositories.  The "staging" tree
36 # is where local work is done.  A set of changes resides there waiting
37 # to be moved upstream.  The "commit" tree is where changes from
38 # "staging" make their way upstream.  Individual incremental changes
39 # from "staging" are applied to "commit" and committed upstream after
40 # a successful build and test run.  A successful build is one in which
41 # testing results in no more failures than seen in the testing of the
42 # "official" tree.
43
44 # A build may be invoked as such:
45 #
46 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official
47 #   --build=debug --build=release --build=paranoid
48 #   --prefix=/home/greened/install --builddir=/home/greened/build
49 #
50 # This will build the LLVM ecosystem, including LLVM, Clangand
51 # dragonegg, putting build results in ~/build and installing tools in
52 # ~/install.  llvm-compilers-check creates separate build and install
53 # directories for each source/build flavor.  In the above example,
54 # llvmbuild will build debug, release and paranoid (debug+checks)
55 # flavors from each source tree (official, staging and commit) for a
56 # total of nine builds.  All builds will be run in parallel.
57 #
58 # The user may control parallelism via the --jobs and --threads
59 # switches.  --jobs tells llvm-compilers-checl the maximum total
60 # number of builds to activate in parallel.  The user may think of it
61 # as equivalent to the GNU make -j switch.  --threads tells
62 # llvm-compilers-check how many worker threads to use to accomplish
63 # those builds.  If --threads is less than --jobs, --threads workers
64 # will be launched and each one will pick a source/flavor combination
65 # to build.  Then llvm-compilers-check will invoke GNU make with -j
66 # (--jobs / --threads) to use up the remaining job capacity.  Once a
67 # worker is finished with a build, it will pick another combination
68 # off the list and start building it.
69 #
70 ##===----------------------------------------------------------------------===##
71
72 import optparse
73 import os
74 import sys
75 import threading
76 import queue
77 import logging
78 import traceback
79 import subprocess
80 import re
81
82 # TODO: Use shutil.which when it is available (3.2 or later)
83 def find_executable(executable, path=None):
84     """Try to find 'executable' in the directories listed in 'path' (a
85     string listing directories separated by 'os.pathsep'; defaults to
86     os.environ['PATH']).  Returns the complete filename or None if not
87     found
88     """
89     if path is None:
90         path = os.environ['PATH']
91     paths = path.split(os.pathsep)
92     extlist = ['']
93     if os.name == 'os2':
94         (base, ext) = os.path.splitext(executable)
95         # executable files on OS/2 can have an arbitrary extension, but
96         # .exe is automatically appended if no dot is present in the name
97         if not ext:
98             executable = executable + ".exe"
99     elif sys.platform == 'win32':
100         pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
101         (base, ext) = os.path.splitext(executable)
102         if ext.lower() not in pathext:
103             extlist = pathext
104     for ext in extlist:
105         execname = executable + ext
106         if os.path.isfile(execname):
107             return execname
108         else:
109             for p in paths:
110                 f = os.path.join(p, execname)
111                 if os.path.isfile(f):
112                     return f
113     else:
114         return None
115
116 def is_executable(fpath):
117     return os.path.exists(fpath) and os.access(fpath, os.X_OK)
118
119 def add_options(parser):
120     parser.add_option("-v", "--verbose", action="store_true",
121                       default=False,
122                       help=("Output informational messages"
123                             " [default: %default]"))
124     parser.add_option("--src", action="append",
125                       help=("Top-level source directory [default: %default]"))
126     parser.add_option("--build", action="append",
127                       help=("Build types to run [default: %default]"))
128     parser.add_option("--cc", default=find_executable("cc"),
129                       help=("The C compiler to use [default: %default]"))
130     parser.add_option("--cxx", default=find_executable("c++"),
131                       help=("The C++ compiler to use [default: %default]"))
132     parser.add_option("--threads", default=4, type="int",
133                       help=("The number of worker threads to use "
134                             "[default: %default]"))
135     parser.add_option("--jobs", "-j", default=8, type="int",
136                       help=("The number of simultaneous build jobs "
137                             "[default: %default]"))
138     parser.add_option("--prefix",
139                       help=("Root install directory [default: %default]"))
140     parser.add_option("--builddir",
141                       help=("Root build directory [default: %default]"))
142     parser.add_option("--extra-llvm-config-flags", default="",
143                       help=("Extra flags to pass to llvm configure [default: %default]"))
144     parser.add_option("--force-configure", default=False, action="store_true",
145                       help=("Force reconfigure of all components"))
146     parser.add_option("--no-dragonegg", default=False, action="store_true",
147                       help=("Do not build dragonegg"))
148     parser.add_option("--no-install", default=False, action="store_true",
149                       help=("Do not do installs"))
150     parser.add_option("--keep-going", default=False, action="store_true",
151                       help=("Keep going after failures"))
152     parser.add_option("--no-flavor-prefix", default=False, action="store_true",
153                       help=("Do not append the build flavor to the install path"))
154     parser.add_option("--enable-werror", default=False, action="store_true",
155                       help=("Build with -Werror"))
156     return
157
158 def check_options(parser, options, valid_builds):
159     # See if we're building valid flavors.
160     for build in options.build:
161         if (build not in valid_builds):
162             parser.error("'" + build + "' is not a valid build flavor "
163                          + str(valid_builds))
164
165     # See if we can find source directories.
166     for src in options.src:
167         for component in components:
168             component = component.rstrip("2")
169             compsrc = src + "/" + component
170             if (not os.path.isdir(compsrc)):
171                 parser.error("'" + compsrc + "' does not exist")
172
173     # See if we can find the compilers
174     options.cc = find_executable(options.cc)
175     options.cxx = find_executable(options.cxx)
176
177     return
178
179 # Find a unique short name for the given set of paths.  This searches
180 # back through path components until it finds unique component names
181 # among all given paths.
182 def get_path_abbrevs(paths):
183     # Find the number of common starting characters in the last component
184     # of the paths.
185     unique_paths = list(paths)
186
187     class NotFoundException(Exception): pass
188
189     # Find a unique component of each path.
190     unique_bases = unique_paths[:]
191     found = 0
192     while len(unique_paths) > 0:
193         bases = [os.path.basename(src) for src in unique_paths]
194         components = { c for c in bases }
195         # Account for single entry in paths.
196         if len(components) > 1 or len(components) == len(bases):
197             # We found something unique.
198             for c in components:
199                 if bases.count(c) == 1:
200                    index = bases.index(c)
201                    unique_bases[index] = c
202                    # Remove the corresponding path from the set under
203                    # consideration.
204                    unique_paths[index] = None
205             unique_paths = [ p for p in unique_paths if p is not None ]
206         unique_paths = [os.path.dirname(src) for src in unique_paths]
207
208     if len(unique_paths) > 0:
209         raise NotFoundException()
210
211     abbrevs = dict(zip(paths, [base for base in unique_bases]))
212
213     return abbrevs
214
215 # Given a set of unique names, find a short character sequence that
216 # uniquely identifies them.
217 def get_short_abbrevs(unique_bases):
218     # Find a unique start character for each path base.
219     my_unique_bases = unique_bases[:]
220     unique_char_starts = unique_bases[:]
221     while len(my_unique_bases) > 0:
222         for start, char_tuple in enumerate(zip(*[base
223                                                  for base in my_unique_bases])):
224             chars = { c for c in char_tuple }
225             # Account for single path.
226             if len(chars) > 1 or len(chars) == len(char_tuple):
227                 # We found something unique.
228                 for c in chars:
229                     if char_tuple.count(c) == 1:
230                         index = char_tuple.index(c)
231                         unique_char_starts[index] = start
232                         # Remove the corresponding path from the set under
233                         # consideration.
234                         my_unique_bases[index] = None
235                 my_unique_bases = [ b for b in my_unique_bases
236                                     if b is not None ]
237                 break
238
239     if len(my_unique_bases) > 0:
240         raise NotFoundException()
241
242     abbrevs = [abbrev[start_index:start_index+3]
243                for abbrev, start_index
244                in zip([base for base in unique_bases],
245                       [index for index in unique_char_starts])]
246
247     abbrevs = dict(zip(unique_bases, abbrevs))
248
249     return abbrevs
250
251 class Builder(threading.Thread):
252     class ExecutableNotFound(Exception): pass
253     class FileNotExecutable(Exception): pass
254
255     def __init__(self, work_queue, jobs,
256                  build_abbrev, source_abbrev,
257                  options):
258         super().__init__()
259         self.work_queue = work_queue
260         self.jobs = jobs
261         self.cc = options.cc
262         self.cxx = options.cxx
263         self.build_abbrev = build_abbrev
264         self.source_abbrev = source_abbrev
265         self.build_prefix = options.builddir
266         self.install_prefix = options.prefix
267         self.options = options
268         self.component_abbrev = dict(
269             llvm="llvm",
270             dragonegg="degg")
271     def run(self):
272         while True:
273             try:
274                 source, build = self.work_queue.get()
275                 self.dobuild(source, build)
276             except:
277                 traceback.print_exc()
278             finally:
279                 self.work_queue.task_done()
280
281     def execute(self, command, execdir, env, component):
282         prefix = self.component_abbrev[component.replace("-", "_")]
283         pwd = os.getcwd()
284         if not os.path.exists(execdir):
285             os.makedirs(execdir)
286
287         execenv = os.environ.copy()
288
289         for key, value in env.items():
290             execenv[key] = value
291
292         self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
293                           + " ".join(command));
294
295         try:
296             proc = subprocess.Popen(command,
297                                     cwd=execdir,
298                                     env=execenv,
299                                     stdout=subprocess.PIPE,
300                                     stderr=subprocess.STDOUT)
301
302             line = proc.stdout.readline()
303             while line:
304                 self.logger.info("[" + prefix + "] "
305                                  + str(line, "utf-8").rstrip())
306                 line = proc.stdout.readline()
307
308             (stdoutdata, stderrdata) = proc.communicate()
309             retcode = proc.wait()
310
311             return retcode
312
313         except:
314             traceback.print_exc()
315
316     # Get a list of C++ include directories to pass to clang.
317     def get_includes(self):
318         # Assume we're building with g++ for now.
319         command = [self.cxx]
320         command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
321         includes = []
322         self.logger.debug(command)
323         try:
324             proc = subprocess.Popen(command,
325                                     stdout=subprocess.PIPE,
326                                     stderr=subprocess.STDOUT)
327
328             gather = False
329             line = proc.stdout.readline()
330             while line:
331                 self.logger.debug(line)
332                 if re.search("End of search list", str(line)) is not None:
333                     self.logger.debug("Stop Gather")
334                     gather = False
335                 if gather:
336                     includes.append(str(line, "utf-8").strip())
337                 if re.search("#include <...> search starts", str(line)) is not None:
338                     self.logger.debug("Start Gather")
339                     gather = True
340                 line = proc.stdout.readline()
341
342         except:
343             traceback.print_exc()
344         self.logger.debug(includes)
345         return includes
346
347     def dobuild(self, source, build):
348         build_suffix = ""
349
350         ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
351
352         prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
353         if (not self.options.no_flavor_prefix):
354             self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
355
356         build_suffix += "/" + self.source_abbrev[source] + "/" + build
357
358         self.logger = logging.getLogger(prefix)
359
360         self.logger.debug(self.install_prefix)
361
362         # Assume we're building with gcc for now.
363         cxxincludes = self.get_includes()
364         cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version
365         cxxroot = os.path.dirname(cxxroot)        # Remove the c++
366         cxxroot = os.path.dirname(cxxroot)        # Remove the include
367
368         configure_flags = dict(
369             llvm=dict(debug=["--prefix=" + self.install_prefix,
370                              "--enable-assertions",
371                              "--disable-optimized",
372                              "--with-gcc-toolchain=" + cxxroot],
373                       release=["--prefix=" + self.install_prefix,
374                                "--enable-optimized",
375                                "--with-gcc-toolchain=" + cxxroot],
376                       paranoid=["--prefix=" + self.install_prefix,
377                                 "--enable-assertions",
378                                 "--enable-expensive-checks",
379                                 "--disable-optimized",
380                                 "--with-gcc-toolchain=" + cxxroot]),
381             dragonegg=dict(debug=[],
382                            release=[],
383                            paranoid=[]))
384
385         if (self.options.enable_werror):
386             configure_flags["llvm"]["debug"].append("--enable-werror")
387             configure_flags["llvm"]["release"].append("--enable-werror")
388             configure_flags["llvm"]["paranoid"].append("--enable-werror")
389
390         configure_env = dict(
391             llvm=dict(debug=dict(CC=self.cc,
392                                  CXX=self.cxx),
393                       release=dict(CC=self.cc,
394                                    CXX=self.cxx),
395                       paranoid=dict(CC=self.cc,
396                                     CXX=self.cxx)),
397             dragonegg=dict(debug=dict(CC=self.cc,
398                                       CXX=self.cxx),
399                            release=dict(CC=self.cc,
400                                         CXX=self.cxx),
401                            paranoid=dict(CC=self.cc,
402                                          CXX=self.cxx)))
403
404         make_flags = dict(
405             llvm=dict(debug=["-j" + str(self.jobs)],
406                       release=["-j" + str(self.jobs)],
407                       paranoid=["-j" + str(self.jobs)]),
408             dragonegg=dict(debug=["-j" + str(self.jobs)],
409                            release=["-j" + str(self.jobs)],
410                            paranoid=["-j" + str(self.jobs)]))
411
412         make_env = dict(
413             llvm=dict(debug=dict(),
414                       release=dict(),
415                       paranoid=dict()),
416             dragonegg=dict(debug=dict(GCC=self.cc,
417                                       LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
418                            release=dict(GCC=self.cc,
419                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
420                            paranoid=dict(GCC=self.cc,
421                                          LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
422
423         make_install_flags = dict(
424             llvm=dict(debug=["install"],
425                       release=["install"],
426                       paranoid=["install"]),
427             dragonegg=dict(debug=["install"],
428                            release=["install"],
429                            paranoid=["install"]))
430
431         make_install_env = dict(
432             llvm=dict(debug=dict(),
433                       release=dict(),
434                       paranoid=dict()),
435             dragonegg=dict(debug=dict(),
436                            release=dict(),
437                            paranoid=dict()))
438
439         make_check_flags = dict(
440             llvm=dict(debug=["check"],
441                       release=["check"],
442                       paranoid=["check"]),
443             dragonegg=dict(debug=["check"],
444                            release=["check"],
445                            paranoid=["check"]))
446
447         make_check_env = dict(
448             llvm=dict(debug=dict(),
449                       release=dict(),
450                       paranoid=dict()),
451             dragonegg=dict(debug=dict(),
452                            release=dict(),
453                            paranoid=dict()))
454
455         for component in components:
456             comp = component[:]
457
458             if (self.options.no_dragonegg):
459                 if (comp == 'dragonegg'):
460                     self.logger.info("Skipping " + component + " in "
461                                      + builddir)
462                     continue
463
464             srcdir = source + "/" + comp.rstrip("2")
465             builddir = self.build_prefix + "/" + comp + "/" + build_suffix
466             installdir = self.install_prefix
467
468             comp_key = comp.replace("-", "_")
469
470             config_args = configure_flags[comp_key][build][:]
471             config_args.extend(getattr(self.options,
472                                        "extra_" + comp_key.rstrip("2")
473                                        + "_config_flags",
474                                        "").split())
475
476             self.logger.info("Configuring " + component + " in " + builddir)
477             configrc = self.configure(component, srcdir, builddir,
478                                       config_args,
479                                       configure_env[comp_key][build])
480
481             if (configrc == None) :
482                 self.logger.info("[None] Failed to configure " + component + " in " + installdir)
483
484             if (configrc == 0 or self.options.keep_going) :
485                 self.logger.info("Building " + component + " in " + builddir)
486                 self.logger.info("Build: make " + str(make_flags[comp_key][build]))
487                 buildrc = self.make(component, srcdir, builddir,
488                                     make_flags[comp_key][build],
489                                     make_env[comp_key][build])
490
491                 if (buildrc == None) :
492                     self.logger.info("[None] Failed to build " + component + " in " + installdir)
493
494                 if (buildrc == 0 or self.options.keep_going) :
495                     self.logger.info("Testing " + component + " in " + builddir)
496                     self.logger.info("Test: make "
497                                      + str(make_check_flags[comp_key][build]))
498                     testrc = self.make(component, srcdir, builddir,
499                                        make_check_flags[comp_key][build],
500                                        make_check_env[comp_key][build])
501
502                     if (testrc == None) :
503                         self.logger.info("[None] Failed to test " + component + " in " + installdir)
504
505                     if ((testrc == 0  or self.options.keep_going)
506                         and not self.options.no_install):
507                         self.logger.info("Installing " + component + " in " + installdir)
508                         self.make(component, srcdir, builddir,
509                                   make_install_flags[comp_key][build],
510                                   make_install_env[comp_key][build])
511                     else :
512                         self.logger.info("Failed testing " + component + " in " + installdir)
513
514                 else :
515                     self.logger.info("Failed to build " + component + " in " + installdir)
516
517             else :
518                 self.logger.info("Failed to configure " + component + " in " + installdir)
519
520     def configure(self, component, srcdir, builddir, flags, env):
521         prefix = self.component_abbrev[component.replace("-", "_")]
522
523         self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
524                           + str(builddir))
525
526         configure_files = dict(
527             llvm=[(srcdir + "/configure", builddir + "/Makefile")],
528             dragonegg=[(None,None)])
529
530
531         doconfig = False
532         for conf, mf in configure_files[component.replace("-", "_")]:
533             if conf is None:
534                 # No configure necessary
535                 return 0
536
537             if not os.path.exists(conf):
538                 self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf)
539                 return -1
540
541             if os.path.exists(conf) and os.path.exists(mf):
542                 confstat = os.stat(conf)
543                 makestat = os.stat(mf)
544                 if confstat.st_mtime > makestat.st_mtime:
545                     doconfig = True
546                     break
547             else:
548                 doconfig = True
549                 break
550
551         if not doconfig and not self.options.force_configure:
552             return 0
553
554         program = srcdir + "/configure"
555         if not is_executable(program):
556             self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program)
557             return -1
558
559         args = [program]
560         args += ["--verbose"]
561         args += flags
562         return self.execute(args, builddir, env, component)
563
564     def make(self, component, srcdir, builddir, flags, env):
565         program = find_executable("make")
566         if program is None:
567             raise ExecutableNotFound
568
569         if not is_executable(program):
570             raise FileNotExecutable
571
572         args = [program]
573         args += flags
574         return self.execute(args, builddir, env, component)
575
576 # Global constants
577 build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
578 components = ["llvm", "dragonegg"]
579
580 # Parse options
581 parser = optparse.OptionParser(version="%prog 1.0")
582 add_options(parser)
583 (options, args) = parser.parse_args()
584 check_options(parser, options, build_abbrev.keys());
585
586 if options.verbose:
587     logging.basicConfig(level=logging.DEBUG,
588                         format='%(name)-13s: %(message)s')
589 else:
590     logging.basicConfig(level=logging.INFO,
591                         format='%(name)-13s: %(message)s')
592
593 source_abbrev = get_path_abbrevs(set(options.src))
594
595 work_queue = queue.Queue()
596
597 jobs = options.jobs // options.threads
598 if jobs == 0:
599     jobs = 1
600
601 numthreads = options.threads
602
603 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
604                          + str(numthreads) + " threads using " + str(jobs)
605                          + " make jobs")
606
607 logging.getLogger().info("CC  = " + str(options.cc))
608 logging.getLogger().info("CXX = " + str(options.cxx))
609
610 for t in range(numthreads):
611     builder = Builder(work_queue, jobs,
612                       build_abbrev, source_abbrev,
613                       options)
614     builder.daemon = True
615     builder.start()
616
617 for build in set(options.build):
618     for source in set(options.src):
619         work_queue.put((source, build))
620
621 work_queue.join()