From: Daniel Dunbar Date: Fri, 9 Aug 2013 00:37:15 +0000 (+0000) Subject: [lit] Rename lit.{TestFormats,Util} to their aliased names {formats,util}. X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=128ce319ec47c46dc7da16aa3a75185899878745;p=oota-llvm.git [lit] Rename lit.{TestFormats,Util} to their aliased names {formats,util}. - With compatibility hack in lit.__init__, so this hopefully shouldn't break anything. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@188040 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/utils/lit/lit/LitConfig.py b/utils/lit/lit/LitConfig.py index bcaea13042a..4b312e29d74 100644 --- a/utils/lit/lit/LitConfig.py +++ b/utils/lit/lit/LitConfig.py @@ -4,9 +4,9 @@ import os import sys import lit.Test -import lit.TestFormats +import lit.formats import lit.TestingConfig -import lit.Util +import lit.util class LitConfig: """LitConfig - Configuration data for a 'lit' test runner instance, shared @@ -22,10 +22,10 @@ class LitConfig: Test = lit.Test # Provide access to built-in formats. - formats = lit.TestFormats + formats = lit.formats # Provide access to built-in utility functions. - util = lit.Util + util = lit.util def __init__(self, progname, path, quiet, useValgrind, valgrindLeakCheck, valgrindArgs, @@ -80,7 +80,7 @@ class LitConfig: if self.bashPath is not None: return self.bashPath - self.bashPath = lit.Util.which('bash', os.pathsep.join(self.path)) + self.bashPath = lit.util.which('bash', os.pathsep.join(self.path)) if self.bashPath is None: # Check some known paths. for path in ('/bin/bash', '/usr/bin/bash', '/usr/local/bin/bash'): @@ -96,13 +96,13 @@ class LitConfig: def getToolsPath(self, dir, paths, tools): if dir is not None and os.path.isabs(dir) and os.path.isdir(dir): - if not lit.Util.checkToolsPath(dir, tools): + if not lit.util.checkToolsPath(dir, tools): return None else: - dir = lit.Util.whichTools(tools, paths) + dir = lit.util.whichTools(tools, paths) # bash - self.bashPath = lit.Util.which('bash', dir) + self.bashPath = lit.util.which('bash', dir) if self.bashPath is None: self.note("Unable to find 'bash.exe'.") self.bashPath = '' diff --git a/utils/lit/lit/ShUtil.py b/utils/lit/lit/ShUtil.py index fb0689b859d..1945ba723bc 100644 --- a/utils/lit/lit/ShUtil.py +++ b/utils/lit/lit/ShUtil.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import itertools -import lit.Util +import lit.util from lit.ShCommands import Command, Pipeline, Seq class ShLexer: @@ -75,7 +75,7 @@ class ShLexer: # Outside of a string, '\\' escapes everything. self.eat() if self.pos == self.end: - lit.Util.warning( + lit.util.warning( "escape at end of quoted argument in: %r" % self.data) return str str += self.eat() @@ -93,7 +93,7 @@ class ShLexer: # Inside a '"' quoted string, '\\' only escapes the quote # character and backslash, otherwise it is preserved. if self.pos == self.end: - lit.Util.warning( + lit.util.warning( "escape at end of quoted argument in: %r" % self.data) return str c = self.eat() @@ -105,7 +105,7 @@ class ShLexer: str += '\\' + c else: str += c - lit.Util.warning("missing quote character in %r" % self.data) + lit.util.warning("missing quote character in %r" % self.data) return str def lex_arg_checked(self, c): diff --git a/utils/lit/lit/TestFormats.py b/utils/lit/lit/TestFormats.py deleted file mode 100644 index 9c43a216b2c..00000000000 --- a/utils/lit/lit/TestFormats.py +++ /dev/null @@ -1,230 +0,0 @@ -from __future__ import absolute_import -import os -import sys - -import lit.Test -import lit.TestRunner -import lit.Util - -kIsWindows = sys.platform in ['win32', 'cygwin'] - -class GoogleTest(object): - def __init__(self, test_sub_dir, test_suffix): - self.test_sub_dir = os.path.normcase(str(test_sub_dir)).split(';') - self.test_suffix = str(test_suffix) - - # On Windows, assume tests will also end in '.exe'. - if kIsWindows: - self.test_suffix += '.exe' - - def getGTestTests(self, path, litConfig, localConfig): - """getGTestTests(path) - [name] - - Return the tests available in gtest executable. - - Args: - path: String path to a gtest executable - litConfig: LitConfig instance - localConfig: TestingConfig instance""" - - try: - lines = lit.Util.capture([path, '--gtest_list_tests'], - env=localConfig.environment) - lines = lines.decode('ascii') - if kIsWindows: - lines = lines.replace('\r', '') - lines = lines.split('\n') - except: - litConfig.error("unable to discover google-tests in %r" % path) - raise StopIteration - - nested_tests = [] - for ln in lines: - if not ln.strip(): - continue - - prefix = '' - index = 0 - while ln[index*2:index*2+2] == ' ': - index += 1 - while len(nested_tests) > index: - nested_tests.pop() - - ln = ln[index*2:] - if ln.endswith('.'): - nested_tests.append(ln) - else: - yield ''.join(nested_tests) + ln - - # Note: path_in_suite should not include the executable name. - def getTestsInExecutable(self, testSuite, path_in_suite, execpath, - litConfig, localConfig): - if not execpath.endswith(self.test_suffix): - return - (dirname, basename) = os.path.split(execpath) - # Discover the tests in this executable. - for testname in self.getGTestTests(execpath, litConfig, localConfig): - testPath = path_in_suite + (basename, testname) - yield lit.Test.Test(testSuite, testPath, localConfig) - - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - source_path = testSuite.getSourcePath(path_in_suite) - for filename in os.listdir(source_path): - filepath = os.path.join(source_path, filename) - if os.path.isdir(filepath): - # Iterate over executables in a directory. - if not os.path.normcase(filename) in self.test_sub_dir: - continue - dirpath_in_suite = path_in_suite + (filename, ) - for subfilename in os.listdir(filepath): - execpath = os.path.join(filepath, subfilename) - for test in self.getTestsInExecutable( - testSuite, dirpath_in_suite, execpath, - litConfig, localConfig): - yield test - elif ('.' in self.test_sub_dir): - for test in self.getTestsInExecutable( - testSuite, path_in_suite, filepath, - litConfig, localConfig): - yield test - - def execute(self, test, litConfig): - testPath,testName = os.path.split(test.getSourcePath()) - while not os.path.exists(testPath): - # Handle GTest parametrized and typed tests, whose name includes - # some '/'s. - testPath, namePrefix = os.path.split(testPath) - testName = os.path.join(namePrefix, testName) - - cmd = [testPath, '--gtest_filter=' + testName] - if litConfig.useValgrind: - cmd = litConfig.valgrindArgs + cmd - - if litConfig.noExecute: - return lit.Test.PASS, '' - - out, err, exitCode = lit.TestRunner.executeCommand( - cmd, env=test.config.environment) - - if not exitCode: - return lit.Test.PASS,'' - - return lit.Test.FAIL, out + err - -### - -class FileBasedTest(object): - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - source_path = testSuite.getSourcePath(path_in_suite) - for filename in os.listdir(source_path): - # Ignore dot files and excluded tests. - if (filename.startswith('.') or - filename in localConfig.excludes): - continue - - filepath = os.path.join(source_path, filename) - if not os.path.isdir(filepath): - base,ext = os.path.splitext(filename) - if ext in localConfig.suffixes: - yield lit.Test.Test(testSuite, path_in_suite + (filename,), - localConfig) - -class ShTest(FileBasedTest): - def __init__(self, execute_external = False): - self.execute_external = execute_external - - def execute(self, test, litConfig): - return lit.TestRunner.executeShTest(test, litConfig, - self.execute_external) - -### - -import re -import tempfile - -class OneCommandPerFileTest: - # FIXME: Refactor into generic test for running some command on a directory - # of inputs. - - def __init__(self, command, dir, recursive=False, - pattern=".*", useTempInput=False): - if isinstance(command, str): - self.command = [command] - else: - self.command = list(command) - if dir is not None: - dir = str(dir) - self.dir = dir - self.recursive = bool(recursive) - self.pattern = re.compile(pattern) - self.useTempInput = useTempInput - - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - dir = self.dir - if dir is None: - dir = testSuite.getSourcePath(path_in_suite) - - for dirname,subdirs,filenames in os.walk(dir): - if not self.recursive: - subdirs[:] = [] - - subdirs[:] = [d for d in subdirs - if (d != '.svn' and - d not in localConfig.excludes)] - - for filename in filenames: - if (filename.startswith('.') or - not self.pattern.match(filename) or - filename in localConfig.excludes): - continue - - path = os.path.join(dirname,filename) - suffix = path[len(dir):] - if suffix.startswith(os.sep): - suffix = suffix[1:] - test = lit.Test.Test( - testSuite, path_in_suite + tuple(suffix.split(os.sep)), - localConfig) - # FIXME: Hack? - test.source_path = path - yield test - - def createTempInput(self, tmp, test): - abstract - - def execute(self, test, litConfig): - if test.config.unsupported: - return (lit.Test.UNSUPPORTED, 'Test is unsupported') - - cmd = list(self.command) - - # If using temp input, create a temporary file and hand it to the - # subclass. - if self.useTempInput: - tmp = tempfile.NamedTemporaryFile(suffix='.cpp') - self.createTempInput(tmp, test) - tmp.flush() - cmd.append(tmp.name) - elif hasattr(test, 'source_path'): - cmd.append(test.source_path) - else: - cmd.append(test.getSourcePath()) - - out, err, exitCode = lit.TestRunner.executeCommand(cmd) - - diags = out + err - if not exitCode and not diags.strip(): - return lit.Test.PASS,'' - - # Try to include some useful information. - report = """Command: %s\n""" % ' '.join(["'%s'" % a - for a in cmd]) - if self.useTempInput: - report += """Temporary File: %s\n""" % tmp.name - report += "--\n%s--\n""" % open(tmp.name).read() - report += """Output:\n--\n%s--""" % diags - - return lit.Test.FAIL, report diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py index 27e29b78ba9..8929c0b5776 100644 --- a/utils/lit/lit/TestRunner.py +++ b/utils/lit/lit/TestRunner.py @@ -10,7 +10,7 @@ except ImportError: import lit.ShUtil as ShUtil import lit.Test as Test -import lit.Util as Util +import lit.util class InternalShellError(Exception): def __init__(self, command, message): @@ -154,7 +154,7 @@ def executeShCmd(cmd, cfg, cwd, results): # Resolve the executable path ourselves. args = list(j.args) - args[0] = Util.which(args[0], cfg.environment['PATH']) + args[0] = lit.util.which(args[0], cfg.environment['PATH']) if not args[0]: raise InternalShellError(j, '%r: command not found' % j.args[0]) @@ -472,7 +472,7 @@ def executeShTest(test, litConfig, useExternalSh, return (Test.PASS, '') # Create the output directory if it does not already exist. - Util.mkdir_p(os.path.dirname(tmpBase)) + lit.util.mkdir_p(os.path.dirname(tmpBase)) if useExternalSh: res = executeScript(test, litConfig, tmpBase, script, execdir) diff --git a/utils/lit/lit/Util.py b/utils/lit/lit/Util.py deleted file mode 100644 index d7d6d7f1632..00000000000 --- a/utils/lit/lit/Util.py +++ /dev/null @@ -1,140 +0,0 @@ -import errno -import itertools -import math -import os -import subprocess -import sys - -def detectCPUs(): - """ - Detects the number of CPUs on a system. Cribbed from pp. - """ - # Linux, Unix and MacOS: - if hasattr(os, "sysconf"): - if "SC_NPROCESSORS_ONLN" in os.sysconf_names: - # Linux & Unix: - ncpus = os.sysconf("SC_NPROCESSORS_ONLN") - if isinstance(ncpus, int) and ncpus > 0: - return ncpus - else: # OSX: - return int(capture(['sysctl', '-n', 'hw.ncpu'])) - # Windows: - if "NUMBER_OF_PROCESSORS" in os.environ: - ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) - if ncpus > 0: - return ncpus - return 1 # Default - -def mkdir_p(path): - """mkdir_p(path) - Make the "path" directory, if it does not exist; this - will also make directories for any missing parent directories.""" - if not path or os.path.exists(path): - return - - parent = os.path.dirname(path) - if parent != path: - mkdir_p(parent) - - try: - os.mkdir(path) - except OSError: - e = sys.exc_info()[1] - # Ignore EEXIST, which may occur during a race condition. - if e.errno != errno.EEXIST: - raise - -def capture(args, env=None): - """capture(command) - Run the given command (or argv list) in a shell and - return the standard output.""" - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env) - out,_ = p.communicate() - return out - -def which(command, paths = None): - """which(command, [paths]) - Look up the given command in the paths string - (or the PATH environment variable, if unspecified).""" - - if paths is None: - paths = os.environ.get('PATH','') - - # Check for absolute match first. - if os.path.isfile(command): - return command - - # Would be nice if Python had a lib function for this. - if not paths: - paths = os.defpath - - # Get suffixes to search. - # On Cygwin, 'PATHEXT' may exist but it should not be used. - if os.pathsep == ';': - pathext = os.environ.get('PATHEXT', '').split(';') - else: - pathext = [''] - - # Search the paths... - for path in paths.split(os.pathsep): - for ext in pathext: - p = os.path.join(path, command + ext) - if os.path.exists(p): - return p - - return None - -def checkToolsPath(dir, tools): - for tool in tools: - if not os.path.exists(os.path.join(dir, tool)): - return False; - return True; - -def whichTools(tools, paths): - for path in paths.split(os.pathsep): - if checkToolsPath(path, tools): - return path - return None - -def printHistogram(items, title = 'Items'): - items.sort(key = lambda item: item[1]) - - maxValue = max([v for _,v in items]) - - # Select first "nice" bar height that produces more than 10 bars. - power = int(math.ceil(math.log(maxValue, 10))) - for inc in itertools.cycle((5, 2, 2.5, 1)): - barH = inc * 10**power - N = int(math.ceil(maxValue / barH)) - if N > 10: - break - elif inc == 1: - power -= 1 - - histo = [set() for i in range(N)] - for name,v in items: - bin = min(int(N * v/maxValue), N-1) - histo[bin].add(name) - - barW = 40 - hr = '-' * (barW + 34) - print('\nSlowest %s:' % title) - print(hr) - for name,value in items[-20:]: - print('%.2fs: %s' % (value, name)) - print('\n%s Times:' % title) - print(hr) - pDigits = int(math.ceil(math.log(maxValue, 10))) - pfDigits = max(0, 3-pDigits) - if pfDigits: - pDigits += pfDigits + 1 - cDigits = int(math.ceil(math.log(len(items), 10))) - print("[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3), - 'Percentage'.center(barW), - 'Count'.center(cDigits*2 + 1))) - print(hr) - for i,row in enumerate(histo): - pct = float(len(row)) / len(items) - w = int(barW * pct) - print("[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" % ( - pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH, - '*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items))) - diff --git a/utils/lit/lit/__init__.py b/utils/lit/lit/__init__.py index 3967fdd020a..7b39cc5ce2a 100644 --- a/utils/lit/lit/__init__.py +++ b/utils/lit/lit/__init__.py @@ -9,3 +9,7 @@ __versioninfo__ = (0, 3, 0) __version__ = '.'.join(str(v) for v in __versioninfo__) + 'dev' __all__ = [] + +# Compatibility hacks for old names. +from . import util as Util +from . import formats as TestFormats diff --git a/utils/lit/lit/formats.py b/utils/lit/lit/formats.py new file mode 100644 index 00000000000..7a93adf7c18 --- /dev/null +++ b/utils/lit/lit/formats.py @@ -0,0 +1,230 @@ +from __future__ import absolute_import +import os +import sys + +import lit.Test +import lit.TestRunner +import lit.util + +kIsWindows = sys.platform in ['win32', 'cygwin'] + +class GoogleTest(object): + def __init__(self, test_sub_dir, test_suffix): + self.test_sub_dir = os.path.normcase(str(test_sub_dir)).split(';') + self.test_suffix = str(test_suffix) + + # On Windows, assume tests will also end in '.exe'. + if kIsWindows: + self.test_suffix += '.exe' + + def getGTestTests(self, path, litConfig, localConfig): + """getGTestTests(path) - [name] + + Return the tests available in gtest executable. + + Args: + path: String path to a gtest executable + litConfig: LitConfig instance + localConfig: TestingConfig instance""" + + try: + lines = lit.util.capture([path, '--gtest_list_tests'], + env=localConfig.environment) + lines = lines.decode('ascii') + if kIsWindows: + lines = lines.replace('\r', '') + lines = lines.split('\n') + except: + litConfig.error("unable to discover google-tests in %r" % path) + raise StopIteration + + nested_tests = [] + for ln in lines: + if not ln.strip(): + continue + + prefix = '' + index = 0 + while ln[index*2:index*2+2] == ' ': + index += 1 + while len(nested_tests) > index: + nested_tests.pop() + + ln = ln[index*2:] + if ln.endswith('.'): + nested_tests.append(ln) + else: + yield ''.join(nested_tests) + ln + + # Note: path_in_suite should not include the executable name. + def getTestsInExecutable(self, testSuite, path_in_suite, execpath, + litConfig, localConfig): + if not execpath.endswith(self.test_suffix): + return + (dirname, basename) = os.path.split(execpath) + # Discover the tests in this executable. + for testname in self.getGTestTests(execpath, litConfig, localConfig): + testPath = path_in_suite + (basename, testname) + yield lit.Test.Test(testSuite, testPath, localConfig) + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for filename in os.listdir(source_path): + filepath = os.path.join(source_path, filename) + if os.path.isdir(filepath): + # Iterate over executables in a directory. + if not os.path.normcase(filename) in self.test_sub_dir: + continue + dirpath_in_suite = path_in_suite + (filename, ) + for subfilename in os.listdir(filepath): + execpath = os.path.join(filepath, subfilename) + for test in self.getTestsInExecutable( + testSuite, dirpath_in_suite, execpath, + litConfig, localConfig): + yield test + elif ('.' in self.test_sub_dir): + for test in self.getTestsInExecutable( + testSuite, path_in_suite, filepath, + litConfig, localConfig): + yield test + + def execute(self, test, litConfig): + testPath,testName = os.path.split(test.getSourcePath()) + while not os.path.exists(testPath): + # Handle GTest parametrized and typed tests, whose name includes + # some '/'s. + testPath, namePrefix = os.path.split(testPath) + testName = os.path.join(namePrefix, testName) + + cmd = [testPath, '--gtest_filter=' + testName] + if litConfig.useValgrind: + cmd = litConfig.valgrindArgs + cmd + + if litConfig.noExecute: + return lit.Test.PASS, '' + + out, err, exitCode = lit.TestRunner.executeCommand( + cmd, env=test.config.environment) + + if not exitCode: + return lit.Test.PASS,'' + + return lit.Test.FAIL, out + err + +### + +class FileBasedTest(object): + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for filename in os.listdir(source_path): + # Ignore dot files and excluded tests. + if (filename.startswith('.') or + filename in localConfig.excludes): + continue + + filepath = os.path.join(source_path, filename) + if not os.path.isdir(filepath): + base,ext = os.path.splitext(filename) + if ext in localConfig.suffixes: + yield lit.Test.Test(testSuite, path_in_suite + (filename,), + localConfig) + +class ShTest(FileBasedTest): + def __init__(self, execute_external = False): + self.execute_external = execute_external + + def execute(self, test, litConfig): + return lit.TestRunner.executeShTest(test, litConfig, + self.execute_external) + +### + +import re +import tempfile + +class OneCommandPerFileTest: + # FIXME: Refactor into generic test for running some command on a directory + # of inputs. + + def __init__(self, command, dir, recursive=False, + pattern=".*", useTempInput=False): + if isinstance(command, str): + self.command = [command] + else: + self.command = list(command) + if dir is not None: + dir = str(dir) + self.dir = dir + self.recursive = bool(recursive) + self.pattern = re.compile(pattern) + self.useTempInput = useTempInput + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + dir = self.dir + if dir is None: + dir = testSuite.getSourcePath(path_in_suite) + + for dirname,subdirs,filenames in os.walk(dir): + if not self.recursive: + subdirs[:] = [] + + subdirs[:] = [d for d in subdirs + if (d != '.svn' and + d not in localConfig.excludes)] + + for filename in filenames: + if (filename.startswith('.') or + not self.pattern.match(filename) or + filename in localConfig.excludes): + continue + + path = os.path.join(dirname,filename) + suffix = path[len(dir):] + if suffix.startswith(os.sep): + suffix = suffix[1:] + test = lit.Test.Test( + testSuite, path_in_suite + tuple(suffix.split(os.sep)), + localConfig) + # FIXME: Hack? + test.source_path = path + yield test + + def createTempInput(self, tmp, test): + abstract + + def execute(self, test, litConfig): + if test.config.unsupported: + return (lit.Test.UNSUPPORTED, 'Test is unsupported') + + cmd = list(self.command) + + # If using temp input, create a temporary file and hand it to the + # subclass. + if self.useTempInput: + tmp = tempfile.NamedTemporaryFile(suffix='.cpp') + self.createTempInput(tmp, test) + tmp.flush() + cmd.append(tmp.name) + elif hasattr(test, 'source_path'): + cmd.append(test.source_path) + else: + cmd.append(test.getSourcePath()) + + out, err, exitCode = lit.TestRunner.executeCommand(cmd) + + diags = out + err + if not exitCode and not diags.strip(): + return lit.Test.PASS,'' + + # Try to include some useful information. + report = """Command: %s\n""" % ' '.join(["'%s'" % a + for a in cmd]) + if self.useTempInput: + report += """Temporary File: %s\n""" % tmp.name + report += "--\n%s--\n""" % open(tmp.name).read() + report += """Output:\n--\n%s--""" % diags + + return lit.Test.FAIL, report diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py index 767fe10506c..fb3614a9e56 100755 --- a/utils/lit/lit/main.py +++ b/utils/lit/lit/main.py @@ -12,7 +12,7 @@ import math, os, platform, random, re, sys, time, threading, traceback import lit.ProgressBar import lit.LitConfig import lit.Test -import lit.Util +import lit.util import lit.discovery @@ -255,7 +255,7 @@ def main(builtinParameters = {}): # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple # threads by default there. if sys.hexversion >= 0x2050200: - opts.numThreads = lit.Util.detectCPUs() + opts.numThreads = lit.util.detectCPUs() else: opts.numThreads = 1 @@ -417,7 +417,7 @@ def main(builtinParameters = {}): byTime = list(times.items()) byTime.sort(key = lambda item: item[1]) if byTime: - lit.Util.printHistogram(byTime, title='Tests') + lit.util.printHistogram(byTime, title='Tests') for name,code in (('Expected Passes ', lit.Test.PASS), ('Expected Failures ', lit.Test.XFAIL), diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py new file mode 100644 index 00000000000..d7d6d7f1632 --- /dev/null +++ b/utils/lit/lit/util.py @@ -0,0 +1,140 @@ +import errno +import itertools +import math +import os +import subprocess +import sys + +def detectCPUs(): + """ + Detects the number of CPUs on a system. Cribbed from pp. + """ + # Linux, Unix and MacOS: + if hasattr(os, "sysconf"): + if "SC_NPROCESSORS_ONLN" in os.sysconf_names: + # Linux & Unix: + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + else: # OSX: + return int(capture(['sysctl', '-n', 'hw.ncpu'])) + # Windows: + if "NUMBER_OF_PROCESSORS" in os.environ: + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + return 1 # Default + +def mkdir_p(path): + """mkdir_p(path) - Make the "path" directory, if it does not exist; this + will also make directories for any missing parent directories.""" + if not path or os.path.exists(path): + return + + parent = os.path.dirname(path) + if parent != path: + mkdir_p(parent) + + try: + os.mkdir(path) + except OSError: + e = sys.exc_info()[1] + # Ignore EEXIST, which may occur during a race condition. + if e.errno != errno.EEXIST: + raise + +def capture(args, env=None): + """capture(command) - Run the given command (or argv list) in a shell and + return the standard output.""" + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out,_ = p.communicate() + return out + +def which(command, paths = None): + """which(command, [paths]) - Look up the given command in the paths string + (or the PATH environment variable, if unspecified).""" + + if paths is None: + paths = os.environ.get('PATH','') + + # Check for absolute match first. + if os.path.isfile(command): + return command + + # Would be nice if Python had a lib function for this. + if not paths: + paths = os.defpath + + # Get suffixes to search. + # On Cygwin, 'PATHEXT' may exist but it should not be used. + if os.pathsep == ';': + pathext = os.environ.get('PATHEXT', '').split(';') + else: + pathext = [''] + + # Search the paths... + for path in paths.split(os.pathsep): + for ext in pathext: + p = os.path.join(path, command + ext) + if os.path.exists(p): + return p + + return None + +def checkToolsPath(dir, tools): + for tool in tools: + if not os.path.exists(os.path.join(dir, tool)): + return False; + return True; + +def whichTools(tools, paths): + for path in paths.split(os.pathsep): + if checkToolsPath(path, tools): + return path + return None + +def printHistogram(items, title = 'Items'): + items.sort(key = lambda item: item[1]) + + maxValue = max([v for _,v in items]) + + # Select first "nice" bar height that produces more than 10 bars. + power = int(math.ceil(math.log(maxValue, 10))) + for inc in itertools.cycle((5, 2, 2.5, 1)): + barH = inc * 10**power + N = int(math.ceil(maxValue / barH)) + if N > 10: + break + elif inc == 1: + power -= 1 + + histo = [set() for i in range(N)] + for name,v in items: + bin = min(int(N * v/maxValue), N-1) + histo[bin].add(name) + + barW = 40 + hr = '-' * (barW + 34) + print('\nSlowest %s:' % title) + print(hr) + for name,value in items[-20:]: + print('%.2fs: %s' % (value, name)) + print('\n%s Times:' % title) + print(hr) + pDigits = int(math.ceil(math.log(maxValue, 10))) + pfDigits = max(0, 3-pDigits) + if pfDigits: + pDigits += pfDigits + 1 + cDigits = int(math.ceil(math.log(len(items), 10))) + print("[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3), + 'Percentage'.center(barW), + 'Count'.center(cDigits*2 + 1))) + print(hr) + for i,row in enumerate(histo): + pct = float(len(row)) / len(items) + w = int(barW * pct) + print("[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" % ( + pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH, + '*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items))) +