[scan-build-py] create decorator for compiler wrapper methods

Differential Revision: https://reviews.llvm.org/D29260


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@296937 91177308-0d34-0410-b5e6-96231b3b80d8

7 files changed, 208 insertions, 114 deletions

diff --git a/tools/scan-build-py/bin/analyze-c++ b/tools/scan-build-py/bin/analyze-c++
index 15186d89aa..df1012dee5 100755
--- a/tools/scan-build-py/bin/analyze-c++
+++ b/tools/scan-build-py/bin/analyze-c++
@@ -10,5 +10,5 @@ import os.path
 this_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.dirname(this_dir))
 
-from libscanbuild.analyze import analyze_build_wrapper
-sys.exit(analyze_build_wrapper(True))
+from libscanbuild.analyze import analyze_compiler_wrapper
+sys.exit(analyze_compiler_wrapper())
diff --git a/tools/scan-build-py/bin/analyze-cc b/tools/scan-build-py/bin/analyze-cc
index 55519fb7b1..df1012dee5 100755
--- a/tools/scan-build-py/bin/analyze-cc
+++ b/tools/scan-build-py/bin/analyze-cc
@@ -10,5 +10,5 @@ import os.path
 this_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.dirname(this_dir))
 
-from libscanbuild.analyze import analyze_build_wrapper
-sys.exit(analyze_build_wrapper(False))
+from libscanbuild.analyze import analyze_compiler_wrapper
+sys.exit(analyze_compiler_wrapper())
diff --git a/tools/scan-build-py/bin/intercept-c++ b/tools/scan-build-py/bin/intercept-c++
index fc422287f8..67e076f39e 100755
--- a/tools/scan-build-py/bin/intercept-c++
+++ b/tools/scan-build-py/bin/intercept-c++
@@ -10,5 +10,5 @@ import os.path
 this_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.dirname(this_dir))
 
-from libscanbuild.intercept import intercept_build_wrapper
-sys.exit(intercept_build_wrapper(True))
+from libscanbuild.intercept import intercept_compiler_wrapper
+sys.exit(intercept_compiler_wrapper())
diff --git a/tools/scan-build-py/bin/intercept-cc b/tools/scan-build-py/bin/intercept-cc
index 69d57aaae1..67e076f39e 100755
--- a/tools/scan-build-py/bin/intercept-cc
+++ b/tools/scan-build-py/bin/intercept-cc
@@ -10,5 +10,5 @@ import os.path
 this_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.dirname(this_dir))
 
-from libscanbuild.intercept import intercept_build_wrapper
-sys.exit(intercept_build_wrapper(False))
+from libscanbuild.intercept import intercept_compiler_wrapper
+sys.exit(intercept_compiler_wrapper())
diff --git a/tools/scan-build-py/libscanbuild/__init__.py b/tools/scan-build-py/libscanbuild/__init__.py
index 4b5582de05..ca75174912 100644
--- a/tools/scan-build-py/libscanbuild/__init__.py
+++ b/tools/scan-build-py/libscanbuild/__init__.py
@@ -4,13 +4,21 @@
 # This file is distributed under the University of Illinois Open Source
 # License. See LICENSE.TXT for details.
 """ This module is a collection of methods commonly used in this project. """
+import collections
 import functools
+import json
 import logging
 import os
 import os.path
+import re
+import shlex
 import subprocess
 import sys
 
+ENVIRONMENT_KEY = 'INTERCEPT_BUILD'
+
+Execution = collections.namedtuple('Execution', ['pid', 'cwd', 'cmd'])
+
 
 def duplicate_check(method):
     """ Predicate to detect duplicated entries.
@@ -75,31 +83,53 @@ def run_command(command, cwd=None):
         raise ex
 
 
-def initialize_logging(verbose_level):
-    """ Output content controlled by the verbosity level. """
+def reconfigure_logging(verbose_level):
+    """ Reconfigure logging level and format based on the verbose flag.
 
-    level = logging.WARNING - min(logging.WARNING, (10 * verbose_level))
+    :param verbose_level: number of `-v` flags received by the command
+    :return: no return value
+    """
+    # Exit when nothing to do.
+    if verbose_level == 0:
+        return
 
+    root = logging.getLogger()
+    # Tune logging level.
+    level = logging.WARNING - min(logging.WARNING, (10 * verbose_level))
+    root.setLevel(level)
+    # Be verbose with messages.
     if verbose_level <= 3:
-        fmt_string = '{0}: %(levelname)s: %(message)s'
+        fmt_string = '%(name)s: %(levelname)s: %(message)s'
     else:
-        fmt_string = '{0}: %(levelname)s: %(funcName)s: %(message)s'
-
-    program = os.path.basename(sys.argv[0])
-    logging.basicConfig(format=fmt_string.format(program), level=level)
+        fmt_string = '%(name)s: %(levelname)s: %(funcName)s: %(message)s'
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(logging.Formatter(fmt=fmt_string))
+    root.handlers = [handler]
 
 
 def command_entry_point(function):
-    """ Decorator for command entry points. """
+    """ Decorator for command entry methods.
+
+    The decorator initialize/shutdown logging and guard on programming
+    errors (catch exceptions).
+
+    The decorated method can have arbitrary parameters, the return value will
+    be the exit code of the process. """
 
     @functools.wraps(function)
     def wrapper(*args, **kwargs):
+        """ Do housekeeping tasks and execute the wrapped method. """
 
-        exit_code = 127
         try:
-            exit_code = function(*args, **kwargs)
+            logging.basicConfig(format='%(name)s: %(message)s',
+                                level=logging.WARNING,
+                                stream=sys.stdout)
+            # This hack to get the executable name as %(name).
+            logging.getLogger().name = os.path.basename(sys.argv[0])
+            return function(*args, **kwargs)
         except KeyboardInterrupt:
-            logging.warning('Keyboard interupt')
+            logging.warning('Keyboard interrupt')
+            return 130  # Signal received exit code for bash.
         except Exception:
             logging.exception('Internal error.')
             if logging.getLogger().isEnabledFor(logging.DEBUG):
@@ -107,8 +137,75 @@ def command_entry_point(function):
                               "to the bug report")
             else:
                 logging.error("Please run this command again and turn on "
-                              "verbose mode (add '-vvv' as argument).")
+                              "verbose mode (add '-vvvv' as argument).")
+            return 64  # Some non used exit code for internal errors.
         finally:
-            return exit_code
+            logging.shutdown()
 
     return wrapper
+
+
+def compiler_wrapper(function):
+    """ Implements compiler wrapper base functionality.
+
+    A compiler wrapper executes the real compiler, then implement some
+    functionality, then returns with the real compiler exit code.
+
+    :param function: the extra functionality what the wrapper want to
+    do on top of the compiler call. If it throws exception, it will be
+    caught and logged.
+    :return: the exit code of the real compiler.
+
+    The :param function: will receive the following arguments:
+
+    :param result:       the exit code of the compilation.
+    :param execution:    the command executed by the wrapper. """
+
+    def is_cxx_compiler():
+        """ Find out was it a C++ compiler call. Compiler wrapper names
+        contain the compiler type. C++ compiler wrappers ends with `c++`,
+        but might have `.exe` extension on windows. """
+
+        wrapper_command = os.path.basename(sys.argv[0])
+        return re.match(r'(.+)c\+\+(.*)', wrapper_command)
+
+    def run_compiler(executable):
+        """ Execute compilation with the real compiler. """
+
+        command = executable + sys.argv[1:]
+        logging.debug('compilation: %s', command)
+        result = subprocess.call(command)
+        logging.debug('compilation exit code: %d', result)
+        return result
+
+    # Get relevant parameters from environment.
+    parameters = json.loads(os.environ[ENVIRONMENT_KEY])
+    reconfigure_logging(parameters['verbose'])
+    # Execute the requested compilation. Do crash if anything goes wrong.
+    cxx = is_cxx_compiler()
+    compiler = parameters['cxx'] if cxx else parameters['cc']
+    result = run_compiler(compiler)
+    # Call the wrapped method and ignore it's return value.
+    try:
+        call = Execution(
+            pid=os.getpid(),
+            cwd=os.getcwd(),
+            cmd=['c++' if cxx else 'cc'] + sys.argv[1:])
+        function(result, call)
+    except:
+        logging.exception('Compiler wrapper failed complete.')
+    finally:
+        # Always return the real compiler exit code.
+        return result
+
+
+def wrapper_environment(args):
+    """ Set up environment for interpose compiler wrapper."""
+
+    return {
+        ENVIRONMENT_KEY: json.dumps({
+            'verbose': args.verbose,
+            'cc': shlex.split(args.cc),
+            'cxx': shlex.split(args.cxx)
+        })
+    }
diff --git a/tools/scan-build-py/libscanbuild/analyze.py b/tools/scan-build-py/libscanbuild/analyze.py
index 8e9f317e69..7d050fa539 100644
--- a/tools/scan-build-py/libscanbuild/analyze.py
+++ b/tools/scan-build-py/libscanbuild/analyze.py
@@ -19,19 +19,18 @@ import json
 import argparse
 import logging
 import tempfile
-import subprocess
 import multiprocessing
 import contextlib
 import datetime
-from libscanbuild import initialize_logging, tempdir, command_entry_point, \
-    run_build
+from libscanbuild import command_entry_point, compiler_wrapper, \
+    wrapper_environment, reconfigure_logging, run_build, tempdir
 from libscanbuild.runner import run
 from libscanbuild.intercept import capture
 from libscanbuild.report import document
 from libscanbuild.clang import get_checkers
 from libscanbuild.compilation import split_command
 
-__all__ = ['analyze_build_main', 'analyze_build_wrapper']
+__all__ = ['analyze_build_main', 'analyze_compiler_wrapper']
 
 COMPILER_WRAPPER_CC = 'analyze-cc'
 COMPILER_WRAPPER_CXX = 'analyze-c++'
@@ -46,8 +45,8 @@ def analyze_build_main(bin_dir, from_build_command):
     validate(parser, args, from_build_command)
 
     # setup logging
-    initialize_logging(args.verbose)
-    logging.debug('Parsed arguments: %s', args)
+    reconfigure_logging(args.verbose)
+    logging.debug('Raw arguments %s', sys.argv)
 
     with report_directory(args.output, args.keep_empty) as target_dir:
         if not from_build_command:
@@ -130,13 +129,11 @@ def setup_environment(args, destination, bin_dir):
     """ Set up environment for build command to interpose compiler wrapper. """
 
     environment = dict(os.environ)
+    environment.update(wrapper_environment(args))
     environment.update({
         'CC': os.path.join(bin_dir, COMPILER_WRAPPER_CC),
         'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX),
-        'ANALYZE_BUILD_CC': args.cc,
-        'ANALYZE_BUILD_CXX': args.cxx,
         'ANALYZE_BUILD_CLANG': args.clang if need_analyzer(args.build) else '',
-        'ANALYZE_BUILD_VERBOSE': 'DEBUG' if args.verbose > 2 else 'WARNING',
         'ANALYZE_BUILD_REPORT_DIR': destination,
         'ANALYZE_BUILD_REPORT_FORMAT': args.output_format,
         'ANALYZE_BUILD_REPORT_FAILURES': 'yes' if args.output_failures else '',
@@ -146,51 +143,45 @@ def setup_environment(args, destination, bin_dir):
     return environment
 
 
-def analyze_build_wrapper(cplusplus):
+@command_entry_point
+def analyze_compiler_wrapper():
     """ Entry point for `analyze-cc` and `analyze-c++` compiler wrappers. """
 
-    # initialize wrapper logging
-    logging.basicConfig(format='analyze: %(levelname)s: %(message)s',
-                        level=os.getenv('ANALYZE_BUILD_VERBOSE', 'INFO'))
-    # execute with real compiler
-    compiler = os.getenv('ANALYZE_BUILD_CXX', 'c++') if cplusplus \
-        else os.getenv('ANALYZE_BUILD_CC', 'cc')
-    compilation = [compiler] + sys.argv[1:]
-    logging.info('execute compiler: %s', compilation)
-    result = subprocess.call(compilation)
-    # exit when it fails, ...
+    return compiler_wrapper(analyze_compiler_wrapper_impl)
+
+
+def analyze_compiler_wrapper_impl(result, execution):
+    """ Implements analyzer compiler wrapper functionality. """
+
+    # don't run analyzer when compilation fails. or when it's not requested.
     if result or not os.getenv('ANALYZE_BUILD_CLANG'):
-        return result
-    # ... and run the analyzer if all went well.
-    try:
-        # check is it a compilation
-        compilation = split_command(sys.argv)
-        if compilation is None:
-            return result
-        # collect the needed parameters from environment, crash when missing
-        parameters = {
-            'clang': os.getenv('ANALYZE_BUILD_CLANG'),
-            'output_dir': os.getenv('ANALYZE_BUILD_REPORT_DIR'),
-            'output_format': os.getenv('ANALYZE_BUILD_REPORT_FORMAT'),
-            'output_failures': os.getenv('ANALYZE_BUILD_REPORT_FAILURES'),
-            'direct_args': os.getenv('ANALYZE_BUILD_PARAMETERS',
-                                     '').split(' '),
-            'force_debug': os.getenv('ANALYZE_BUILD_FORCE_DEBUG'),
-            'directory': os.getcwd(),
-            'command': [sys.argv[0], '-c'] + compilation.flags
-        }
-        # call static analyzer against the compilation
-        for source in compilation.files:
-            parameters.update({'file': source})
-            logging.debug('analyzer parameters %s', parameters)
-            current = run(parameters)
-            # display error message from the static analyzer
-            if current is not None:
-                for line in current['error_output']:
-                    logging.info(line.rstrip())
-    except Exception:
-        logging.exception("run analyzer inside compiler wrapper failed.")
-    return result
+        return
+
+    # check is it a compilation?
+    compilation = split_command(execution.cmd)
+    if compilation is None:
+        return
+    # collect the needed parameters from environment, crash when missing
+    parameters = {
+        'clang': os.getenv('ANALYZE_BUILD_CLANG'),
+        'output_dir': os.getenv('ANALYZE_BUILD_REPORT_DIR'),
+        'output_format': os.getenv('ANALYZE_BUILD_REPORT_FORMAT'),
+        'output_failures': os.getenv('ANALYZE_BUILD_REPORT_FAILURES'),
+        'direct_args': os.getenv('ANALYZE_BUILD_PARAMETERS',
+                                 '').split(' '),
+        'force_debug': os.getenv('ANALYZE_BUILD_FORCE_DEBUG'),
+        'directory': execution.cwd,
+        'command': [execution.cmd[0], '-c'] + compilation.flags
+    }
+    # call static analyzer against the compilation
+    for source in compilation.files:
+        parameters.update({'file': source})
+        logging.debug('analyzer parameters %s', parameters)
+        current = run(parameters)
+        # display error message from the static analyzer
+        if current is not None:
+            for line in current['error_output']:
+                logging.info(line.rstrip())
 
 
 @contextlib.contextmanager
diff --git a/tools/scan-build-py/libscanbuild/intercept.py b/tools/scan-build-py/libscanbuild/intercept.py
index b3c55e0417..2d1e825d44 100644
--- a/tools/scan-build-py/libscanbuild/intercept.py
+++ b/tools/scan-build-py/libscanbuild/intercept.py
@@ -29,14 +29,14 @@ import json
 import glob
 import argparse
 import logging
-import subprocess
 from libear import build_libear, TemporaryDirectory
-from libscanbuild import command_entry_point, run_build, run_command
-from libscanbuild import duplicate_check, tempdir, initialize_logging
+from libscanbuild import command_entry_point, compiler_wrapper, \
+    wrapper_environment, run_command, run_build, reconfigure_logging
+from libscanbuild import duplicate_check, tempdir
 from libscanbuild.compilation import split_command
 from libscanbuild.shell import encode, decode
 
-__all__ = ['capture', 'intercept_build_main', 'intercept_build_wrapper']
+__all__ = ['capture', 'intercept_build_main', 'intercept_compiler_wrapper']
 
 GS = chr(0x1d)
 RS = chr(0x1e)
@@ -44,6 +44,7 @@ US = chr(0x1f)
 
 COMPILER_WRAPPER_CC = 'intercept-cc'
 COMPILER_WRAPPER_CXX = 'intercept-c++'
+TRACE_FILE_EXTENSION = '.cmd'  # same as in ear.c
 WRAPPER_ONLY_PLATFORMS = frozenset({'win32', 'cygwin'})
 
 
@@ -54,8 +55,8 @@ def intercept_build_main(bin_dir):
     parser = create_parser()
     args = parser.parse_args()
 
-    initialize_logging(args.verbose)
-    logging.debug('Parsed arguments: %s', args)
+    reconfigure_logging(args.verbose)
+    logging.debug('Raw arguments %s', sys.argv)
 
     if not args.build:
         parser.print_help()
@@ -126,12 +127,10 @@ def setup_environment(args, destination, bin_dir):
 
     if not libear_path:
         logging.debug('intercept gonna use compiler wrappers')
+        environment.update(wrapper_environment(args))
         environment.update({
             'CC': os.path.join(bin_dir, COMPILER_WRAPPER_CC),
-            'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX),
-            'INTERCEPT_BUILD_CC': c_compiler,
-            'INTERCEPT_BUILD_CXX': cxx_compiler,
-            'INTERCEPT_BUILD_VERBOSE': 'DEBUG' if args.verbose > 2 else 'INFO'
+            'CXX': os.path.join(bin_dir, COMPILER_WRAPPER_CXX)
         })
     elif sys.platform == 'darwin':
         logging.debug('intercept gonna preload libear on OSX')
@@ -146,42 +145,49 @@ def setup_environment(args, destination, bin_dir):
     return environment
 
 
-def intercept_build_wrapper(cplusplus):
-    """ Entry point for `intercept-cc` and `intercept-c++` compiler wrappers.
+@command_entry_point
+def intercept_compiler_wrapper():
+    """ Entry point for `intercept-cc` and `intercept-c++`. """
+
+    return compiler_wrapper(intercept_compiler_wrapper_impl)
+
 
-    It does generate execution report into target directory. And execute
-    the wrapped compilation with the real compiler. The parameters for
-    report and execution are from environment variables.
+def intercept_compiler_wrapper_impl(_, execution):
+    """ Implement intercept compiler wrapper functionality.
 
-    Those parameters which for 'libear' library can't have meaningful
-    values are faked. """
+    It does generate execution report into target directory.
+    The target directory name is from environment variables. """
 
-    # initialize wrapper logging
-    logging.basicConfig(format='intercept: %(levelname)s: %(message)s',
-                        level=os.getenv('INTERCEPT_BUILD_VERBOSE', 'INFO'))
-    # write report
+    message_prefix = 'execution report might be incomplete: %s'
+
+    target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR')
+    if not target_dir:
+        logging.warning(message_prefix, 'missing target directory')
+        return
+    # write current execution info to the pid file
     try:
-        target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR')
-        if not target_dir:
-            raise UserWarning('exec report target directory not found')
-        pid = str(os.getpid())
-        target_file = os.path.join(target_dir, pid + '.cmd')
-        logging.debug('writing exec report to: %s', target_file)
-        with open(target_file, 'ab') as handler:
-            working_dir = os.getcwd()
-            command = US.join(sys.argv) + US
-            content = RS.join([pid, pid, 'wrapper', working_dir, command]) + GS
-            handler.write(content.encode('utf-8'))
+        target_file_name = str(os.getpid()) + TRACE_FILE_EXTENSION
+        target_file = os.path.join(target_dir, target_file_name)
+        logging.debug('writing execution report to: %s', target_file)
+        write_exec_trace(target_file, execution)
     except IOError:
-        logging.exception('writing exec report failed')
-    except UserWarning as warning:
-        logging.warning(warning)
-    # execute with real compiler
-    compiler = os.getenv('INTERCEPT_BUILD_CXX', 'c++') if cplusplus \
-        else os.getenv('INTERCEPT_BUILD_CC', 'cc')
-    compilation = [compiler] + sys.argv[1:]
-    logging.debug('execute compiler: %s', compilation)
-    return subprocess.call(compilation)
+        logging.warning(message_prefix, 'io problem')
+
+
+def write_exec_trace(filename, entry):
+    """ Write execution report file.
+
+    This method shall be sync with the execution report writer in interception
+    library. The entry in the file is a JSON objects.
+
+    :param filename:    path to the output execution trace file,
+    :param entry:       the Execution object to append to that file. """
+
+    with open(filename, 'ab') as handler:
+        pid = str(entry.pid)
+        command = US.join(entry.cmd) + US
+        content = RS.join([pid, pid, 'wrapper', entry.cwd, command]) + GS
+        handler.write(content.encode('utf-8'))
 
 
 def parse_exec_trace(filename):