iwyu_test_util.py

#!/usr/bin/env python3

##===--- iwyu_test_util.py - include-what-you-use test framework ----------===##
#
#                     The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
##===----------------------------------------------------------------------===##

"""Utilities for writing tests for IWYU.

This script has been tested with python 2.7, 3.1.3 and 3.2.
In order to support all of these platforms there are a few unusual constructs:
 * print statements require parentheses
 * standard output must be decoded as utf-8
 * range() must be used in place of xrange()
 * _PortableNext() is used to obtain next iterator value

There is more detail on some of these issues at:
http://diveintopython3.org/porting-code-to-python-3-with-2to3.html
"""

__author__ = 'wan@google.com (Zhanyong Wan)'

import difflib
import operator
import os
import re
import subprocess
import sys

# These are the warning/error lines that iwyu.cc produces when --verbose >= 3
_EXPECTED_DIAGNOSTICS_RE = re.compile(r'^\s*//\s*IWYU:\s*(.*)$')
_ACTUAL_DIAGNOSTICS_RE = re.compile(r'^(.*?):(\d+):\d+:\s*'
                                    r'(?:warning|error|fatal error):\s*(.*)$')

# This is the final summary output that iwyu.cc produces when --verbose >= 1
# The summary for a given source file should appear in that source file,
# surrounded by '/**** IWYU_SUMMARY' and '***** IWYU_SUMMARY */'.
# The leading summary line may also have an expected exit-code in parentheses
# after the summary marker: '/**** IWYU_SUMMARY(10)'.
_EXPECTED_SUMMARY_START_RE = re.compile(r'/\*+\s*IWYU_SUMMARY')
_EXPECTED_SUMMARY_EXIT_CODE_RE = re.compile(r'/\*+\s*IWYU_SUMMARY\((\d+)\)')
_EXPECTED_SUMMARY_END_RE = re.compile(r'\**\s*IWYU_SUMMARY\s*\*+/')
_ACTUAL_SUMMARY_START_RE = re.compile(r'^(.*?) should add these lines:$')
_ACTUAL_SUMMARY_END_RE = re.compile(r'^---$')
_ACTUAL_REMOVAL_LIST_START_RE = re.compile(r'.* should remove these lines:$')
_NODIFFS_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')

# This is an IWYU_ARGS line that specifies launch arguments for a test in its
# source file. Example:
# // IWYU_ARGS: -Xiwyu --mapping_file=... -I .
_IWYU_TEST_RUN_ARGS_RE = re.compile(r'^//\sIWYU_ARGS:\s(.*)$')


def _PortableNext(iterator):
  if hasattr(iterator, 'next'):
    iterator.next()  # Python 2.4-2.6
  else:
    next(iterator)   # Python 3


def _Which(program, paths):
    """Searches specified paths for program."""
    if sys.platform == 'win32' and not program.lower().endswith('.exe'):
        program += '.exe'

    for path in paths:
        candidate = os.path.join(os.path.normpath(path), program)
        if os.path.isfile(candidate):
            return candidate

    return None


_IWYU_PATH = None
_SYSTEM_PATHS = [p.strip('"') for p in os.environ["PATH"].split(os.pathsep)]
_IWYU_PATHS = [
    '../../../../Debug+Asserts/bin',
    '../../../../Release+Asserts/bin',
    '../../../../Release/bin',
    '../../../../build/Debug+Asserts/bin',
    '../../../../build/Release+Asserts/bin',
    '../../../../build/Release/bin',
    # Linux/Mac OS X default out-of-tree paths.
    '../../../../../build/Debug+Asserts/bin',
    '../../../../../build/Release+Asserts/bin',
    '../../../../../build/Release/bin',
    # Windows default out-of-tree paths.
    '../../../../../build/bin/Debug',
    '../../../../../build/bin/Release',
    '../../../../../build/bin/MinSizeRel',
    '../../../../../build/bin/RelWithDebInfo',
    ]


def SetIwyuPath(iwyu_path):
  """Set the path to the IWYU executable under test.
  """
  global _IWYU_PATH
  _IWYU_PATH = iwyu_path


def _GetIwyuPath():
  """Returns the path to IWYU or raises IOError if it cannot be found."""
  global _IWYU_PATH

  if not _IWYU_PATH:
    iwyu_paths = _IWYU_PATHS + _SYSTEM_PATHS
    _IWYU_PATH = _Which('include-what-you-use', iwyu_paths)
    if not _IWYU_PATH:
      raise IOError('Failed to locate IWYU.\nSearched\n %s' %
                    '\n '.join(iwyu_paths))

  return _IWYU_PATH


def _ShellQuote(arg):
  if ' ' in arg:
    arg = '"' + arg + '"'
  return arg


def _GetCommandOutput(command):
  p = subprocess.Popen(command,
                       shell=True,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT)
  stdout, _ = p.communicate()
  lines = stdout.decode("utf-8").splitlines(True)
  lines = [line.replace(os.linesep, '\n') for line in lines]
  return p.returncode, lines


def _GetMatchingLines(regex, file_names):
  """Returns a map: file location => string matching `regex`.
  
  File location is a tuple (file_name, line number starting from 1)."""

  loc_to_line = {}
  for file_name in file_names:
    with open(file_name) as fileobj:
      for line_num, line in enumerate(fileobj):
        m = regex.match(line)
        if m:
          loc_to_line[file_name, line_num + 1] = m.group()
  return loc_to_line


def _GetExpectedDiagnosticRegexes(spec_loc_to_line):
  """Returns a map: source file location => list of regexes for that line."""

  # Maps a source file line location to a list of regexes for diagnostics
  # that should be generated for that line.
  expected_diagnostic_regexes = {}
  regexes = []
  for loc in sorted(spec_loc_to_line.keys()):
    line = spec_loc_to_line[loc]
    m = _EXPECTED_DIAGNOSTICS_RE.match(line.strip())
    assert m is not None, "Input should contain only matching lines."
    regex = m.group(1)
    if not regex:
      # Allow the regex to be omitted if we are uninterested in the
      # diagnostic message.
      regex = r'.*'
    regexes.append(re.compile(regex))
    # Do we have a spec on the next line?
    path, line_num = loc
    next_line_loc = path, line_num + 1
    if next_line_loc not in spec_loc_to_line:
      expected_diagnostic_regexes[next_line_loc] = regexes
      regexes = []

  return expected_diagnostic_regexes


def _GetActualDiagnostics(actual_output):
  """Returns a map: source file location => list of diagnostics on that line.

  The elements of the list are unique and sorted."""

  actual_diagnostics = {}
  for line in actual_output:
    m = _ACTUAL_DIAGNOSTICS_RE.match(line.strip())
    if m:
      path, line_num, message = m.groups()
      loc = path, int(line_num)
      actual_diagnostics[loc] = actual_diagnostics.get(loc, []) + [message]

  locs = actual_diagnostics.keys()
  for loc in locs:
    actual_diagnostics[loc] = sorted(set(actual_diagnostics[loc]))

  return actual_diagnostics


def _StripCommentFromLine(line):
  """Removes the "// ..." comment at the end of the given line."""
  return re.sub(r'\s*//.*$', '', line)


def _NormalizeSummaryLineNumbers(line):
  """Replaces the comment '// lines <number>-<number>' with '// lines XX-YY'.

  Because line numbers in the source code often change, it's a pain to
  keep the '// lines <number>-<number>' comments accurate in our
  'golden' output.  Instead, we normalize these iwyu comments to just
  say how many line numbers are listed by mapping the output to
  '// lines XX-XX' (for one-line spans) or '// lines XX-XX+<number>'.
  For instance, '// lines 12-12' would map to '// lines XX-XX', while
  '// lines 12-14' would map to '//lines XX-XX+2'.

  Arguments:
    line: the line to be normalized.

  Returns:
    A new line with the '// lines' comment, if any, normalized as
    described above.  If no '// lines' comment is present, returns
    the original line.
  """
  m = re.search('// lines ([0-9]+)-([0-9]+)', line)
  if not m:
    return line
  if m.group(1) == m.group(2):
    return line[:m.start()] + '// lines XX-XX\n'
  else:
    num_lines = int(m.group(2)) - int(m.group(1))
    return line[:m.start()] + '// lines XX-XX+%d\n' % num_lines


def _NormalizeSummaryLine(line):
  """Alphabetically sorts the symbols in the '// for XXX, YYY, ZZZ' comments.

  Most iwyu summary lines have the form
     #include <foo.h>   // for XXX, YYY, ZZZ
  XXX, YYY, ZZZ are symbols that this file uses from foo.h.  They are
  sorted in frequency order, but that changes so often as the test is
  augmented, that it's impractical to test.  We just sort the symbols
  alphabetically and compare that way.  This means we never test the
  frequency ordering here, but that's a small price to pay for easier
  testing development.

  We also always move the '// for' comment to be exactly two spaces
  after the '#include' text.  Again, this means we don't test the
  indenting correctly (though iwyu_output_test.cc does), but allows us
  to rename filenames without having to reformat each test.  This is
  particularly important when opensourcing, since the filenames will
  be different in opensource-land than they are inside google.

  Arguments:
    line: one line of the summary output

  Returns:
    A normalized form of 'line', with the 'why' symbols sorted and
    whitespace before the 'why' comment collapsed.
  """
  m = re.match(r'(.*?)\s*  // for (.*)', line)
  if not m:
    return line
  symbols = m.group(2).strip().split(', ')
  symbols.sort()
  return '%s  // for %s\n' % (m.group(1), ', '.join(symbols))


def _GetExpectedSummaries(files):
  """Returns a map: source file => list of iwyu summary lines."""

  expected_summaries = {}
  for f in files:
    in_summary = False
    fh = open(f)
    for line in fh:
      if _EXPECTED_SUMMARY_START_RE.match(line):
        in_summary = True
        expected_summaries[f] = []
      elif _EXPECTED_SUMMARY_END_RE.match(line):
        in_summary = False
      elif re.match(r'^\s*//', line):
        pass   # ignore comment lines
      elif in_summary:
        expected_summaries[f].append(line)
    fh.close()

  # Get rid of blank lines at the beginning and end of the each summary.
  for loc in expected_summaries:
    while expected_summaries[loc] and expected_summaries[loc][-1] == '\n':
      expected_summaries[loc].pop()
    while expected_summaries[loc] and expected_summaries[loc][0] == '\n':
      expected_summaries[loc].pop(0)

  return expected_summaries


def _GetExpectedExitCode(main_file):
  with open(main_file, 'r') as fh:
    for line in fh:
      m = _EXPECTED_SUMMARY_EXIT_CODE_RE.match(line)
      if m:
        res = int(m.group(1))
        return res
  return None


def _GetActualSummaries(output):
  """Returns a map: source file => list of iwyu summary lines."""

  actual_summaries = {}
  file_being_summarized = None
  in_addition_section = False  # Are we in the "should add these lines" section?
  for line in output:
    # For files with no diffs, we print a different (one-line) summary.
    m = _NODIFFS_RE.match(line)
    if m:
      actual_summaries[m.group(1)] = [line]
      continue

    m = _ACTUAL_SUMMARY_START_RE.match(line)
    if m:
      file_being_summarized = m.group(1)
      in_addition_section = True
      actual_summaries[file_being_summarized] = [line]
    elif _ACTUAL_SUMMARY_END_RE.match(line):
      file_being_summarized = None
    elif file_being_summarized:
      if _ACTUAL_REMOVAL_LIST_START_RE.match(line):
        in_addition_section = False
      # Replace any line numbers in comments with something more stable.
      line = _NormalizeSummaryLineNumbers(line)
      if in_addition_section:
        # Each #include in the "should add" list will appear later in
        # the full include list.  There's no need to verify its symbol
        # list twice.  Therefore we remove the symbol list here for
        # easy test maintenance.
        line = _StripCommentFromLine(line)
      else:
        line = _NormalizeSummaryLine(line)
      actual_summaries[file_being_summarized].append(line)

  return actual_summaries


def _VerifyDiagnosticsAtLoc(loc_str, regexes, diagnostics):
  """Verify the diagnostics at the given location; return a list of failures."""

  # Find out which regexes match a diagnostic and vice versa.
  matching_regexes = [[] for unused_i in range(len(diagnostics))]
  matched_diagnostics = [[] for unused_i in range(len(regexes))]
  for (r_index, regex) in enumerate(regexes):
    for (d_index, diagnostic) in enumerate(diagnostics):
      if regex.search(diagnostic):
        matching_regexes[d_index].append(r_index)
        matched_diagnostics[r_index].append(d_index)

  failure_messages = []

  # Collect unmatched diagnostics and multiply matched diagnostics.
  for (d_index, r_indexes) in enumerate(matching_regexes):
    if not r_indexes:
      failure_messages.append('Unexpected diagnostic:\n%s\n'
                              % diagnostics[d_index])
    elif len(r_indexes) > 1:
      failure_messages.append(
          'The diagnostic message:\n%s\n'
          'matches multiple regexes:\n%s'
          % (diagnostics[d_index],
             '\n'.join([regexes[r_index].pattern for r_index in r_indexes])))

  # Collect unmatched regexes and regexes with multiple matches.
  for (r_index, d_indexes) in enumerate(matched_diagnostics):
    if not d_indexes:
      failure_messages.append('Unmatched regex:\n%s\n'
                              % regexes[r_index].pattern)
    elif len(d_indexes) > 1:
      failure_messages.append(
          'The regex:\n%s\n'
          'matches multiple diagnostics:\n%s'
          % (regexes[r_index].pattern,
             '\n'.join([diagnostics[d_index] for d_index in d_indexes])))

  return ['%s %s' % (loc_str, message) for message in failure_messages]


def _CompareExpectedAndActualDiagnostics(expected_diagnostic_regexes,
                                         actual_diagnostics):
  """Verify that the diagnostics are as expected; return a list of failures."""

  failures = []
  for loc in sorted(set(actual_diagnostics.keys()) |
                    set(expected_diagnostic_regexes.keys())):
    # Find all regexes and actual diagnostics for the given location.
    regexes = expected_diagnostic_regexes.get(loc, [])
    diagnostics = actual_diagnostics.get(loc, [])
    failures += _VerifyDiagnosticsAtLoc('\n%s:%s:' % loc, regexes, diagnostics)

  return failures


def _CompareExpectedAndActualSummaries(expected_summaries, actual_summaries):
  """Verify that the summaries are as expected; return a list of failures."""

  failures = []
  for loc in sorted(set(actual_summaries.keys()) |
                    set(expected_summaries.keys())):
    this_failure = difflib.unified_diff(expected_summaries.get(loc, []),
                                        actual_summaries.get(loc, []))
    try:
      _PortableNext(this_failure)     # read past the 'what files are this' header
      failures.append('\n')
      failures.append('Unexpected summary diffs for %s:\n' % loc)
      failures.extend(this_failure)
      failures.append('---\n')
    except StopIteration:
      pass                    # empty diff
  return failures


def _GetLaunchArguments(cc_file):
  """Gets IWYU launch arguments for a source file from its contents."""
  args = ''
  with open(cc_file) as it:
    # Find the first '// IWYU_ARGS: ' line.
    for lineno, line in enumerate(it):
      m = _IWYU_TEST_RUN_ARGS_RE.match(line)
      if m:
        args = m.group(1)
        break

    for line in it:
      # Consume all comment lines until we hit one that doesn't have a
      # multi-line continuation.
      if not line.startswith('// ') or not args.endswith('\\'):
         break
      line = line[3:].strip()
      args = args[:-1] + ' ' + line

    if args.endswith('\\'):
      raise SyntaxError('%s:%s syntax error in multiline IWYU_ARGS' % 
          (cc_file, lineno))

  return args


def TestIwyuOnRelativeFile(cc_file, cpp_files_to_check, verbose=False):
  """Checks running IWYU on the given .cc file.

  Args:
    cc_file: The name of the file to test, relative to the current dir.
    cpp_files_to_check: A list of filenames for the files
              to check the diagnostics on, relative to the current dir.
    verbose: Whether to display verbose output.
  """
  verbosity_flags = []
  env_verbose_level = os.getenv('IWYU_VERBOSE')
  if env_verbose_level:
    verbosity_flags = ['-Xiwyu', '--verbose=' + env_verbose_level]

  cmd = '%s %s %s %s %s' % (
    _ShellQuote(_GetIwyuPath()),
    # Require verbose level 3 so that we can verify the individual diagnostics.
    # We allow the level to be overriden by
    # * IWYU_ARGS comment in a test file;
    # * iwyu_flags;
    # * IWYU_VERBOSE environment variable;
    '-Xiwyu --verbose=3',
    _GetLaunchArguments(cc_file),
    ' '.join(verbosity_flags),
    cc_file)
  if verbose:
    print('>>> Running %s' % cmd)
  exit_code, output = _GetCommandOutput(cmd)
  print(''.join(output))
  sys.stdout.flush()      # don't commingle this output with the failure output

  # Verify exit code if requested
  expected_exit_code = _GetExpectedExitCode(cc_file)
  if expected_exit_code is not None and exit_code != expected_exit_code:
    raise AssertionError('Unexpected exit code, wanted %d, was %d' %
                         (expected_exit_code, exit_code))

  expected_diagnostics = _GetMatchingLines(
      _EXPECTED_DIAGNOSTICS_RE, cpp_files_to_check)
  failures = _CompareExpectedAndActualDiagnostics(
      _GetExpectedDiagnosticRegexes(expected_diagnostics),
      _GetActualDiagnostics(output))

  # Also figure out if the end-of-parsing suggestions match up.
  failures += _CompareExpectedAndActualSummaries(
      _GetExpectedSummaries(cpp_files_to_check),
      _GetActualSummaries(output))

  if failures:
    raise AssertionError(''.join(failures))