Add FileCheck -implicit-check-not option to allow stricter tests without adding too...

Summary:
Add FileCheck -implicit-check-not option which allows specifying a
pattern that should only occur in the input when explicitly matched by a
positive check. This feature allows checking tool diagnostics in a way
clang -verify does it for compiler diagnostics.

The option has been tested on a number of clang-tidy checks, I'll post a link to
the clang-tidy patch to this thread.

Once there's an agreement on the general direction, I can add tests and
documentation.

Reviewers: djasper, bkramer

Reviewed By: bkramer

Subscribers: llvm-commits

Differential Revision: http://reviews.llvm.org/D4462

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@212810 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/docs/CommandGuide/FileCheck.rst b/docs/CommandGuide/FileCheck.rst
index 5a60d60ae3df0aaa183091be2d3972bd5c59a45f..5c60b2ad08b82fbcb83673ff80649b47f6351271 100644 (file)
--- a/docs/CommandGuide/FileCheck.rst
+++ b/docs/CommandGuide/FileCheck.rst
@@ -49,6 +49,17 @@ OPTIONS
  The :option:`--strict-whitespace` argument disables this behavior. End-of-line
  sequences are canonicalized to UNIX-style ``\n`` in all modes.
 
+.. option:: --implicit-check-not check-pattern
+
+  Adds implicit negative checks for the specified patterns between positive
+  checks. The option allows writing stricter tests without stuffing them with
+  ``CHECK-NOT``s.
+
+  For example, "``--implicit-check-not warning:``" can be useful when testing
+  diagnostic messages from tools that don't have an option similar to ``clang
+  -verify``. With this option FileCheck will verify that input does not contain
+  warnings not covered by any ``CHECK:`` patterns.
+
 .. option:: -version
 
  Show the version number of this program.

diff --git a/test/FileCheck/implicit-check-not.txt b/test/FileCheck/implicit-check-not.txt
new file mode 100644 (file)
index 0000000..4267736
--- /dev/null
+++ b/test/FileCheck/implicit-check-not.txt
@@ -0,0 +1,44 @@
+; RUN: sed 's#^;.*##' %s | FileCheck -check-prefix=CHECK-PASS -implicit-check-not=warning: %s
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL1 -implicit-check-not=warning: %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR1
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL2 -implicit-check-not=warning: %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR2
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL3 -implicit-check-not=warning: %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR3
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL1 -implicit-check-not='{{aaa|bbb|ccc}}' %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR4
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL1 -implicit-check-not=aaa -implicit-check-not=bbb -implicit-check-not=ccc %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR5
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL2 -implicit-check-not=aaa -implicit-check-not=bbb -implicit-check-not=ccc %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR6
+; RUN: sed 's#^;.*##' %s | not FileCheck -check-prefix=CHECK-FAIL3 -implicit-check-not=aaa -implicit-check-not=bbb -implicit-check-not=ccc %s 2>&1 | FileCheck %s -check-prefix CHECK-ERROR7
+
+warning: aaa
+; CHECK-PASS: warning: aaa
+; CHECK-ERROR1: error: CHECK-FAIL1-NOT: string occurred!
+; CHECK-ERROR1: command line:1:22: note: CHECK-FAIL1-NOT: pattern specified here
+; CHECK-ERROR1-NEXT: -implicit-check-not='warning:'
+; CHECK-FAIL2: warning: aaa
+; CHECK-FAIL3: warning: aaa
+; CHECK-ERROR4: error: CHECK-FAIL1-NOT: string occurred!
+; CHECK-ERROR4: command line:1:22: note: CHECK-FAIL1-NOT: pattern specified here
+; CHECK-ERROR4-NEXT: {{-implicit-check-not='\{\{aaa\|bbb\|ccc\}\}'}}
+; CHECK-ERROR5: error: CHECK-FAIL1-NOT: string occurred!
+; CHECK-ERROR5: command line:1:22: note: CHECK-FAIL1-NOT: pattern specified here
+; CHECK-ERROR5-NEXT: -implicit-check-not='aaa'
+
+warning: bbb
+; CHECK-PASS: warning: bbb
+; CHECK-FAIL1: warning: bbb
+; CHECK-ERROR2: error: CHECK-FAIL2-NOT: string occurred!
+; CHECK-ERROR2: command line:1:22: note: CHECK-FAIL2-NOT: pattern specified here
+; CHECK-ERROR2-NEXT: -implicit-check-not='warning:'
+; CHECK-FAIL3: warning: bbb
+; CHECK-ERROR6: error: CHECK-FAIL2-NOT: string occurred!
+; CHECK-ERROR6: command line:1:22: note: CHECK-FAIL2-NOT: pattern specified here
+; CHECK-ERROR6-NEXT: -implicit-check-not='bbb'
+
+warning: ccc
+; CHECK-PASS: warning: ccc
+; CHECK-FAIL1: warning: ccc
+; CHECK-FAIL2: warning: ccc
+; CHECK-ERROR3: error: CHECK-FAIL3-NOT: string occurred!
+; CHECK-ERROR3: command line:1:22: note: CHECK-FAIL3-NOT: pattern specified here
+; CHECK-ERROR3-NEXT: -implicit-check-not='warning:'
+; CHECK-ERROR7: error: CHECK-FAIL3-NOT: string occurred!
+; CHECK-ERROR7: command line:1:22: note: CHECK-FAIL3-NOT: pattern specified here
+; CHECK-ERROR7-NEXT: -implicit-check-not='ccc'

diff --git a/utils/FileCheck/FileCheck.cpp b/utils/FileCheck/FileCheck.cpp
index d88cf367c9d4bd52d98f6ed0e1911e0105dbc630..9e6a6a2c8ab4df2fad392458945a3383d76275d9 100644 (file)
--- a/utils/FileCheck/FileCheck.cpp
+++ b/utils/FileCheck/FileCheck.cpp
@@ -50,6 +50,13 @@ static cl::opt<bool>
 NoCanonicalizeWhiteSpace("strict-whitespace",
               cl::desc("Do not treat all horizontal whitespace as equivalent"));
 
+static cl::list<std::string> ImplicitCheckNot(
+    "implicit-check-not",
+    cl::desc("Add an implicit negative check with this pattern to every\n"
+             "positive check. This can be used to ensure that no instances of\n"
+             "this pattern occur which are not matched by a positive pattern"),
+    cl::value_desc("pattern"));
+
 typedef cl::list<std::string>::const_iterator prefix_iterator;
 
 //===----------------------------------------------------------------------===//
@@ -837,7 +844,26 @@ static bool ReadCheckFile(SourceMgr &SM,
 
   // Find all instances of CheckPrefix followed by : in the file.
   StringRef Buffer = F->getBuffer();
-  std::vector<Pattern> DagNotMatches;
+
+  std::vector<Pattern> ImplicitNegativeChecks;
+  for (const auto &PatternString : ImplicitCheckNot) {
+    // Create a buffer with fake command line content in order to display the
+    // command line option responsible for the specific implicit CHECK-NOT.
+    std::string Prefix = std::string("-") + ImplicitCheckNot.ArgStr + "='";
+    std::string Suffix = "'";
+    MemoryBuffer *CmdLine = MemoryBuffer::getMemBufferCopy(
+        Prefix + PatternString + Suffix, "command line");
+    StringRef PatternInBuffer =
+        CmdLine->getBuffer().substr(Prefix.size(), PatternString.size());
+    SM.AddNewSourceBuffer(CmdLine, SMLoc());
+
+    ImplicitNegativeChecks.push_back(Pattern(Check::CheckNot));
+    ImplicitNegativeChecks.back().ParsePattern(PatternInBuffer,
+                                               "IMPLICIT-CHECK", SM, 0);
+  }
+
+
+  std::vector<Pattern> DagNotMatches = ImplicitNegativeChecks;
 
   // LineNumber keeps track of the line on which CheckPrefix instances are
   // found.
@@ -910,6 +936,7 @@ static bool ReadCheckFile(SourceMgr &SM,
                                        PatternLoc,
                                        CheckTy));
     std::swap(DagNotMatches, CheckStrings.back().DagNotStrings);
+    DagNotMatches = ImplicitNegativeChecks;
   }
 
   // Add an EOF pattern for any trailing CHECK-DAG/-NOTs, and use the first