[clangd] Switch xrefs and documenthighlight to annotated-code unit tests. NFC

Summary:
The goal here is again to make it easier to read and write the tests.

I've extracted `parseTextMarker` from CodeCompleteTests into an `Annotations`
class, adding features to it:
  - as well as points `^s` it allows ranges `[[...]]`
  - multiple points and ranges are supported
  - points and ranges may be named: `$name^` and `$name[[...]]`

These features are used for the xrefs tests. This also paves the way for
replacing the lit diagnostics.test with more readable unit tests, using named
ranges.

Alternative considered: `TestSelectionRange` in clang-refactor/TestSupport
Main problems were:
 - delimiting the end of ranges is awkward, requiring counting
 - comment syntax is long and at least as cryptic for most cases
 - no separate syntax for point vs range, which keeps xrefs tests concise
 - Still need to convert to Position everywhere
 - Still need helpers for common case of expecting exactly one point/range

(I'll probably promote the extra `PrintTo`s from some of the core Protocol types
into `operator<<` in `Protocol.h` itself in a separate, prior patch...)

Reviewers: ioeric

Subscribers: klimek, mgorny, ilya-biryukov, cfe-commits

Differential Revision: https://reviews.llvm.org/D41432

git-svn-id: https://llvm.org/svn/llvm-project/clang-tools-extra/trunk@321184 91177308-0d34-0410-b5e6-96231b3b80d8

6 files changed, 395 insertions, 37 deletions

diff --git a/unittests/clangd/Annotations.cpp b/unittests/clangd/Annotations.cpp
new file mode 100644
index 00000000..69532214
--- /dev/null
+++ b/unittests/clangd/Annotations.cpp
@@ -0,0 +1,87 @@
+//===--- Annotations.cpp - Annotated source code for unit tests -*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+#include "Annotations.h"
+#include "SourceCode.h"
+
+namespace clang {
+namespace clangd {
+using namespace llvm;
+
+// Crash if the assertion fails, printing the message and testcase.
+// More elegant error handling isn't needed for unit tests.
+static void require(bool Assertion, const char *Msg, llvm::StringRef Code) {
+  if (!Assertion) {
+    llvm::errs() << "Annotated testcase: " << Msg << "\n" << Code << "\n";
+    llvm_unreachable("Annotated testcase assertion failed!");
+  }
+}
+
+Annotations::Annotations(StringRef Text) {
+  auto Here = [this] { return offsetToPosition(Code, Code.size()); };
+  auto Require = [this, Text](bool Assertion, const char *Msg) {
+    require(Assertion, Msg, Text);
+  };
+  Optional<StringRef> Name;
+  SmallVector<std::pair<StringRef, Position>, 8> OpenRanges;
+
+  Code.reserve(Text.size());
+  while (!Text.empty()) {
+    if (Text.consume_front("^")) {
+      Points[Name.getValueOr("")].push_back(Here());
+      Name = None;
+      continue;
+    }
+    if (Text.consume_front("[[")) {
+      OpenRanges.emplace_back(Name.getValueOr(""), Here());
+      Name = None;
+      continue;
+    }
+    Require(!Name, "$name should be followed by ^ or [[");
+    if (Text.consume_front("]]")) {
+      Require(!OpenRanges.empty(), "unmatched ]]");
+      Ranges[OpenRanges.back().first].push_back(
+          {OpenRanges.back().second, Here()});
+      OpenRanges.pop_back();
+      continue;
+    }
+    if (Text.consume_front("$")) {
+      Name = Text.take_while(llvm::isAlnum);
+      Text = Text.drop_front(Name->size());
+      continue;
+    }
+    Code.push_back(Text.front());
+    Text = Text.drop_front();
+  }
+  Require(!Name, "unterminated $name");
+  Require(OpenRanges.empty(), "unmatched [[");
+}
+
+Position Annotations::point(llvm::StringRef Name) const {
+  auto I = Points.find(Name);
+  require(I != Points.end() && I->getValue().size() == 1,
+          "expected exactly one point", Code);
+  return I->getValue()[0];
+}
+std::vector<Position> Annotations::points(llvm::StringRef Name) const {
+  auto P = Points.lookup(Name);
+  return {P.begin(), P.end()};
+}
+Range Annotations::range(llvm::StringRef Name) const {
+  auto I = Ranges.find(Name);
+  require(I != Ranges.end() && I->getValue().size() == 1,
+          "expected exactly one range", Code);
+  return I->getValue()[0];
+}
+std::vector<Range> Annotations::ranges(llvm::StringRef Name) const {
+  auto R = Ranges.lookup(Name);
+  return {R.begin(), R.end()};
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/unittests/clangd/Annotations.h b/unittests/clangd/Annotations.h
new file mode 100644
index 00000000..b376dd3f
--- /dev/null
+++ b/unittests/clangd/Annotations.h
@@ -0,0 +1,69 @@
+//===--- Annotations.h - Annotated source code for tests --------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// Annotations lets you mark points and ranges inside source code, for tests:
+//
+//    Annotations Example(R"cpp(
+//       int complete() { x.pri^ }          // ^ indicates a point
+//       void err() { [["hello" == 42]]; }  // [[this is a range]]
+//       $definition^class Foo{};           // points can be named: "definition"
+//       $fail[[static_assert(false, "")]]  // ranges can be named too: "fail"
+//    )cpp");
+//
+//    StringRef Code = Example.code();              // annotations stripped.
+//    std::vector<Position> PP = Example.points();  // all unnamed points
+//    Position P = Example.point();                 // there must be exactly one
+//    Range R = Example.range("fail");              // find named ranges
+//
+// Points/ranges are coordinates into `code()` which is stripped of annotations.
+//
+// Ranges may be nested (and points can be inside ranges), but there's no way
+// to define general overlapping ranges.
+//
+//===---------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_ANNOTATIONS_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_ANNOTATIONS_H
+#include "Protocol.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+namespace clangd {
+
+class Annotations {
+public:
+  // Parses the annotations from Text. Crashes if it's malformed.
+  Annotations(llvm::StringRef Text);
+
+  // The input text with all annotations stripped.
+  // All points and ranges are relative to this stripped text.
+  llvm::StringRef code() const { return Code; }
+
+  // Returns the position of the point marked by ^ (or $name^) in the text.
+  // Crashes if there isn't exactly one.
+  Position point(llvm::StringRef Name = "") const;
+  // Returns the position of all points marked by ^ (or $name^) in the text.
+  std::vector<Position> points(llvm::StringRef Name = "") const;
+
+  // Returns the location of the range marked by [[ ]] (or $name[[ ]]).
+  // Crashes if there isn't exactly one.
+  Range range(llvm::StringRef Name = "") const;
+  // Returns the location of all ranges marked by [[ ]] (or $name[[ ]]).
+  std::vector<Range> ranges(llvm::StringRef Name = "") const;
+
+private:
+  std::string Code;
+  llvm::StringMap<llvm::SmallVector<Position, 1>> Points;
+  llvm::StringMap<llvm::SmallVector<Range, 1>> Ranges;
+};
+
+} // namespace clangd
+} // namespace clang
+#endif
diff --git a/unittests/clangd/CMakeLists.txt b/unittests/clangd/CMakeLists.txt
index 36853a17..ffe11b77 100644
--- a/unittests/clangd/CMakeLists.txt
+++ b/unittests/clangd/CMakeLists.txt
@@ -9,6 +9,7 @@ include_directories(
   )
 
 add_extra_unittest(ClangdTests
+  Annotations.cpp
   ClangdTests.cpp
   CodeCompleteTests.cpp
   ContextTests.cpp
@@ -20,6 +21,7 @@ add_extra_unittest(ClangdTests
   TraceTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
+  XRefsTests.cpp
   )
 
 target_link_libraries(ClangdTests
diff --git a/unittests/clangd/CodeCompleteTests.cpp b/unittests/clangd/CodeCompleteTests.cpp
index 312725fc..7411e1c3 100644
--- a/unittests/clangd/CodeCompleteTests.cpp
+++ b/unittests/clangd/CodeCompleteTests.cpp
@@ -7,7 +7,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Annotations.h"
 #include "ClangdServer.h"
+#include "CodeComplete.h"
 #include "Compiler.h"
 #include "Context.h"
 #include "Matchers.h"
@@ -60,27 +62,6 @@ class IgnoreDiagnostics : public DiagnosticsConsumer {
       PathRef File, Tagged<std::vector<DiagWithFixIts>> Diagnostics) override {}
 };
 
-struct StringWithPos {
-  std::string Text;
-  clangd::Position MarkerPos;
-};
-
-/// Accepts a source file with a cursor marker ^.
-/// Returns the source file with the marker removed, and the marker position.
-StringWithPos parseTextMarker(StringRef Text) {
-  std::size_t MarkerOffset = Text.find('^');
-  assert(MarkerOffset != StringRef::npos && "^ wasn't found in Text.");
-
-  std::string WithoutMarker;
-  WithoutMarker += Text.take_front(MarkerOffset);
-  WithoutMarker += Text.drop_front(MarkerOffset + 1);
-  assert(StringRef(WithoutMarker).find('^') == StringRef::npos &&
-         "There were multiple occurences of ^ inside Text");
-
-  auto MarkerPos = offsetToPosition(WithoutMarker, MarkerOffset);
-  return {std::move(WithoutMarker), MarkerPos};
-}
-
 // GMock helpers for matching completion items.
 MATCHER_P(Named, Name, "") { return arg.insertText == Name; }
 MATCHER_P(Labeled, Label, "") { return arg.label == Label; }
@@ -112,9 +93,9 @@ CompletionList completions(StringRef Text,
   ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
                       /*StorePreamblesInMemory=*/true);
   auto File = getVirtualTestFilePath("foo.cpp");
-  auto Test = parseTextMarker(Text);
-  Server.addDocument(Context::empty(), File, Test.Text);
-  return Server.codeComplete(Context::empty(), File, Test.MarkerPos, Opts)
+  Annotations Test(Text);
+  Server.addDocument(Context::empty(), File, Test.code());
+  return Server.codeComplete(Context::empty(), File, Test.point(), Opts)
       .get()
       .second.Value;
 }
@@ -291,13 +272,13 @@ TEST(CompletionTest, CheckContentsOverride) {
   auto File = getVirtualTestFilePath("foo.cpp");
   Server.addDocument(Context::empty(), File, "ignored text!");
 
-  auto Example = parseTextMarker("int cbc; int b = ^;");
-  auto Results =
-      Server
-          .codeComplete(Context::empty(), File, Example.MarkerPos,
-                        clangd::CodeCompleteOptions(), StringRef(Example.Text))
-          .get()
-          .second.Value;
+  Annotations Example("int cbc; int b = ^;");
+  auto Results = Server
+                     .codeComplete(Context::empty(), File, Example.point(),
+                                   clangd::CodeCompleteOptions(),
+                                   StringRef(Example.code()))
+                     .get()
+                     .second.Value;
   EXPECT_THAT(Results.items, Contains(Named("cbc")));
 }
 
@@ -392,9 +373,9 @@ SignatureHelp signatures(StringRef Text) {
   ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
                       /*StorePreamblesInMemory=*/true);
   auto File = getVirtualTestFilePath("foo.cpp");
-  auto Test = parseTextMarker(Text);
-  Server.addDocument(Context::empty(), File, Test.Text);
-  auto R = Server.signatureHelp(Context::empty(), File, Test.MarkerPos);
+  Annotations Test(Text);
+  Server.addDocument(Context::empty(), File, Test.code());
+  auto R = Server.signatureHelp(Context::empty(), File, Test.point());
   assert(R);
   return R.get().Value;
 }
@@ -573,13 +554,13 @@ TEST(CompletionTest, ASTIndexMultiFile) {
       .wait();
 
   auto File = getVirtualTestFilePath("bar.cpp");
-  auto Test = parseTextMarker(R"cpp(
+  Annotations Test(R"cpp(
       namespace ns { class XXX {}; void fooooo() {} }
       void f() { ns::^ }
   )cpp");
-  Server.addDocument(Context::empty(), File, Test.Text).wait();
+  Server.addDocument(Context::empty(), File, Test.code()).wait();
 
-  auto Results = Server.codeComplete(Context::empty(), File, Test.MarkerPos, {})
+  auto Results = Server.codeComplete(Context::empty(), File, Test.point(), {})
                      .get()
                      .second.Value;
   // "XYZ" and "foo" are not included in the file being completed but are still
diff --git a/unittests/clangd/Matchers.h b/unittests/clangd/Matchers.h
index 073a5525..81bc110b 100644
--- a/unittests/clangd/Matchers.h
+++ b/unittests/clangd/Matchers.h
@@ -13,6 +13,7 @@
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_MATCHERS_H
 #define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_MATCHERS_H
+#include "Protocol.h"
 #include "gmock/gmock.h"
 
 namespace clang {
diff --git a/unittests/clangd/XRefsTests.cpp b/unittests/clangd/XRefsTests.cpp
new file mode 100644
index 00000000..db158e7b
--- /dev/null
+++ b/unittests/clangd/XRefsTests.cpp
@@ -0,0 +1,218 @@
+//===-- XRefsTests.cpp  ---------------------------*- C++ -*--------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#include "Annotations.h"
+#include "ClangdUnit.h"
+#include "Matchers.h"
+#include "XRefs.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/Frontend/Utils.h"
+#include "llvm/Support/Path.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+using namespace llvm;
+
+void PrintTo(const DocumentHighlight &V, std::ostream *O) {
+  llvm::raw_os_ostream OS(*O);
+  OS << V.range;
+  if (V.kind == DocumentHighlightKind::Read)
+    OS << "(r)";
+  if (V.kind == DocumentHighlightKind::Write)
+    OS << "(w)";
+}
+
+namespace {
+using testing::ElementsAre;
+using testing::Field;
+using testing::Matcher;
+using testing::UnorderedElementsAreArray;
+
+// FIXME: this is duplicated with FileIndexTests. Share it.
+ParsedAST build(StringRef Code) {
+  auto CI = createInvocationFromCommandLine({"clang", "-xc++", "Foo.cpp"});
+  auto Buf = MemoryBuffer::getMemBuffer(Code);
+  auto AST = ParsedAST::Build(
+      Context::empty(), std::move(CI), nullptr, std::move(Buf),
+      std::make_shared<PCHContainerOperations>(), vfs::getRealFileSystem());
+  assert(AST.hasValue());
+  return std::move(*AST);
+}
+
+// Extracts ranges from an annotated example, and constructs a matcher for a
+// highlight set. Ranges should be named $read/$write as appropriate.
+Matcher<const std::vector<DocumentHighlight> &>
+HighlightsFrom(const Annotations &Test) {
+  std::vector<DocumentHighlight> Expected;
+  auto Add = [&](const Range &R, DocumentHighlightKind K) {
+    Expected.emplace_back();
+    Expected.back().range = R;
+    Expected.back().kind = K;
+  };
+  for (const auto &Range : Test.ranges())
+    Add(Range, DocumentHighlightKind::Text);
+  for (const auto &Range : Test.ranges("read"))
+    Add(Range, DocumentHighlightKind::Read);
+  for (const auto &Range : Test.ranges("write"))
+    Add(Range, DocumentHighlightKind::Write);
+  return UnorderedElementsAreArray(Expected);
+}
+
+TEST(HighlightsTest, All) {
+  const char *Tests[] = {
+      R"cpp(// Local variable
+        int main() {
+          int [[bonjour]];
+          $write[[^bonjour]] = 2;
+          int test1 = $read[[bonjour]];
+        }
+      )cpp",
+
+      R"cpp(// Struct
+        namespace ns1 {
+        struct [[MyClass]] {
+          static void foo([[MyClass]]*) {}
+        };
+        } // namespace ns1
+        int main() {
+          ns1::[[My^Class]]* Params;
+        }
+      )cpp",
+
+      R"cpp(// Function
+        int [[^foo]](int) {}
+        int main() {
+          [[foo]]([[foo]](42));
+          auto *X = &[[foo]];
+        }
+      )cpp",
+  };
+  for (const char *Test : Tests) {
+    Annotations T(Test);
+    auto AST = build(T.code());
+    EXPECT_THAT(findDocumentHighlights(Context::empty(), AST, T.point()),
+                HighlightsFrom(T))
+        << Test;
+  }
+}
+
+MATCHER_P(RangeIs, R, "") { return arg.range == R; }
+
+TEST(GoToDefinition, All) {
+  const char *Tests[] = {
+      R"cpp(// Local variable
+        int main() {
+          [[int bonjour]];
+          ^bonjour = 2;
+          int test1 = bonjour;
+        }
+      )cpp",
+
+      R"cpp(// Struct
+        namespace ns1 {
+        [[struct MyClass {}]];
+        } // namespace ns1
+        int main() {
+          ns1::My^Class* Params;
+        }
+      )cpp",
+
+      R"cpp(// Function definition via pointer
+        [[int foo(int) {}]]
+        int main() {
+          auto *X = &^foo;
+        }
+      )cpp",
+
+      R"cpp(// Function declaration via call
+        [[int foo(int)]];
+        int main() {
+          return ^foo(42);
+        }
+      )cpp",
+
+      R"cpp(// Field
+        struct Foo { [[int x]]; };
+        int main() {
+          Foo bar;
+          bar.^x;
+        }
+      )cpp",
+
+      R"cpp(// Field, member initializer
+        struct Foo {
+          [[int x]];
+          Foo() : ^x(0) {}
+        };
+      )cpp",
+
+      R"cpp(// Field, GNU old-style field designator
+        struct Foo { [[int x]]; };
+        int main() {
+          Foo bar = { ^x : 1 };
+        }
+      )cpp",
+
+      R"cpp(// Field, field designator
+        struct Foo { [[int x]]; };
+        int main() {
+          Foo bar = { .^x = 2 };
+        }
+      )cpp",
+
+      R"cpp(// Method call
+        struct Foo { [[int x()]]; };
+        int main() {
+          Foo bar;
+          bar.^x();
+        }
+      )cpp",
+
+      R"cpp(// Typedef
+        [[typedef int Foo]];
+        int main() {
+          ^Foo bar;
+        }
+      )cpp",
+
+      /* FIXME: clangIndex doesn't handle template type parameters
+      R"cpp(// Template type parameter
+        template <[[typename T]]>
+        void foo() { ^T t; }
+      )cpp", */
+
+      R"cpp(// Namespace
+        [[namespace ns {
+        struct Foo { static void bar(); }
+        }]] // namespace ns
+        int main() { ^ns::Foo::bar(); }
+      )cpp",
+
+      R"cpp(// Macro
+        #define MACRO 0
+        #define [[MACRO 1]]
+        int main() { return ^MACRO; }
+        #define MACRO 2
+        #undef macro
+      )cpp",
+  };
+  for (const char *Test : Tests) {
+    Annotations T(Test);
+    auto AST = build(T.code());
+    EXPECT_THAT(findDefinitions(Context::empty(), AST, T.point()),
+                ElementsAre(RangeIs(T.range())))
+        << Test;
+  }
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang