Major refactoring of BreakableToken.

Unify handling of whitespace when breaking protruding tokens with other
whitespace replacements.

As a side effect, the BreakableToken structure changed significantly:
- have a common base class for single-line breakable tokens, as they are
  much more similar
- revamp handling of multi-line comments; we now calculate the
  information about lines in multi-line comments similar to normal
  tokens, and always issue replacements

As a result, we were able to get rid of special casing of trailing
whitespace deletion for comments in the whitespace manager and the
BreakableToken and fixed bugs related to tab handling and escaped
newlines.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@182738 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 8 insertions, 6 deletions

diff --git a/lib/Format/UnwrappedLineParser.h b/lib/Format/UnwrappedLineParser.h
index c738ef13c6..3359745a9f 100644
--- a/lib/Format/UnwrappedLineParser.h
+++ b/lib/Format/UnwrappedLineParser.h
@@ -28,9 +28,8 @@ namespace format {
 /// whitespace characters preceeding it.
 struct FormatToken {
   FormatToken()
-      : NewlinesBefore(0), HasUnescapedNewline(false),
-        LastNewlineOffset(0), TokenLength(0), IsFirst(false),
-        MustBreakBefore(false), TrailingWhiteSpaceLength(0) {}
+      : NewlinesBefore(0), HasUnescapedNewline(false), LastNewlineOffset(0),
+        TokenLength(0), IsFirst(false), MustBreakBefore(false) {}
 
   /// \brief The \c Token.
   Token Tok;
@@ -66,9 +65,6 @@ struct FormatToken {
   /// before the token.
   bool MustBreakBefore;
 
-  /// \brief Number of characters of trailing whitespace.
-  unsigned TrailingWhiteSpaceLength;
-
   /// \brief Returns actual token start location without leading escaped
   /// newlines and whitespace.
   ///
@@ -77,6 +73,12 @@ struct FormatToken {
   SourceLocation getStartOfNonWhitespace() const {
     return WhitespaceRange.getEnd();
   }
+
+  /// \brief The raw text of the token.
+  ///
+  /// Contains the raw token text without leading whitespace and without leading
+  /// escaped newlines.
+  StringRef TokenText;
 };
 
 /// \brief An unwrapped line is a sequence of \c Token, that we would like to