[clang-format] Refactor WhitespaceManager and friends

The main motivation behind this is to cleanup the WhitespaceManager and
make it more extensible for future alignment etc. features.
Specifically, WhitespaceManager has started to copy more and more code
that is already present in FormatToken. Instead, I think it makes more
sense to actually store a reference to each FormatToken for each change.

This has as a consequence led to a change in the calculation of indent
levels. Now, we actually compute them for each Token ahead of time,
which should be more efficient as it removes an unsigned value for the
ParenState, which is used during the combinatorial exploration of the
solution space.

No functional changes intended.

Review: https://reviews.llvm.org/D29300

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@293616 91177308-0d34-0410-b5e6-96231b3b80d8

10 files changed, 182 insertions, 211 deletions

diff --git a/lib/Format/BreakableToken.cpp b/lib/Format/BreakableToken.cpp
index 4a29faded1..95d1a63dfa 100644
--- a/lib/Format/BreakableToken.cpp
+++ b/lib/Format/BreakableToken.cpp
@@ -181,10 +181,10 @@ unsigned BreakableSingleLineToken::getLineLengthAfterSplit(
 }
 
 BreakableSingleLineToken::BreakableSingleLineToken(
-    const FormatToken &Tok, unsigned IndentLevel, unsigned StartColumn,
-    StringRef Prefix, StringRef Postfix, bool InPPDirective,
-    encoding::Encoding Encoding, const FormatStyle &Style)
-    : BreakableToken(Tok, IndentLevel, InPPDirective, Encoding, Style),
+    const FormatToken &Tok, unsigned StartColumn, StringRef Prefix,
+    StringRef Postfix, bool InPPDirective, encoding::Encoding Encoding,
+    const FormatStyle &Style)
+    : BreakableToken(Tok, InPPDirective, Encoding, Style),
       StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix) {
   assert(Tok.TokenText.endswith(Postfix));
   Line = Tok.TokenText.substr(
@@ -192,11 +192,11 @@ BreakableSingleLineToken::BreakableSingleLineToken(
 }
 
 BreakableStringLiteral::BreakableStringLiteral(
-    const FormatToken &Tok, unsigned IndentLevel, unsigned StartColumn,
-    StringRef Prefix, StringRef Postfix, bool InPPDirective,
-    encoding::Encoding Encoding, const FormatStyle &Style)
-    : BreakableSingleLineToken(Tok, IndentLevel, StartColumn, Prefix, Postfix,
-                               InPPDirective, Encoding, Style) {}
+    const FormatToken &Tok, unsigned StartColumn, StringRef Prefix,
+    StringRef Postfix, bool InPPDirective, encoding::Encoding Encoding,
+    const FormatStyle &Style)
+    : BreakableSingleLineToken(Tok, StartColumn, Prefix, Postfix, InPPDirective,
+                               Encoding, Style) {}
 
 BreakableToken::Split
 BreakableStringLiteral::getSplit(unsigned LineIndex, unsigned TailOffset,
@@ -218,16 +218,16 @@ void BreakableStringLiteral::insertBreak(unsigned LineIndex,
     --LeadingSpaces;
   Whitespaces.replaceWhitespaceInToken(
       Tok, Prefix.size() + TailOffset + Split.first, Split.second, Postfix,
-      Prefix, InPPDirective, 1, IndentLevel, LeadingSpaces);
+      Prefix, InPPDirective, 1, LeadingSpaces);
 }
 
 BreakableComment::BreakableComment(const FormatToken &Token,
-                                   unsigned IndentLevel, unsigned StartColumn,
+                                   unsigned StartColumn,
                                    unsigned OriginalStartColumn,
                                    bool FirstInLine, bool InPPDirective,
                                    encoding::Encoding Encoding,
                                    const FormatStyle &Style)
-    : BreakableToken(Token, IndentLevel, InPPDirective, Encoding, Style),
+    : BreakableToken(Token, InPPDirective, Encoding, Style),
       StartColumn(StartColumn), OriginalStartColumn(OriginalStartColumn),
       FirstInLine(FirstInLine) {}
 
@@ -254,8 +254,7 @@ void BreakableComment::compressWhitespace(unsigned LineIndex,
   unsigned CharsToRemove = Split.second;
   Whitespaces.replaceWhitespaceInToken(
       tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove, "", "",
-      /*InPPDirective=*/false,
-      /*Newlines=*/0, /*IndentLevel=*/0, /*Spaces=*/1);
+      /*InPPDirective=*/false, /*Newlines=*/0, /*Spaces=*/1);
 }
 
 BreakableToken::Split
@@ -319,11 +318,11 @@ bool BreakableComment::mayReflow(unsigned LineIndex) const {
 }
 
 BreakableBlockComment::BreakableBlockComment(
-    const FormatToken &Token, unsigned IndentLevel, unsigned StartColumn,
+    const FormatToken &Token, unsigned StartColumn,
     unsigned OriginalStartColumn, bool FirstInLine, bool InPPDirective,
     encoding::Encoding Encoding, const FormatStyle &Style)
-    : BreakableComment(Token, IndentLevel, StartColumn, OriginalStartColumn,
-                       FirstInLine, InPPDirective, Encoding, Style) {
+    : BreakableComment(Token, StartColumn, OriginalStartColumn, FirstInLine,
+                       InPPDirective, Encoding, Style) {
   assert(Tok.is(TT_BlockComment) &&
          "block comment section must start with a block comment");
 
@@ -484,7 +483,7 @@ void BreakableBlockComment::insertBreak(unsigned LineIndex, unsigned TailOffset,
   assert(LocalIndentAtLineBreak >= Prefix.size());
   Whitespaces.replaceWhitespaceInToken(
       tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove, "", Prefix,
-      InPPDirective, /*Newlines=*/1, IndentLevel,
+      InPPDirective, /*Newlines=*/1,
       /*Spaces=*/LocalIndentAtLineBreak - Prefix.size());
 }
 
@@ -555,10 +554,9 @@ void BreakableBlockComment::replaceWhitespaceBefore(
         WhitespaceOffsetInToken;
     Whitespaces.replaceWhitespaceInToken(
         tokenAt(LineIndex), WhitespaceOffsetInToken,
-        /*ReplaceChars=*/WhitespaceLength,
-        /*PreviousPostfix=*/"",
-        /*CurrentPrefix=*/ReflowPrefix, InPPDirective,
-        /*Newlines=*/0, IndentLevel, /*Spaces=*/0);
+        /*ReplaceChars=*/WhitespaceLength, /*PreviousPostfix=*/"",
+        /*CurrentPrefix=*/ReflowPrefix, InPPDirective, /*Newlines=*/0,
+        /*Spaces=*/0);
     // Check if we need to also insert a break at the whitespace range.
     // For this we first adapt the reflow split relative to the beginning of the
     // content.
@@ -604,9 +602,8 @@ void BreakableBlockComment::replaceWhitespaceBefore(
                               tokenAt(LineIndex).TokenText.data() -
                               WhitespaceOffsetInToken;
   Whitespaces.replaceWhitespaceInToken(
-      tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength, "",
-      Prefix, InPPDirective, /*Newlines=*/1, IndentLevel,
-      ContentColumn[LineIndex] - Prefix.size());
+      tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength, "", Prefix,
+      InPPDirective, /*Newlines=*/1, ContentColumn[LineIndex] - Prefix.size());
 }
 
 unsigned
@@ -619,11 +616,11 @@ BreakableBlockComment::getContentStartColumn(unsigned LineIndex,
 }
 
 BreakableLineCommentSection::BreakableLineCommentSection(
-    const FormatToken &Token, unsigned IndentLevel, unsigned StartColumn,
+    const FormatToken &Token, unsigned StartColumn,
     unsigned OriginalStartColumn, bool FirstInLine, bool InPPDirective,
     encoding::Encoding Encoding, const FormatStyle &Style)
-    : BreakableComment(Token, IndentLevel, StartColumn, OriginalStartColumn,
-                       FirstInLine, InPPDirective, Encoding, Style) {
+    : BreakableComment(Token, StartColumn, OriginalStartColumn, FirstInLine,
+                       InPPDirective, Encoding, Style) {
   assert(Tok.is(TT_LineComment) &&
          "line comment section must start with a line comment");
   FormatToken *LineTok = nullptr;
@@ -713,7 +710,7 @@ void BreakableLineCommentSection::insertBreak(unsigned LineIndex,
   assert(IndentAtLineBreak >= Prefix[LineIndex].size());
   Whitespaces.replaceWhitespaceInToken(
       tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove, "",
-      Prefix[LineIndex], InPPDirective, /*Newlines=*/1, IndentLevel,
+      Prefix[LineIndex], InPPDirective, /*Newlines=*/1,
       /*Spaces=*/IndentAtLineBreak - Prefix[LineIndex].size());
 }
 
@@ -758,12 +755,9 @@ void BreakableLineCommentSection::replaceWhitespaceBefore(
     if (SplitBefore.first != StringRef::npos) {
       // Reflow happens between tokens. Replace the whitespace between the
       // tokens by the empty string.
-      Whitespaces.replaceWhitespace(*Tokens[LineIndex],
-                                    /*Newlines=*/0,
-                                    /*IndentLevel=*/IndentLevel,
-                                    /*Spaces=*/0,
-                                    /*StartOfTokenColumn=*/StartColumn,
-                                    /*InPPDirective=*/false);
+      Whitespaces.replaceWhitespace(
+          *Tokens[LineIndex], /*Newlines=*/0, /*Spaces=*/0,
+          /*StartOfTokenColumn=*/StartColumn, /*InPPDirective=*/false);
       // Replace the indent and prefix of the token with the reflow prefix.
       unsigned WhitespaceLength =
           Content[LineIndex].data() - tokenAt(LineIndex).TokenText.data();
@@ -774,7 +768,6 @@ void BreakableLineCommentSection::replaceWhitespaceBefore(
                                            /*CurrentPrefix=*/ReflowPrefix,
                                            /*InPPDirective=*/false,
                                            /*Newlines=*/0,
-                                           /*IndentLevel=*/IndentLevel,
                                            /*Spaces=*/0);
     } else {
       // This is the first line for the current token, but no reflow with the
@@ -786,7 +779,6 @@ void BreakableLineCommentSection::replaceWhitespaceBefore(
       if (tokenAt(LineIndex).OriginalColumn != LineColumn) {
         Whitespaces.replaceWhitespace(*Tokens[LineIndex],
                                       /*Newlines=*/1,
-                                      /*IndentLevel=*/IndentLevel,
                                       /*Spaces=*/LineColumn,
                                       /*StartOfTokenColumn=*/LineColumn,
                                       /*InPPDirective=*/false);
@@ -806,9 +798,7 @@ void BreakableLineCommentSection::replaceWhitespaceBefore(
            "at most a space");
     Whitespaces.replaceWhitespaceInToken(
         tokenAt(LineIndex), OriginalPrefix[LineIndex].size(), 0, "", "",
-        /*InPPDirective=*/false,
-        /*Newlines=*/0, /*IndentLevel=*/0,
-        /*Spaces=*/1);
+        /*InPPDirective=*/false, /*Newlines=*/0, /*Spaces=*/1);
   }
   // Add a break after a reflow split has been introduced, if necessary.
   // Note that this break doesn't need to be penalized, since it doesn't change
diff --git a/lib/Format/BreakableToken.h b/lib/Format/BreakableToken.h
index 848a76ddca..cf8105347a 100644
--- a/lib/Format/BreakableToken.h
+++ b/lib/Format/BreakableToken.h
@@ -149,14 +149,12 @@ public:
   virtual void updateNextToken(LineState &State) const {}
 
 protected:
-  BreakableToken(const FormatToken &Tok, unsigned IndentLevel,
-                 bool InPPDirective, encoding::Encoding Encoding,
-                 const FormatStyle &Style)
-      : Tok(Tok), IndentLevel(IndentLevel), InPPDirective(InPPDirective),
-        Encoding(Encoding), Style(Style) {}
+  BreakableToken(const FormatToken &Tok, bool InPPDirective,
+                 encoding::Encoding Encoding, const FormatStyle &Style)
+      : Tok(Tok), InPPDirective(InPPDirective), Encoding(Encoding),
+        Style(Style) {}
 
   const FormatToken &Tok;
-  const unsigned IndentLevel;
   const bool InPPDirective;
   const encoding::Encoding Encoding;
   const FormatStyle &Style;
@@ -172,10 +170,9 @@ public:
                                    StringRef::size_type Length) const override;
 
 protected:
-  BreakableSingleLineToken(const FormatToken &Tok, unsigned IndentLevel,
-                           unsigned StartColumn, StringRef Prefix,
-                           StringRef Postfix, bool InPPDirective,
-                           encoding::Encoding Encoding,
+  BreakableSingleLineToken(const FormatToken &Tok, unsigned StartColumn,
+                           StringRef Prefix, StringRef Postfix,
+                           bool InPPDirective, encoding::Encoding Encoding,
                            const FormatStyle &Style);
 
   // The column in which the token starts.
@@ -194,10 +191,10 @@ public:
   ///
   /// \p StartColumn specifies the column in which the token will start
   /// after formatting.
-  BreakableStringLiteral(const FormatToken &Tok, unsigned IndentLevel,
-                         unsigned StartColumn, StringRef Prefix,
-                         StringRef Postfix, bool InPPDirective,
-                         encoding::Encoding Encoding, const FormatStyle &Style);
+  BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn,
+                         StringRef Prefix, StringRef Postfix,
+                         bool InPPDirective, encoding::Encoding Encoding,
+                         const FormatStyle &Style);
 
   Split getSplit(unsigned LineIndex, unsigned TailOffset,
                  unsigned ColumnLimit) const override;
@@ -215,10 +212,10 @@ protected:
   /// after formatting, while \p OriginalStartColumn specifies in which
   /// column the comment started before formatting.
   /// If the comment starts a line after formatting, set \p FirstInLine to true.
-  BreakableComment(const FormatToken &Token, unsigned IndentLevel,
-                   unsigned StartColumn, unsigned OriginalStartColumn,
-                   bool FirstInLine, bool InPPDirective,
-                   encoding::Encoding Encoding, const FormatStyle &Style);
+  BreakableComment(const FormatToken &Token, unsigned StartColumn,
+                   unsigned OriginalStartColumn, bool FirstInLine,
+                   bool InPPDirective, encoding::Encoding Encoding,
+                   const FormatStyle &Style);
 
 public:
   unsigned getLineCount() const override;
@@ -299,10 +296,10 @@ protected:
 
 class BreakableBlockComment : public BreakableComment {
 public:
-  BreakableBlockComment(const FormatToken &Token, unsigned IndentLevel,
-                        unsigned StartColumn, unsigned OriginalStartColumn,
-                        bool FirstInLine, bool InPPDirective,
-                        encoding::Encoding Encoding, const FormatStyle &Style);
+  BreakableBlockComment(const FormatToken &Token, unsigned StartColumn,
+                        unsigned OriginalStartColumn, bool FirstInLine,
+                        bool InPPDirective, encoding::Encoding Encoding,
+                        const FormatStyle &Style);
 
   unsigned getLineLengthAfterSplit(unsigned LineIndex,
                                    unsigned TailOffset,
@@ -363,8 +360,7 @@ private:
 
 class BreakableLineCommentSection : public BreakableComment {
 public:
-  BreakableLineCommentSection(const FormatToken &Token, unsigned IndentLevel,
-                              unsigned StartColumn,
+  BreakableLineCommentSection(const FormatToken &Token, unsigned StartColumn,
                               unsigned OriginalStartColumn, bool FirstInLine,
                               bool InPPDirective, encoding::Encoding Encoding,
                               const FormatStyle &Style);
diff --git a/lib/Format/ContinuationIndenter.cpp b/lib/Format/ContinuationIndenter.cpp
index 05408bfa4f..13a6177669 100644
--- a/lib/Format/ContinuationIndenter.cpp
+++ b/lib/Format/ContinuationIndenter.cpp
@@ -80,7 +80,7 @@ LineState ContinuationIndenter::getInitialState(unsigned FirstIndent,
   State.Column = FirstIndent;
   State.Line = Line;
   State.NextToken = Line->First;
-  State.Stack.push_back(ParenState(FirstIndent, Line->Level, FirstIndent,
+  State.Stack.push_back(ParenState(FirstIndent, FirstIndent,
                                    /*AvoidBinPacking=*/false,
                                    /*NoLineBreak=*/false));
   State.LineContainsContinuedForLoopSection = false;
@@ -347,8 +347,8 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
   unsigned Spaces = Current.SpacesRequiredBefore + ExtraSpaces;
 
   if (!DryRun)
-    Whitespaces.replaceWhitespace(Current, /*Newlines=*/0, /*IndentLevel=*/0,
-                                  Spaces, State.Column + Spaces);
+    Whitespaces.replaceWhitespace(Current, /*Newlines=*/0, Spaces,
+                                  State.Column + Spaces);
 
   if (Current.is(TT_SelectorName) &&
       !State.Stack.back().ObjCSelectorNameFound) {
@@ -574,9 +574,8 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
   if (!DryRun) {
     unsigned Newlines = std::max(
         1u, std::min(Current.NewlinesBefore, Style.MaxEmptyLinesToKeep + 1));
-    Whitespaces.replaceWhitespace(Current, Newlines,
-                                  State.Stack.back().IndentLevel, State.Column,
-                                  State.Column, State.Line->InPPDirective);
+    Whitespaces.replaceWhitespace(Current, Newlines, State.Column, State.Column,
+                                  State.Line->InPPDirective);
   }
 
   if (!Current.isTrailingComment())
@@ -953,7 +952,6 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
   }
 
   unsigned NewIndent;
-  unsigned NewIndentLevel = State.Stack.back().IndentLevel;
   unsigned LastSpace = State.Stack.back().LastSpace;
   bool AvoidBinPacking;
   bool BreakBeforeParameter = false;
@@ -963,7 +961,6 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
     if (Current.opensBlockOrBlockTypeList(Style)) {
       NewIndent = State.Stack.back().NestedBlockIndent + Style.IndentWidth;
       NewIndent = std::min(State.Column + 2, NewIndent);
-      ++NewIndentLevel;
     } else {
       NewIndent = State.Stack.back().LastSpace + Style.ContinuationIndentWidth;
     }
@@ -1032,8 +1029,8 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
        State.Stack.back().NoLineBreakInOperand ||
        (Current.is(TT_TemplateOpener) &&
         State.Stack.back().ContainsUnwrappedBuilder));
-  State.Stack.push_back(ParenState(NewIndent, NewIndentLevel, LastSpace,
-                                   AvoidBinPacking, NoLineBreak));
+  State.Stack.push_back(
+      ParenState(NewIndent, LastSpace, AvoidBinPacking, NoLineBreak));
   State.Stack.back().NestedBlockIndent = NestedBlockIndent;
   State.Stack.back().BreakBeforeParameter = BreakBeforeParameter;
   State.Stack.back().HasMultipleNestedBlocks = Current.BlockParameterCount > 1;
@@ -1067,10 +1064,9 @@ void ContinuationIndenter::moveStateToNewBlock(LineState &State) {
       NestedBlockIndent + (State.NextToken->is(TT_ObjCBlockLBrace)
                                ? Style.ObjCBlockIndentWidth
                                : Style.IndentWidth);
-  State.Stack.push_back(ParenState(
-      NewIndent, /*NewIndentLevel=*/State.Stack.back().IndentLevel + 1,
-      State.Stack.back().LastSpace, /*AvoidBinPacking=*/true,
-      /*NoLineBreak=*/false));
+  State.Stack.push_back(ParenState(NewIndent, State.Stack.back().LastSpace,
+                                   /*AvoidBinPacking=*/true,
+                                   /*NoLineBreak=*/false));
   State.Stack.back().NestedBlockIndent = NestedBlockIndent;
   State.Stack.back().BreakBeforeParameter = true;
 }
@@ -1153,9 +1149,9 @@ unsigned ContinuationIndenter::breakProtrudingToken(const FormatToken &Current,
           Text.startswith(Prefix = "u8\"") ||
           Text.startswith(Prefix = "L\""))) ||
         (Text.startswith(Prefix = "_T(\"") && Text.endswith(Postfix = "\")"))) {
-      Token.reset(new BreakableStringLiteral(
-          Current, State.Line->Level, StartColumn, Prefix, Postfix,
-          State.Line->InPPDirective, Encoding, Style));
+      Token.reset(new BreakableStringLiteral(Current, StartColumn, Prefix,
+                                             Postfix, State.Line->InPPDirective,
+                                             Encoding, Style));
     } else {
       return 0;
     }
@@ -1168,8 +1164,8 @@ unsigned ContinuationIndenter::breakProtrudingToken(const FormatToken &Current,
         switchesFormatting(Current))
       return addMultilineToken(Current, State);
     Token.reset(new BreakableBlockComment(
-        Current, State.Line->Level, StartColumn, Current.OriginalColumn,
-        !Current.Previous, State.Line->InPPDirective, Encoding, Style));
+        Current, StartColumn, Current.OriginalColumn, !Current.Previous,
+        State.Line->InPPDirective, Encoding, Style));
   } else if (Current.is(TT_LineComment) &&
              (Current.Previous == nullptr ||
               Current.Previous->isNot(TT_ImplicitStringLiteral))) {
@@ -1178,8 +1174,7 @@ unsigned ContinuationIndenter::breakProtrudingToken(const FormatToken &Current,
         switchesFormatting(Current))
       return 0;
     Token.reset(new BreakableLineCommentSection(
-        Current, State.Line->Level, StartColumn, Current.OriginalColumn,
-        !Current.Previous,
+        Current, StartColumn, Current.OriginalColumn, !Current.Previous,
         /*InPPDirective=*/false, Encoding, Style));
     // We don't insert backslashes when breaking line comments.
     ColumnLimit = Style.ColumnLimit;
diff --git a/lib/Format/ContinuationIndenter.h b/lib/Format/ContinuationIndenter.h
index 704b2f891f..9a06aa6f62 100644
--- a/lib/Format/ContinuationIndenter.h
+++ b/lib/Format/ContinuationIndenter.h
@@ -146,24 +146,20 @@ private:
 };
 
 struct ParenState {
-  ParenState(unsigned Indent, unsigned IndentLevel, unsigned LastSpace,
-             bool AvoidBinPacking, bool NoLineBreak)
-      : Indent(Indent), IndentLevel(IndentLevel), LastSpace(LastSpace),
-        NestedBlockIndent(Indent), BreakBeforeClosingBrace(false),
-        AvoidBinPacking(AvoidBinPacking), BreakBeforeParameter(false),
-        NoLineBreak(NoLineBreak), NoLineBreakInOperand(false),
-        LastOperatorWrapped(true), ContainsLineBreak(false),
-        ContainsUnwrappedBuilder(false), AlignColons(true),
-        ObjCSelectorNameFound(false), HasMultipleNestedBlocks(false),
-        NestedBlockInlined(false) {}
+  ParenState(unsigned Indent, unsigned LastSpace, bool AvoidBinPacking,
+             bool NoLineBreak)
+      : Indent(Indent), LastSpace(LastSpace), NestedBlockIndent(Indent),
+        BreakBeforeClosingBrace(false), AvoidBinPacking(AvoidBinPacking),
+        BreakBeforeParameter(false), NoLineBreak(NoLineBreak),
+        NoLineBreakInOperand(false), LastOperatorWrapped(true),
+        ContainsLineBreak(false), ContainsUnwrappedBuilder(false),
+        AlignColons(true), ObjCSelectorNameFound(false),
+        HasMultipleNestedBlocks(false), NestedBlockInlined(false) {}
 
   /// \brief The position to which a specific parenthesis level needs to be
   /// indented.
   unsigned Indent;
 
-  /// \brief The number of indentation levels of the block.
-  unsigned IndentLevel;
-
   /// \brief The position of the last space on each level.
   ///
   /// Used e.g. to break like:
diff --git a/lib/Format/FormatToken.h b/lib/Format/FormatToken.h
index ea3bbe368d..b22e4c5d7c 100644
--- a/lib/Format/FormatToken.h
+++ b/lib/Format/FormatToken.h
@@ -220,6 +220,9 @@ struct FormatToken {
   /// [], {} or <>.
   unsigned NestingLevel = 0;
 
+  /// \brief The indent level of this token. Copied from the surrounding line.
+  unsigned IndentLevel = 0;
+
   /// \brief Penalty for inserting a line break before this token.
   unsigned SplitPenalty = 0;
 
diff --git a/lib/Format/TokenAnnotator.cpp b/lib/Format/TokenAnnotator.cpp
index 2e8f3ec920..61d1443256 100644
--- a/lib/Format/TokenAnnotator.cpp
+++ b/lib/Format/TokenAnnotator.cpp
@@ -1759,8 +1759,6 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) {
 
   Line.First->TotalLength =
       Line.First->IsMultiline ? Style.ColumnLimit : Line.First->ColumnWidth;
-  if (!Line.First->Next)
-    return;
   FormatToken *Current = Line.First->Next;
   bool InFunctionDecl = Line.MightBeFunctionDecl;
   while (Current) {
@@ -1836,9 +1834,18 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) {
   }
 
   calculateUnbreakableTailLengths(Line);
+  unsigned IndentLevel = Line.Level;
   for (Current = Line.First; Current != nullptr; Current = Current->Next) {
     if (Current->Role)
       Current->Role->precomputeFormattingInfos(Current);
+    if (Current->MatchingParen &&
+        Current->MatchingParen->opensBlockOrBlockTypeList(Style)) {
+      assert(IndentLevel > 0);
+      --IndentLevel;
+    }
+    Current->IndentLevel = IndentLevel;
+    if (Current->opensBlockOrBlockTypeList(Style))
+      ++IndentLevel;
   }
 
   DEBUG({ printDebugInfo(Line); });
diff --git a/lib/Format/UnwrappedLineFormatter.cpp b/lib/Format/UnwrappedLineFormatter.cpp
index d7f1c4232d..957256965f 100644
--- a/lib/Format/UnwrappedLineFormatter.cpp
+++ b/lib/Format/UnwrappedLineFormatter.cpp
@@ -530,34 +530,33 @@ protected:
     if (Previous.Children[0]->First->MustBreakBefore)
       return false;
 
-    // Cannot merge multiple statements into a single line.
-    if (Previous.Children.size() > 1)
-      return false;
-
     // Cannot merge into one line if this line ends on a comment.
     if (Previous.is(tok::comment))
       return false;
 
+    // Cannot merge multiple statements into a single line.
+    if (Previous.Children.size() > 1)
+      return false;
+
+    const AnnotatedLine *Child = Previous.Children[0];
     // We can't put the closing "}" on a line with a trailing comment.
-    if (Previous.Children[0]->Last->isTrailingComment())
+    if (Child->Last->isTrailingComment())
       return false;
 
     // If the child line exceeds the column limit, we wouldn't want to merge it.
     // We add +2 for the trailing " }".
     if (Style.ColumnLimit > 0 &&
-        Previous.Children[0]->Last->TotalLength + State.Column + 2 >
-            Style.ColumnLimit)
+        Child->Last->TotalLength + State.Column + 2 > Style.ColumnLimit)
       return false;
 
     if (!DryRun) {
       Whitespaces->replaceWhitespace(
-          *Previous.Children[0]->First,
-          /*Newlines=*/0, /*IndentLevel=*/0, /*Spaces=*/1,
+          *Child->First, /*Newlines=*/0, /*Spaces=*/1,
           /*StartOfTokenColumn=*/State.Column, State.Line->InPPDirective);
     }
-    Penalty += formatLine(*Previous.Children[0], State.Column + 1, DryRun);
+    Penalty += formatLine(*Child, State.Column + 1, DryRun);
 
-    State.Column += 1 + Previous.Children[0]->Last->TotalLength;
+    State.Column += 1 + Child->Last->TotalLength;
     return true;
   }
 
@@ -841,8 +840,7 @@ UnwrappedLineFormatter::format(const SmallVectorImpl<AnnotatedLine *> &Lines,
 
     if (ShouldFormat && TheLine.Type != LT_Invalid) {
       if (!DryRun)
-        formatFirstToken(*TheLine.First, PreviousLine, TheLine.Level, Indent,
-                         TheLine.InPPDirective);
+        formatFirstToken(TheLine, PreviousLine, Indent);
 
       NextLine = Joiner.getNextMergedLine(DryRun, IndentTracker);
       unsigned ColumnLimit = getColumnLimit(TheLine.InPPDirective, NextLine);
@@ -882,9 +880,8 @@ UnwrappedLineFormatter::format(const SmallVectorImpl<AnnotatedLine *> &Lines,
                               TheLine.LeadingEmptyLinesAffected);
         // Format the first token.
         if (ReformatLeadingWhitespace)
-          formatFirstToken(*TheLine.First, PreviousLine, TheLine.Level,
-                           TheLine.First->OriginalColumn,
-                           TheLine.InPPDirective);
+          formatFirstToken(TheLine, PreviousLine,
+                           TheLine.First->OriginalColumn);
         else
           Whitespaces->addUntouchableToken(*TheLine.First,
                                            TheLine.InPPDirective);
@@ -904,15 +901,14 @@ UnwrappedLineFormatter::format(const SmallVectorImpl<AnnotatedLine *> &Lines,
   return Penalty;
 }
 
-void UnwrappedLineFormatter::formatFirstToken(FormatToken &RootToken,
+void UnwrappedLineFormatter::formatFirstToken(const AnnotatedLine &Line,
                                               const AnnotatedLine *PreviousLine,
-                                              unsigned IndentLevel,
-                                              unsigned Indent,
-                                              bool InPPDirective) {
+                                              unsigned Indent) {
+  FormatToken& RootToken = *Line.First;
   if (RootToken.is(tok::eof)) {
     unsigned Newlines = std::min(RootToken.NewlinesBefore, 1u);
-    Whitespaces->replaceWhitespace(RootToken, Newlines, /*IndentLevel=*/0,
-                                   /*Spaces=*/0, /*TargetColumn=*/0);
+    Whitespaces->replaceWhitespace(RootToken, Newlines, /*Spaces=*/0,
+                                   /*TargetColumn=*/0);
     return;
   }
   unsigned Newlines =
@@ -944,9 +940,9 @@ void UnwrappedLineFormatter::formatFirstToken(FormatToken &RootToken,
       (!PreviousLine->InPPDirective || !RootToken.HasUnescapedNewline))
     Newlines = std::min(1u, Newlines);
 
-  Whitespaces->replaceWhitespace(RootToken, Newlines, IndentLevel, Indent,
-                                 Indent, InPPDirective &&
-                                             !RootToken.HasUnescapedNewline);
+  Whitespaces->replaceWhitespace(RootToken, Newlines, Indent, Indent,
+                                 Line.InPPDirective &&
+                                     !RootToken.HasUnescapedNewline);
 }
 
 unsigned
diff --git a/lib/Format/UnwrappedLineFormatter.h b/lib/Format/UnwrappedLineFormatter.h
index 7bcead9d25..93247f71d6 100644
--- a/lib/Format/UnwrappedLineFormatter.h
+++ b/lib/Format/UnwrappedLineFormatter.h
@@ -44,9 +44,8 @@ public:
 private:
   /// \brief Add a new line and the required indent before the first Token
   /// of the \c UnwrappedLine if there was no structural parsing error.
-  void formatFirstToken(FormatToken &RootToken,
-                        const AnnotatedLine *PreviousLine, unsigned IndentLevel,
-                        unsigned Indent, bool InPPDirective);
+  void formatFirstToken(const AnnotatedLine &Line,
+                        const AnnotatedLine *PreviousLine, unsigned Indent);
 
   /// \brief Returns the column limit for a line, taking into account whether we
   /// need an escaped newline due to a continued preprocessor directive.
@@ -57,7 +56,8 @@ private:
   // starting from a specific additional offset. Improves performance if there
   // are many nested blocks.
   std::map<std::pair<const SmallVectorImpl<AnnotatedLine *> *, unsigned>,
-           unsigned> PenaltyCache;
+           unsigned>
+      PenaltyCache;
 
   ContinuationIndenter *Indenter;
   WhitespaceManager *Whitespaces;
diff --git a/lib/Format/WhitespaceManager.cpp b/lib/Format/WhitespaceManager.cpp
index 6b14af3f75..3fd92eaf78 100644
--- a/lib/Format/WhitespaceManager.cpp
+++ b/lib/Format/WhitespaceManager.cpp
@@ -25,64 +25,60 @@ operator()(const Change &C1, const Change &C2) const {
       C2.OriginalWhitespaceRange.getBegin());
 }
 
-WhitespaceManager::Change::Change(
-    bool CreateReplacement, SourceRange OriginalWhitespaceRange,
-    unsigned IndentLevel, int Spaces, unsigned StartOfTokenColumn,
-    unsigned NewlinesBefore, StringRef PreviousLinePostfix,
-    StringRef CurrentLinePrefix, tok::TokenKind Kind, bool ContinuesPPDirective,
-    bool IsStartOfDeclName, bool IsInsideToken)
-    : CreateReplacement(CreateReplacement),
+WhitespaceManager::Change::Change(const FormatToken &Tok,
+                                  bool CreateReplacement,
+                                  SourceRange OriginalWhitespaceRange,
+                                  int Spaces, unsigned StartOfTokenColumn,
+                                  unsigned NewlinesBefore,
+                                  StringRef PreviousLinePostfix,
+                                  StringRef CurrentLinePrefix,
+                                  bool ContinuesPPDirective, bool IsInsideToken)
+    : Tok(&Tok), CreateReplacement(CreateReplacement),
       OriginalWhitespaceRange(OriginalWhitespaceRange),
       StartOfTokenColumn(StartOfTokenColumn), NewlinesBefore(NewlinesBefore),
       PreviousLinePostfix(PreviousLinePostfix),
-      CurrentLinePrefix(CurrentLinePrefix), Kind(Kind),
-      ContinuesPPDirective(ContinuesPPDirective),
-      IsStartOfDeclName(IsStartOfDeclName), IndentLevel(IndentLevel),
-      Spaces(Spaces), IsInsideToken(IsInsideToken), IsTrailingComment(false),
-      TokenLength(0), PreviousEndOfTokenColumn(0), EscapedNewlineColumn(0),
+      CurrentLinePrefix(CurrentLinePrefix),
+      ContinuesPPDirective(ContinuesPPDirective), Spaces(Spaces),
+      IsInsideToken(IsInsideToken), IsTrailingComment(false), TokenLength(0),
+      PreviousEndOfTokenColumn(0), EscapedNewlineColumn(0),
       StartOfBlockComment(nullptr), IndentationOffset(0) {}
 
 void WhitespaceManager::replaceWhitespace(FormatToken &Tok, unsigned Newlines,
-                                          unsigned IndentLevel, unsigned Spaces,
+                                          unsigned Spaces,
                                           unsigned StartOfTokenColumn,
                                           bool InPPDirective) {
   if (Tok.Finalized)
     return;
   Tok.Decision = (Newlines > 0) ? FD_Break : FD_Continue;
-  Changes.push_back(
-      Change(/*CreateReplacement=*/true, Tok.WhitespaceRange, IndentLevel,
-             Spaces, StartOfTokenColumn, Newlines, "", "", Tok.Tok.getKind(),
-             InPPDirective && !Tok.IsFirst,
-             Tok.is(TT_StartOfName) || Tok.is(TT_FunctionDeclarationName),
-             /*IsInsideToken=*/false));
+  Changes.push_back(Change(Tok, /*CreateReplacement=*/true,
+                           Tok.WhitespaceRange, Spaces, StartOfTokenColumn,
+                           Newlines, "", "", InPPDirective && !Tok.IsFirst,
+                           /*IsInsideToken=*/false));
 }
 
 void WhitespaceManager::addUntouchableToken(const FormatToken &Tok,
                                             bool InPPDirective) {
   if (Tok.Finalized)
     return;
-  Changes.push_back(Change(
-      /*CreateReplacement=*/false, Tok.WhitespaceRange, /*IndentLevel=*/0,
-      /*Spaces=*/0, Tok.OriginalColumn, Tok.NewlinesBefore, "", "",
-      Tok.Tok.getKind(), InPPDirective && !Tok.IsFirst,
-      Tok.is(TT_StartOfName) || Tok.is(TT_FunctionDeclarationName),
-      /*IsInsideToken=*/false));
+  Changes.push_back(Change(Tok, /*CreateReplacement=*/false,
+                           Tok.WhitespaceRange, /*Spaces=*/0,
+                           Tok.OriginalColumn, Tok.NewlinesBefore, "", "",
+                           InPPDirective && !Tok.IsFirst,
+                           /*IsInsideToken=*/false));
 }
 
 void WhitespaceManager::replaceWhitespaceInToken(
     const FormatToken &Tok, unsigned Offset, unsigned ReplaceChars,
     StringRef PreviousPostfix, StringRef CurrentPrefix, bool InPPDirective,
-    unsigned Newlines, unsigned IndentLevel, int Spaces) {
+    unsigned Newlines, int Spaces) {
   if (Tok.Finalized)
     return;
   SourceLocation Start = Tok.getStartOfNonWhitespace().getLocWithOffset(Offset);
-  Changes.push_back(Change(
-      true, SourceRange(Start, Start.getLocWithOffset(ReplaceChars)),
-      IndentLevel, Spaces, std::max(0, Spaces), Newlines, PreviousPostfix,
-      CurrentPrefix, Tok.is(TT_LineComment) ? tok::comment : tok::unknown,
-      InPPDirective && !Tok.IsFirst,
-      Tok.is(TT_StartOfName) || Tok.is(TT_FunctionDeclarationName),
-      /*IsInsideToken=*/Newlines == 0));
+  Changes.push_back(
+      Change(Tok, /*CreateReplacement=*/true,
+             SourceRange(Start, Start.getLocWithOffset(ReplaceChars)), Spaces,
+             std::max(0, Spaces), Newlines, PreviousPostfix, CurrentPrefix,
+             InPPDirective && !Tok.IsFirst, /*IsInsideToken=*/true));
 }
 
 const tooling::Replacements &WhitespaceManager::generateReplacements() {
@@ -125,9 +121,9 @@ void WhitespaceManager::calculateLineBreakInformation() {
         Changes[i - 1].StartOfTokenColumn + Changes[i - 1].TokenLength;
 
     Changes[i - 1].IsTrailingComment =
-        (Changes[i].NewlinesBefore > 0 || Changes[i].Kind == tok::eof ||
-         (Changes[i].IsInsideToken && Changes[i].Kind == tok::comment)) &&
-        Changes[i - 1].Kind == tok::comment &&
+        (Changes[i].NewlinesBefore > 0 || Changes[i].Tok->is(tok::eof) ||
+         (Changes[i].IsInsideToken && Changes[i].Tok->is(tok::comment))) &&
+        Changes[i - 1].Tok->is(tok::comment) &&
         // FIXME: This is a dirty hack. The problem is that
         // BreakableLineCommentSection does comment reflow changes and here is
         // the aligning of trailing comments. Consider the case where we reflow
@@ -163,7 +159,7 @@ void WhitespaceManager::calculateLineBreakInformation() {
   // FIXME: The last token is currently not always an eof token; in those
   // cases, setting TokenLength of the last token to 0 is wrong.
   Changes.back().TokenLength = 0;
-  Changes.back().IsTrailingComment = Changes.back().Kind == tok::comment;
+  Changes.back().IsTrailingComment = Changes.back().Tok->is(tok::comment);
 
   const WhitespaceManager::Change *LastBlockComment = nullptr;
   for (auto &Change : Changes) {
@@ -173,13 +169,15 @@ void WhitespaceManager::calculateLineBreakInformation() {
       Change.IsTrailingComment = false;
     Change.StartOfBlockComment = nullptr;
     Change.IndentationOffset = 0;
-    if (Change.Kind == tok::comment) {
-      LastBlockComment = &Change;
-    } else if (Change.Kind == tok::unknown) {
-      if ((Change.StartOfBlockComment = LastBlockComment))
-        Change.IndentationOffset =
-            Change.StartOfTokenColumn -
-            Change.StartOfBlockComment->StartOfTokenColumn;
+    if (Change.Tok->is(tok::comment)) {
+      if (Change.Tok->is(TT_LineComment) || !Change.IsInsideToken)
+        LastBlockComment = &Change;
+      else {
+        if ((Change.StartOfBlockComment = LastBlockComment))
+          Change.IndentationOffset =
+              Change.StartOfTokenColumn -
+              Change.StartOfBlockComment->StartOfTokenColumn;
+      }
     } else {
       LastBlockComment = nullptr;
     }
@@ -278,15 +276,13 @@ static void AlignTokens(const FormatStyle &Style, F &&Matches,
       FoundMatchOnLine = false;
     }
 
-    if (Changes[i].Kind == tok::comma) {
+    if (Changes[i].Tok->is(tok::comma)) {
       ++CommasBeforeMatch;
-    } else if (Changes[i].Kind == tok::r_brace ||
-               Changes[i].Kind == tok::r_paren ||
-               Changes[i].Kind == tok::r_square) {
+    } else if (Changes[i].Tok->isOneOf(tok::r_brace, tok::r_paren,
+                                       tok::r_square)) {
       --NestingLevel;
-    } else if (Changes[i].Kind == tok::l_brace ||
-               Changes[i].Kind == tok::l_paren ||
-               Changes[i].Kind == tok::l_square) {
+    } else if (Changes[i].Tok->isOneOf(tok::l_brace, tok::l_paren,
+                                       tok::l_square)) {
       // We want sequences to skip over child scopes if possible, but not the
       // other way around.
       NestingLevelOfLastMatch = std::min(NestingLevelOfLastMatch, NestingLevel);
@@ -345,7 +341,7 @@ void WhitespaceManager::alignConsecutiveAssignments() {
                 if (&C != &Changes.back() && (&C + 1)->NewlinesBefore > 0)
                   return false;
 
-                return C.Kind == tok::equal;
+                return C.Tok->is(tok::equal);
               },
               Changes);
 }
@@ -361,7 +357,11 @@ void WhitespaceManager::alignConsecutiveDeclarations() {
   //   float const* v2;
   //   SomeVeryLongType const& v3;
 
-  AlignTokens(Style, [](Change const &C) { return C.IsStartOfDeclName; },
+  AlignTokens(Style,
+              [](Change const &C) {
+                return C.Tok->isOneOf(TT_StartOfName,
+                                      TT_FunctionDeclarationName);
+              },
               Changes);
 }
 
@@ -391,17 +391,14 @@ void WhitespaceManager::alignTrailingComments() {
     // If this comment follows an } in column 0, it probably documents the
     // closing of a namespace and we don't want to align it.
     bool FollowsRBraceInColumn0 = i > 0 && Changes[i].NewlinesBefore == 0 &&
-                                  Changes[i - 1].Kind == tok::r_brace &&
+                                  Changes[i - 1].Tok->is(tok::r_brace) &&
                                   Changes[i - 1].StartOfTokenColumn == 0;
     bool WasAlignedWithStartOfNextLine = false;
     if (Changes[i].NewlinesBefore == 1) { // A comment on its own line.
       unsigned CommentColumn = SourceMgr.getSpellingColumnNumber(
           Changes[i].OriginalWhitespaceRange.getEnd());
       for (unsigned j = i + 1; j != e; ++j) {
-        if (Changes[j].Kind == tok::comment ||
-            Changes[j].Kind == tok::unknown)
-          // Skip over comments and unknown tokens. "unknown tokens are used for
-          // the continuation of multiline comments.
+        if (Changes[j].Tok->is(tok::comment))
           continue;
 
         unsigned NextColumn = SourceMgr.getSpellingColumnNumber(
@@ -512,7 +509,8 @@ void WhitespaceManager::generateChanges() {
                           C.PreviousEndOfTokenColumn, C.EscapedNewlineColumn);
       else
         appendNewlineText(ReplacementText, C.NewlinesBefore);
-      appendIndentText(ReplacementText, C.IndentLevel, std::max(0, C.Spaces),
+      appendIndentText(ReplacementText, C.Tok->IndentLevel,
+                       std::max(0, C.Spaces),
                        C.StartOfTokenColumn - std::max(0, C.Spaces));
       ReplacementText.append(C.CurrentLinePrefix);
       storeReplacement(C.OriginalWhitespaceRange, ReplacementText);
diff --git a/lib/Format/WhitespaceManager.h b/lib/Format/WhitespaceManager.h
index f42e371830..0a2a031dd6 100644
--- a/lib/Format/WhitespaceManager.h
+++ b/lib/Format/WhitespaceManager.h
@@ -43,8 +43,7 @@ public:
 
   /// \brief Replaces the whitespace in front of \p Tok. Only call once for
   /// each \c AnnotatedToken.
-  void replaceWhitespace(FormatToken &Tok, unsigned Newlines,
-                         unsigned IndentLevel, unsigned Spaces,
+  void replaceWhitespace(FormatToken &Tok, unsigned Newlines, unsigned Spaces,
                          unsigned StartOfTokenColumn,
                          bool InPPDirective = false);
 
@@ -72,8 +71,7 @@ public:
                                 unsigned ReplaceChars,
                                 StringRef PreviousPostfix,
                                 StringRef CurrentPrefix, bool InPPDirective,
-                                unsigned Newlines, unsigned IndentLevel,
-                                int Spaces);
+                                unsigned Newlines, int Spaces);
 
   /// \brief Returns all the \c Replacements created during formatting.
   const tooling::Replacements &generateReplacements();
@@ -91,8 +89,6 @@ public:
       const SourceManager &SourceMgr;
     };
 
-    Change() {}
-
     /// \brief Creates a \c Change.
     ///
     /// The generated \c Change will replace the characters at
@@ -102,12 +98,17 @@ public:
     ///
     /// \p StartOfTokenColumn and \p InPPDirective will be used to lay out
     /// trailing comments and escaped newlines.
-    Change(bool CreateReplacement, SourceRange OriginalWhitespaceRange,
-           unsigned IndentLevel, int Spaces, unsigned StartOfTokenColumn,
-           unsigned NewlinesBefore, StringRef PreviousLinePostfix,
-           StringRef CurrentLinePrefix, tok::TokenKind Kind,
-           bool ContinuesPPDirective, bool IsStartOfDeclName,
-           bool IsInsideToken);
+    Change(const FormatToken &Tok, bool CreateReplacement,
+           SourceRange OriginalWhitespaceRange, int Spaces,
+           unsigned StartOfTokenColumn, unsigned NewlinesBefore,
+           StringRef PreviousLinePostfix, StringRef CurrentLinePrefix,
+           bool ContinuesPPDirective, bool IsInsideToken);
+
+    // The kind of the token whose whitespace this change replaces, or in which
+    // this change inserts whitespace.
+    // FIXME: Currently this is not set correctly for breaks inside comments, as
+    // the \c BreakableToken is still doing its own alignment.
+    const FormatToken *Tok;
 
     bool CreateReplacement;
     // Changes might be in the middle of a token, so we cannot just keep the
@@ -117,18 +118,7 @@ public:
     unsigned NewlinesBefore;
     std::string PreviousLinePostfix;
     std::string CurrentLinePrefix;
-    // The kind of the token whose whitespace this change replaces, or in which
-    // this change inserts whitespace.
-    // FIXME: Currently this is not set correctly for breaks inside comments, as
-    // the \c BreakableToken is still doing its own alignment.
-    tok::TokenKind Kind;
     bool ContinuesPPDirective;
-    bool IsStartOfDeclName;
-
-    // The number of nested blocks the token is in. This is used to add tabs
-    // only for the indentation, and not for alignment, when
-    // UseTab = US_ForIndentation.
-    unsigned IndentLevel;
 
     // The number of spaces in front of the token or broken part of the token.
     // This will be adapted when aligning tokens.