PTH-- Remove feature entirely-

When debugging a boost build with a modified
version of Clang, I discovered that the PTH implementation
stores TokenKind in 8 bits. However, we currently have 368
TokenKinds.

The result is that the value gets truncated and the wrong token
gets picked up when including PTH files. It seems that this will
go wrong every time someone uses a token that uses the 9th bit.

Upon asking on IRC, it was brought up that this was a highly
experimental features that was considered a failure. I discovered
via googling that BoostBuild (mostly Boost.Math) is the only user of
this
feature, using the CC1 flag directly. I believe that this can be
transferred over to normal PCH with minimal effort:
https://github.com/boostorg/build/issues/367

Based on advice on IRC and research showing that this is a nearly
completely unused feature, this patch removes it entirely.

Note: I considered leaving the build-flags in place and making them
emit an error/warning, however since I've basically identified and
warned the only user, it seemed better to just remove them.

Differential Revision: https://reviews.llvm.org/D54547

Change-Id: If32744275ef1f585357bd6c1c813d96973c4d8d9

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@348266 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 6 insertions, 301 deletions

diff --git a/include/clang/Lex/PTHLexer.h b/include/clang/Lex/PTHLexer.h
deleted file mode 100644
index 0b84df1434..0000000000
--- a/include/clang/Lex/PTHLexer.h
+++ /dev/null
@@ -1,107 +0,0 @@
-//===- PTHLexer.h - Lexer based on Pre-tokenized input ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the PTHLexer interface.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LEX_PTHLEXER_H
-#define LLVM_CLANG_LEX_PTHLEXER_H
-
-#include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/TokenKinds.h"
-#include "clang/Lex/PreprocessorLexer.h"
-#include "clang/Lex/Token.h"
-
-namespace clang {
-
-class Preprocessor;
-class PTHManager;
-
-class PTHLexer : public PreprocessorLexer {
-  SourceLocation FileStartLoc;
-
-  /// TokBuf - Buffer from PTH file containing raw token data.
-  const unsigned char* TokBuf;
-
-  /// CurPtr - Pointer into current offset of the token buffer where
-  ///  the next token will be read.
-  const unsigned char* CurPtr;
-
-  /// LastHashTokPtr - Pointer into TokBuf of the last processed '#'
-  ///  token that appears at the start of a line.
-  const unsigned char* LastHashTokPtr = nullptr;
-
-  /// PPCond - Pointer to a side table in the PTH file that provides a
-  ///  a concise summary of the preprocessor conditional block structure.
-  ///  This is used to perform quick skipping of conditional blocks.
-  const unsigned char* PPCond;
-
-  /// CurPPCondPtr - Pointer inside PPCond that refers to the next entry
-  ///  to process when doing quick skipping of preprocessor blocks.
-  const unsigned char* CurPPCondPtr;
-
-  /// ReadToken - Used by PTHLexer to read tokens TokBuf.
-  void ReadToken(Token &T);
-
-  bool LexEndOfFile(Token &Result);
-
-  /// PTHMgr - The PTHManager object that created this PTHLexer.
-  PTHManager& PTHMgr;
-
-  Token EofToken;
-
-protected:
-  friend class PTHManager;
-
-  /// Create a PTHLexer for the specified token stream.
-  PTHLexer(Preprocessor &pp, FileID FID, const unsigned char *D,
-           const unsigned char* ppcond, PTHManager &PM);
-
-public:
-  PTHLexer(const PTHLexer &) = delete;
-  PTHLexer &operator=(const PTHLexer &) = delete;
-  ~PTHLexer() override = default;
-
-  /// Lex - Return the next token.
-  bool Lex(Token &Tok);
-
-  void getEOF(Token &Tok);
-
-  /// DiscardToEndOfLine - Read the rest of the current preprocessor line as an
-  /// uninterpreted string.  This switches the lexer out of directive mode.
-  void DiscardToEndOfLine();
-
-  /// isNextPPTokenLParen - Return 1 if the next unexpanded token will return a
-  /// tok::l_paren token, 0 if it is something else and 2 if there are no more
-  /// tokens controlled by this lexer.
-  unsigned isNextPPTokenLParen() {
-    // isNextPPTokenLParen is not on the hot path, and all we care about is
-    // whether or not we are at a token with kind tok::eof or tok::l_paren.
-    // Just read the first byte from the current token pointer to determine
-    // its kind.
-    tok::TokenKind x = (tok::TokenKind)*CurPtr;
-    return x == tok::eof ? 2 : x == tok::l_paren;
-  }
-
-  /// IndirectLex - An indirect call to 'Lex' that can be invoked via
-  ///  the PreprocessorLexer interface.
-  void IndirectLex(Token &Result) override { Lex(Result); }
-
-  /// getSourceLocation - Return a source location for the token in
-  /// the current file.
-  SourceLocation getSourceLocation() override;
-
-  /// SkipBlock - Used by Preprocessor to skip the current conditional block.
-  bool SkipBlock();
-};
-
-} // namespace clang
-
-#endif // LLVM_CLANG_LEX_PTHLEXER_H
diff --git a/include/clang/Lex/PTHManager.h b/include/clang/Lex/PTHManager.h
deleted file mode 100644
index 483b69f23a..0000000000
--- a/include/clang/Lex/PTHManager.h
+++ /dev/null
@@ -1,149 +0,0 @@
-//===- PTHManager.h - Manager object for PTH processing ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the PTHManager interface.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LEX_PTHMANAGER_H
-#define LLVM_CLANG_LEX_PTHMANAGER_H
-
-#include "clang/Basic/IdentifierTable.h"
-#include "clang/Basic/SourceLocation.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/OnDiskHashTable.h"
-#include <memory>
-
-namespace llvm {
-
-class MemoryBuffer;
-
-} // namespace llvm
-
-namespace clang {
-
-class DiagnosticsEngine;
-class FileSystemStatCache;
-class Preprocessor;
-class PTHLexer;
-
-class PTHManager : public IdentifierInfoLookup {
-  friend class PTHLexer;
-  friend class PTHStatCache;
-
-  class PTHFileLookupTrait;
-  class PTHStringLookupTrait;
-
-  using PTHStringIdLookup = llvm::OnDiskChainedHashTable<PTHStringLookupTrait>;
-  using PTHFileLookup = llvm::OnDiskChainedHashTable<PTHFileLookupTrait>;
-
-  /// The memory mapped PTH file.
-  std::unique_ptr<const llvm::MemoryBuffer> Buf;
-
-  /// Alloc - Allocator used for IdentifierInfo objects.
-  llvm::BumpPtrAllocator Alloc;
-
-  /// IdMap - A lazily generated cache mapping from persistent identifiers to
-  ///  IdentifierInfo*.
-  std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> PerIDCache;
-
-  /// FileLookup - Abstract data structure used for mapping between files
-  ///  and token data in the PTH file.
-  std::unique_ptr<PTHFileLookup> FileLookup;
-
-  /// IdDataTable - Array representing the mapping from persistent IDs to the
-  ///  data offset within the PTH file containing the information to
-  ///  reconsitute an IdentifierInfo.
-  const unsigned char* const IdDataTable;
-
-  /// SortedIdTable - Abstract data structure mapping from strings to
-  ///  persistent IDs.  This is used by get().
-  std::unique_ptr<PTHStringIdLookup> StringIdLookup;
-
-  /// NumIds - The number of identifiers in the PTH file.
-  const unsigned NumIds;
-
-  /// PP - The Preprocessor object that will use this PTHManager to create
-  ///  PTHLexer objects.
-  Preprocessor* PP = nullptr;
-
-  /// SpellingBase - The base offset within the PTH memory buffer that
-  ///  contains the cached spellings for literals.
-  const unsigned char* const SpellingBase;
-
-  /// OriginalSourceFile - A null-terminated C-string that specifies the name
-  ///  if the file (if any) that was to used to generate the PTH cache.
-  const char* OriginalSourceFile;
-
-  /// This constructor is intended to only be called by the static 'Create'
-  /// method.
-  PTHManager(std::unique_ptr<const llvm::MemoryBuffer> buf,
-             std::unique_ptr<PTHFileLookup> fileLookup,
-             const unsigned char *idDataTable,
-             std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> perIDCache,
-             std::unique_ptr<PTHStringIdLookup> stringIdLookup, unsigned numIds,
-             const unsigned char *spellingBase, const char *originalSourceFile);
-
-  /// getSpellingAtPTHOffset - Used by PTHLexer classes to get the cached
-  ///  spelling for a token.
-  unsigned getSpellingAtPTHOffset(unsigned PTHOffset, const char*& Buffer);
-
-  /// GetIdentifierInfo - Used to reconstruct IdentifierInfo objects from the
-  ///  PTH file.
-  IdentifierInfo *GetIdentifierInfo(unsigned PersistentID) {
-    // Check if the IdentifierInfo has already been resolved.
-    if (IdentifierInfo* II = PerIDCache[PersistentID])
-      return II;
-    return LazilyCreateIdentifierInfo(PersistentID);
-  }
-  IdentifierInfo* LazilyCreateIdentifierInfo(unsigned PersistentID);
-
-public:
-  // The current PTH version.
-  enum { Version = 10 };
-
-  PTHManager(const PTHManager &) = delete;
-  PTHManager &operator=(const PTHManager &) = delete;
-  ~PTHManager() override;
-
-  /// getOriginalSourceFile - Return the full path to the original header
-  ///  file name that was used to generate the PTH cache.
-  const char* getOriginalSourceFile() const {
-    return OriginalSourceFile;
-  }
-
-  /// get - Return the identifier token info for the specified named identifier.
-  ///  Unlike the version in IdentifierTable, this returns a pointer instead
-  ///  of a reference.  If the pointer is nullptr then the IdentifierInfo cannot
-  ///  be found.
-  IdentifierInfo *get(StringRef Name) override;
-
-  /// Create - This method creates PTHManager objects.  The 'file' argument
-  ///  is the name of the PTH file.  This method returns nullptr upon failure.
-  static PTHManager *Create(StringRef file, DiagnosticsEngine &Diags);
-
-  void setPreprocessor(Preprocessor *pp) { PP = pp; }
-
-  /// CreateLexer - Return a PTHLexer that "lexes" the cached tokens for the
-  ///  specified file.  This method returns nullptr if no cached tokens exist.
-  ///  It is the responsibility of the caller to 'delete' the returned object.
-  PTHLexer *CreateLexer(FileID FID);
-
-  /// createStatCache - Returns a FileSystemStatCache object for use with
-  ///  FileManager objects.  These objects use the PTH data to speed up
-  ///  calls to stat by memoizing their results from when the PTH file
-  ///  was generated.
-  std::unique_ptr<FileSystemStatCache> createStatCache();
-};
-
-} // namespace clang
-
-#endif // LLVM_CLANG_LEX_PTHMANAGER_H
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index 75910011d0..792f8af7da 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -29,7 +29,6 @@
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/ModuleMap.h"
 #include "clang/Lex/PPCallbacks.h"
-#include "clang/Lex/PTHLexer.h"
 #include "clang/Lex/Token.h"
 #include "clang/Lex/TokenLexer.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -79,7 +78,6 @@ class PragmaNamespace;
 class PreprocessingRecord;
 class PreprocessorLexer;
 class PreprocessorOptions;
-class PTHManager;
 class ScratchBuffer;
 class TargetInfo;
 
@@ -143,10 +141,6 @@ class Preprocessor {
   /// External source of macros.
   ExternalPreprocessorSource *ExternalSource;
 
-  /// An optional PTHManager object used for getting tokens from
-  /// a token cache rather than lexing the original source file.
-  std::unique_ptr<PTHManager> PTH;
-
   /// A BumpPtrAllocator object used to quickly allocate and release
   /// objects internal to the Preprocessor.
   llvm::BumpPtrAllocator BP;
@@ -392,19 +386,13 @@ private:
   /// The current top of the stack that we're lexing from if
   /// not expanding a macro and we are lexing directly from source code.
   ///
-  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
+  /// Only one of CurLexer, or CurTokenLexer will be non-null.
   std::unique_ptr<Lexer> CurLexer;
 
-  /// The current top of stack that we're lexing from if
-  /// not expanding from a macro and we are lexing from a PTH cache.
-  ///
-  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
-  std::unique_ptr<PTHLexer> CurPTHLexer;
-
   /// The current top of the stack what we're lexing from
   /// if not expanding a macro.
   ///
-  /// This is an alias for either CurLexer or  CurPTHLexer.
+  /// This is an alias for CurLexer.
   PreprocessorLexer *CurPPLexer = nullptr;
 
   /// Used to find the current FileEntry, if CurLexer is non-null
@@ -422,7 +410,6 @@ private:
   /// The kind of lexer we're currently working with.
   enum CurLexerKind {
     CLK_Lexer,
-    CLK_PTHLexer,
     CLK_TokenLexer,
     CLK_CachingLexer,
     CLK_LexAfterModuleImport
@@ -439,7 +426,6 @@ private:
     enum CurLexerKind           CurLexerKind;
     Module                     *TheSubmodule;
     std::unique_ptr<Lexer>      TheLexer;
-    std::unique_ptr<PTHLexer>   ThePTHLexer;
     PreprocessorLexer          *ThePPLexer;
     std::unique_ptr<TokenLexer> TheTokenLexer;
     const DirectoryLookup      *TheDirLookup;
@@ -448,13 +434,11 @@ private:
     // versions, only needed to pacify MSVC.
     IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
                      std::unique_ptr<Lexer> &&TheLexer,
-                     std::unique_ptr<PTHLexer> &&ThePTHLexer,
                      PreprocessorLexer *ThePPLexer,
                      std::unique_ptr<TokenLexer> &&TheTokenLexer,
                      const DirectoryLookup *TheDirLookup)
         : CurLexerKind(std::move(CurLexerKind)),
           TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
-          ThePTHLexer(std::move(ThePTHLexer)),
           ThePPLexer(std::move(ThePPLexer)),
           TheTokenLexer(std::move(TheTokenLexer)),
           TheDirLookup(std::move(TheDirLookup)) {}
@@ -838,10 +822,6 @@ public:
   Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
   llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
 
-  void setPTHManager(PTHManager* pm);
-
-  PTHManager *getPTHManager() { return PTH.get(); }
-
   void setExternalSource(ExternalPreprocessorSource *Source) {
     ExternalSource = Source;
   }
@@ -1456,7 +1436,7 @@ public:
       CachedTokens[CachedLexPos-1] = Tok;
   }
 
-  /// Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
+  /// Recompute the current lexer kind based on the CurLexer/
   /// CurTokenLexer pointers.
   void recomputeCurLexerKind();
 
@@ -1915,15 +1895,13 @@ private:
   void PushIncludeMacroStack() {
     assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
     IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
-                                   std::move(CurLexer), std::move(CurPTHLexer),
-                                   CurPPLexer, std::move(CurTokenLexer),
-                                   CurDirLookup);
+                                   std::move(CurLexer), CurPPLexer,
+                                   std::move(CurTokenLexer), CurDirLookup);
     CurPPLexer = nullptr;
   }
 
   void PopIncludeMacroStack() {
     CurLexer = std::move(IncludeMacroStack.back().TheLexer);
-    CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
     CurPPLexer = IncludeMacroStack.back().ThePPLexer;
     CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
     CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
@@ -1992,9 +1970,6 @@ private:
                                     bool FoundNonSkipPortion, bool FoundElse,
                                     SourceLocation ElseLoc = SourceLocation());
 
-  /// A fast PTH version of SkipExcludedConditionalBlock.
-  void PTHSkipExcludedConditionalBlock();
-
   /// Information about the result for evaluating an expression for a
   /// preprocessor directive.
   struct DirectiveEvalResult {
@@ -2060,10 +2035,6 @@ private:
   /// start lexing tokens from it instead of the current buffer.
   void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
 
-  /// Add a lexer to the top of the include stack and
-  /// start getting tokens from it using the PTH cache.
-  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
-
   /// Set the FileID for the preprocessor predefines.
   void setPredefinesFileID(FileID FID) {
     assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
@@ -2094,8 +2065,7 @@ private:
   bool InCachingLexMode() const {
     // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
     // that we are past EOF, not that we are in CachingLex mode.
-    return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
-           !IncludeMacroStack.empty();
+    return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
   }
 
   void EnterCachingLexMode();
diff --git a/include/clang/Lex/PreprocessorOptions.h b/include/clang/Lex/PreprocessorOptions.h
index 6b3823911c..f1ac72c474 100644
--- a/include/clang/Lex/PreprocessorOptions.h
+++ b/include/clang/Lex/PreprocessorOptions.h
@@ -110,13 +110,6 @@ public:
   /// clients don't use them.
   bool WriteCommentListToPCH = true;
 
-  /// The implicit PTH input included at the start of the translation unit, or
-  /// empty.
-  std::string ImplicitPTHInclude;
-
-  /// If given, a PTH cache file to use for speeding up header parsing.
-  std::string TokenCache;
-
   /// When enabled, preprocessor is in a mode for parsing a single file only.
   ///
   /// Disables #includes of other files and if there are unresolved identifiers
@@ -204,8 +197,6 @@ public:
     ChainedIncludes.clear();
     DumpDeserializedPCHDecls = false;
     ImplicitPCHInclude.clear();
-    ImplicitPTHInclude.clear();
-    TokenCache.clear();
     SingleFileParseMode = false;
     LexEditorPlaceholders = true;
     RetainRemappedFileBuffers = true;