Modules: Use hash of PCM content for SIGNATURE

Change ASTFileSignature from a random 32-bit number to the hash of the
PCM content.

  - Move definition ASTFileSignature to Basic/Module.h so Module and
    ASTSourceDescriptor can use it.

  - Change the signature from uint64_t to std::array<uint32_t,5>.

  - Stop using (saving/reading) the size and modification time of PCM
    files when there is a valid SIGNATURE.

  - Add UNHASHED_CONTROL_BLOCK, and use it to store the SIGNATURE record
    and other records that shouldn't affect the hash.  Because implicit
    modules reuses the same file for multiple levels of -Werror, this
    includes DIAGNOSTIC_OPTIONS and DIAG_PRAGMA_MAPPINGS.

This helps to solve a PCH + implicit Modules dependency issue: PCH files
are handled by the external build system, whereas implicit modules are
handled by internal compiler build system.  This prevents invalidating a
PCH when the compiler overwrites a PCM file with the same content
(modulo the diagnostic differences).

Design and original patch by Manman Ren!

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@297655 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 60 insertions, 24 deletions

diff --git a/include/clang/Serialization/ASTBitCodes.h b/include/clang/Serialization/ASTBitCodes.h
index 91e5734451..de8e2a8183 100644
--- a/include/clang/Serialization/ASTBitCodes.h
+++ b/include/clang/Serialization/ASTBitCodes.h
@@ -226,7 +226,7 @@ namespace clang {
 
       /// \brief The block containing the detailed preprocessing record.
       PREPROCESSOR_DETAIL_BLOCK_ID,
-      
+
       /// \brief The block containing the submodule structure.
       SUBMODULE_BLOCK_ID,
 
@@ -253,6 +253,12 @@ namespace clang {
 
       /// \brief A block containing a module file extension.
       EXTENSION_BLOCK_ID,
+
+      /// A block with unhashed content.
+      ///
+      /// These records should not change the \a ASTFileSignature.  See \a
+      /// UnhashedControlBlockRecordTypes for the list of records.
+      UNHASHED_CONTROL_BLOCK_ID,
     };
 
     /// \brief Record types that occur within the control block.
@@ -288,9 +294,6 @@ namespace clang {
       /// AST file.
       MODULE_MAP_FILE,
 
-      /// \brief Record code for the signature that identifiers this AST file.
-      SIGNATURE,
-
       /// \brief Record code for the module build directory.
       MODULE_DIRECTORY,
     };
@@ -309,9 +312,6 @@ namespace clang {
       /// \brief Record code for the target options table.
       TARGET_OPTIONS,
 
-      /// \brief Record code for the diagnostic options table.
-      DIAGNOSTIC_OPTIONS,
-
       /// \brief Record code for the filesystem options table.
       FILE_SYSTEM_OPTIONS,
 
@@ -322,6 +322,18 @@ namespace clang {
       PREPROCESSOR_OPTIONS,
     };
 
+    /// Record codes for the unhashed control block.
+    enum UnhashedControlBlockRecordTypes {
+      /// Record code for the signature that identifiers this AST file.
+      SIGNATURE = 1,
+
+      /// Record code for the diagnostic options table.
+      DIAGNOSTIC_OPTIONS,
+
+      /// Record code for \#pragma diagnostic mappings.
+      DIAG_PRAGMA_MAPPINGS,
+    };
+
     /// \brief Record code for extension blocks.
     enum ExtensionBlockRecordTypes {
       /// Metadata describing this particular extension.
@@ -493,8 +505,7 @@ namespace clang {
 
       // ID 31 used to be a list of offsets to DECL_CXX_BASE_SPECIFIERS records.
 
-      /// \brief Record code for \#pragma diagnostic mappings.
-      DIAG_PRAGMA_MAPPINGS = 32,
+      // ID 32 used to be the code for \#pragma diagnostic mappings.
 
       /// \brief Record code for special CUDA declarations.
       CUDA_SPECIAL_DECL_REFS = 33,
diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h
index edceb64847..5a1514ad80 100644
--- a/include/clang/Serialization/ASTReader.h
+++ b/include/clang/Serialization/ASTReader.h
@@ -1174,7 +1174,7 @@ private:
                             SourceLocation ImportLoc, ModuleFile *ImportedBy,
                             SmallVectorImpl<ImportedModule> &Loaded,
                             off_t ExpectedSize, time_t ExpectedModTime,
-                            serialization::ASTFileSignature ExpectedSignature,
+                            ASTFileSignature ExpectedSignature,
                             unsigned ClientLoadCapabilities);
   ASTReadResult ReadControlBlock(ModuleFile &F,
                                  SmallVectorImpl<ImportedModule> &Loaded,
@@ -1183,7 +1183,22 @@ private:
   static ASTReadResult ReadOptionsBlock(
       llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities,
       bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener,
-      std::string &SuggestedPredefines, bool ValidateDiagnosticOptions);
+      std::string &SuggestedPredefines);
+
+  /// Read the unhashed control block.
+  ///
+  /// This has no effect on \c F.Stream, instead creating a fresh cursor from
+  /// \c F.Data and reading ahead.
+  ASTReadResult readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy,
+                                         unsigned ClientLoadCapabilities);
+
+  static ASTReadResult
+  readUnhashedControlBlockImpl(ModuleFile *F, llvm::StringRef StreamData,
+                               unsigned ClientLoadCapabilities,
+                               bool AllowCompatibleConfigurationMismatch,
+                               ASTReaderListener *Listener,
+                               bool ValidateDiagnosticOptions);
+
   ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities);
   ASTReadResult ReadExtensionBlock(ModuleFile &F);
   void ReadModuleOffsetMap(ModuleFile &F) const;
diff --git a/include/clang/Serialization/ASTWriter.h b/include/clang/Serialization/ASTWriter.h
index 23afa639de..f3c644b9e2 100644
--- a/include/clang/Serialization/ASTWriter.h
+++ b/include/clang/Serialization/ASTWriter.h
@@ -106,6 +106,9 @@ private:
   /// \brief The bitstream writer used to emit this precompiled header.
   llvm::BitstreamWriter &Stream;
 
+  /// The buffer associated with the bitstream.
+  const SmallVectorImpl<char> &Buffer;
+
   /// \brief The ASTContext we're writing.
   ASTContext *Context = nullptr;
 
@@ -425,8 +428,16 @@ private:
   void WriteSubStmt(Stmt *S);
 
   void WriteBlockInfoBlock();
-  uint64_t WriteControlBlock(Preprocessor &PP, ASTContext &Context,
-                             StringRef isysroot, const std::string &OutputFile);
+  void WriteControlBlock(Preprocessor &PP, ASTContext &Context,
+                         StringRef isysroot, const std::string &OutputFile);
+
+  /// Write out the signature and diagnostic options, and return the signature.
+  ASTFileSignature writeUnhashedControlBlock(Preprocessor &PP,
+                                             ASTContext &Context);
+
+  /// Calculate hash of the pcm content.
+  static ASTFileSignature createSignature(StringRef Bytes);
+
   void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts,
                        bool Modules);
   void WriteSourceManagerBlock(SourceManager &SourceMgr,
@@ -493,14 +504,14 @@ private:
   void WriteDeclAbbrevs();
   void WriteDecl(ASTContext &Context, Decl *D);
 
-  uint64_t WriteASTCore(Sema &SemaRef,
-                        StringRef isysroot, const std::string &OutputFile,
-                        Module *WritingModule);
+  ASTFileSignature WriteASTCore(Sema &SemaRef, StringRef isysroot,
+                                const std::string &OutputFile,
+                                Module *WritingModule);
 
 public:
   /// \brief Create a new precompiled header writer that outputs to
   /// the given bitstream.
-  ASTWriter(llvm::BitstreamWriter &Stream,
+  ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
             ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
             bool IncludeTimestamps = true);
   ~ASTWriter() override;
@@ -526,9 +537,9 @@ public:
   ///
   /// \return the module signature, which eventually will be a hash of
   /// the module but currently is merely a random 32-bit number.
-  uint64_t WriteAST(Sema &SemaRef, const std::string &OutputFile,
-                    Module *WritingModule, StringRef isysroot,
-                    bool hasErrors = false);
+  ASTFileSignature WriteAST(Sema &SemaRef, const std::string &OutputFile,
+                            Module *WritingModule, StringRef isysroot,
+                            bool hasErrors = false);
 
   /// \brief Emit a token.
   void AddToken(const Token &Tok, RecordDataImpl &Record);
diff --git a/include/clang/Serialization/Module.h b/include/clang/Serialization/Module.h
index 2fd156ceb4..c1d4a9fd2d 100644
--- a/include/clang/Serialization/Module.h
+++ b/include/clang/Serialization/Module.h
@@ -16,6 +16,7 @@
 #define LLVM_CLANG_SERIALIZATION_MODULE_H
 
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/Module.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ContinuousRangeMap.h"
@@ -89,8 +90,6 @@ public:
   bool isNotFound() const { return Val.getInt() == NotFound; }
 };
 
-typedef unsigned ASTFileSignature;
-
 /// \brief Information about a module that has been loaded by the ASTReader.
 ///
 /// Each instance of the Module class corresponds to a single AST file, which
@@ -153,9 +152,9 @@ public:
   /// \brief The file entry for the module file.
   const FileEntry *File = nullptr;
 
-  /// \brief The signature of the module file, which may be used along with size
+  /// The signature of the module file, which may be used instead of the size
   /// and modification time to identify this particular file.
-  ASTFileSignature Signature = 0;
+  ASTFileSignature Signature;
 
   /// \brief Whether this module has been directly imported by the
   /// user.