From d15b1db69ad01b6fe6a145e1f31d3ea5c469de96 Mon Sep 17 00:00:00 2001 From: "Duncan P. N. Exon Smith" Date: Mon, 13 Mar 2017 18:45:08 +0000 Subject: Modules: Use hash of PCM content for SIGNATURE Change ASTFileSignature from a random 32-bit number to the hash of the PCM content. - Move definition ASTFileSignature to Basic/Module.h so Module and ASTSourceDescriptor can use it. - Change the signature from uint64_t to std::array. - Stop using (saving/reading) the size and modification time of PCM files when there is a valid SIGNATURE. - Add UNHASHED_CONTROL_BLOCK, and use it to store the SIGNATURE record and other records that shouldn't affect the hash. Because implicit modules reuses the same file for multiple levels of -Werror, this includes DIAGNOSTIC_OPTIONS and DIAG_PRAGMA_MAPPINGS. This helps to solve a PCH + implicit Modules dependency issue: PCH files are handled by the external build system, whereas implicit modules are handled by internal compiler build system. This prevents invalidating a PCH when the compiler overwrites a PCM file with the same content (modulo the diagnostic differences). Design and original patch by Manman Ren! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@297655 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Serialization/ASTBitCodes.h | 29 ++++++++++++++++++++--------- include/clang/Serialization/ASTReader.h | 19 +++++++++++++++++-- include/clang/Serialization/ASTWriter.h | 29 ++++++++++++++++++++--------- include/clang/Serialization/Module.h | 7 +++---- 4 files changed, 60 insertions(+), 24 deletions(-) (limited to 'include/clang/Serialization') diff --git a/include/clang/Serialization/ASTBitCodes.h b/include/clang/Serialization/ASTBitCodes.h index 91e5734451..de8e2a8183 100644 --- a/include/clang/Serialization/ASTBitCodes.h +++ b/include/clang/Serialization/ASTBitCodes.h @@ -226,7 +226,7 @@ namespace clang { /// \brief The block containing the detailed preprocessing record. PREPROCESSOR_DETAIL_BLOCK_ID, - + /// \brief The block containing the submodule structure. SUBMODULE_BLOCK_ID, @@ -253,6 +253,12 @@ namespace clang { /// \brief A block containing a module file extension. EXTENSION_BLOCK_ID, + + /// A block with unhashed content. + /// + /// These records should not change the \a ASTFileSignature. See \a + /// UnhashedControlBlockRecordTypes for the list of records. + UNHASHED_CONTROL_BLOCK_ID, }; /// \brief Record types that occur within the control block. @@ -288,9 +294,6 @@ namespace clang { /// AST file. MODULE_MAP_FILE, - /// \brief Record code for the signature that identifiers this AST file. - SIGNATURE, - /// \brief Record code for the module build directory. MODULE_DIRECTORY, }; @@ -309,9 +312,6 @@ namespace clang { /// \brief Record code for the target options table. TARGET_OPTIONS, - /// \brief Record code for the diagnostic options table. - DIAGNOSTIC_OPTIONS, - /// \brief Record code for the filesystem options table. FILE_SYSTEM_OPTIONS, @@ -322,6 +322,18 @@ namespace clang { PREPROCESSOR_OPTIONS, }; + /// Record codes for the unhashed control block. + enum UnhashedControlBlockRecordTypes { + /// Record code for the signature that identifiers this AST file. + SIGNATURE = 1, + + /// Record code for the diagnostic options table. + DIAGNOSTIC_OPTIONS, + + /// Record code for \#pragma diagnostic mappings. + DIAG_PRAGMA_MAPPINGS, + }; + /// \brief Record code for extension blocks. enum ExtensionBlockRecordTypes { /// Metadata describing this particular extension. @@ -493,8 +505,7 @@ namespace clang { // ID 31 used to be a list of offsets to DECL_CXX_BASE_SPECIFIERS records. - /// \brief Record code for \#pragma diagnostic mappings. - DIAG_PRAGMA_MAPPINGS = 32, + // ID 32 used to be the code for \#pragma diagnostic mappings. /// \brief Record code for special CUDA declarations. CUDA_SPECIAL_DECL_REFS = 33, diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h index edceb64847..5a1514ad80 100644 --- a/include/clang/Serialization/ASTReader.h +++ b/include/clang/Serialization/ASTReader.h @@ -1174,7 +1174,7 @@ private: SourceLocation ImportLoc, ModuleFile *ImportedBy, SmallVectorImpl &Loaded, off_t ExpectedSize, time_t ExpectedModTime, - serialization::ASTFileSignature ExpectedSignature, + ASTFileSignature ExpectedSignature, unsigned ClientLoadCapabilities); ASTReadResult ReadControlBlock(ModuleFile &F, SmallVectorImpl &Loaded, @@ -1183,7 +1183,22 @@ private: static ASTReadResult ReadOptionsBlock( llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities, bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, - std::string &SuggestedPredefines, bool ValidateDiagnosticOptions); + std::string &SuggestedPredefines); + + /// Read the unhashed control block. + /// + /// This has no effect on \c F.Stream, instead creating a fresh cursor from + /// \c F.Data and reading ahead. + ASTReadResult readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy, + unsigned ClientLoadCapabilities); + + static ASTReadResult + readUnhashedControlBlockImpl(ModuleFile *F, llvm::StringRef StreamData, + unsigned ClientLoadCapabilities, + bool AllowCompatibleConfigurationMismatch, + ASTReaderListener *Listener, + bool ValidateDiagnosticOptions); + ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities); ASTReadResult ReadExtensionBlock(ModuleFile &F); void ReadModuleOffsetMap(ModuleFile &F) const; diff --git a/include/clang/Serialization/ASTWriter.h b/include/clang/Serialization/ASTWriter.h index 23afa639de..f3c644b9e2 100644 --- a/include/clang/Serialization/ASTWriter.h +++ b/include/clang/Serialization/ASTWriter.h @@ -106,6 +106,9 @@ private: /// \brief The bitstream writer used to emit this precompiled header. llvm::BitstreamWriter &Stream; + /// The buffer associated with the bitstream. + const SmallVectorImpl &Buffer; + /// \brief The ASTContext we're writing. ASTContext *Context = nullptr; @@ -425,8 +428,16 @@ private: void WriteSubStmt(Stmt *S); void WriteBlockInfoBlock(); - uint64_t WriteControlBlock(Preprocessor &PP, ASTContext &Context, - StringRef isysroot, const std::string &OutputFile); + void WriteControlBlock(Preprocessor &PP, ASTContext &Context, + StringRef isysroot, const std::string &OutputFile); + + /// Write out the signature and diagnostic options, and return the signature. + ASTFileSignature writeUnhashedControlBlock(Preprocessor &PP, + ASTContext &Context); + + /// Calculate hash of the pcm content. + static ASTFileSignature createSignature(StringRef Bytes); + void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts, bool Modules); void WriteSourceManagerBlock(SourceManager &SourceMgr, @@ -493,14 +504,14 @@ private: void WriteDeclAbbrevs(); void WriteDecl(ASTContext &Context, Decl *D); - uint64_t WriteASTCore(Sema &SemaRef, - StringRef isysroot, const std::string &OutputFile, - Module *WritingModule); + ASTFileSignature WriteASTCore(Sema &SemaRef, StringRef isysroot, + const std::string &OutputFile, + Module *WritingModule); public: /// \brief Create a new precompiled header writer that outputs to /// the given bitstream. - ASTWriter(llvm::BitstreamWriter &Stream, + ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, ArrayRef> Extensions, bool IncludeTimestamps = true); ~ASTWriter() override; @@ -526,9 +537,9 @@ public: /// /// \return the module signature, which eventually will be a hash of /// the module but currently is merely a random 32-bit number. - uint64_t WriteAST(Sema &SemaRef, const std::string &OutputFile, - Module *WritingModule, StringRef isysroot, - bool hasErrors = false); + ASTFileSignature WriteAST(Sema &SemaRef, const std::string &OutputFile, + Module *WritingModule, StringRef isysroot, + bool hasErrors = false); /// \brief Emit a token. void AddToken(const Token &Tok, RecordDataImpl &Record); diff --git a/include/clang/Serialization/Module.h b/include/clang/Serialization/Module.h index 2fd156ceb4..c1d4a9fd2d 100644 --- a/include/clang/Serialization/Module.h +++ b/include/clang/Serialization/Module.h @@ -16,6 +16,7 @@ #define LLVM_CLANG_SERIALIZATION_MODULE_H #include "clang/Basic/FileManager.h" +#include "clang/Basic/Module.h" #include "clang/Basic/SourceLocation.h" #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" @@ -89,8 +90,6 @@ public: bool isNotFound() const { return Val.getInt() == NotFound; } }; -typedef unsigned ASTFileSignature; - /// \brief Information about a module that has been loaded by the ASTReader. /// /// Each instance of the Module class corresponds to a single AST file, which @@ -153,9 +152,9 @@ public: /// \brief The file entry for the module file. const FileEntry *File = nullptr; - /// \brief The signature of the module file, which may be used along with size + /// The signature of the module file, which may be used instead of the size /// and modification time to identify this particular file. - ASTFileSignature Signature = 0; + ASTFileSignature Signature; /// \brief Whether this module has been directly imported by the /// user. -- cgit v1.2.3