diff options
Diffstat (limited to 'lib/Index')
-rw-r--r-- | lib/Index/CMakeLists.txt | 3 | ||||
-rw-r--r-- | lib/Index/FileIndexData.cpp | 52 | ||||
-rw-r--r-- | lib/Index/FileIndexData.h | 56 | ||||
-rw-r--r-- | lib/Index/IndexingAction.cpp | 697 | ||||
-rw-r--r-- | lib/Index/IndexingContext.cpp | 85 | ||||
-rw-r--r-- | lib/Index/IndexingContext.h | 32 | ||||
-rw-r--r-- | lib/Index/UnitIndexDataRecorder.cpp | 52 | ||||
-rw-r--r-- | lib/Index/UnitIndexDataRecorder.h | 52 |
8 files changed, 952 insertions, 77 deletions
diff --git a/lib/Index/CMakeLists.txt b/lib/Index/CMakeLists.txt index c9fbfafcf9..38d84e4c19 100644 --- a/lib/Index/CMakeLists.txt +++ b/lib/Index/CMakeLists.txt @@ -6,12 +6,14 @@ set(LLVM_LINK_COMPONENTS add_clang_library(clangIndex CodegenNameGenerator.cpp CommentToXML.cpp + FileIndexData.cpp IndexBody.cpp IndexDecl.cpp IndexingAction.cpp IndexingContext.cpp IndexSymbol.cpp IndexTypeSourceInfo.cpp + UnitIndexDataRecorder.cpp USRGeneration.cpp ADDITIONAL_HEADERS @@ -23,6 +25,7 @@ add_clang_library(clangIndex clangBasic clangFormat clangFrontend + clangLex clangRewrite clangSerialization clangToolingCore diff --git a/lib/Index/FileIndexData.cpp b/lib/Index/FileIndexData.cpp new file mode 100644 index 0000000000..b68319a777 --- /dev/null +++ b/lib/Index/FileIndexData.cpp @@ -0,0 +1,52 @@ +//===--- FileIndexData.cpp - Index data per file ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FileIndexData.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/DeclTemplate.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/Path.h" + +using namespace clang; +using namespace clang::index; + +void FileIndexData::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, + const Decl *D, + ArrayRef<SymbolRelation> Relations) { + assert(D->isCanonicalDecl() && + "Occurrences should be associated with their canonical decl"); + + Decls.emplace_back(Roles, Offset, D, Relations); +} + +std::vector<DeclOccurrence> +FileIndexData::getDeclOccurrencesSortedByOffset() const { + std::vector<DeclOccurrence> Sorted(Decls); + std::sort(Sorted.begin(), Sorted.end()); + return Sorted; +} + +void FileIndexData::print(llvm::raw_ostream &OS) const { + OS << "DECLS BEGIN ---\n"; + for (auto &DclInfo : Decls) { + auto D = DclInfo.Dcl; + SourceManager &SM = D->getASTContext().getSourceManager(); + SourceLocation Loc = SM.getFileLoc(D->getLocation()); + PresumedLoc PLoc = SM.getPresumedLoc(Loc); + OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine() + << ':' << PLoc.getColumn(); + + if (auto ND = dyn_cast<NamedDecl>(D)) { + OS << ' ' << ND->getNameAsString(); + } + + OS << '\n'; + } + OS << "DECLS END ---\n"; +} diff --git a/lib/Index/FileIndexData.h b/lib/Index/FileIndexData.h new file mode 100644 index 0000000000..386b785b51 --- /dev/null +++ b/lib/Index/FileIndexData.h @@ -0,0 +1,56 @@ +//===--- FileIndexData.h - Index data per file --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H +#define LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H + +#include "clang/Basic/SourceLocation.h" +#include "clang/Index/DeclOccurrence.h" +#include "clang/Index/IndexSymbol.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include <vector> + +namespace clang { +class IdentifierInfo; + +namespace index { + +/// Stores the declaration occurrences seen in a particular source or header +/// file of a translation unit +class FileIndexData { +private: + FileID FID; + bool IsSystem; + std::vector<DeclOccurrence> Decls; + +public: + FileIndexData(FileID FID, bool IsSystem) : FID(FID), IsSystem(IsSystem) {} + + std::vector<DeclOccurrence> getDeclOccurrencesSortedByOffset() const; + + FileID getFileID() const { return FID; } + bool isSystem() const { return IsSystem; } + + /// Adds an occurrence of the canonical declaration \c D at the supplied + /// \c Offset + /// + /// \param Roles the roles the occurrence fulfills in this position. + /// \param Offset the offset in the file of this occurrence. + /// \param D the canonical declaration this is an occurrence of. + /// \param Relations the set of symbols related to this occurrence. + void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D, + ArrayRef<SymbolRelation> Relations); + void print(llvm::raw_ostream &OS) const; +}; + +} // end namespace index +} // end namespace clang + +#endif diff --git a/lib/Index/IndexingAction.cpp b/lib/Index/IndexingAction.cpp index 411657bf3d..e17b369e9d 100644 --- a/lib/Index/IndexingAction.cpp +++ b/lib/Index/IndexingAction.cpp @@ -8,35 +8,50 @@ //===----------------------------------------------------------------------===// #include "clang/Index/IndexingAction.h" +#include "clang/Index/RecordingAction.h" +#include "clang/Index/UnitIndexingAction.h" + +#include "FileIndexData.h" #include "IndexingContext.h" +#include "UnitIndexDataRecorder.h" +#include "clang/Basic/FileManager.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/MultiplexConsumer.h" +#include "clang/Frontend/Utils.h" #include "clang/Index/IndexDataConsumer.h" +#include "clang/Index/IndexDiagnostic.h" +#include "clang/Index/UnitIndexDataConsumer.h" #include "clang/Lex/Preprocessor.h" #include "clang/Serialization/ASTReader.h" +#include "llvm/Support/Path.h" using namespace clang; using namespace clang::index; +void UnitIndexDataConsumer::_anchor() {} void IndexDataConsumer::_anchor() {} bool IndexDataConsumer::handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, ArrayRef<SymbolRelation> Relations, FileID FID, unsigned Offset, + bool IsInSystemFile, ASTNodeInfo ASTNode) { return true; } bool IndexDataConsumer::handleMacroOccurence(const IdentifierInfo *Name, const MacroInfo *MI, SymbolRoleSet Roles, - FileID FID, unsigned Offset) { + FileID FID, unsigned Offset, + bool IsInSystemFile) { return true; } bool IndexDataConsumer::handleModuleOccurence(const ImportDecl *ImportD, SymbolRoleSet Roles, - FileID FID, unsigned Offset) { + FileID FID, unsigned Offset, + bool IsInSystemFile) { return true; } @@ -73,96 +88,116 @@ protected: } }; -class IndexActionBase { -protected: - std::shared_ptr<IndexDataConsumer> DataConsumer; - IndexingContext IndexCtx; - - IndexActionBase(std::shared_ptr<IndexDataConsumer> dataConsumer, - IndexingOptions Opts) - : DataConsumer(std::move(dataConsumer)), - IndexCtx(Opts, *DataConsumer) {} - - std::unique_ptr<IndexASTConsumer> - createIndexASTConsumer(CompilerInstance &CI) { - return llvm::make_unique<IndexASTConsumer>(CI.getPreprocessorPtr(), - IndexCtx); - } - - void finish() { - DataConsumer->finish(); - } +/// Abstracts the core logic shared between \c IndexAction and +/// \c WrappingIndexAction frontend actions. +class IndexActionImpl { +public: + virtual ~IndexActionImpl() = default; + + /// Called at the beginning of processing a single input, this creates the + /// IndexASTConsumer object to use. + /// + /// \param CI The compiler instance used to process the input + /// \returns the created IndexASTConsumer. + virtual std::unique_ptr<IndexASTConsumer> + createIndexASTConsumer(CompilerInstance &CI) = 0; + + /// Callback at the end of processing a single input. + /// + /// \param CI The compiler instance used to process the input. It will be the + /// same instance as provided in \c createIndexASTConsumer. + virtual void finish(CompilerInstance &CI) = 0; }; -class IndexAction : public ASTFrontendAction, IndexActionBase { +class IndexAction : public ASTFrontendAction { + std::unique_ptr<IndexActionImpl> Impl; + public: - IndexAction(std::shared_ptr<IndexDataConsumer> DataConsumer, - IndexingOptions Opts) - : IndexActionBase(std::move(DataConsumer), Opts) {} + IndexAction(std::unique_ptr<IndexActionImpl> Impl) : Impl(std::move(Impl)) {} protected: std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override { - return createIndexASTConsumer(CI); + return Impl->createIndexASTConsumer(CI); } void EndSourceFileAction() override { FrontendAction::EndSourceFileAction(); - finish(); + Impl->finish(getCompilerInstance()); } }; -class WrappingIndexAction : public WrapperFrontendAction, IndexActionBase { - bool IndexActionFailed = false; +class WrappingIndexAction : public WrapperFrontendAction { + std::unique_ptr<IndexActionImpl> Impl; + bool CreatedASTConsumer = false; public: WrappingIndexAction(std::unique_ptr<FrontendAction> WrappedAction, - std::shared_ptr<IndexDataConsumer> DataConsumer, - IndexingOptions Opts) - : WrapperFrontendAction(std::move(WrappedAction)), - IndexActionBase(std::move(DataConsumer), Opts) {} + std::unique_ptr<IndexActionImpl> Impl) + : WrapperFrontendAction(std::move(WrappedAction)), Impl(std::move(Impl)) { + } protected: std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI, - StringRef InFile) override; - void EndSourceFileAction() override; + StringRef InFile) override { + auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); + if (!OtherConsumer) + return nullptr; + + std::vector<std::unique_ptr<ASTConsumer>> Consumers; + Consumers.push_back(std::move(OtherConsumer)); + Consumers.push_back(Impl->createIndexASTConsumer(CI)); + CreatedASTConsumer = true; + + return llvm::make_unique<MultiplexConsumer>(std::move(Consumers)); + }; + + void EndSourceFileAction() override { + // Invoke wrapped action's method. + WrapperFrontendAction::EndSourceFileAction(); + if (CreatedASTConsumer) { + CreatedASTConsumer = false; + Impl->finish(getCompilerInstance()); + } + }; }; -} // anonymous namespace +/// An implementation for \c IndexAction or \c WrappingIndexAction that provides +/// decl ocurrences information from the AST. +class DataConsumerActionImpl : public IndexActionImpl { +protected: + std::shared_ptr<IndexDataConsumer> DataConsumer; + IndexingContext IndexCtx; -void WrappingIndexAction::EndSourceFileAction() { - // Invoke wrapped action's method. - WrapperFrontendAction::EndSourceFileAction(); - if (!IndexActionFailed) - finish(); -} +public: + DataConsumerActionImpl(std::shared_ptr<IndexDataConsumer> Consumer, + IndexingOptions Opts) + : DataConsumer(std::move(Consumer)), IndexCtx(Opts, *DataConsumer) {} -std::unique_ptr<ASTConsumer> -WrappingIndexAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { - auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); - if (!OtherConsumer) { - IndexActionFailed = true; - return nullptr; + std::unique_ptr<IndexASTConsumer> + createIndexASTConsumer(CompilerInstance &CI) override { + IndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); + return llvm::make_unique<IndexASTConsumer>(CI.getPreprocessorPtr(), + IndexCtx); } - std::vector<std::unique_ptr<ASTConsumer>> Consumers; - Consumers.push_back(std::move(OtherConsumer)); - Consumers.push_back(createIndexASTConsumer(CI)); - return llvm::make_unique<MultiplexConsumer>(std::move(Consumers)); -} + void finish(CompilerInstance &CI) override { DataConsumer->finish(); } +}; + +} // anonymous namespace std::unique_ptr<FrontendAction> index::createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer, IndexingOptions Opts, std::unique_ptr<FrontendAction> WrappedAction) { + auto ActionImpl = + llvm::make_unique<DataConsumerActionImpl>(std::move(DataConsumer), Opts); if (WrappedAction) return llvm::make_unique<WrappingIndexAction>(std::move(WrappedAction), - std::move(DataConsumer), - Opts); - return llvm::make_unique<IndexAction>(std::move(DataConsumer), Opts); + std::move(ActionImpl)); + return llvm::make_unique<IndexAction>(std::move(ActionImpl)); } - static bool topLevelDeclVisitor(void *context, const Decl *D) { IndexingContext &IndexCtx = *static_cast<IndexingContext*>(context); return IndexCtx.indexTopLevelDecl(D); @@ -209,3 +244,551 @@ void index::indexModuleFile(serialization::ModuleFile &Mod, } DataConsumer->finish(); } + +//===----------------------------------------------------------------------===// +// Index Data Recording +//===----------------------------------------------------------------------===// + +/// Construct a \c UnitDetails for a translation unit with the provided root +/// \c FileEntry or \c Module and with the provided sysroot path. +static index::UnitDetails getUnitDetails(const CompilerInstance &CI, + std::string OutputFile, + const FileEntry *RootFile, + Module *UnitMod, + StringRef SysrootPath) { + std::string ModuleName = + UnitMod ? UnitMod->getFullModuleName() : std::string(); + bool IsSystemUnit = UnitMod ? UnitMod->IsSystem : false; + bool IsModuleUnit = UnitMod != nullptr; + bool IsDebugCompilation = CI.getCodeGenOpts().OptimizationLevel == 0; + + // Ignore sysroot path if it points to root, otherwise every header will be + // treated as system one. + if (llvm::sys::path::root_path(SysrootPath) == SysrootPath) + SysrootPath = ""; + + return {CI, UnitMod, ModuleName, + RootFile, OutputFile, SysrootPath, + IsSystemUnit, IsModuleUnit, IsDebugCompilation}; +} + +/// Construct a \c UnitDetails from the invocation associated with the provided +/// \c CompilerInstance and the provided sysroot path. +static index::UnitDetails getUnitDetails(const CompilerInstance &CI, + StringRef SysrootPath) { + SourceManager &SM = CI.getASTContext().getSourceManager(); + + std::string OutputFile = CI.getFrontendOpts().OutputFile; + if (OutputFile.empty()) { + OutputFile = CI.getFrontendOpts().Inputs[0].getFile(); + OutputFile += ".o"; + } + + const FileEntry *RootFile = nullptr; + Module *UnitMod = nullptr; + bool IsModuleGeneration = CI.getLangOpts().isCompilingModule(); + if (!IsModuleGeneration && + CI.getFrontendOpts().ProgramAction != frontend::GeneratePCH) + RootFile = SM.getFileEntryForID(SM.getMainFileID()); + + if (IsModuleGeneration) { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + UnitMod = HS.lookupModule(CI.getLangOpts().CurrentModule, + /*AllowSearch=*/false); + assert(UnitMod && "only loaded modules should be indexed"); + } + return getUnitDetails(CI, std::move(OutputFile), RootFile, UnitMod, + SysrootPath); +} + +/// Construct a \c UnitDetails for the given module file. +static index::UnitDetails getUnitDetails(serialization::ModuleFile &Mod, + const CompilerInstance &CI, + StringRef SysrootPath) { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + Module *UnitMod = HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false); + assert(UnitMod && "only loaded modules should be indexed"); + + return getUnitDetails(CI, /*OutputFile=*/Mod.FileName, /*RootFile=*/nullptr, + UnitMod, SysrootPath); +} + +namespace { + +/// Collects and groups consumed index data by \c FileID. +class FileIndexDataCollector : public IndexDataConsumer { + std::shared_ptr<Preprocessor> PP; + typedef llvm::DenseMap<FileID, std::unique_ptr<FileIndexData>> + IndexDataByFileTy; + IndexDataByFileTy IndexDataByFile; + +public: + void setPreprocessor(std::shared_ptr<Preprocessor> PreProc) override { + PP = PreProc; + } + + IndexDataByFileTy::const_iterator begin() const { + return IndexDataByFile.begin(); + } + + IndexDataByFileTy::const_iterator end() const { + return IndexDataByFile.end(); + } + + bool empty() const { return IndexDataByFile.empty(); } + + bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, + ArrayRef<SymbolRelation> Relations, + FileID FID, unsigned Offset, bool IsInSystemFile, + ASTNodeInfo ASTNode) override { + ASTContext &Ctx = D->getASTContext(); + SourceManager &SM = Ctx.getSourceManager(); + + // Ignore occurrences in the predefines buffer + if (FID == PP->getPredefinesFileID()) + return true; + + FileIndexData &FileData = getFileIndexData(FID, IsInSystemFile); + FileData.addDeclOccurence(Roles, Offset, D, Relations); + return true; + } + +private: + FileIndexData &getFileIndexData(FileID FID, bool IsInSystemFile) { + auto &Entry = IndexDataByFile[FID]; + if (!Entry) { + Entry.reset(new FileIndexData(FID, IsInSystemFile)); + } + return *Entry; + } +}; + +struct IncludeLocation { + const FileEntry *Source; + const FileEntry *Target; + unsigned Line; +}; + +/// Preprocessor callbacks to collect file to file inclusion information +class IncludePPCallbacks : public PPCallbacks { + SystemFileCache &SystemCache; + UnitIndexingOptions::FileIncludeFilterKind FileIncludeFilter; + std::vector<IncludeLocation> &Includes; + SourceManager &SourceMgr; + +public: + IncludePPCallbacks(SystemFileCache &SystemCache, + UnitIndexingOptions::FileIncludeFilterKind IncludeFilter, + std::vector<IncludeLocation> &IncludesForFile, + SourceManager &SourceMgr) + : SystemCache(SystemCache), FileIncludeFilter(IncludeFilter), + Includes(IncludesForFile), SourceMgr(SourceMgr) {} + + virtual void InclusionDirective(SourceLocation HashLoc, + const Token &IncludeTok, + StringRef FileName, + bool IsAngled, + CharSourceRange FilenameRange, + const FileEntry *File, + StringRef SearchPath, + StringRef RelativePath, + const Module *Imported) override { + if (HashLoc.isFileID() && File && File->isValid()) + addInclude(HashLoc, File); + } + +private: + void addInclude(SourceLocation From, const FileEntry *To) { + assert(To); + if (FileIncludeFilter == UnitIndexingOptions::FileIncludeFilterKind::None) + return; + + std::pair<FileID, unsigned> LocInfo = + SourceMgr.getDecomposedExpansionLoc(From); + + if (LocInfo.first.isInvalid()) + return; // Ignore invalid locations. + + if (FileIncludeFilter == + UnitIndexingOptions::FileIncludeFilterKind::UserOnly) + if (SystemCache.isSystem(LocInfo.first, SourceMgr)) + return; // Ignore includes of system headers. + + if (auto *FE = SourceMgr.getFileEntryForID(LocInfo.first)) { + auto lineNo = SourceMgr.getLineNumber(LocInfo.first, LocInfo.second); + Includes.push_back({FE, To, lineNo}); + } + } +}; + +/// Abstract interface for providing the file and module dependencies of a +/// translation unit, as well as the set of file to file inclusions +class IndexDependencyProvider { +public: + virtual ~IndexDependencyProvider() {} + + virtual void forEachFileDependency( + const CompilerInstance &CI, + llvm::function_ref<void(const FileEntry *FE, bool IsSystem)> Callback) + const = 0; + + virtual void + forEachInclude(llvm::function_ref<void(const FileEntry *Source, unsigned Line, + const FileEntry *Target)> + Callback) const = 0; + virtual void forEachModuleImport( + const CompilerInstance &CI, + llvm::function_ref<void(serialization::ModuleFile &Mod, bool IsSystem)> + Callback) const = 0; +}; + +/// An IndexDependencyProvider for the index data collected by +/// \c FileIndexDependencyCollector. +class FileIndexDependencyProvider : public IndexDependencyProvider { + llvm::SetVector<const FileEntry *> Files; + llvm::BitVector IsSystemByUID; + std::vector<IncludeLocation> Includes; + bool IncludeSysModules; + +public: + FileIndexDependencyProvider(llvm::SetVector<const FileEntry *> Entries, + llvm::BitVector IsSystemByUID, + std::vector<IncludeLocation> Includes, + bool IncludeSysDeps) + : Files(std::move(Entries)), IsSystemByUID(std::move(IsSystemByUID)), + Includes(std::move(Includes)), IncludeSysModules(IncludeSysDeps) {} + + void forEachFileDependency( + const CompilerInstance &CI, + llvm::function_ref<void(const FileEntry *FE, bool IsSystem)> Callback) + const override { + for (auto *FE : Files) + Callback(FE, isSystemFile(FE)); + } + + void + forEachInclude(llvm::function_ref<void(const FileEntry *Source, unsigned Line, + const FileEntry *Target)> + Callback) const override { + for (auto &Include : Includes) + Callback(Include.Source, Include.Line, Include.Target); + } + + void forEachModuleImport( + const CompilerInstance &CI, + llvm::function_ref<void(serialization::ModuleFile &Mod, bool IsSystem)> + Callback) const override { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + + if (auto Reader = CI.getModuleManager()) { + Reader->getModuleManager().visit( + [&](serialization::ModuleFile &Mod) -> bool { + bool IsSystemMod = false; + if (Mod.isModule()) { + if (auto *M = + HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false)) + IsSystemMod = M->IsSystem; + } + if (!IsSystemMod || IncludeSysModules) + Callback(Mod, IsSystemMod); + return true; // skip module dependencies. + }); + } + } + +private: + bool isSystemFile(const FileEntry *FE) const { + auto UID = FE->getUID(); + return IsSystemByUID.size() > UID && IsSystemByUID[UID]; + } +}; + +/// Collects file and module dependency information for a translation unit, +/// including file to file inclusions. +class FileIndexDependencyCollector : public DependencyCollector { + SystemFileCache &SystemCache; + UnitIndexingOptions IndexOpts; + llvm::SetVector<const FileEntry *> SeenFiles; + llvm::BitVector IsSystemByUID; + std::vector<IncludeLocation> Includes; + SourceManager *SourceMgr = nullptr; + +public: + FileIndexDependencyCollector(SystemFileCache &SystemCache, + UnitIndexingOptions IndexOpts) + : SystemCache(SystemCache), IndexOpts(IndexOpts) {} + + void attachToPreprocessor(Preprocessor &PP) override { + DependencyCollector::attachToPreprocessor(PP); + PP.addPPCallbacks(llvm::make_unique<IncludePPCallbacks>( + SystemCache, IndexOpts.FileIncludeFilter, Includes, + PP.getSourceManager())); + } + + void setSourceManager(SourceManager *SourceMgr) { + this->SourceMgr = SourceMgr; + } + + FileIndexDependencyProvider consume() { + return FileIndexDependencyProvider( + std::move(SeenFiles), std::move(IsSystemByUID), std::move(Includes), + IndexOpts.IncludeSystemDependencies); + } + +private: + bool needSystemDependencies() override { + return IndexOpts.IncludeSystemDependencies; + } + + bool sawDependency(StringRef Filename, bool FromModule, bool IsSystem, + bool IsModuleFile, bool IsMissing) override { + bool SawIt = DependencyCollector::sawDependency( + Filename, FromModule, IsSystem, IsModuleFile, IsMissing); + if (auto *FE = SourceMgr->getFileManager().getFile(Filename)) { + if (SawIt) + SeenFiles.insert(FE); + + // Record system-ness for all files that we pass through. + if (IsSystemByUID.size() < FE->getUID() + 1) + IsSystemByUID.resize(FE->getUID() + 1); + IsSystemByUID[FE->getUID()] = IsSystem || isInSysroot(Filename); + } + return SawIt; + } + + bool isInSysroot(StringRef Filename) { + StringRef SysrootPath = SystemCache.getSysrootPath(); + return !SysrootPath.empty() && Filename.startswith(SysrootPath); + } +}; +} // anonymous namespace + +static void reportData(const CompilerInstance &CI, + const FileIndexDataCollector &Collector, + const IndexDependencyProvider &DepProvider, + UnitDetails UnitInfo, + const IndexUnitDataConsumerFactory &UnitConsumerFactory, + const UnitIndexingOptions &IndexOpts) { + + std::unique_ptr<UnitIndexDataConsumer> Consumer = + UnitConsumerFactory(UnitInfo); + if (!Consumer) + return; + + DepProvider.forEachFileDependency( + CI, [&](const FileEntry *FE, bool IsSystemFile) { + Consumer->handleFileDependency(FE, IsSystemFile); + }); + DepProvider.forEachInclude( + [&](const FileEntry *Source, unsigned Line, const FileEntry *Target) { + Consumer->handleInclude(Source, Line, Target); + }); + DepProvider.forEachModuleImport( + CI, [&](serialization::ModuleFile &Mod, bool IsSystemMod) { + Consumer->handleModuleImport(Mod, IsSystemMod); + if (Mod.isModule() && Consumer->shouldIndexModuleDependency(Mod)) + indexModuleFile(Mod, CI, UnitConsumerFactory, IndexOpts); + }); + + for (auto I = Collector.begin(), E = Collector.end(); I != E; ++I) { + FileID FID = I->first; + const FileIndexData &FileData = *I->second; + if (Consumer->handleFileOccurrences( + FID, FileData.getDeclOccurrencesSortedByOffset(), + FileData.isSystem())) + return; + } + + Consumer->finish(); +} + +namespace { + +/// An implementation for IndexAction or WrappingIndexAction that gathers decl +/// occurrence, file inclusion and dependency information for the translation +/// unit and, optionally, its module dependencies. +class UnitDataConsumerActionImpl : public IndexActionImpl { + UnitIndexingOptions IndexOpts; + FileIndexDataCollector Collector; + IndexingContext IndexCtx; + FileIndexDependencyCollector DepCollector; + IndexUnitDataConsumerFactory UnitConsumerFactory; + +public: + UnitDataConsumerActionImpl(UnitIndexingOptions UnitIndexOpts, + IndexUnitDataConsumerFactory UnitConsumerFactory) + : IndexOpts(UnitIndexOpts), IndexCtx(UnitIndexOpts, Collector), + DepCollector(IndexCtx.getSystemCache(), IndexOpts), + UnitConsumerFactory(std::move(UnitConsumerFactory)) {} + + std::unique_ptr<IndexASTConsumer> + createIndexASTConsumer(CompilerInstance &CI) override { + IndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); + + std::shared_ptr<Preprocessor> PP = CI.getPreprocessorPtr(); + Collector.setPreprocessor(PP); + DepCollector.setSourceManager(&CI.getSourceManager()); + DepCollector.attachToPreprocessor(CI.getPreprocessor()); + + return llvm::make_unique<IndexASTConsumer>(PP, IndexCtx); + } + + /// Provides the collected indexing info to the \c IndexUnitDataConsumer + void finish(CompilerInstance &CI) override { + // The consumer may emit more diagnostics so do the begin/end source file + // invocations on the diagnostic client. + // FIXME: FrontendAction::EndSourceFile() should probably not call + // CI.getDiagnosticClient().EndSourceFile()' until after it has called + // 'EndSourceFileAction()', so that code executing during + // EndSourceFileAction() can emit diagnostics. If this is fixed, + // DiagClientBeginEndRAII can go away. + struct DiagClientBeginEndRAII { + CompilerInstance &CI; + DiagClientBeginEndRAII(CompilerInstance &CI) : CI(CI) { + CI.getDiagnosticClient().BeginSourceFile(CI.getLangOpts()); + } + ~DiagClientBeginEndRAII() { CI.getDiagnosticClient().EndSourceFile(); } + } diagClientBeginEndRAII(CI); + + Collector.finish(); + reportData(CI, Collector, DepCollector.consume(), + getUnitDetails(CI, IndexCtx.getSysrootPath()), + UnitConsumerFactory, IndexOpts); + } +}; + +/// Provides the file and module dependency information for a \c ModuleFile +class ModuleFileIndexDependencyCollector : public IndexDependencyProvider { + serialization::ModuleFile &ModFile; + bool CollectSystemDependencies; + +public: + ModuleFileIndexDependencyCollector(serialization::ModuleFile &Mod, + bool CollectSystemDependencies) + : ModFile(Mod), CollectSystemDependencies(CollectSystemDependencies) {} + + void forEachFileDependency( + const CompilerInstance &CI, + llvm::function_ref<void(const FileEntry *FE, bool IsSystem)> Callback) + const override { + auto Reader = CI.getModuleManager(); + Reader->visitInputFiles( + ModFile, CollectSystemDependencies, /*Complain=*/false, + [&](const serialization::InputFile &IF, bool IsSystem) { + auto *FE = IF.getFile(); + if (!FE) + return; + // Ignore module map files, they are not as important to track as + // source files and they may be auto-generated which would create an + // undesirable dependency on an intermediate build byproduct. + if (FE->getName().endswith("module.modulemap")) + return; + + Callback(FE, IsSystem); + }); + } + + void + forEachInclude(llvm::function_ref<void(const FileEntry *Source, unsigned Line, + const FileEntry *Target)> + Callback) const override { + // FIXME: Module files without a preprocessing record do not have info about + // include locations. Serialize enough data to be able to retrieve such + // info. + } + + void forEachModuleImport( + const CompilerInstance &CI, + llvm::function_ref<void(serialization::ModuleFile &Mod, bool IsSystem)> + Callback) const override { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + for (auto *Mod : ModFile.Imports) { + bool IsSystemMod = false; + if (auto *M = HS.lookupModule(Mod->ModuleName, /*AllowSearch=*/false)) + IsSystemMod = M->IsSystem; + if (!IsSystemMod || CollectSystemDependencies) + Callback(*Mod, IsSystemMod); + } + } +}; +} // anonymous namespace. + +void index::indexModuleFile(serialization::ModuleFile &Mod, + const CompilerInstance &CI, + IndexUnitDataConsumerFactory UnitConsumerFactory, + UnitIndexingOptions IndexOpts) { + + DiagnosticsEngine &Diag = CI.getDiagnostics(); + Diag.Report(Mod.ImportLoc, diag::remark_index_producing_module_file_data) + << Mod.FileName; + + FileIndexDataCollector Collector; + IndexingContext ModIndexCtx(IndexOpts, Collector); + + auto &ASTCtx = CI.getASTContext(); + Collector.initialize(ASTCtx); + Collector.setPreprocessor(CI.getPreprocessorPtr()); + ModIndexCtx.setASTContext(ASTCtx); + ModIndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); + + for (const Decl *D : CI.getModuleManager()->getModuleFileLevelDecls(Mod)) + ModIndexCtx.indexTopLevelDecl(D); + + Collector.finish(); + + ModuleFileIndexDependencyCollector DepCollector( + Mod, IndexOpts.IncludeSystemDependencies); + + reportData(CI, Collector, DepCollector, + getUnitDetails(Mod, CI, ModIndexCtx.getSysrootPath()), + UnitConsumerFactory, IndexOpts); +} + +std::unique_ptr<FrontendAction> +index::createUnitIndexingAction(IndexUnitDataConsumerFactory ConsumerFactory, + UnitIndexingOptions IndexOpts, + std::unique_ptr<FrontendAction> WrappedAction) { + auto ActionImpl = llvm::make_unique<UnitDataConsumerActionImpl>( + std::move(IndexOpts), ConsumerFactory); + if (WrappedAction) + return llvm::make_unique<WrappingIndexAction>(std::move(WrappedAction), + std::move(ActionImpl)); + return llvm::make_unique<IndexAction>(std::move(ActionImpl)); +}; + +std::unique_ptr<FrontendAction> index::createIndexDataRecordingAction( + RecordingOptions RecordOpts, + std::unique_ptr<FrontendAction> WrappedAction) { + + auto ConsumerFactory = + [RecordOpts]( + UnitDetails UnitInfo) -> std::unique_ptr<UnitIndexDataConsumer> { + return llvm::make_unique<UnitIndexDataRecorder>(std::move(UnitInfo), + RecordOpts); + }; + return createUnitIndexingAction(ConsumerFactory, std::move(RecordOpts), + std::move(WrappedAction)); +}; + +RecordingOptions +index::getRecordingOptionsFromFrontendOptions(const FrontendOptions &FEOpts) { + RecordingOptions RecordOpts; + RecordOpts.DataDirPath = FEOpts.IndexStorePath; + if (FEOpts.IndexIgnoreSystemSymbols) { + RecordOpts.SystemSymbolFilter = + index::IndexingOptions::SystemSymbolFilterKind::None; + } + RecordOpts.RecordSymbolCodeGenName = FEOpts.IndexRecordCodegenName; + return RecordOpts; +} + +void index::recordIndexDataForModuleFile(serialization::ModuleFile *ModFile, + RecordingOptions RecordOpts, + const CompilerInstance &CI) { + auto UnitConsumerFactory = [RecordOpts](UnitDetails UnitInfo) { + return llvm::make_unique<UnitIndexDataRecorder>(std::move(UnitInfo), + RecordOpts); + }; + return indexModuleFile(*ModFile, CI, UnitConsumerFactory, + std::move(RecordOpts)); +} diff --git a/lib/Index/IndexingContext.cpp b/lib/Index/IndexingContext.cpp index de9fe39df0..17690487a1 100644 --- a/lib/Index/IndexingContext.cpp +++ b/lib/Index/IndexingContext.cpp @@ -8,14 +8,16 @@ //===----------------------------------------------------------------------===// #include "IndexingContext.h" -#include "clang/Index/IndexDataConsumer.h" #include "clang/AST/ASTContext.h" -#include "clang/AST/DeclTemplate.h" #include "clang/AST/DeclObjC.h" +#include "clang/AST/DeclTemplate.h" #include "clang/Basic/SourceManager.h" +#include "clang/Index/IndexDataConsumer.h" +#include "llvm/Support/Path.h" using namespace clang; using namespace index; +using namespace llvm; static bool isGeneratedDecl(const Decl *D) { if (auto *attr = D->getAttr<ExternalSourceSymbolAttr>()) { @@ -24,6 +26,59 @@ static bool isGeneratedDecl(const Decl *D) { return false; } +void SystemFileCache::setSysrootPath(llvm::StringRef Path) { + // Ignore sysroot path if it points to root, otherwise every header will be + // treated as system one. + SysrootPath = sys::path::root_path(Path) == Path ? StringRef() : Path; + + // Invalidate existing results + LastFileCheck = {FileID(), false}; + DirEntries.clear(); +} + +SystemFileCache::SystemFileCache(std::string Path) { setSysrootPath(Path); } + +bool SystemFileCache::isSystem(clang::FileID FID, clang::SourceManager &SM) { + if (LastFileCheck.first == FID) + return LastFileCheck.second; + + auto Result = [&](bool Res) -> bool { + LastFileCheck = {FID, Res}; + return Res; + }; + + bool Invalid = false; + const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); + if (Invalid || !SEntry.isFile()) + return Result(false); + + const SrcMgr::FileInfo &FI = SEntry.getFile(); + if (FI.getFileCharacteristic() != SrcMgr::C_User) + return Result(true); + + auto *CC = FI.getContentCache(); + if (!CC) + return Result(false); + auto *FE = CC->OrigEntry; + if (!FE) + return Result(false); + + if (SysrootPath.empty()) + return Result(false); + + // Check if directory is in sysroot so that we can consider system headers + // even the headers found via a user framework search path, pointing inside + // sysroot. + auto DirEntry = FE->getDir(); + auto Pair = DirEntries.insert(std::make_pair(DirEntry, false)); + bool &IsSystemDir = Pair.first->second; + bool WasInserted = Pair.second; + if (WasInserted) { + IsSystemDir = StringRef(DirEntry->getName()).startswith(SysrootPath); + } + return Result(IsSystemDir); +} + bool IndexingContext::shouldIndex(const Decl *D) { return !isGeneratedDecl(D); } @@ -93,12 +148,8 @@ bool IndexingContext::importedModule(const ImportDecl *ImportD) { if (FID.isInvalid()) return true; - bool Invalid = false; - const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); - if (Invalid || !SEntry.isFile()) - return true; - - if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) { + bool IsInSystemFile = SystemCache.isSystem(FID, SM); + if (IsInSystemFile) { switch (IndexOpts.SystemSymbolFilter) { case IndexingOptions::SystemSymbolFilterKind::None: return true; @@ -112,7 +163,8 @@ bool IndexingContext::importedModule(const ImportDecl *ImportD) { if (ImportD->isImplicit()) Roles |= (unsigned)SymbolRole::Implicit; - return DataConsumer.handleModuleOccurence(ImportD, Roles, FID, Offset); + return DataConsumer.handleModuleOccurence(ImportD, Roles, FID, Offset, + IsInSystemFile); } bool IndexingContext::isTemplateImplicitInstantiation(const Decl *D) { @@ -205,7 +257,8 @@ static const Decl *adjustTemplateImplicitInstantiation(const Decl *D) { return nullptr; } -static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC, ASTContext &Ctx) { +static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC, + ASTContext &Ctx) { if (auto VD = dyn_cast<VarDecl>(D)) return VD->isThisDeclarationADefinition(Ctx); @@ -321,7 +374,7 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc, const Expr *OrigE, const Decl *OrigD, const DeclContext *ContainerDC) { - if (D->isImplicit() && !isa<ObjCMethodDecl>(D)) + if (D->isImplicit() && !(isa<ObjCMethodDecl>(D) || isa<ObjCIvarDecl>(D))) return true; if (!isa<NamedDecl>(D) || shouldSkipNamelessDecl(cast<NamedDecl>(D))) return true; @@ -337,12 +390,8 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc, if (FID.isInvalid()) return true; - bool Invalid = false; - const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); - if (Invalid || !SEntry.isFile()) - return true; - - if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) { + bool IsInSystemFile = SystemCache.isSystem(FID, SM); + if (IsInSystemFile) { switch (IndexOpts.SystemSymbolFilter) { case IndexingOptions::SystemSymbolFilterKind::None: return true; @@ -416,5 +465,5 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc, IndexDataConsumer::ASTNodeInfo Node{ OrigE, OrigD, Parent, ContainerDC }; return DataConsumer.handleDeclOccurence(D, Roles, FinalRelations, FID, Offset, - Node); + IsInSystemFile, Node); } diff --git a/lib/Index/IndexingContext.h b/lib/Index/IndexingContext.h index 566651c83a..33e90e85bd 100644 --- a/lib/Index/IndexingContext.h +++ b/lib/Index/IndexingContext.h @@ -11,9 +11,11 @@ #define LLVM_CLANG_LIB_INDEX_INDEXINGCONTEXT_H #include "clang/Basic/LLVM.h" +#include "clang/Basic/SourceLocation.h" #include "clang/Index/IndexSymbol.h" #include "clang/Index/IndexingAction.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" namespace clang { class ASTContext; @@ -29,25 +31,51 @@ namespace clang { class Stmt; class Expr; class TypeLoc; - class SourceLocation; + class DirectoryEntry; namespace index { class IndexDataConsumer; +/// Tracks the current system root path and computes and caches whether a +/// file is considered a system file or not +class SystemFileCache { + std::string SysrootPath; + // Records whether a directory entry is system or not. + llvm::DenseMap<const DirectoryEntry *, bool> DirEntries; + // Keeps track of the last check for whether a FileID is system or + // not. This is used to speed up isSystemFile() call. + std::pair<FileID, bool> LastFileCheck; + +public: + SystemFileCache() = default; + SystemFileCache(std::string SysrootPath); + + void setSysrootPath(StringRef path); + StringRef getSysrootPath() const { return SysrootPath; } + bool isSystem(FileID FID, SourceManager &SM); +}; + +/// Generates and reports indexing data to the provided \c IndexDataConsumer +/// for any AST nodes passed to its various \c index* methods. class IndexingContext { IndexingOptions IndexOpts; + SystemFileCache SystemCache; IndexDataConsumer &DataConsumer; ASTContext *Ctx = nullptr; public: IndexingContext(IndexingOptions IndexOpts, IndexDataConsumer &DataConsumer) - : IndexOpts(IndexOpts), DataConsumer(DataConsumer) {} + : IndexOpts(IndexOpts), DataConsumer(DataConsumer) {} const IndexingOptions &getIndexOpts() const { return IndexOpts; } + SystemFileCache &getSystemCache() { return SystemCache; } IndexDataConsumer &getDataConsumer() { return DataConsumer; } void setASTContext(ASTContext &ctx) { Ctx = &ctx; } + void setSysrootPath(StringRef path) { SystemCache.setSysrootPath(path); } + StringRef getSysrootPath() const { return SystemCache.getSysrootPath(); } + bool shouldIndex(const Decl *D); const LangOptions &getLangOpts() const; diff --git a/lib/Index/UnitIndexDataRecorder.cpp b/lib/Index/UnitIndexDataRecorder.cpp new file mode 100644 index 0000000000..f7f2afe843 --- /dev/null +++ b/lib/Index/UnitIndexDataRecorder.cpp @@ -0,0 +1,52 @@ +//===--- UnitIndexDataRecorder.cpp - Persist index data to the file system ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "UnitIndexDataRecorder.h" +#include "clang/Frontend/FrontendOptions.h" + +using namespace clang; +using namespace clang::index; + +UnitIndexDataRecorder::UnitIndexDataRecorder(UnitDetails UnitDetails, + RecordingOptions RecordOpts) + : UnitInfo(UnitDetails) { + // TODO +} + +void UnitIndexDataRecorder::handleFileDependency(const FileEntry *FE, + bool IsSystem) { + // TODO +} + +void UnitIndexDataRecorder::handleInclude(const FileEntry *Source, + unsigned int Line, + const FileEntry *Target) { + // TODO +} + +void UnitIndexDataRecorder::handleModuleImport( + const serialization::ModuleFile &Mod, bool IsSystem) { + // TODO +} + +bool UnitIndexDataRecorder::shouldIndexModuleDependency( + const serialization::ModuleFile &Mod) { + // TODO + return true; +}; + +bool UnitIndexDataRecorder::handleFileOccurrences( + FileID FID, ArrayRef<DeclOccurrence> Occurs, bool IsSystem) { + // TODO + return false; +}; + +void UnitIndexDataRecorder::finish(){ + // TODO +}; diff --git a/lib/Index/UnitIndexDataRecorder.h b/lib/Index/UnitIndexDataRecorder.h new file mode 100644 index 0000000000..6f81d61270 --- /dev/null +++ b/lib/Index/UnitIndexDataRecorder.h @@ -0,0 +1,52 @@ +//===--- UnitIndexDataRecorder.h - Persist index data to the file system --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_UNITINDEXDATARECORDER_H +#define LLVM_CLANG_LIB_INDEX_UNITINDEXDATARECORDER_H + +#include "clang/Index/RecordingAction.h" +#include "clang/Index/UnitIndexDataConsumer.h" + +namespace clang { +class DiagnosticsEngine; +class FrontendOptions; + +namespace index { + +/// Persists the provided index data for a single translation unit out to the +/// file system. +class UnitIndexDataRecorder : public UnitIndexDataConsumer { +protected: + UnitDetails UnitInfo; + +public: + UnitIndexDataRecorder(UnitDetails UnitInfo, RecordingOptions RecordOpts); + + void handleFileDependency(const FileEntry *FE, bool IsSystem) override; + + void handleInclude(const FileEntry *Source, unsigned Line, + const FileEntry *Target) override; + + void handleModuleImport(const serialization::ModuleFile &Mod, + bool IsSystem) override; + + bool + shouldIndexModuleDependency(const serialization::ModuleFile &Mod) override; + + bool handleFileOccurrences(FileID FID, + ArrayRef<DeclOccurrence> OccurrencesSortedByOffset, + bool IsSystem) override; + + void finish() override; +}; + +} // end namespace index +} // end namespace clang + +#endif |