1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
|
//===--- FileIndex.cpp - Indexes for files. ------------------------ C++-*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#include "FileIndex.h"
#include "ClangdUnit.h"
#include "Logger.h"
#include "SymbolCollector.h"
#include "index/Index.h"
#include "index/MemIndex.h"
#include "index/Merge.h"
#include "index/dex/Dex.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include <memory>
namespace clang {
namespace clangd {
static std::pair<SymbolSlab, RefSlab>
indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
llvm::ArrayRef<Decl *> DeclsToIndex, bool IsIndexMainAST) {
SymbolCollector::Options CollectorOpts;
// FIXME(ioeric): we might also want to collect include headers. We would need
// to make sure all includes are canonicalized (with CanonicalIncludes), which
// is not trivial given the current way of collecting symbols: we only have
// AST at this point, but we also need preprocessor callbacks (e.g.
// CommentHandler for IWYU pragma) to canonicalize includes.
CollectorOpts.CollectIncludePath = false;
CollectorOpts.CountReferences = false;
CollectorOpts.Origin = SymbolOrigin::Dynamic;
index::IndexingOptions IndexOpts;
// We only need declarations, because we don't count references.
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
IndexOpts.IndexFunctionLocals = false;
if (IsIndexMainAST) {
// We only collect refs when indexing main AST.
CollectorOpts.RefFilter = RefKind::All;
}else {
IndexOpts.IndexMacrosInPreprocessor = true;
CollectorOpts.CollectMacro = true;
}
SymbolCollector Collector(std::move(CollectorOpts));
Collector.setPreprocessor(PP);
index::indexTopLevelDecls(AST, *PP, DeclsToIndex, Collector, IndexOpts);
const auto &SM = AST.getSourceManager();
const auto *MainFileEntry = SM.getFileEntryForID(SM.getMainFileID());
std::string FileName = MainFileEntry ? MainFileEntry->getName() : "";
auto Syms = Collector.takeSymbols();
auto Refs = Collector.takeRefs();
vlog("index AST for {0} (main={1}): \n"
" symbol slab: {2} symbols, {3} bytes\n"
" ref slab: {4} symbols, {5} refs, {6} bytes",
FileName, IsIndexMainAST, Syms.size(), Syms.bytes(), Refs.size(),
Refs.numRefs(), Refs.bytes());
return {std::move(Syms), std::move(Refs)};
}
std::pair<SymbolSlab, RefSlab> indexMainDecls(ParsedAST &AST) {
return indexSymbols(AST.getASTContext(), AST.getPreprocessorPtr(),
AST.getLocalTopLevelDecls(),
/*IsIndexMainAST=*/true);
}
SymbolSlab indexHeaderSymbols(ASTContext &AST,
std::shared_ptr<Preprocessor> PP) {
std::vector<Decl *> DeclsToIndex(
AST.getTranslationUnitDecl()->decls().begin(),
AST.getTranslationUnitDecl()->decls().end());
return indexSymbols(AST, std::move(PP), DeclsToIndex,
/*IsIndexMainAST=*/false)
.first;
}
void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Symbols,
std::unique_ptr<RefSlab> Refs) {
std::lock_guard<std::mutex> Lock(Mutex);
if (!Symbols)
FileToSymbols.erase(Path);
else
FileToSymbols[Path] = std::move(Symbols);
if (!Refs)
FileToRefs.erase(Path);
else
FileToRefs[Path] = std::move(Refs);
}
std::unique_ptr<SymbolIndex>
FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) {
std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
std::vector<std::shared_ptr<RefSlab>> RefSlabs;
{
std::lock_guard<std::mutex> Lock(Mutex);
for (const auto &FileAndSymbols : FileToSymbols)
SymbolSlabs.push_back(FileAndSymbols.second);
for (const auto &FileAndRefs : FileToRefs)
RefSlabs.push_back(FileAndRefs.second);
}
std::vector<const Symbol *> AllSymbols;
std::vector<Symbol> SymsStorage;
switch (DuplicateHandle) {
case DuplicateHandling::Merge: {
llvm::DenseMap<SymbolID, Symbol> Merged;
for (const auto &Slab : SymbolSlabs) {
for (const auto &Sym : *Slab) {
auto I = Merged.try_emplace(Sym.ID, Sym);
if (!I.second)
I.first->second = mergeSymbol(I.first->second, Sym);
}
}
SymsStorage.reserve(Merged.size());
for (auto &Sym : Merged) {
SymsStorage.push_back(std::move(Sym.second));
AllSymbols.push_back(&SymsStorage.back());
}
// FIXME: aggregate symbol reference count based on references.
break;
}
case DuplicateHandling::PickOne: {
llvm::DenseSet<SymbolID> AddedSymbols;
for (const auto &Slab : SymbolSlabs)
for (const auto &Sym : *Slab)
if (AddedSymbols.insert(Sym.ID).second)
AllSymbols.push_back(&Sym);
break;
}
}
std::vector<Ref> RefsStorage; // Contiguous ranges for each SymbolID.
llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
{
llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
size_t Count = 0;
for (const auto &RefSlab : RefSlabs)
for (const auto &Sym : *RefSlab) {
MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
Count += Sym.second.size();
}
RefsStorage.reserve(Count);
AllRefs.reserve(MergedRefs.size());
for (auto &Sym : MergedRefs) {
auto &SymRefs = Sym.second;
// Sorting isn't required, but yields more stable results over rebuilds.
llvm::sort(SymRefs);
llvm::copy(SymRefs, back_inserter(RefsStorage));
AllRefs.try_emplace(
Sym.first,
llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
SymRefs.size()));
}
}
size_t StorageSize =
RefsStorage.size() * sizeof(Ref) + SymsStorage.size() * sizeof(Symbol);
for (const auto &Slab : SymbolSlabs)
StorageSize += Slab->bytes();
for (const auto &RefSlab : RefSlabs)
StorageSize += RefSlab->bytes();
// Index must keep the slabs and contiguous ranges alive.
switch (Type) {
case IndexType::Light:
return llvm::make_unique<MemIndex>(
llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
std::move(RefsStorage), std::move(SymsStorage)),
StorageSize);
case IndexType::Heavy:
return llvm::make_unique<dex::Dex>(
llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
std::move(RefsStorage), std::move(SymsStorage)),
StorageSize);
}
llvm_unreachable("Unknown clangd::IndexType");
}
FileIndex::FileIndex(bool UseDex)
: MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
PreambleIndex(llvm::make_unique<MemIndex>()),
MainFileIndex(llvm::make_unique<MemIndex>()) {}
void FileIndex::updatePreamble(PathRef Path, ASTContext &AST,
std::shared_ptr<Preprocessor> PP) {
auto Symbols = indexHeaderSymbols(AST, std::move(PP));
PreambleSymbols.update(Path,
llvm::make_unique<SymbolSlab>(std::move(Symbols)),
llvm::make_unique<RefSlab>());
PreambleIndex.reset(
PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
DuplicateHandling::PickOne));
}
void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
auto Contents = indexMainDecls(AST);
MainFileSymbols.update(
Path, llvm::make_unique<SymbolSlab>(std::move(Contents.first)),
llvm::make_unique<RefSlab>(std::move(Contents.second)));
MainFileIndex.reset(
MainFileSymbols.buildIndex(IndexType::Light, DuplicateHandling::PickOne));
}
} // namespace clangd
} // namespace clang
|
