1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
//===--- Background.h - Build an index in a background thread ----*- C++-*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
#include "Context.h"
#include "FSProvider.h"
#include "GlobalCompilationDatabase.h"
#include "Threading.h"
#include "index/FileIndex.h"
#include "index/Index.h"
#include "index/Serialization.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/SHA1.h"
#include "llvm/Support/Threading.h"
#include <atomic>
#include <condition_variable>
#include <deque>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
namespace clang {
namespace clangd {
// Handles storage and retrieval of index shards. Both store and load
// operations can be called from multiple-threads concurrently.
class BackgroundIndexStorage {
public:
virtual ~BackgroundIndexStorage() = default;
// Shards of the index are stored and retrieved independently, keyed by shard
// identifier - in practice this is a source file name
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
IndexFileOut Shard) const = 0;
// Tries to load shard with given identifier, returns nullptr if shard
// couldn't be loaded.
virtual std::unique_ptr<IndexFileIn>
loadShard(llvm::StringRef ShardIdentifier) const = 0;
// The factory provides storage for each CDB.
// It keeps ownership of the storage instances, and should manage caching
// itself. Factory must be threadsafe and never returns nullptr.
using Factory =
llvm::unique_function<BackgroundIndexStorage *(llvm::StringRef)>;
// Creates an Index Storage that saves shards into disk. Index storage uses
// CDBDirectory + ".clangd-index/" as the folder to save shards.
static Factory createDiskBackedStorageFactory();
};
// Builds an in-memory index by by running the static indexer action over
// all commands in a compilation database. Indexing happens in the background.
// FIXME: it should also persist its state on disk for fast start.
// FIXME: it should watch for changes to files on disk.
class BackgroundIndex : public SwapIndex {
public:
/// If BuildIndexPeriodMs is greater than 0, the symbol index will only be
/// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is
/// rebuilt for each indexed file.
// FIXME: resource-dir injection should be hoisted somewhere common.
BackgroundIndex(Context BackgroundContext, llvm::StringRef ResourceDir,
const FileSystemProvider &,
const GlobalCompilationDatabase &CDB,
BackgroundIndexStorage::Factory IndexStorageFactory,
size_t BuildIndexPeriodMs = 0,
size_t ThreadPoolSize = llvm::hardware_concurrency());
~BackgroundIndex(); // Blocks while the current task finishes.
// Enqueue translation units for indexing.
// The indexing happens in a background thread, so the symbols will be
// available sometime later.
void enqueue(const std::vector<std::string> &ChangedFiles);
// Cause background threads to stop after ther current task, any remaining
// tasks will be discarded.
void stop();
// Wait until the queue is empty, to allow deterministic testing.
LLVM_NODISCARD bool
blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10);
private:
/// Given index results from a TU, only update symbols coming from files with
/// different digests than \p DigestsSnapshot. Also stores new index
/// information on IndexStorage.
void update(llvm::StringRef MainFile, IndexFileIn Index,
const llvm::StringMap<FileDigest> &DigestsSnapshot,
BackgroundIndexStorage *IndexStorage);
// configuration
std::string ResourceDir;
const FileSystemProvider &FSProvider;
const GlobalCompilationDatabase &CDB;
Context BackgroundContext;
// index state
llvm::Error index(tooling::CompileCommand,
BackgroundIndexStorage *IndexStorage);
void buildIndex(); // Rebuild index periodically every BuildIndexPeriodMs.
const size_t BuildIndexPeriodMs;
std::atomic<bool> SymbolsUpdatedSinceLastIndex;
std::mutex IndexMu;
std::condition_variable IndexCV;
FileSymbols IndexedSymbols;
llvm::StringMap<FileDigest> IndexedFileDigests; // Key is absolute file path.
std::mutex DigestsMu;
BackgroundIndexStorage::Factory IndexStorageFactory;
struct Source {
std::string Path;
bool NeedsReIndexing;
Source(llvm::StringRef Path, bool NeedsReIndexing)
: Path(Path), NeedsReIndexing(NeedsReIndexing) {}
};
// Loads the shards for a single TU and all of its dependencies. Returns the
// list of sources and whether they need to be re-indexed.
std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
BackgroundIndexStorage *IndexStorage,
llvm::StringSet<> &LoadedShards);
// Tries to load shards for the ChangedFiles.
std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
loadShards(std::vector<std::string> ChangedFiles);
void enqueue(tooling::CompileCommand Cmd, BackgroundIndexStorage *Storage);
// queue management
using Task = std::function<void()>;
void run(); // Main loop executed by Thread. Runs tasks from Queue.
void enqueueTask(Task T, ThreadPriority Prioirty);
void enqueueLocked(tooling::CompileCommand Cmd,
BackgroundIndexStorage *IndexStorage);
std::mutex QueueMu;
unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
std::condition_variable QueueCV;
bool ShouldStop = false;
std::deque<std::pair<Task, ThreadPriority>> Queue;
std::vector<std::thread> ThreadPool; // FIXME: Abstract this away.
GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
};
} // namespace clangd
} // namespace clang
#endif
|
