From 32d38789f9bb322ef9510cf79c1ce0de017e07b6 Mon Sep 17 00:00:00 2001 From: Ivan Donchevskii Date: Thu, 27 Jul 2017 13:20:22 +0200 Subject: Clang: implement followSymbol in TranslationUnit Follow symbol in current TU or dependent files Current algorithm tries to do the same as built-in follow symbol but better. Currently clang-based follow symbol has some limitations: - following function usage may return the declaration instead of definition because we don't have header dependencies in backend - overrides are not searched because of the same reason and the amount of dependent files (parsing 250 files takes a while) - some includes are not handled correctly, in that case we return failure and ask built-in code model to follow (example: or other qt includes) Change-Id: If35028ee0b5e818fdba29363c9520c5cca996348 Reviewed-by: Nikolai Kosjar --- src/shared/qbs | 2 +- .../ipcsource/clangbackendclangipc-source.pri | 10 +- .../clangbackend/ipcsource/clangfollowsymbol.cpp | 316 +++++++++++++++++++++ .../clangbackend/ipcsource/clangfollowsymbol.h | 51 ++++ .../ipcsource/clangfollowsymboljob.cpp | 22 +- .../clangbackend/ipcsource/clangfollowsymboljob.h | 10 +- .../ipcsource/clangtranslationunit.cpp | 12 + .../clangbackend/ipcsource/clangtranslationunit.h | 10 +- src/tools/clangbackend/ipcsource/cursor.cpp | 5 + src/tools/clangbackend/ipcsource/cursor.h | 1 + 10 files changed, 418 insertions(+), 21 deletions(-) create mode 100644 src/tools/clangbackend/ipcsource/clangfollowsymbol.cpp create mode 100644 src/tools/clangbackend/ipcsource/clangfollowsymbol.h diff --git a/src/shared/qbs b/src/shared/qbs index 4b58033621..998c698980 160000 --- a/src/shared/qbs +++ b/src/shared/qbs @@ -1 +1 @@ -Subproject commit 4b5803362114eaea3edbb57c9b47e03547ece20a +Subproject commit 998c69898058a7917a35875b7c7591bba6cf9f48 diff --git a/src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri b/src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri index c89d84c591..c12ba23254 100644 --- a/src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri +++ b/src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri @@ -16,6 +16,8 @@ HEADERS += \ $$PWD/clangexceptions.h \ $$PWD/clangfilepath.h \ $$PWD/clangfilesystemwatcher.h \ + $$PWD/clangfollowsymboljob.h \ + $$PWD/clangfollowsymbol.h \ $$PWD/clangiasyncjob.h \ $$PWD/clangjobcontext.h \ $$PWD/clangjobqueue.h \ @@ -55,8 +57,7 @@ HEADERS += \ $$PWD/sourcerange.h \ $$PWD/unsavedfile.h \ $$PWD/unsavedfiles.h \ - $$PWD/utf8positionfromlinecolumn.h \ - $$PWD/clangfollowsymboljob.h + $$PWD/utf8positionfromlinecolumn.h SOURCES += \ $$PWD/clangcodecompleteresults.cpp \ @@ -71,6 +72,8 @@ SOURCES += \ $$PWD/clangexceptions.cpp \ $$PWD/clangfilepath.cpp \ $$PWD/clangfilesystemwatcher.cpp \ + $$PWD/clangfollowsymboljob.cpp \ + $$PWD/clangfollowsymbol.cpp \ $$PWD/clangiasyncjob.cpp \ $$PWD/clangjobcontext.cpp \ $$PWD/clangjobqueue.cpp \ @@ -107,5 +110,4 @@ SOURCES += \ $$PWD/sourcerange.cpp \ $$PWD/unsavedfile.cpp \ $$PWD/unsavedfiles.cpp \ - $$PWD/utf8positionfromlinecolumn.cpp \ - $$PWD/clangfollowsymboljob.cpp + $$PWD/utf8positionfromlinecolumn.cpp diff --git a/src/tools/clangbackend/ipcsource/clangfollowsymbol.cpp b/src/tools/clangbackend/ipcsource/clangfollowsymbol.cpp new file mode 100644 index 0000000000..b4d3fdb333 --- /dev/null +++ b/src/tools/clangbackend/ipcsource/clangfollowsymbol.cpp @@ -0,0 +1,316 @@ +/**************************************************************************** +** +** Copyright (C) 2017 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include "clangfollowsymbol.h" +#include "clangfollowsymboljob.h" +#include "commandlinearguments.h" +#include "cursor.h" +#include "clangstring.h" +#include "sourcerange.h" +#include "clangbackendipcdebugutils.h" + +#include + +#include + +namespace ClangBackEnd { + +namespace { + +struct Tokens +{ + Tokens(const Cursor &cursor) { + tu = cursor.cxTranslationUnit(); + clang_tokenize(tu, cursor.cxSourceRange(), &data, &tokenCount); + } + ~Tokens() { + clang_disposeTokens(tu, data, tokenCount); + } + + CXToken *data = nullptr; + uint tokenCount = 0; +private: + CXTranslationUnit tu; +}; + +class FollowSymbolData { +public: + FollowSymbolData() = delete; + FollowSymbolData(const Utf8String &usr, const Utf8String &tokenSpelling, bool isFunctionLike, + std::atomic &ready) + : m_usr(usr) + , m_spelling(tokenSpelling) + , m_isFunctionLike(isFunctionLike) + , m_ready(ready) + {} + FollowSymbolData(const FollowSymbolData &other) + : m_usr(other.m_usr) + , m_spelling(other.m_spelling) + , m_isFunctionLike(other.m_isFunctionLike) + , m_ready(other.m_ready) + {} + + const Utf8String &usr() const { return m_usr; } + const Utf8String &spelling() const { return m_spelling; } + bool isFunctionLike() const { return m_isFunctionLike; } + bool ready() const { return m_ready; } + const SourceRangeContainer &result() const { return m_result; } + + void setReady(bool ready = true) { m_ready = ready; } + void setResult(const SourceRangeContainer &result) { m_result = result; } +private: + const Utf8String &m_usr; + const Utf8String &m_spelling; + SourceRangeContainer m_result; + bool m_isFunctionLike; + std::atomic &m_ready; +}; + +} // anonymous namespace + +static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor, + const Utf8String &tokenStr) +{ + Tokens tokens(cursor); + const CXTranslationUnit tu = cursor.cxTranslationUnit(); + for (uint i = 0; i < tokens.tokenCount; ++i) { + if (!(tokenStr == ClangString(clang_getTokenSpelling(tu, tokens.data[i])))) + continue; + + if (cursor.isFunctionLike() + && (i+1 > tokens.tokenCount + || !(ClangString(clang_getTokenSpelling(tu, tokens.data[i+1])) == "("))) { + continue; + } + return SourceRange(clang_getTokenExtent(tu, tokens.data[i])); + } + return SourceRangeContainer(); +} + +static void handleDeclaration(CXClientData client_data, const CXIdxDeclInfo *declInfo) +{ + if (!declInfo || !declInfo->isDefinition) + return; + + const Cursor currentCursor(declInfo->cursor); + auto* data = reinterpret_cast(client_data); + if (data->ready()) + return; + + if (data->usr() != currentCursor.canonical().unifiedSymbolResolution()) + return; + + QString str = Utf8String(currentCursor.displayName()); + if (currentCursor.isFunctionLike() || currentCursor.isConstructorOrDestructor()) { + if (!data->isFunctionLike()) + return; + str = str.mid(0, str.indexOf('(')); + } else if (data->isFunctionLike()) { + return; + } + if (!str.endsWith(data->spelling())) + return; + const CXTranslationUnit tu = clang_Cursor_getTranslationUnit(declInfo->cursor); + Tokens tokens(currentCursor); + + for (uint i = 0; i < tokens.tokenCount; ++i) { + Utf8String curSpelling = ClangString(clang_getTokenSpelling(tu, tokens.data[i])); + if (data->spelling() == curSpelling) { + if (data->isFunctionLike() + && (i+1 >= tokens.tokenCount + || !(ClangString(clang_getTokenSpelling(tu, tokens.data[i+1])) == "("))) { + continue; + } + data->setResult(SourceRange(clang_getTokenExtent(tu, tokens.data[i]))); + data->setReady(); + return; + } + } +} + +static int getTokenIndex(CXTranslationUnit tu, const Tokens &tokens, uint line, uint column) +{ + int tokenIndex = -1; + for (int i = static_cast(tokens.tokenCount - 1); i >= 0; --i) { + const SourceRange range = clang_getTokenExtent(tu, tokens.data[i]); + if (range.contains(line, column)) { + tokenIndex = i; + break; + } + } + return tokenIndex; +} + +static IndexerCallbacks createIndexerCallbacks() +{ + return { + [](CXClientData client_data, void *) { + auto* data = reinterpret_cast(client_data); + return data->ready() ? 1 : 0; + }, + [](CXClientData, CXDiagnosticSet, void *) {}, + [](CXClientData, CXFile, void *) { return CXIdxClientFile(); }, + [](CXClientData, const CXIdxIncludedFileInfo *) { return CXIdxClientFile(); }, + [](CXClientData, const CXIdxImportedASTFileInfo *) { return CXIdxClientASTFile(); }, + [](CXClientData, void *) { return CXIdxClientContainer(); }, + handleDeclaration, + [](CXClientData, const CXIdxEntityRefInfo *) {} + }; +} + +static FollowSymbolResult followSymbolInDependentFiles(CXIndex index, + const Cursor &cursor, + const Utf8String &tokenSpelling, + const QVector &dependentFiles, + const CommandLineArguments ¤tArgs) +{ + int argsCount = 0; + if (currentArgs.data()) + argsCount = currentArgs.count() - 1; + + const Utf8String usr = cursor.canonical().unifiedSymbolResolution(); + + // ready is shared for all data in vector + std::atomic ready {false}; + std::vector dataVector( + dependentFiles.size(), + FollowSymbolData(usr, tokenSpelling, + cursor.isFunctionLike() || cursor.isConstructorOrDestructor(), + ready)); + + std::vector> indexFutures; + + for (int i = 0; i < dependentFiles.size(); ++i) { + if (i > 0 && ready) + break; + indexFutures.emplace_back(std::async([&, i]() { + TIME_SCOPE_DURATION("Dependent file " + dependentFiles.at(i) + " indexer runner"); + + const CXIndexAction indexAction = clang_IndexAction_create(index); + IndexerCallbacks callbacks = createIndexerCallbacks(); + clang_indexSourceFile(indexAction, + &dataVector[i], + &callbacks, + sizeof(callbacks), + CXIndexOpt_SkipParsedBodiesInSession + | CXIndexOpt_SuppressRedundantRefs + | CXIndexOpt_SuppressWarnings, + dependentFiles.at(i).constData(), + currentArgs.data(), + argsCount, + nullptr, + 0, + nullptr, + CXTranslationUnit_SkipFunctionBodies + | CXTranslationUnit_KeepGoing); + clang_IndexAction_dispose(indexAction); + })); + } + + for (const std::future &future: indexFutures) + future.wait(); + + FollowSymbolResult result; + for (const FollowSymbolData &data: dataVector) { + if (!data.result().start().filePath().isEmpty()) { + result.range = data.result(); + break; + } + } + return result; +} + +FollowSymbolResult FollowSymbol::followSymbol(CXIndex index, + const Cursor &fullCursor, + uint line, + uint column, + const QVector &dependentFiles, + const CommandLineArguments ¤tArgs) +{ + FollowSymbolResult result; + Tokens tokens(fullCursor); + if (!tokens.tokenCount) { + result.failedToFollow = true; + return result; + } + + const CXTranslationUnit tu = fullCursor.cxTranslationUnit(); + + QVector cursors(static_cast(tokens.tokenCount)); + clang_annotateTokens(tu, tokens.data, tokens.tokenCount, cursors.data()); + int tokenIndex = getTokenIndex(tu, tokens, line, column); + QTC_ASSERT(tokenIndex >= 0, return result); + + const Utf8String tokenSpelling = ClangString(clang_getTokenSpelling(tu, tokens.data[tokenIndex])); + if (tokenSpelling.isEmpty()) + return result; + + Cursor cursor{cursors[tokenIndex]}; + if (cursor.kind() == CXCursor_InclusionDirective) { + CXFile file = clang_getIncludedFile(cursors[tokenIndex]); + const ClangString filename(clang_getFileName(file)); + const SourceLocation loc(tu, filename, 1, 1); + result.range = SourceRange(loc, loc); + return result; + } + + if (cursor.isDefinition()) { + // For definitions we can always find a declaration in current TU + result.range = extractMatchingTokenRange(cursor.canonical(), tokenSpelling); + return result; + } + + if (!cursor.isDeclaration()) { + // This is the symbol usage + // We want to return definition or at least declaration of this symbol + const Cursor referencedCursor = cursor.referenced(); + if (referencedCursor.isNull() || referencedCursor == cursor) + return result; + result.range = extractMatchingTokenRange(referencedCursor, tokenSpelling); + + // We've already found what we need + if (referencedCursor.isDefinition()) + return result; + cursor = referencedCursor; + } + + const Cursor definitionCursor = cursor.definition(); + if (!definitionCursor.isNull() && definitionCursor != cursor) { + // If we are able to find a definition in current TU + result.range = extractMatchingTokenRange(definitionCursor, tokenSpelling); + return result; + } + + // Search for the definition in the dependent files + FollowSymbolResult dependentFilesResult = followSymbolInDependentFiles(index, + cursor, + tokenSpelling, + dependentFiles, + currentArgs); + return dependentFilesResult.range.start().filePath().isEmpty() ? + result : dependentFilesResult; +} + +} // namespace ClangBackEnd diff --git a/src/tools/clangbackend/ipcsource/clangfollowsymbol.h b/src/tools/clangbackend/ipcsource/clangfollowsymbol.h new file mode 100644 index 0000000000..c3c0c5c209 --- /dev/null +++ b/src/tools/clangbackend/ipcsource/clangfollowsymbol.h @@ -0,0 +1,51 @@ +/**************************************************************************** +** +** Copyright (C) 2017 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#pragma once + +#include + +#include + +class Utf8String; + +namespace ClangBackEnd { + +class Cursor; +class FollowSymbolResult; +class CommandLineArguments; + +class FollowSymbol +{ +public: + static FollowSymbolResult followSymbol(CXIndex index, + const Cursor &fullCursor, + uint line, + uint column, + const QVector &dependentFiles, + const CommandLineArguments ¤tArgs); +}; + +} // namespace ClangBackEnd diff --git a/src/tools/clangbackend/ipcsource/clangfollowsymboljob.cpp b/src/tools/clangbackend/ipcsource/clangfollowsymboljob.cpp index 813f6f839f..ffac66b65f 100644 --- a/src/tools/clangbackend/ipcsource/clangfollowsymboljob.cpp +++ b/src/tools/clangbackend/ipcsource/clangfollowsymboljob.cpp @@ -37,11 +37,11 @@ static FollowSymbolJob::AsyncResult runAsyncHelperFollow(const TranslationUnit & quint32 line, quint32 column, const QVector &dependentFiles, - bool resolveTarget) + const CommandLineArguments ¤tArgs) { TIME_SCOPE_DURATION("FollowSymbolJobRunner"); - return FollowSymbolResult(); + return translationUnit.followSymbol(line, column, dependentFiles, currentArgs); } IAsyncJob::AsyncPrepareResult FollowSymbolJob::prepareAsyncRun() @@ -49,6 +49,8 @@ IAsyncJob::AsyncPrepareResult FollowSymbolJob::prepareAsyncRun() const JobRequest jobRequest = context().jobRequest; QTC_ASSERT(jobRequest.type == JobRequest::Type::FollowSymbol, return AsyncPrepareResult()); + // Is too slow because of IPC timings, no implementation for now + QTC_ASSERT(jobRequest.resolveTarget, return AsyncPrepareResult()); try { m_pinnedDocument = context().documentForJobRequest(); @@ -56,12 +58,18 @@ IAsyncJob::AsyncPrepareResult FollowSymbolJob::prepareAsyncRun() const TranslationUnit translationUnit = m_pinnedDocument.translationUnit(jobRequest.preferredTranslationUnit); + + const TranslationUnitUpdateInput updateInput = m_pinnedDocument.createUpdateInput(); + const CommandLineArguments currentArgs(updateInput.filePath.constData(), + updateInput.projectArguments, + updateInput.fileArguments, + false); + const quint32 line = jobRequest.line; const quint32 column = jobRequest.column; const QVector &dependentFiles = jobRequest.dependentFiles; - const bool resolveTarget = jobRequest.resolveTarget; - setRunner([translationUnit, line, column, dependentFiles, resolveTarget]() { - return runAsyncHelperFollow(translationUnit, line, column, dependentFiles, resolveTarget); + setRunner([translationUnit, line, column, dependentFiles, currentArgs]() { + return runAsyncHelperFollow(translationUnit, line, column, dependentFiles, currentArgs); }); return AsyncPrepareResult{translationUnit.id()}; @@ -77,8 +85,8 @@ void FollowSymbolJob::finalizeAsyncRun() const AsyncResult result = asyncResult(); const FollowSymbolMessage message(m_pinnedFileContainer, - result.m_range, - result.m_failedToFollow, + result.range, + result.failedToFollow, context().jobRequest.ticketNumber); context().client->followSymbol(message); } diff --git a/src/tools/clangbackend/ipcsource/clangfollowsymboljob.h b/src/tools/clangbackend/ipcsource/clangfollowsymboljob.h index 3c0086f4ec..cd96b7158d 100644 --- a/src/tools/clangbackend/ipcsource/clangfollowsymboljob.h +++ b/src/tools/clangbackend/ipcsource/clangfollowsymboljob.h @@ -36,13 +36,13 @@ class FollowSymbolResult { public: FollowSymbolResult() = default; - FollowSymbolResult(SourceRangeContainer &range, bool failedToFollow = false) - : m_range(range) - , m_failedToFollow(failedToFollow) + FollowSymbolResult(const SourceRangeContainer &range, bool failedToFollow = false) + : range(range) + , failedToFollow(failedToFollow) {} - SourceRangeContainer m_range; - bool m_failedToFollow = false; + SourceRangeContainer range; + bool failedToFollow = false; }; class FollowSymbolJob : public AsyncJob diff --git a/src/tools/clangbackend/ipcsource/clangtranslationunit.cpp b/src/tools/clangbackend/ipcsource/clangtranslationunit.cpp index 0bc0509a5e..a0d2fc932e 100644 --- a/src/tools/clangbackend/ipcsource/clangtranslationunit.cpp +++ b/src/tools/clangbackend/ipcsource/clangtranslationunit.cpp @@ -28,6 +28,8 @@ #include "clangbackend_global.h" #include "clangreferencescollector.h" #include "clangtranslationunitupdater.h" +#include "clangfollowsymbol.h" +#include "clangfollowsymboljob.h" #include #include @@ -38,6 +40,7 @@ #include #include #include +#include #include @@ -236,4 +239,13 @@ void TranslationUnit::extractDiagnostics(DiagnosticContainer &firstHeaderErrorDi } } +FollowSymbolResult TranslationUnit::followSymbol(uint line, + uint column, + const QVector &dependentFiles, + const CommandLineArguments ¤tArgs) const +{ + return FollowSymbol::followSymbol(m_cxIndex, cursorAt(line, column), line, column, + dependentFiles, currentArgs); +} + } // namespace ClangBackEnd diff --git a/src/tools/clangbackend/ipcsource/clangtranslationunit.h b/src/tools/clangbackend/ipcsource/clangtranslationunit.h index cc398e9a72..8bb317a8cd 100644 --- a/src/tools/clangbackend/ipcsource/clangtranslationunit.h +++ b/src/tools/clangbackend/ipcsource/clangtranslationunit.h @@ -27,12 +27,8 @@ #include -#include - #include -class Utf8String; - namespace ClangBackEnd { class Cursor; @@ -41,6 +37,7 @@ class DiagnosticSet; class HighlightingMarkContainer; class HighlightingMarks; class ReferencesResult; +class FollowSymbolResult; class SkippedSourceRanges; class SourceLocation; class SourceRange; @@ -48,6 +45,7 @@ class SourceRangeContainer; class TranslationUnitUpdateInput; class TranslationUnitUpdateResult; class UnsavedFiles; +class CommandLineArguments; class TranslationUnit { @@ -102,6 +100,10 @@ public: HighlightingMarks highlightingMarksInRange(const SourceRange &range) const; SkippedSourceRanges skippedSourceRanges() const; + FollowSymbolResult followSymbol(uint line, + uint column, + const QVector &dependentFiles, + const CommandLineArguments ¤tArgs) const; private: const Utf8String m_id; diff --git a/src/tools/clangbackend/ipcsource/cursor.cpp b/src/tools/clangbackend/ipcsource/cursor.cpp index 518fd013d2..6ad50307cd 100644 --- a/src/tools/clangbackend/ipcsource/cursor.cpp +++ b/src/tools/clangbackend/ipcsource/cursor.cpp @@ -273,6 +273,11 @@ CXSourceRange Cursor::cxSourceRange() const return clang_getCursorExtent(cxCursor); } +CXTranslationUnit Cursor::cxTranslationUnit() const +{ + return clang_Cursor_getTranslationUnit(cxCursor); +} + SourceRange Cursor::commentRange() const { return clang_Cursor_getCommentRange(cxCursor); diff --git a/src/tools/clangbackend/ipcsource/cursor.h b/src/tools/clangbackend/ipcsource/cursor.h index 0187d8448b..6520366b36 100644 --- a/src/tools/clangbackend/ipcsource/cursor.h +++ b/src/tools/clangbackend/ipcsource/cursor.h @@ -86,6 +86,7 @@ public: CXSourceLocation cxSourceLocation() const; SourceRange sourceRange() const; CXSourceRange cxSourceRange() const; + CXTranslationUnit cxTranslationUnit() const; SourceRange commentRange() const; bool hasSameSourceLocationAs(const Cursor &other) const; -- cgit v1.2.3