diff options
Diffstat (limited to 'src/linguist/lupdate')
-rw-r--r-- | src/linguist/lupdate/CMakeLists.txt | 40 | ||||
-rw-r--r-- | src/linguist/lupdate/clangtoolastreader.cpp | 128 | ||||
-rw-r--r-- | src/linguist/lupdate/clangtoolastreader.h | 78 | ||||
-rw-r--r-- | src/linguist/lupdate/cpp.cpp | 360 | ||||
-rw-r--r-- | src/linguist/lupdate/cpp.h | 29 | ||||
-rw-r--r-- | src/linguist/lupdate/cpp_clang.cpp | 145 | ||||
-rw-r--r-- | src/linguist/lupdate/cpp_clang.h | 55 | ||||
-rw-r--r-- | src/linguist/lupdate/filesignificancecheck.cpp | 70 | ||||
-rw-r--r-- | src/linguist/lupdate/filesignificancecheck.h | 63 | ||||
-rw-r--r-- | src/linguist/lupdate/java.cpp | 31 | ||||
-rw-r--r-- | src/linguist/lupdate/lupdate.1 | 25 | ||||
-rw-r--r-- | src/linguist/lupdate/lupdate.h | 30 | ||||
-rw-r--r-- | src/linguist/lupdate/lupdatepreprocessoraction.cpp | 83 | ||||
-rw-r--r-- | src/linguist/lupdate/lupdatepreprocessoraction.h | 63 | ||||
-rw-r--r-- | src/linguist/lupdate/main.cpp | 189 | ||||
-rw-r--r-- | src/linguist/lupdate/merge.cpp | 243 | ||||
-rw-r--r-- | src/linguist/lupdate/python.cpp | 293 | ||||
-rw-r--r-- | src/linguist/lupdate/qdeclarative.cpp | 94 | ||||
-rw-r--r-- | src/linguist/lupdate/synchronized.h | 29 | ||||
-rw-r--r-- | src/linguist/lupdate/ui.cpp | 33 |
20 files changed, 945 insertions, 1136 deletions
diff --git a/src/linguist/lupdate/CMakeLists.txt b/src/linguist/lupdate/CMakeLists.txt index c008fbd0e..56d4ad301 100644 --- a/src/linguist/lupdate/CMakeLists.txt +++ b/src/linguist/lupdate/CMakeLists.txt @@ -1,6 +1,7 @@ -# Generated from lupdate.pro. +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + -# special case begin if(CMAKE_VERSION VERSION_LESS "3.19" AND MSVC AND CMAKE_GENERATOR STREQUAL "Ninja Multi-Config") message(WARNING "lupdate will not be built in this configuration.") return() @@ -9,7 +10,6 @@ endif() if (MINGW) set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY _qt_skip_separate_debug_info ON) endif() -# special case end ##################################################################### ## lupdate Tool: @@ -18,7 +18,7 @@ endif() qt_get_tool_target_name(target_name lupdate) qt_internal_add_tool(${target_name} TARGET_DESCRIPTION "Qt Translation File Update Tool" - TOOLS_TARGET Linguist # special case + TOOLS_TARGET Linguist EXTRA_CMAKE_FILES "${CMAKE_CURRENT_LIST_DIR}/../GenerateLUpdateProject.cmake" SOURCES ../shared/numerus.cpp @@ -46,18 +46,14 @@ qt_internal_add_tool(${target_name} QT_NO_CAST_TO_ASCII INCLUDE_DIRECTORIES ../shared - PUBLIC_LIBRARIES + LIBRARIES Qt::CorePrivate Qt::Tools ) -qt_internal_return_unless_building_tools() -#### Keys ignored in scope 1:.:.:lupdate.pro:<TRUE>: -# QMAKE_TARGET_DESCRIPTION = "Qt Translation File Update Tool" -# QT_TOOL_ENV = "qmake" -# _OPTION = "host_build" -# qmake.name = "QMAKE" -# qmake.value = "$$shell_path($$QMAKE_QMAKE)" +set_source_files_properties(python.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + +qt_internal_return_unless_building_tools() ## Scopes: ##################################################################### @@ -78,6 +74,7 @@ qt_internal_extend_target(${target_name} CONDITION QT_FEATURE_clangcpp SOURCES clangtoolastreader.cpp clangtoolastreader.h cpp_clang.cpp cpp_clang.h + filesignificancecheck.cpp filesignificancecheck.h lupdatepreprocessoraction.cpp lupdatepreprocessoraction.h synchronized.h DEFINES @@ -94,18 +91,19 @@ qt_internal_extend_target(${target_name} CONDITION QT_FEATURE_clangcpp LUPDATE_CLANG_VERSION_MINOR=${QT_LIB_CLANG_VERSION_MINOR} LUPDATE_CLANG_VERSION_PATCH=${QT_LIB_CLANG_VERSION_PATCH} # special case end - LIBRARIES # special case - WrapLibClang::WrapLibClang # special case + LIBRARIES + WrapLibClang::WrapLibClang ) -# special case begin if(QT_FEATURE_clangcpp) - set_property(SOURCE clangtoolastreader.cpp PROPERTY SKIP_AUTOMOC ON) + # If libclangTooling.a is not built with -fPIE enabled we cannot link it to lupdate. + # TODO: Re-enable PIE once clang is built with PIE in provisioning. + set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE FALSE) endif() -# special case end -#### Keys ignored in scope 6:.:.:lupdate.pro:NOT QMAKE_DEFAULT_LIBDIRS___contains____ss_CLANG_LIBDIR AND NOT disable_external_rpath: -# QMAKE_RPATHDIR = "$$CLANG_LIBDIR" +qt_internal_extend_target(${target_name} CONDITION MSVC + DEFINES _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING) -#### Keys ignored in scope 9:.:.:lupdate.pro:MINGW: -# RC_FILE = "lupdate.rc" +if(QT_FEATURE_clangcpp) + set_property(SOURCE clangtoolastreader.cpp PROPERTY SKIP_AUTOMOC ON) +endif() diff --git a/src/linguist/lupdate/clangtoolastreader.cpp b/src/linguist/lupdate/clangtoolastreader.cpp index 8aac00ddf..6b85c6ccb 100644 --- a/src/linguist/lupdate/clangtoolastreader.cpp +++ b/src/linguist/lupdate/clangtoolastreader.cpp @@ -1,32 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2019 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2019 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "clangtoolastreader.h" +#include "filesignificancecheck.h" #include "translator.h" #include <QLibraryInfo> @@ -347,7 +323,8 @@ bool LupdateVisitor::VisitCallExpr(clang::CallExpr *callExpression) const auto fileLoc = sm.getFileLoc(callExpression->getBeginLoc()); if (fileLoc.isInvalid() || !fileLoc.isFileID()) return true; - auto presumedLoc = sm.getPresumedLoc(fileLoc); + // not using line directive (# line) + auto presumedLoc = sm.getPresumedLoc(fileLoc, false); if (presumedLoc.isInvalid()) return true; info = { presumedLoc.getLine(), presumedLoc.getFilename() }; @@ -357,7 +334,7 @@ bool LupdateVisitor::VisitCallExpr(clang::CallExpr *callExpression) } // Checking that the CallExpression is from the input file we're interested in - if (info.Filename != m_inputFile) + if (!LupdatePrivate::isFileSignificant(info.Filename)) return true; qCDebug(lcClang) << "************************** VisitCallExpr ****************"; @@ -436,16 +413,21 @@ bool LupdateVisitor::VisitCallExpr(clang::CallExpr *callExpression) return true; } +void LupdateVisitor::processIsolatedComments() +{ + auto &sourceMgr = m_context->getSourceManager(); + processIsolatedComments(sourceMgr.getMainFileID()) ; +} + /* Retrieve the comments not associated with tr calls. */ -void LupdateVisitor::processIsolatedComments() +void LupdateVisitor::processIsolatedComments(const clang::FileID file) { qCDebug(lcClang) << "==== processIsolatedComments ===="; auto &sourceMgr = m_context->getSourceManager(); #if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(10,0,0)) - const clang::FileID file = sourceMgr.getMainFileID(); const auto commentsInThisFile = m_context->Comments.getCommentsInFile(file); if (!commentsInThisFile) return; @@ -455,6 +437,7 @@ void LupdateVisitor::processIsolatedComments() tmp.emplace_back(commentInFile.second); clang::ArrayRef<clang::RawComment *> rawComments = tmp; #else + Q_UNUSED(file); clang::ArrayRef<clang::RawComment *> rawComments = m_context->getRawCommentList().getComments(); #endif @@ -468,13 +451,15 @@ void LupdateVisitor::processIsolatedComments() // They are not associated to any tr calls // Each one needs its own entry in the m_stores->AST translation store for (const auto &rawComment : rawComments) { - if (sourceMgr.getFilename(rawComment->getBeginLoc()).str() != m_inputFile) + if (!LupdatePrivate::isFileSignificant(sourceMgr.getFilename(rawComment->getBeginLoc()).str())) continue; // Comments not separated by an empty line will be part of the same Raw comments // Each one needs to be saved with its line number. // The store is used here only to pass this information. TranslationRelatedStore store; - store.lupdateLocationLine = sourceMgr.getPresumedLoc(rawComment->getBeginLoc()).getLine(); + store.lupdateLocationLine = sourceMgr.getPresumedLoc(rawComment->getBeginLoc(), false).getLine(); + store.lupdateLocationFile = QString::fromStdString( + sourceMgr.getPresumedLoc(rawComment->getBeginLoc(), false).getFilename()); QString comment = toQt(rawComment->getRawText(sourceMgr)); qCDebug(lcClang) << " raw Comment : \n" << comment; setInfoFromRawComment(comment, &store); @@ -718,7 +703,7 @@ void LupdateVisitor::setInfoFromRawComment(const QString &commentString, newStore.contextArg = comment.left(index).trimmed(); newStore.lupdateComment = comment.mid(index).trimmed(); } - newStore.lupdateLocationFile = QString::fromStdString(m_inputFile); + newStore.lupdateLocationFile = store->lupdateLocationFile; newStore.lupdateLocationLine = storeLine; newStore.locationCol = 0; newStore.printStore(); @@ -733,18 +718,53 @@ void LupdateVisitor::setInfoFromRawComment(const QString &commentString, void LupdateVisitor::processPreprocessorCalls() { - m_macro = (m_stores->Preprocessor.size() > 0); - for (const auto &store : m_stores->Preprocessor) - processPreprocessorCall(store); + QString inputFile = toQt(m_inputFile); + for (const auto &store : m_stores->Preprocessor) { + if (store.lupdateInputFile == inputFile) + processPreprocessorCall(store); + } + + // Processing the isolated comments (TRANSLATOR) in the files included in the main input file. +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + for (const clang::FileEntry *file : m_preprocessor->getIncludedFiles()) { + auto &sourceMgr = m_context->getSourceManager(); + + clang::StringRef fileNameRealPath = file->tryGetRealPathName(); + if (!LupdatePrivate::isFileSignificant(fileNameRealPath.str()) + || fileNameRealPath.str() == m_inputFile) + continue; + + auto sourceFile = sourceMgr.getFileManager() + .getFile(fileNameRealPath); + auto sourceLocation = sourceMgr.translateFileLineCol(sourceFile.get(), 1, 1); + const clang::FileID fileId = sourceMgr.getDecomposedLoc(sourceLocation).first; + processIsolatedComments(fileId); + } +#endif + + if (m_qDeclareTrMacroAll.size() > 0 || m_noopTranslationMacroAll.size() > 0) + m_macro = true; } void LupdateVisitor::processPreprocessorCall(TranslationRelatedStore store) { + // To get the comments around the macros const std::vector<QString> rawComments = rawCommentsFromSourceLocation(store .callLocation(m_context->getSourceManager())); + // to pick up the raw comments in the files collected from the preprocessing. for (const auto &rawComment : rawComments) setInfoFromRawComment(rawComment, &store); + // Processing the isolated comments (TRANSLATOR) in the files included in the main input file. +#if (LUPDATE_CLANG_VERSION < LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + if (store.callType.contains(QStringLiteral("InclusionDirective"))) { + auto &sourceMgr = m_context->getSourceManager(); + const clang::FileID file = sourceMgr.getDecomposedLoc(store.callLocation(sourceMgr)).first; + processIsolatedComments(file); + return; + } +#endif + if (store.isValid()) { if (store.funcName.contains(QStringLiteral("Q_DECLARE_TR_FUNCTIONS"))) m_qDeclareTrMacroAll.emplace_back(std::move(store)); @@ -762,12 +782,12 @@ bool LupdateVisitor::VisitNamedDecl(clang::NamedDecl *namedDeclaration) if (!fullLocation.isValid() || !fullLocation.getFileEntry()) return true; - if (fullLocation.getFileEntry()->getName() != m_inputFile) + if (!LupdatePrivate::isFileSignificant(fullLocation.getFileEntry()->getName().str())) return true; - qCDebug(lcClang) << "NamedDecl Name: " << namedDeclaration->getQualifiedNameAsString(); - qCDebug(lcClang) << "NamedDecl source: " << namedDeclaration->getSourceRange().printToString( - m_context->getSourceManager()); + qCDebug(lcClang) << "NamedDecl Name: " << QString::fromStdString(namedDeclaration->getQualifiedNameAsString()); + qCDebug(lcClang) << "NamedDecl source: " << QString::fromStdString(namedDeclaration->getSourceRange().printToString( + m_context->getSourceManager())); // Checks if there is a macro located within the range of this NamedDeclaration // in order to find a context for the macro findContextForTranslationStoresFromPP(namedDeclaration); @@ -795,15 +815,15 @@ void LupdateVisitor::findContextForTranslationStoresFromPP(clang::NamedDecl *nam store.contextRetrieved = LupdatePrivate::contextForNoopMacro(namedDeclaration, sm); qCDebug(lcClang) << "------------------------------------------NOOP Macro in range ---"; - qCDebug(lcClang) << "Range " << namedDeclaration->getSourceRange().printToString(sm); - qCDebug(lcClang) << "Point " << sourceLoc.printToString(sm); + qCDebug(lcClang) << "Range " << QString::fromStdString(namedDeclaration->getSourceRange().printToString(sm)); + qCDebug(lcClang) << "Point " << QString::fromStdString(sourceLoc.printToString(sm)); qCDebug(lcClang) << "=========== Visit Named Declaration ============================="; qCDebug(lcClang) << " Declaration Location " << - namedDeclaration->getSourceRange().printToString(sm); + QString::fromStdString(namedDeclaration->getSourceRange().printToString(sm)); qCDebug(lcClang) << " Macro Location " - << sourceLoc.printToString(sm); + << QString::fromStdString(sourceLoc.printToString(sm)); qCDebug(lcClang) << " Context namedDeclaration->getQualifiedNameAsString() " - << namedDeclaration->getQualifiedNameAsString(); + << QString::fromStdString(namedDeclaration->getQualifiedNameAsString()); qCDebug(lcClang) << " Context LupdatePrivate::contextForNoopMacro " << store.contextRetrieved; qCDebug(lcClang) << " Context Retrieved " << store.contextRetrieved; @@ -820,13 +840,13 @@ void LupdateVisitor::findContextForTranslationStoresFromPP(clang::NamedDecl *nam store.contextRetrieved = QString::fromStdString( namedDeclaration->getQualifiedNameAsString()); qCDebug(lcClang) << "------------------------------------------DECL Macro in range ---"; - qCDebug(lcClang) << "Range " << namedDeclaration->getSourceRange().printToString(sm); - qCDebug(lcClang) << "Point " << sourceLoc.printToString(sm); + qCDebug(lcClang) << "Range " << QString::fromStdString(namedDeclaration->getSourceRange().printToString(sm)); + qCDebug(lcClang) << "Point " << QString::fromStdString(sourceLoc.printToString(sm)); qCDebug(lcClang) << "=========== Visit Named Declaration ============================="; qCDebug(lcClang) << " Declaration Location " << - namedDeclaration->getSourceRange().printToString(sm); + QString::fromStdString(namedDeclaration->getSourceRange().printToString(sm)); qCDebug(lcClang) << " Macro Location " - << sourceLoc.printToString(sm); + << QString::fromStdString(sourceLoc.printToString(sm)); qCDebug(lcClang) << " Context namedDeclaration->getQualifiedNameAsString() " << store.contextRetrieved; qCDebug(lcClang) << " Context Retrieved " << store.contextRetrieved; @@ -836,16 +856,16 @@ void LupdateVisitor::findContextForTranslationStoresFromPP(clang::NamedDecl *nam } } -void LupdateVisitor::generateOuput() +void LupdateVisitor::generateOutput() { - qCDebug(lcClang) << "=================m_trCallserateOuput============================"; + qCDebug(lcClang) << "=================generateOutput============================"; m_noopTranslationMacroAll.erase(std::remove_if(m_noopTranslationMacroAll.begin(), - m_noopTranslationMacroAll.end(), [this](const TranslationRelatedStore &store) { + m_noopTranslationMacroAll.end(), [](const TranslationRelatedStore &store) { // Macros not located in the currently visited file are missing context (and it's normal), // so an output is only generated for macros present in the currently visited file. // If context could not be found, it is warned against in ClangCppParser::collectMessages // (where it is possible to order the warnings and print them consistantly) - if ( m_inputFile != qPrintable(store.lupdateLocationFile)) + if (!LupdatePrivate::isFileSignificant(store.lupdateLocationFile.toStdString())) return true; return false; }), m_noopTranslationMacroAll.end()); diff --git a/src/linguist/lupdate/clangtoolastreader.h b/src/linguist/lupdate/clangtoolastreader.h index 3a5ea7dfb..a8483219d 100644 --- a/src/linguist/lupdate/clangtoolastreader.h +++ b/src/linguist/lupdate/clangtoolastreader.h @@ -1,51 +1,24 @@ -/**************************************************************************** -** -** Copyright (C) 2019 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2019 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #ifndef CLANG_TOOL_AST_READER_H #define CLANG_TOOL_AST_READER_H #include "cpp_clang.h" -#if defined(Q_CC_MSVC) -# pragma warning(push) -# pragma warning(disable: 4100) -# pragma warning(disable: 4146) -# pragma warning(disable: 4267) -# pragma warning(disable: 4624) -#endif + +QT_WARNING_PUSH +QT_WARNING_DISABLE_MSVC(4100) +QT_WARNING_DISABLE_MSVC(4146) +QT_WARNING_DISABLE_MSVC(4267) +QT_WARNING_DISABLE_MSVC(4624) +QT_WARNING_DISABLE_GCC("-Wnonnull") #include <clang/AST/RecursiveASTVisitor.h> #include <clang/Frontend/CompilerInstance.h> #include <clang/Frontend/FrontendActions.h> #include <clang/Tooling/Tooling.h> -#if defined(Q_CC_MSVC) -# pragma warning(pop) -#endif +QT_WARNING_POP #include <iostream> #include <memory> @@ -57,11 +30,19 @@ class Translator; class LupdateVisitor : public clang::RecursiveASTVisitor<LupdateVisitor> { public: +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + explicit LupdateVisitor(clang::ASTContext *context, + clang::Preprocessor *preprocessor, Stores *stores) + : m_context(context) + , m_preprocessor(preprocessor) + , m_stores(stores) +#else explicit LupdateVisitor(clang::ASTContext *context, Stores *stores) : m_context(context) , m_stores(stores) +#endif { - m_inputFile = m_context->getSourceManager().getFileEntryForID( + m_inputFile = m_context->getSourceManager().getFileEntryRefForID( m_context->getSourceManager().getMainFileID())->getName(); } @@ -69,7 +50,7 @@ public: void processPreprocessorCalls(); bool VisitNamedDecl(clang::NamedDecl *namedDeclaration); void findContextForTranslationStoresFromPP(clang::NamedDecl *namedDeclaration); - void generateOuput(); + void generateOutput(); private: std::vector<QString> rawCommentsForCallExpr(const clang::CallExpr *callExpr) const; @@ -79,8 +60,12 @@ private: void processPreprocessorCall(TranslationRelatedStore store); void processIsolatedComments(); + void processIsolatedComments(const clang::FileID file); clang::ASTContext *m_context = nullptr; +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + clang::Preprocessor *m_preprocessor = nullptr; +#endif std::string m_inputFile; Stores *m_stores = nullptr; @@ -94,8 +79,14 @@ private: class LupdateASTConsumer : public clang::ASTConsumer { public: +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + explicit LupdateASTConsumer(clang::ASTContext *context, clang::Preprocessor *preprocessor, + Stores *stores) + : m_visitor(context, preprocessor, stores) +#else explicit LupdateASTConsumer(clang::ASTContext *context, Stores *stores) : m_visitor(context, stores) +#endif {} // This method is called when the ASTs for entire translation unit have been @@ -105,7 +96,7 @@ public: m_visitor.processPreprocessorCalls(); bool traverse = m_visitor.TraverseAST(context); qCDebug(lcClang) << "TraverseAST: " << traverse; - m_visitor.generateOuput(); + m_visitor.generateOutput(); } private: @@ -122,7 +113,12 @@ public: std::unique_ptr<clang::ASTConsumer> CreateASTConsumer( clang::CompilerInstance &compiler, llvm::StringRef /* inFile */) override { - auto consumer = new LupdateASTConsumer(&compiler.getASTContext(), m_stores); + #if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + auto consumer = new LupdateASTConsumer(&compiler.getASTContext(), + &compiler.getPreprocessor(), m_stores); + #else + auto consumer = new LupdateASTConsumer(&compiler.getASTContext(), m_stores); + #endif return std::unique_ptr<clang::ASTConsumer>(consumer); } diff --git a/src/linguist/lupdate/cpp.cpp b/src/linguist/lupdate/cpp.cpp index 22891c6aa..00c36d9fa 100644 --- a/src/linguist/lupdate/cpp.cpp +++ b/src/linguist/lupdate/cpp.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "cpp.h" @@ -39,11 +14,9 @@ QT_BEGIN_NAMESPACE /* qmake ignore Q_OBJECT */ -static QString MagicComment(QLatin1String("TRANSLATOR")); +using namespace Qt::StringLiterals; -#define STRING(s) static QString str##s(QLatin1String(#s)) - -//#define DIAGNOSE_RETRANSLATABILITY // FIXME: should make a runtime option of this +static const QString CppMagicComment = u"TRANSLATOR"_s; size_t qHash(const HashString &str) { @@ -84,7 +57,16 @@ private: QBitArray m_ba; }; -class CppParser { +struct CppParserState +{ + NamespaceList namespaces; + QStack<qsizetype> namespaceDepths; + NamespaceList functionContext; + QString functionContextUnresolved; + QString pendingContext; +}; + +class CppParser : private CppParserState { public: CppParser(ParseResults *results = 0); @@ -96,14 +78,6 @@ public: const ParseResults *recordResults(bool isHeader); void deleteResults() { delete results; } - struct SavedState { - NamespaceList namespaces; - QStack<int> namespaceDepths; - NamespaceList functionContext; - QString functionContextUnresolved; - QString pendingContext; - }; - private: struct IfdefState { IfdefState() {} @@ -114,7 +88,7 @@ private: elseLine(-1) {} - SavedState state; + CppParserState state; int bracketDepth, bracketDepth1st; int braceDepth, braceDepth1st; int parenDepth, parenDepth1st; @@ -122,14 +96,13 @@ private: }; enum TokenType { - Tok_Eof, Tok_class, Tok_friend, Tok_namespace, Tok_using, Tok_return, - Tok_Q_OBJECT, Tok_Access, Tok_Cancel, + Tok_Eof, Tok_class, Tok_enum, Tok_friend, Tok_namespace, Tok_using, Tok_return, + Tok_decltype, Tok_Q_OBJECT, Tok_Access, Tok_Cancel, Tok_Ident, Tok_String, Tok_RawString, Tok_Arrow, Tok_Colon, Tok_ColonColon, - Tok_Equals, Tok_LeftBracket, Tok_RightBracket, Tok_QuestionMark, + Tok_Equals, Tok_LeftBracket, Tok_RightBracket, Tok_AngleBracket, Tok_QuestionMark, Tok_LeftBrace, Tok_RightBrace, Tok_LeftParen, Tok_RightParen, Tok_Comma, Tok_Semicolon, Tok_Null, Tok_Integer, - Tok_QuotedInclude, Tok_AngledInclude, - Tok_Other + Tok_QuotedInclude, Tok_AngledInclude }; std::ostream &yyMsg(int line = 0); @@ -159,8 +132,8 @@ private: void processInclude(const QString &file, ConversionData &cd, const QStringList &includeStack, QSet<QString> &inclusions); - void saveState(SavedState *state); - void loadState(const SavedState *state); + void saveState(CppParserState *state); + void loadState(const CppParserState &state); static QString stringifyNamespace(int start, const NamespaceList &namespaces); static QString stringifyNamespace(const NamespaceList &namespaces) @@ -226,17 +199,12 @@ private: QString sourcetext; TranslatorMessage::ExtraData extra; - NamespaceList namespaces; - QStack<int> namespaceDepths; - NamespaceList functionContext; - QString functionContextUnresolved; QString prospectiveContext; - QString pendingContext; ParseResults *results; Translator *tor; bool directInclude; - SavedState savedState; + CppParserState savedState; int yyMinBraceDepth; bool inDefine; }; @@ -364,25 +332,44 @@ CppParser::TokenType CppParser::lookAheadToSemicolonOrLeftBrace() } } -STRING(Q_OBJECT); -STRING(class); -STRING(final); -STRING(friend); -STRING(namespace); -STRING(nullptr); -STRING(Q_NULLPTR); -STRING(NULL); -STRING(operator); -STRING(return); -STRING(struct); -STRING(using); -STRING(private); -STRING(protected); -STRING(public); -STRING(slots); -STRING(signals); -STRING(Q_SLOTS); -STRING(Q_SIGNALS); +static bool isStringLiteralPrefix(const QStringView s) +{ + return s == u"L"_s + || s == u"U"_s + || s == u"u"_s + || s == u"u8"_s; +} + +static bool isRawStringLiteralPrefix(QStringView s) +{ + if (s.endsWith(u'R')) { + s.chop(1); + return s.isEmpty() || isStringLiteralPrefix(s); + } + return false; +} + +static const QString strQ_OBJECT = u"Q_OBJECT"_s; +static const QString strclass = u"class"_s; +static const QString strdecltype = u"decltype"_s; +static const QString strenum = u"enum"_s; +static const QString strfinal = u"final"_s; +static const QString strfriend = u"friend"_s; +static const QString strnamespace = u"namespace"_s; +static const QString strnullptr = u"nullptr"_s; +static const QString strQ_NULLPTR = u"Q_NULLPTR"_s; +static const QString strNULL = u"NULL"_s; +static const QString stroperator = u"operator"_s; +static const QString strreturn = u"return"_s; +static const QString strstruct = u"struct"_s; +static const QString strusing = u"using"_s; +static const QString strprivate = u"private"_s; +static const QString strprotected = u"protected"_s; +static const QString strpublic = u"public"_s; +static const QString strslots = u"slots"_s; +static const QString strsignals = u"signals"_s; +static const QString strQ_SLOTS = u"Q_SLOTS"_s; +static const QString strQ_SIGNALS = u"Q_SIGNALS"_s; CppParser::TokenType CppParser::getToken() { @@ -540,7 +527,7 @@ CppParser::TokenType CppParser::getToken() yyBracketDepth = is.bracketDepth1st; yyBraceDepth = is.braceDepth1st; yyParenDepth = is.parenDepth1st; - loadState(&is.state); + loadState(is.state); } } yyCh = getChar(); @@ -592,6 +579,11 @@ CppParser::TokenType CppParser::getToken() //qDebug() << "IDENT: " << yyWord; + if (yyCh == '"' && isStringLiteralPrefix(yyWord)) { + // Handle prefixed string literals as ordinary string literals. + continue; + } + switch (yyWord.unicode()[0].unicode()) { case 'N': if (yyWord == strNULL) @@ -609,6 +601,14 @@ CppParser::TokenType CppParser::getToken() if (yyWord == strclass) return Tok_class; break; + case 'd': + if (yyWord == strdecltype) + return Tok_decltype; + break; + case 'e': + if (yyWord == strenum) + return Tok_enum; + break; case 'f': if (yyWord == strfriend) return Tok_friend; @@ -654,10 +654,7 @@ CppParser::TokenType CppParser::getToken() } // a C++11 raw string literal? - if (yyCh == '"' && ( - yyWord == QLatin1String("R") || yyWord == QLatin1String("LR") || yyWord == QLatin1String("u8R") || - yyWord == QLatin1String("uR") || yyWord == QLatin1String("UR") - )) { + if (yyCh == '"' && isRawStringLiteralPrefix(yyWord)) { ptr = reinterpret_cast<ushort *>(const_cast<QChar *>(yyWord.unicode())); //get delimiter QString delimiter; @@ -712,7 +709,7 @@ CppParser::TokenType CppParser::getToken() switch (yyCh) { case '\n': if (inDefine) { - loadState(&savedState); + loadState(savedState); prospectiveContext.clear(); yyBraceDepth = yyMinBraceDepth; yyMinBraceDepth = 0; @@ -805,7 +802,7 @@ CppParser::TokenType CppParser::getToken() case '>': case '<': yyCh = getChar(); - return Tok_Other; + return Tok_AngleBracket; case '\'': yyCh = getChar(); if (yyCh == '\\') @@ -883,10 +880,10 @@ CppParser::TokenType CppParser::getToken() return Tok_QuestionMark; case '0': yyCh = getChar(); - if (yyCh == 'x') { + if (yyCh == 'x' || yyCh == 'X') { do { yyCh = getChar(); - } while ((yyCh >= '0' && yyCh <= '9') + } while ((yyCh >= '0' && yyCh <= '9') || yyCh == '\'' || (yyCh >= 'a' && yyCh <= 'f') || (yyCh >= 'A' && yyCh <= 'F')); return Tok_Integer; } @@ -904,7 +901,7 @@ CppParser::TokenType CppParser::getToken() case '9': do { yyCh = getChar(); - } while (yyCh >= '0' && yyCh <= '9'); + } while ((yyCh >= '0' && yyCh <= '9') || yyCh == '\''); return Tok_Integer; default: yyCh = getChar(); @@ -920,38 +917,30 @@ CppParser::TokenType CppParser::getToken() utilities for the third part. */ -void CppParser::saveState(SavedState *state) +void CppParser::saveState(CppParserState *state) { - state->namespaces = namespaces; - state->namespaceDepths = namespaceDepths; - state->functionContext = functionContext; - state->functionContextUnresolved = functionContextUnresolved; - state->pendingContext = pendingContext; + *state = *this; } -void CppParser::loadState(const SavedState *state) +void CppParser::loadState(const CppParserState &state) { - namespaces = state->namespaces; - namespaceDepths = state->namespaceDepths; - functionContext = state->functionContext; - functionContextUnresolved = state->functionContextUnresolved; - pendingContext = state->pendingContext; + *static_cast<CppParserState *>(this) = state; } Namespace *CppParser::modifyNamespace(NamespaceList *namespaces, bool haveLast) { Namespace *pns, *ns = &results->rootNamespace; - for (int i = 1; i < namespaces->count(); ++i) { + for (int i = 1; i < namespaces->size(); ++i) { pns = ns; if (!(ns = pns->children.value(namespaces->at(i)))) { do { ns = new Namespace; - if (haveLast || i < namespaces->count() - 1) + if (haveLast || i < namespaces->size() - 1) if (const Namespace *ons = findNamespace(*namespaces, i + 1)) ns->classDef = ons->classDef; pns->children.insert(namespaces->at(i), ns); pns = ns; - } while (++i < namespaces->count()); + } while (++i < namespaces->size()); break; } } @@ -962,10 +951,10 @@ QString CppParser::stringifyNamespace(int start, const NamespaceList &namespaces { QString ret; int l = 0; - for (int j = start; j < namespaces.count(); ++j) - l += namespaces.at(j).value().length(); - ret.reserve(l + qMax(0, (namespaces.count() - start - 1)) * 2); - for (int i = start; i < namespaces.count(); ++i) { + for (int j = start; j < namespaces.size(); ++j) + l += namespaces.at(j).value().size(); + ret.reserve(l + qMax(0, (namespaces.size() - start - 1)) * 2); + for (int i = start; i < namespaces.size(); ++i) { if (i > start) ret += QLatin1String("::"); ret += namespaces.at(i).value(); @@ -1049,7 +1038,7 @@ bool CppParser::qualifyOneCallbackUsing(const Namespace *ns, void *context) cons for (const HashStringList &use : ns->usings) if (!data->visitedUsings->contains(use)) { data->visitedUsings->insert(use); - if (qualifyOne(use.value(), use.value().count(), data->segment, data->resolved, + if (qualifyOne(use.value(), use.value().size(), data->segment, data->resolved, data->visitedUsings)) return true; } @@ -1084,7 +1073,7 @@ bool CppParser::fullyQualify(const NamespaceList &namespaces, int nsCnt, if (segments.first().value().isEmpty()) { // fully qualified - if (segments.count() == 1) { + if (segments.size() == 1) { resolved->clear(); *resolved << HashString(QString()); return true; @@ -1099,8 +1088,8 @@ bool CppParser::fullyQualify(const NamespaceList &namespaces, int nsCnt, do { if (qualifyOne(namespaces, nsIdx + 1, segments[initSegIdx], resolved)) { int segIdx = initSegIdx; - while (++segIdx < segments.count()) { - if (!qualifyOne(*resolved, resolved->count(), segments[segIdx], resolved)) { + while (++segIdx < segments.size()) { + if (!qualifyOne(*resolved, resolved->size(), segments[segIdx], resolved)) { if (unresolved) *unresolved = segments.mid(segIdx); return false; @@ -1120,7 +1109,7 @@ bool CppParser::fullyQualify(const NamespaceList &namespaces, const NamespaceList &segments, bool isDeclaration, NamespaceList *resolved, NamespaceList *unresolved) const { - return fullyQualify(namespaces, namespaces.count(), + return fullyQualify(namespaces, namespaces.size(), segments, isDeclaration, resolved, unresolved); } @@ -1146,7 +1135,7 @@ const Namespace *CppParser::findNamespace(const NamespaceList &namespaces, int n { const Namespace *ns = 0; if (nsCount == -1) - nsCount = namespaces.count(); + nsCount = namespaces.size(); visitNamespace(namespaces, nsCount, &CppParser::findNamespaceCallback, &ns); return ns; } @@ -1160,10 +1149,11 @@ void CppParser::enterNamespace(NamespaceList *namespaces, const HashString &name void CppParser::truncateNamespaces(NamespaceList *namespaces, int length) { - if (namespaces->count() > length) + if (namespaces->size() > length) namespaces->erase(namespaces->begin() + length, namespaces->end()); } + /* Functions for processing include files. */ @@ -1250,7 +1240,7 @@ void CppFiles::addIncludeCycle(const QSet<QString> &fileNames) } qDeleteAll(intersectingCycles); - for (const QString &fileName : qAsConst(cycle->fileNames)) + for (const QString &fileName : std::as_const(cycle->fileNames)) includeCycles().insert(fileName, cycle); } @@ -1265,7 +1255,7 @@ void CppParser::processInclude(const QString &file, ConversionData &cd, const QS { QString cleanFile = QDir::cleanPath(file); - for (const QString &ex : qAsConst(cd.m_excludes)) { + for (const QString &ex : std::as_const(cd.m_excludes)) { QRegularExpression rx(QRegularExpression::wildcardToRegularExpression(ex)); if (rx.match(cleanFile).hasMatch()) return; @@ -1282,7 +1272,7 @@ void CppParser::processInclude(const QString &file, ConversionData &cd, const QS // it. Otherwise it is safe to process it stand-alone and re-use the parsed // namespace data for inclusion into other files. bool isIndirect = false; - if (namespaces.count() == 1 && functionContext.count() == 1 + if (namespaces.size() == 1 && functionContext.size() == 1 && functionContextUnresolved.isEmpty() && pendingContext.isEmpty() && !CppFiles::isBlacklisted(cleanFile) && isHeader(cleanFile)) { @@ -1310,7 +1300,7 @@ void CppParser::processInclude(const QString &file, ConversionData &cd, const QS inclusions.insert(cleanFile); if (isIndirect) { CppParser parser; - for (const QString &projectRoot : qAsConst(cd.m_projectRoots)) + for (const QString &projectRoot : std::as_const(cd.m_projectRoots)) if (cleanFile.startsWith(projectRoot)) { parser.setTranslator(new Translator); break; @@ -1374,12 +1364,12 @@ bool CppParser::matchString(QString *s) } } -STRING(QApplication); -STRING(QCoreApplication); -STRING(UnicodeUTF8); -STRING(DefaultCodec); -STRING(CodecForTr); -STRING(Latin1); +static const QString strQApplication = u"QApplication"_s; +static const QString strQCoreApplication = u"QCoreApplication"_s; +static const QString strUnicodeUTF8 = u"UnicodeUTF8"_s; +static const QString strDefaultCodec = u"DefaultCodec"_s; +static const QString strCodecForTr = u"CodecForTr"_s; +static const QString strLatin1 = u"Latin1"_s; bool CppParser::matchEncoding() { @@ -1490,7 +1480,7 @@ void CppParser::handleTr(QString &prefix, bool plural) } if (prefix.isEmpty()) { if (functionContextUnresolved.isEmpty()) { - int idx = functionContext.length(); + int idx = functionContext.size(); if (idx < 2) { yyMsg() << "tr() cannot be called without context\n"; return; @@ -1525,14 +1515,6 @@ void CppParser::handleTr(QString &prefix, bool plural) context = joinNamespaces(stringifyNamespace(functionContext), functionContextUnresolved); } } else { -#ifdef DIAGNOSE_RETRANSLATABILITY - int last = prefix.lastIndexOf(QLatin1String("::")); - QString className = prefix.mid(last == -1 ? 0 : last + 2); - if (!className.isEmpty() && className == functionName) { - yyMsg() << qPrintable(QStringLiteral("It is not recommended to call tr() from within a constructor '%1::%2'\n") - .arg(className).arg(functionName)); - } -#endif prefix.chop(2); NamespaceList nsl; NamespaceList unresolved; @@ -1673,11 +1655,9 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac static QString strColons(QLatin1String("::")); QString prefix; -#ifdef DIAGNOSE_RETRANSLATABILITY - QString functionName; -#endif bool yyTokColonSeen = false; // Start of c'tor's initializer list bool yyTokIdentSeen = false; // Start of initializer (member or base class) + bool maybeInTrailingReturnType = false; metaExpected = true; prospectiveContext.clear(); @@ -1693,7 +1673,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac // so they don't confuse our scoping of static initializers. // we enter the loop by either reading a left bracket or by an // #else popping the state. - if (yyBracketDepth && yyBraceDepth == namespaceDepths.count()) { + if (yyBracketDepth && yyBraceDepth == namespaceDepths.size()) { yyTok = getToken(); continue; } @@ -1716,7 +1696,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac processInclude(cSource, cd, includeStack, inclusions); goto incOk; } - for (const QString &incPath : qAsConst(cd.m_includePath)) { + for (const QString &incPath : std::as_const(cd.m_includePath)) { text = QDir(incPath).absoluteFilePath(yyWord); text.detach(); if (QFileInfo(text).isFile()) { @@ -1739,7 +1719,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac Partial support for inlined functions. */ yyTok = getToken(); - if (yyBraceDepth == namespaceDepths.count() && yyParenDepth == 0) { + if (yyBraceDepth == namespaceDepths.size() && yyParenDepth == 0) { NamespaceList quali; HashString fct; @@ -1769,7 +1749,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac } } - if (yyTok == Tok_Colon || yyTok == Tok_Other) { + if (yyTok == Tok_Colon || yyTok == Tok_AngleBracket) { // Skip any token until '{' or ';' since we might do things wrong if we find // a '::' or ':' token here. do { @@ -1794,10 +1774,10 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac yyMsg() << "Ignoring definition of undeclared qualified class\n"; break; } - namespaceDepths.push(namespaces.count()); + namespaceDepths.push(namespaces.size()); namespaces = nsl; } else { - namespaceDepths.push(namespaces.count()); + namespaceDepths.push(namespaces.size()); } enterNamespace(&namespaces, fct); @@ -1830,7 +1810,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac ns = HashString(text); } if (yyTok == Tok_LeftBrace) { - namespaceDepths.push(namespaces.count()); + namespaceDepths.push(namespaces.size()); for (const auto &nns : nestedNamespaces) enterNamespace(&namespaces, nns); enterNamespace(&namespaces, ns); @@ -1863,7 +1843,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac } } else if (yyTok == Tok_LeftBrace) { // Anonymous namespace - namespaceDepths.push(namespaces.count()); + namespaceDepths.push(namespaces.size()); metaExpected = true; yyTok = getToken(); } @@ -1916,7 +1896,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac break; case Tok_Ident: if (yyTokColonSeen && - yyBraceDepth == namespaceDepths.count() && yyParenDepth == 0) { + yyBraceDepth == namespaceDepths.size() && yyParenDepth == 0) { // member or base class identifier yyTokIdentSeen = true; } @@ -1964,18 +1944,18 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac yyTok = getToken(); break; } - if (yyTok == Tok_ColonColon) { + if (yyTok == Tok_ColonColon && !maybeInTrailingReturnType) { prefix += yyWord; prefix.detach(); } else { notrfunc: prefix.clear(); - if (yyTok == Tok_Ident && !yyParenDepth) - prospectiveContext.clear(); } metaExpected = false; break; case Tok_Arrow: + if (yyParenDepth == 0 && yyBraceDepth == namespaceDepths.size()) + maybeInTrailingReturnType = true; yyTok = getToken(); if (yyTok == Tok_Ident) { switch (trFunctionAliasManager.trFunctionByName(yyWord)) { @@ -1987,29 +1967,23 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac } break; case Tok_ColonColon: - if (yyTokIdentSeen) { + if (yyTokIdentSeen || maybeInTrailingReturnType) { // member or base class identifier yyTok = getToken(); break; } - if (yyBraceDepth == namespaceDepths.count() && yyParenDepth == 0 && !yyTokColonSeen) + if (yyBraceDepth == namespaceDepths.size() && yyParenDepth == 0 && !yyTokColonSeen) prospectiveContext = prefix; prefix += strColons; yyTok = getToken(); -#ifdef DIAGNOSE_RETRANSLATABILITY - if (yyTok == Tok_Ident && yyBraceDepth == namespaceDepths.count() && yyParenDepth == 0) { - functionName = yyWord; - functionName.detach(); - } -#endif break; case Tok_RightBrace: if (!yyTokColonSeen) { - if (yyBraceDepth + 1 == namespaceDepths.count()) { + if (yyBraceDepth + 1 == namespaceDepths.size()) { // class or namespace truncateNamespaces(&namespaces, namespaceDepths.pop()); } - if (yyBraceDepth == namespaceDepths.count()) { + if (yyBraceDepth == namespaceDepths.size()) { // function, class or namespace if (!yyBraceDepth && !directInclude) truncateNamespaces(&functionContext, 1); @@ -2021,6 +1995,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac } Q_FALLTHROUGH(); case Tok_Semicolon: + maybeInTrailingReturnType = false; prospectiveContext.clear(); prefix.clear(); if (!sourcetext.isEmpty() || !extracomment.isEmpty() || !msgid.isEmpty() || !extra.isEmpty()) { @@ -2044,7 +2019,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac break; case Tok_Colon: case Tok_Equals: - if (yyBraceDepth == namespaceDepths.count() && yyParenDepth == 0) { + if (yyBraceDepth == namespaceDepths.size() && yyParenDepth == 0) { if (!prospectiveContext.isEmpty()) { pendingContext = prospectiveContext; prospectiveContext.clear(); @@ -2059,7 +2034,7 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac yyTok = getToken(); break; case Tok_LeftBrace: - if (yyBraceDepth == namespaceDepths.count() + 1 && yyParenDepth == 0) { + if (yyBraceDepth == namespaceDepths.size() + 1 && yyParenDepth == 0) { if (!prospectiveContext.isEmpty()) { pendingContext = prospectiveContext; prospectiveContext.clear(); @@ -2069,24 +2044,61 @@ void CppParser::parseInternal(ConversionData &cd, const QStringList &includeStac yyTokColonSeen = false; } } - Q_FALLTHROUGH(); + maybeInTrailingReturnType = false; + yyTokIdentSeen = false; + metaExpected = true; + yyTok = getToken(); + break; case Tok_LeftParen: + if (!yyTokColonSeen && yyBraceDepth == namespaceDepths.size() && yyParenDepth == 1 + && !prospectiveContext.isEmpty()) { + pendingContext = prospectiveContext; + prospectiveContext.clear(); + } yyTokIdentSeen = false; - Q_FALLTHROUGH(); + metaExpected = true; + yyTok = getToken(); + break; case Tok_Comma: case Tok_QuestionMark: metaExpected = true; yyTok = getToken(); break; case Tok_RightParen: - if (yyParenDepth == 0) + if (yyParenDepth == 0) { + if (!yyTokColonSeen && !pendingContext.isEmpty() + && yyBraceDepth == namespaceDepths.size()) { + // Demote the pendingContext to prospectiveContext. + prospectiveContext = pendingContext; + pendingContext.clear(); + } metaExpected = true; - else + } else { metaExpected = false; + } + yyTok = getToken(); + break; + case Tok_decltype: + { + // Save the parentheses depth outside the 'decltype' specifier. + auto initialParenDepth = yyParenDepth; + + // Eat the opening parenthesis that follows 'decltype'. + yyTok = getToken(); + + // Skip over everything within the parentheses that follow 'decltype'. + while (yyParenDepth != initialParenDepth && yyTok != Tok_Eof) + yyTok = getToken(); + } + break; + case Tok_enum: yyTok = getToken(); + // If it is an enum class then ignore + if (yyTok == Tok_class) + yyTok = getToken(); break; default: - if (!yyParenDepth) + if (!yyParenDepth && !maybeInTrailingReturnType) prospectiveContext.clear(); Q_FALLTHROUGH(); case Tok_RightBracket: // ignoring indexing; for static initializers @@ -2128,15 +2140,21 @@ void CppParser::processComment() yyWord.remove(0, 2); text = yyWord.trimmed(); int k = text.indexOf(QLatin1Char(' ')); - if (k > -1) - extra.insert(text.left(k), text.mid(k + 1).trimmed()); + if (k > -1) { + QString commentvalue = text.mid(k + 1).trimmed(); + if (commentvalue.startsWith(QLatin1Char('"')) && commentvalue.endsWith(QLatin1Char('"')) + && commentvalue.size() != 1) { + commentvalue = commentvalue.sliced(1, commentvalue.size() - 2); + } + extra.insert(text.left(k), commentvalue); + } text.clear(); } else if (*ptr == QLatin1Char('%') && ptr[1].isSpace()) { - sourcetext.reserve(sourcetext.length() + yyWord.length() - 2); - ushort *ptr = (ushort *)sourcetext.data() + sourcetext.length(); + sourcetext.reserve(sourcetext.size() + yyWord.size() - 2); + ushort *ptr = (ushort *)sourcetext.data() + sourcetext.size(); int p = 2, c; forever { - if (p >= yyWord.length()) + if (p >= yyWord.size()) break; c = yyWord.unicode()[p++].unicode(); if (isspace(c)) @@ -2146,7 +2164,7 @@ void CppParser::processComment() break; } forever { - if (p >= yyWord.length()) { + if (p >= yyWord.size()) { whoops: yyMsg() << "Unterminated meta string\n"; break; @@ -2155,7 +2173,7 @@ void CppParser::processComment() if (c == '"') break; if (c == '\\') { - if (p >= yyWord.length()) + if (p >= yyWord.size()) goto whoops; c = yyWord.unicode()[p++].unicode(); if (c == '\n') @@ -2172,10 +2190,10 @@ void CppParser::processComment() ushort c; while ((c = uc[idx]) == ' ' || c == '\t' || c == '\n') ++idx; - if (!memcmp(uc + idx, MagicComment.unicode(), MagicComment.length() * 2)) { - idx += MagicComment.length(); + if (!memcmp(uc + idx, CppMagicComment.unicode(), CppMagicComment.size() * 2)) { + idx += CppMagicComment.size(); comment = QString::fromRawData(yyWord.unicode() + idx, - yyWord.length() - idx).simplified(); + yyWord.size() - idx).simplified(); int k = comment.indexOf(QLatin1Char(' ')); if (k == -1) { context = comment; @@ -2209,7 +2227,7 @@ const ParseResults *CppParser::recordResults(bool isHeader) } if (isHeader) { const ParseResults *pr; - if (!tor && results->includes.count() == 1 + if (!tor && results->includes.size() == 1 && results->rootNamespace.children.isEmpty() && results->rootNamespace.aliases.isEmpty() && results->rootNamespace.usings.isEmpty()) { diff --git a/src/linguist/lupdate/cpp.h b/src/linguist/lupdate/cpp.h index d12c5e924..ca3cd72f9 100644 --- a/src/linguist/lupdate/cpp.h +++ b/src/linguist/lupdate/cpp.h @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #ifndef CPP_H #define CPP_H diff --git a/src/linguist/lupdate/cpp_clang.cpp b/src/linguist/lupdate/cpp_clang.cpp index 695280c21..180bbb4cb 100644 --- a/src/linguist/lupdate/cpp_clang.cpp +++ b/src/linguist/lupdate/cpp_clang.cpp @@ -1,33 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2019 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2019 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "cpp_clang.h" #include "clangtoolastreader.h" +#include "filesignificancecheck.h" #include "lupdatepreprocessoraction.h" #include "synchronized.h" #include "translator.h" @@ -38,6 +14,7 @@ #include <QtCore/qjsonarray.h> #include <QtCore/qjsondocument.h> #include <QtCore/qjsonobject.h> +#include <QtCore/qscopeguard.h> #include <QtCore/QProcess> #include <QStandardPaths> #include <QtTools/private/qttools-config_p.h> @@ -60,6 +37,8 @@ using clang::tooling::CompilationDatabase; QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; + Q_LOGGING_CATEGORY(lcClang, "qt.lupdate.clang"); static QString getSysCompiler() @@ -99,6 +78,21 @@ static QByteArrayList getMSVCIncludePathsFromEnvironment() return pathList; } +static QStringList getProjectDirsFromEnvironment() +{ + QList<QByteArray> dirList; + QStringList rootdirs; + if (const char* includeEnv = std::getenv("LUPDATE_ROOT_DIRS")) { + QByteArray includeList = QByteArray::fromRawData(includeEnv, strlen(includeEnv)); + dirList = includeList.split(';'); + + for (auto dir : dirList) { + rootdirs.append(QString::fromStdString(dir.toStdString())); + } + } + return rootdirs; +} + static QByteArray frameworkSuffix() { @@ -245,51 +239,46 @@ std::vector<std::string> ClangCppParser::getAliasFunctionDefinition() } static std::vector<std::string> aliasDefinition; -// Makes sure all the comments will be parsed and part of the AST -// Clang will run with the flag -fparse-all-comments -clang::tooling::ArgumentsAdjuster getClangArgumentAdjuster() + +static clang::tooling::ArgumentsAdjuster getClangArgumentAdjuster() { - return [](const clang::tooling::CommandLineArguments &args, llvm::StringRef /*unused*/) { - clang::tooling::CommandLineArguments adjustedArgs; - for (size_t i = 0, e = args.size(); i < e; ++i) { - llvm::StringRef arg = args[i]; - // FIXME: Remove options that generate output. - if (!arg.startswith("-fcolor-diagnostics") && !arg.startswith("-fdiagnostics-color")) - adjustedArgs.push_back(args[i]); - } - adjustedArgs.push_back("-fparse-all-comments"); - adjustedArgs.push_back("-nostdinc"); + const QByteArrayList compilerIncludeFlags = getIncludePathsFromCompiler(); + return [=](const clang::tooling::CommandLineArguments &args, llvm::StringRef /*unused*/) { + clang::tooling::CommandLineArguments adjustedArgs(args); + clang::tooling::CommandLineArguments adjustedArgsTemp; + + adjustedArgsTemp.push_back("-fparse-all-comments"); + adjustedArgsTemp.push_back("-nostdinc"); // Turn off SSE support to avoid usage of gcc builtins. // TODO: Look into what Qt Creator does. // Pointers: HeaderPathFilter::removeGccInternalIncludePaths() // and gccInstallDir() in gcctoolchain.cpp // Also needed for Mac, No need for CLANG_RESOURCE_DIR when this is part of the argument. - adjustedArgs.push_back("-mno-sse"); + adjustedArgsTemp.push_back("-mno-sse"); - adjustedArgs.push_back("-fsyntax-only"); #ifdef Q_OS_WIN - adjustedArgs.push_back("-fms-compatibility-version=19"); - adjustedArgs.push_back("-DQ_COMPILER_UNIFORM_INIT"); // qtbase + clang-cl hack + adjustedArgsTemp.push_back("-fms-compatibility-version=19"); + adjustedArgsTemp.push_back("-DQ_COMPILER_UNIFORM_INIT"); // qtbase + clang-cl hack + // avoid constexpr error connected with offsetof (QTBUG-97380) + adjustedArgsTemp.push_back("-D_CRT_USE_BUILTIN_OFFSETOF"); #endif - adjustedArgs.push_back("-Wno-everything"); - adjustedArgs.push_back("-std=gnu++17"); + adjustedArgsTemp.push_back("-Wno-everything"); - for (QByteArray line : getIncludePathsFromCompiler()) { - line = line.trimmed(); - if (line.isEmpty()) - continue; - adjustedArgs.push_back(line.data()); - } + for (const QByteArray &flag : compilerIncludeFlags) + adjustedArgsTemp.push_back(flag.data()); for (auto alias : aliasDefinition) { - adjustedArgs.push_back(alias); + adjustedArgsTemp.push_back(alias); } + + clang::tooling::CommandLineArguments::iterator it = llvm::find(adjustedArgs, "--"); + adjustedArgs.insert(it, adjustedArgsTemp.begin(), adjustedArgsTemp.end()); return adjustedArgs; }; } -bool ClangCppParser::containsTranslationInformation(llvm::StringRef ba) +bool ClangCppParser::stringContainsTranslationInformation(llvm::StringRef ba) { // pre-process the files by a simple text search if there is any occurrence // of things we are interested in @@ -345,6 +334,7 @@ static bool generateCompilationDatabase(const QString &outputFilePath, const Con obj[QLatin1String("directory")] = buildDir; QJsonArray args = { QLatin1String("clang++"), + QLatin1String("-std=gnu++17"), #ifndef Q_OS_WIN QLatin1String("-fPIC"), #endif @@ -410,32 +400,24 @@ bool ClangCppParser::hasAliases() void ClangCppParser::loadCPP(Translator &translator, const QStringList &files, ConversionData &cd, bool *fail) { + FileSignificanceCheck::create(); + auto cleanup = qScopeGuard(FileSignificanceCheck::destroy); + FileSignificanceCheck::the()->setExclusionPatterns(cd.m_excludes); + if (cd.m_rootDirs.size() > 0) + FileSignificanceCheck::the()->setRootDirectories(cd.m_rootDirs); + else + FileSignificanceCheck::the()->setRootDirectories(getProjectDirsFromEnvironment()); + if (hasAliases()) aliasDefinition = getAliasFunctionDefinition(); // pre-process the files by a simple text search if there is any occurrence // of things we are interested in qCDebug(lcClang) << "Load CPP \n"; - std::vector<std::string> sourcesAst, sourcesPP; + std::vector<std::string> sources; for (const QString &filename : files) { - QFile file(filename); qCDebug(lcClang) << "File: " << filename << " \n"; - if (file.open(QIODevice::ReadOnly)) { - if (const uchar *memory = file.map(0, file.size())) { - const auto ba = llvm::StringRef((const char*) (memory), file.size()); - if (containsTranslationInformation(ba)) { - sourcesPP.emplace_back(filename.toStdString()); - sourcesAst.emplace_back(sourcesPP.back()); - } - } else { - QByteArray mem = file.readAll(); - const auto ba = llvm::StringRef((const char*) (mem), file.size()); - if (containsTranslationInformation(ba)) { - sourcesPP.emplace_back(filename.toStdString()); - sourcesAst.emplace_back(sourcesPP.back()); - } - } - } + sources.emplace_back(filename.toStdString()); } std::string errorMessage; @@ -456,7 +438,7 @@ void ClangCppParser::loadCPP(Translator &translator, const QStringList &files, C qCDebug(lcClang) << "Generating compilation database" << dbFilePath; if (!generateCompilationDatabase(dbFilePath, cd)) { *fail = true; - cd.appendError(u"Cannot generate compilation database."_qs); + cd.appendError(u"Cannot generate compilation database."_s); return; } errorMessage.clear(); @@ -473,16 +455,21 @@ void ClangCppParser::loadCPP(Translator &translator, const QStringList &files, C Stores stores(ast, qdecl, qnoop); std::vector<std::thread> producers; - ReadSynchronizedRef<std::string> ppSources(sourcesPP); + ReadSynchronizedRef<std::string> ppSources(sources); WriteSynchronizedRef<TranslationRelatedStore> ppStore(stores.Preprocessor); size_t idealProducerCount = std::min(ppSources.size(), size_t(std::thread::hardware_concurrency())); + clang::tooling::ArgumentsAdjuster argumentsAdjusterSyntaxOnly = + clang::tooling::getClangSyntaxOnlyAdjuster(); + clang::tooling::ArgumentsAdjuster argumentsAdjusterLocal = getClangArgumentAdjuster(); + clang::tooling::ArgumentsAdjuster argumentsAdjuster = + clang::tooling::combineAdjusters(argumentsAdjusterLocal, argumentsAdjusterSyntaxOnly); for (size_t i = 0; i < idealProducerCount; ++i) { - std::thread producer([&ppSources, &db, &ppStore]() { + std::thread producer([&ppSources, &db, &ppStore, &argumentsAdjuster]() { std::string file; while (ppSources.next(&file)) { clang::tooling::ClangTool tool(*db, file); - tool.appendArgumentsAdjuster(getClangArgumentAdjuster()); + tool.appendArgumentsAdjuster(argumentsAdjuster); tool.run(new LupdatePreprocessorActionFactory(&ppStore)); } }); @@ -492,14 +479,14 @@ void ClangCppParser::loadCPP(Translator &translator, const QStringList &files, C producer.join(); producers.clear(); - ReadSynchronizedRef<std::string> astSources(sourcesAst); + ReadSynchronizedRef<std::string> astSources(sources); idealProducerCount = std::min(astSources.size(), size_t(std::thread::hardware_concurrency())); for (size_t i = 0; i < idealProducerCount; ++i) { - std::thread producer([&astSources, &db, &stores]() { + std::thread producer([&astSources, &db, &stores, &argumentsAdjuster]() { std::string file; while (astSources.next(&file)) { clang::tooling::ClangTool tool(*db, file); - tool.appendArgumentsAdjuster(getClangArgumentAdjuster()); + tool.appendArgumentsAdjuster(argumentsAdjuster); tool.run(new LupdateToolActionFactory(&stores)); } }); diff --git a/src/linguist/lupdate/cpp_clang.h b/src/linguist/lupdate/cpp_clang.h index f27ee2bb0..88e25facd 100644 --- a/src/linguist/lupdate/cpp_clang.h +++ b/src/linguist/lupdate/cpp_clang.h @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2019 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2019 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #ifndef CLANG_CPP_H #define CLANG_CPP_H @@ -36,22 +11,19 @@ #include <QtCore/qregularexpression.h> #include <QtCore/qstring.h> -#if defined(Q_CC_MSVC) -# pragma warning(push) -# pragma warning(disable: 4100) -# pragma warning(disable: 4146) -# pragma warning(disable: 4267) -# pragma warning(disable: 4624) -#endif +QT_WARNING_PUSH +QT_WARNING_DISABLE_MSVC(4100) +QT_WARNING_DISABLE_MSVC(4146) +QT_WARNING_DISABLE_MSVC(4267) +QT_WARNING_DISABLE_MSVC(4624) +QT_WARNING_DISABLE_GCC("-Wnonnull") #include <llvm/ADT/StringRef.h> #include <clang/Basic/SourceLocation.h> #include <clang/Basic/SourceManager.h> #include <clang/Basic/FileManager.h> -#if defined(Q_CC_MSVC) -# pragma warning(pop) -#endif +QT_WARNING_POP #include <vector> #include <iostream> @@ -59,12 +31,6 @@ QT_BEGIN_NAMESPACE -inline QDebug operator<<(QDebug out, const std::string& str) -{ - out << QString::fromStdString(str); - return out; -} - Q_DECLARE_LOGGING_CATEGORY(lcClang) inline QString toQt(llvm::StringRef str) @@ -87,6 +53,7 @@ struct TranslationRelatedStore QString contextRetrieved; QString lupdateSource; QString lupdateLocationFile; + QString lupdateInputFile; // file associated to the running of the tool qint64 lupdateLocationLine = -1; QString lupdateId; QString lupdateSourceWhenId; @@ -338,7 +305,7 @@ namespace ClangCppParser void finalize(ReadSynchronizedRef<TranslationRelatedStore> &ast, WriteSynchronizedRef<TranslationRelatedStore> &newAst); - bool containsTranslationInformation(llvm::StringRef ba); + bool stringContainsTranslationInformation(llvm::StringRef ba); bool hasAliases(); std::vector<std::string> getAliasFunctionDefinition(); diff --git a/src/linguist/lupdate/filesignificancecheck.cpp b/src/linguist/lupdate/filesignificancecheck.cpp new file mode 100644 index 000000000..9babe6898 --- /dev/null +++ b/src/linguist/lupdate/filesignificancecheck.cpp @@ -0,0 +1,70 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include "filesignificancecheck.h" + +#include <mutex> + +QT_BEGIN_NAMESPACE + +FileSignificanceCheck *FileSignificanceCheck::m_instance = nullptr; + +void FileSignificanceCheck::setRootDirectories(const QStringList &paths) +{ + const size_t pathsSize = static_cast<size_t>(paths.size()); + m_rootDirs.resize(pathsSize); + for (size_t i = 0; i < pathsSize; ++i) + m_rootDirs[i].setPath(paths.at(i)); +} + +void FileSignificanceCheck::setExclusionPatterns(const QStringList &patterns) +{ + const size_t patternsSize = static_cast<size_t>(patterns.size()); + m_exclusionRegExes.resize(patternsSize); + for (size_t i = 0; i < patternsSize; ++i) + m_exclusionRegExes[i] = QRegularExpression::fromWildcard(patterns.at(i)); +} + +/* + * Return true if the given source file is significant for lupdate. + * A file is considered insignificant if + * - it's not within any project root + * - it's excluded + * + * This method is called from multiple threads. + * Results are cached. + */ +bool FileSignificanceCheck::isFileSignificant(const std::string &filePath) const +{ + // cache lookup + std::shared_lock<std::shared_mutex> readLock(m_cacheMutex); + auto it = m_cache.find(filePath); + if (it != m_cache.end()) + return it->second; + + // cache miss + readLock.unlock(); + std::unique_lock<std::shared_mutex> writeLock(m_cacheMutex); + QString file = QString::fromUtf8(filePath); + QString cleanFile = QDir::cleanPath(file); + for (const QRegularExpression &rx : m_exclusionRegExes) { + if (rx.match(cleanFile).hasMatch()) { + m_cache.insert({filePath, false}); + return false; + } + } + + for (const QDir &rootDir : m_rootDirs) { + QString relativeFilePath = rootDir.relativeFilePath(file); + if (!relativeFilePath.startsWith(QLatin1String("../")) + && QFileInfo(relativeFilePath).isRelative()) { + m_cache.insert({filePath, true}); + return true; + } + } + + m_cache.insert({filePath, false}); + return false; +} + +QT_END_NAMESPACE diff --git a/src/linguist/lupdate/filesignificancecheck.h b/src/linguist/lupdate/filesignificancecheck.h new file mode 100644 index 000000000..15a947133 --- /dev/null +++ b/src/linguist/lupdate/filesignificancecheck.h @@ -0,0 +1,63 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#ifndef FILESIGNIFICANCECHECK_H +#define FILESIGNIFICANCECHECK_H + +#include <QtCore/qdir.h> +#include <QtCore/qregularexpression.h> +#include <QtCore/qstringlist.h> + +#include <shared_mutex> +#include <string> +#include <unordered_map> +#include <vector> + +QT_BEGIN_NAMESPACE + +class FileSignificanceCheck +{ +public: + FileSignificanceCheck() = default; + + static void create() + { + m_instance = new FileSignificanceCheck; + } + + static void destroy() + { + delete m_instance; + m_instance = nullptr; + } + + static FileSignificanceCheck *the() + { + return m_instance; + } + + void setRootDirectories(const QStringList &paths); + void setExclusionPatterns(const QStringList &patterns); + + bool isFileSignificant(const std::string &filePath) const; + +private: + static FileSignificanceCheck *m_instance; + std::vector<QDir> m_rootDirs; + std::vector<QRegularExpression> m_exclusionRegExes; + mutable std::unordered_map<std::string, bool> m_cache; + mutable std::shared_mutex m_cacheMutex; +}; + +namespace LupdatePrivate { + +inline bool isFileSignificant(const std::string &filePath) +{ + return FileSignificanceCheck::the()->isFileSignificant(filePath); +} + +} // namespace LupdatePrivate + +QT_END_NAMESPACE + +#endif // header guard diff --git a/src/linguist/lupdate/java.cpp b/src/linguist/lupdate/java.cpp index 7a1c6b6dc..22a226200 100644 --- a/src/linguist/lupdate/java.cpp +++ b/src/linguist/lupdate/java.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "lupdate.h" @@ -36,6 +11,8 @@ #include <QtCore/QStack> #include <QtCore/QString> #include <QtCore/QCoreApplication> +#include <QtCore/QStringConverter> +#include <QtCore/QTextStream> #include <iostream> diff --git a/src/linguist/lupdate/lupdate.1 b/src/linguist/lupdate/lupdate.1 index 1eaf03aed..a866965ca 100644 --- a/src/linguist/lupdate/lupdate.1 +++ b/src/linguist/lupdate/lupdate.1 @@ -1,28 +1,7 @@ .TH lupdate 1 "18 October 2001" "Digia Plc and/or its subsidiary(-ies)" \" -*- nroff -*- .\" .\" Copyright (C) 2016 The Qt Company Ltd. -.\" Contact: https://www.qt.io/licensing/ -.\" -.\" This file is part of the QtGui module of the Qt Toolkit. -.\" -.\" $QT_BEGIN_LICENSE:GPL-EXCEPT$ -.\" Commercial License Usage -.\" Licensees holding valid commercial Qt licenses may use this file in -.\" accordance with the commercial license agreement provided with the -.\" Software or, alternatively, in accordance with the terms contained in -.\" a written agreement between you and The Qt Company. For licensing terms -.\" and conditions see https://www.qt.io/terms-conditions. For further -.\" information use the contact form at https://www.qt.io/contact-us. -.\" -.\" GNU General Public License Usage -.\" Alternatively, this file may be used under the terms of the GNU -.\" General Public License version 3 as published by the Free Software -.\" Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -.\" included in the packaging of this file. Please review the following -.\" information to ensure the GNU General Public License requirements will -.\" be met: https://www.gnu.org/licenses/gpl-3.0.html. -.\" -.\" $QT_END_LICENSE$ +.\" SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 .\" .SH NAME lupdate \- update Qt Linguist translation files @@ -49,7 +28,7 @@ used with version control systems if required. .PP .SH OPTIONS .TP -.I "-disable-heuristic {sametext|similartext|number}" +.I "-disable-heuristic {sametext|similartext}" Disable the named merge heuristic. Can be specified multiple times. .TP .I "-extensions <ext>[,<ext>...]" diff --git a/src/linguist/lupdate/lupdate.h b/src/linguist/lupdate/lupdate.h index 4aaac1e78..5cc0f8c2b 100644 --- a/src/linguist/lupdate/lupdate.h +++ b/src/linguist/lupdate/lupdate.h @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #ifndef LUPDATE_H #define LUPDATE_H @@ -52,7 +27,6 @@ enum UpdateOption { NoSort = 8, HeuristicSameText = 16, HeuristicSimilarText = 32, - HeuristicNumber = 64, AbsoluteLocations = 256, RelativeLocations = 512, NoLocations = 1024, diff --git a/src/linguist/lupdate/lupdatepreprocessoraction.cpp b/src/linguist/lupdate/lupdatepreprocessoraction.cpp index 7946ac7a5..4d5b9a3b4 100644 --- a/src/linguist/lupdate/lupdatepreprocessoraction.cpp +++ b/src/linguist/lupdate/lupdatepreprocessoraction.cpp @@ -1,32 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "lupdatepreprocessoraction.h" +#include "filesignificancecheck.h" #include <clang/Lex/MacroArgs.h> #include <clang/Basic/TokenKinds.h> @@ -41,7 +17,7 @@ void LupdatePPCallbacks::MacroExpands(const clang::Token &token, const auto &sm = m_preprocessor.getSourceManager(); llvm::StringRef fileName = sm.getFilename(sourceRange.getBegin()); - if (fileName != m_inputFile) + if (!LupdatePrivate::isFileSignificant(fileName.str())) return; const QString funcName = QString::fromStdString(m_preprocessor.getSpelling(token)); @@ -67,6 +43,7 @@ void LupdatePPCallbacks::MacroExpands(const clang::Token &token, store.callType = QStringLiteral("MacroExpands"); store.funcName = funcName; store.lupdateLocationFile = toQt(fileName); + store.lupdateInputFile = toQt(m_inputFile); store.lupdateLocationLine = sm.getExpansionLineNumber(sourceRange.getBegin()); store.locationCol = sm.getExpansionColumnNumber(sourceRange.getBegin()); @@ -159,20 +136,62 @@ void LupdatePPCallbacks::SourceRangeSkipped(clang::SourceRange sourceRange, const auto &sm = m_preprocessor.getSourceManager(); llvm::StringRef fileName = sm.getFilename(sourceRange.getBegin()); - if (fileName != m_inputFile) + + if (!LupdatePrivate::isFileSignificant(fileName.str())) return; + const char *begin = sm.getCharacterData(sourceRange.getBegin()); const char *end = sm.getCharacterData(sourceRange.getEnd()); llvm::StringRef skippedText = llvm::StringRef(begin, end - begin); - if (ClangCppParser::containsTranslationInformation(skippedText)) { - qCDebug(lcClang) << "SourceRangeSkipped: skipped text:" << skippedText.str(); + if (ClangCppParser::stringContainsTranslationInformation(skippedText)) { + qCDebug(lcClang) << "SourceRangeSkipped: skipped text:" << QString::fromStdString(skippedText.str()); unsigned int beginLine = sm.getExpansionLineNumber(sourceRange.getBegin()); unsigned int endLine = sm.getExpansionLineNumber(sourceRange.getEnd()); qWarning("%s Code with translation information has been skipped " "between lines %d and %d", - m_inputFile.c_str(), beginLine, endLine); + fileName.str().c_str(), beginLine, endLine); } +} + +// To list the included files +#if (LUPDATE_CLANG_VERSION < LUPDATE_CLANG_VERSION_CHECK(14,0,0)) +void LupdatePPCallbacks::InclusionDirective(clang::SourceLocation /*hashLoc*/, + const clang::Token & /*includeTok*/, clang::StringRef /*fileName*/, bool /*isAngled*/, + clang::CharSourceRange /*filenameRange*/, +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(16,0,0)) + const clang::OptionalFileEntryRef file, +#elif (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(15,0,0)) + const clang::Optional<clang::FileEntryRef> file, +#else + const clang::FileEntry *file, +#endif + clang::StringRef /*searchPath*/, clang::StringRef /*relativePath*/, + const clang::Module */*imported*/, clang::SrcMgr::CharacteristicKind /*fileType*/) +{ + if (!file) + return; + clang::StringRef fileNameRealPath = file-> +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(15,0,0)) + getFileEntry(). +#endif + tryGetRealPathName(); + if (!LupdatePrivate::isFileSignificant(fileNameRealPath.str())) + return; + + TranslationRelatedStore store; + store.callType = QStringLiteral("InclusionDirective"); + store.lupdateLocationFile = toQt(fileNameRealPath); + store.lupdateLocationLine = 1; + store.locationCol = 1; + store.lupdateInputFile = toQt(m_inputFile); + // do not fill the store.funcName. There is no function at this point + // the information is retrieved here to look for TRANSLATOR comments in header files + // when traversing the AST + + if (store.isValid()) + m_ppStores.emplace_back(std::move(store)); } +#endif QT_END_NAMESPACE diff --git a/src/linguist/lupdate/lupdatepreprocessoraction.h b/src/linguist/lupdate/lupdatepreprocessoraction.h index 01caed52d..f373248b3 100644 --- a/src/linguist/lupdate/lupdatepreprocessoraction.h +++ b/src/linguist/lupdate/lupdatepreprocessoraction.h @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #ifndef LUPDATEPREPROCESSORACTION_H #define LUPDATEPREPROCESSORACTION_H @@ -32,13 +7,12 @@ #include "cpp_clang.h" #include "synchronized.h" -#if defined(Q_CC_MSVC) -# pragma warning(push) -# pragma warning(disable: 4100) -# pragma warning(disable: 4146) -# pragma warning(disable: 4267) -# pragma warning(disable: 4624) -#endif +QT_WARNING_PUSH +QT_WARNING_DISABLE_MSVC(4100) +QT_WARNING_DISABLE_MSVC(4146) +QT_WARNING_DISABLE_MSVC(4267) +QT_WARNING_DISABLE_MSVC(4624) +QT_WARNING_DISABLE_GCC("-Wnonnull") #include <clang/Frontend/CompilerInstance.h> #include <clang/Frontend/FrontendActions.h> @@ -46,9 +20,7 @@ #include <clang/Lex/PPCallbacks.h> #include <clang/Lex/Preprocessor.h> -#if defined(Q_CC_MSVC) -# pragma warning(pop) -#endif +QT_WARNING_POP #include <memory> @@ -62,7 +34,7 @@ public: , m_stores(stores) { const auto &sm = m_preprocessor.getSourceManager(); - m_inputFile = sm.getFileEntryForID(sm.getMainFileID())->getName(); + m_inputFile = sm.getFileEntryRefForID(sm.getMainFileID())->getName(); } ~LupdatePPCallbacks() override @@ -77,6 +49,21 @@ private: void storeMacroArguments(const std::vector<QString> &args, TranslationRelatedStore *store); void SourceRangeSkipped(clang::SourceRange sourceRange, clang::SourceLocation endifLoc) override; +#if (LUPDATE_CLANG_VERSION < LUPDATE_CLANG_VERSION_CHECK(14,0,0)) + void InclusionDirective(clang::SourceLocation /*hashLoc*/, const clang::Token &/*includeTok*/, + clang::StringRef /*fileName*/, bool /*isAngled*/, + clang::CharSourceRange /*filenameRange*/, +#if (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(16,0,0)) + const clang::OptionalFileEntryRef file, +#elif (LUPDATE_CLANG_VERSION >= LUPDATE_CLANG_VERSION_CHECK(15,0,0)) + const clang::Optional<clang::FileEntryRef> file, +#else + const clang::FileEntry *file, +#endif + clang::StringRef /*searchPath*/, clang::StringRef /*relativePath*/, + const clang::Module */*imported*/, + clang::SrcMgr::CharacteristicKind /*fileType*/) override; +#endif std::string m_inputFile; clang::Preprocessor &m_preprocessor; diff --git a/src/linguist/lupdate/main.cpp b/src/linguist/lupdate/main.cpp index 2d594ba4d..820f750de 100644 --- a/src/linguist/lupdate/main.cpp +++ b/src/linguist/lupdate/main.cpp @@ -1,31 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Copyright (C) 2016 Klarälvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author Marc Mutz <marc.mutz@kdab.com> -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// Copyright (C) 2016 Klarälvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author Marc Mutz <marc.mutz@kdab.com> +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "lupdate.h" #if QT_CONFIG(clangcpp) @@ -43,15 +18,19 @@ #include <QtCore/QFile> #include <QtCore/QFileInfo> #include <QtCore/QLibraryInfo> +#include <QtCore/QRegularExpression> #include <QtCore/QString> #include <QtCore/QStringList> #include <QtCore/QTranslator> #include <iostream> +using namespace Qt::StringLiterals; + bool useClangToParseCpp = false; QString commandLineCompilationDatabaseDir; // for the path to the json file passed as a command line argument. // Has priority over what is in the .pro file and passed to the project. +QStringList rootDirs; // Can't have an array of QStaticStringData<N> for different N, so // use QString, which requires constructor calls. Doesn't matter @@ -166,11 +145,11 @@ QString ParserTool::transcode(const QString &str) const QByteArray in = str.toUtf8(); QByteArray out; - out.reserve(in.length()); - for (int i = 0; i < in.length();) { + out.reserve(in.size()); + for (int i = 0; i < in.size();) { uchar c = in[i++]; if (c == '\\') { - if (i >= in.length()) + if (i >= in.size()) break; c = in[i++]; @@ -180,7 +159,7 @@ QString ParserTool::transcode(const QString &str) if (c == 'x' || c == 'u' || c == 'U') { const bool unicode = (c != 'x'); QByteArray hex; - while (i < in.length() && isxdigit((c = in[i]))) { + while (i < in.size() && isxdigit((c = in[i]))) { hex += c; i++; } @@ -192,7 +171,7 @@ QString ParserTool::transcode(const QString &str) QByteArray oct; int n = 0; oct += c; - while (n < 2 && i < in.length() && (c = in[i]) >= '0' && c < '8') { + while (n < 2 && i < in.size() && (c = in[i]) >= '0' && c < '8') { i++; n++; oct += c; @@ -206,7 +185,7 @@ QString ParserTool::transcode(const QString &str) out += c; } } - return QString::fromUtf8(out.constData(), out.length()); + return QString::fromUtf8(out.constData(), out.size()); } static QString m_defaultExtensions; @@ -267,11 +246,16 @@ static void printUsage() " May be specified multiple times.\n" " -locations {absolute|relative|none}\n" " Specify/override how source code references are saved in TS files.\n" + " absolute: Source file path is relative to target file. Absolute line\n" + " number is stored.\n" + " relative: Source file path is relative to target file. Line number is\n" + " relative to other entries in the same source file.\n" + " none: no information about source location is stored.\n" " Guessed from existing TS files if not specified.\n" " Default is absolute for new files.\n" " -no-ui-lines\n" " Do not record line numbers in references to UI files.\n" - " -disable-heuristic {sametext|similartext|number}\n" + " -disable-heuristic {sametext|similartext}\n" " Disable the named merge heuristic. Can be specified multiple times.\n" " -project <filename>\n" " Name of a file containing the project's description in JSON format.\n" @@ -309,6 +293,10 @@ static void printUsage() " A directory specified on the command line takes precedence.\n" " If no path is given, the compilation database will be searched\n" " in all parent paths of the first input file.\n" + " -project-roots <directory>...\n" + " Specify one or more project root directories.\n" + " Only files below a project root are considered for translation when using\n" + " the -clang-parser option.\n" " @lst-file\n" " Read additional file names (one per line) or includepaths (one per\n" " line, and prefixed with -I) from lst-file.\n" @@ -399,8 +387,8 @@ static void updateTsFiles(const Translator &fetchedTor, const QStringList &tsFil // (when the language is not recognized, plural translations are lost) if (tor.translationsExist()) { QLocale::Language l; - QLocale::Country c; - tor.languageAndCountry(tor.languageCode(), &l, &c); + QLocale::Territory c; + tor.languageAndTerritory(tor.languageCode(), &l, &c); QStringList forms; if (!getNumerusInfo(l, c, 0, &forms, 0)) { printErr(QStringLiteral("File %1 won't be updated: it contains translation but the" @@ -476,6 +464,22 @@ static bool readFileContent(const QString &filePath, QString *content, QString * return true; } +static void removeExcludedSources(Projects &projects) +{ + for (Project &project : projects) { + for (const QString &ex : project.excluded) { + QRegularExpression rx(QRegularExpression::wildcardToRegularExpression(ex)); + for (auto it = project.sources.begin(); it != project.sources.end(); ) { + if (rx.match(*it).hasMatch()) + it = project.sources.erase(it); + else + ++it; + } + } + removeExcludedSources(project.subProjects); + } +} + static QStringList getResources(const QString &resourceFile) { if (!QFile::exists(resourceFile)) @@ -494,9 +498,34 @@ static QStringList getResources(const QString &resourceFile) return rqr.files; } +// Remove .qrc files from the project and return them as absolute paths. +static QStringList extractQrcFiles(Project &project) +{ + auto it = project.sources.begin(); + QStringList qrcFiles; + while (it != project.sources.end()) { + QFileInfo fi(*it); + QString fn = QDir::cleanPath(fi.absoluteFilePath()); + if (fn.endsWith(QLatin1String(".qrc"), Qt::CaseInsensitive)) { + qrcFiles += fn; + it = project.sources.erase(it); + } else { + ++it; + } + } + return qrcFiles; +} + +// Replace all .qrc files in the project with their content. +static void expandQrcFiles(Project &project) +{ + for (const QString &qrcFile : extractQrcFiles(project)) + project.sources << getResources(qrcFile); +} + static bool processTs(Translator &fetchedTor, const QString &file, ConversionData &cd) { - for (const Translator::FileFormat &fmt : qAsConst(Translator::registeredFileFormats())) { + for (const Translator::FileFormat &fmt : std::as_const(Translator::registeredFileFormats())) { if (file.endsWith(QLatin1Char('.') + fmt.extension, Qt::CaseInsensitive)) { Translator tor; if (tor.load(file, cd, fmt.extension)) { @@ -573,7 +602,7 @@ static QSet<QString> projectRoots(const QString &projectFile, const QStringList sourceDirs.insert(sf.left(sf.lastIndexOf(QLatin1Char('/')) + 1)); QStringList rootList = sourceDirs.values(); rootList.sort(); - for (int prev = 0, curr = 1; curr < rootList.length(); ) + for (int prev = 0, curr = 1; curr < rootList.size(); ) if (rootList.at(curr).startsWith(rootList.at(prev))) rootList.removeAt(curr); else @@ -622,6 +651,10 @@ private: ConversionData cd; cd.m_noUiLines = options & NoUiLines; cd.m_projectRoots = projectRoots(projectFile, sources); + QStringList projectRootDirs; + for (auto dir : cd.m_projectRoots) + projectRootDirs.append(dir); + cd.m_rootDirs = projectRootDirs; cd.m_includePath = prj.includePaths; cd.m_excludes = prj.excluded; cd.m_sourceIsUtf16 = options & SourceIsUtf16; @@ -712,8 +745,7 @@ int main(int argc, char **argv) UpdateOptions options = Verbose | // verbose is on by default starting with Qt 4.2 - HeuristicSameText | HeuristicSimilarText | HeuristicNumber; - int proDebug = 0; + HeuristicSameText | HeuristicSimilarText; int numFiles = 0; bool metTsFlag = false; bool metXTsFlag = false; @@ -743,16 +775,15 @@ int main(int argc, char **argv) options &= ~Verbose; continue; } else if (arg == QLatin1String("-pro-debug")) { - proDebug++; continue; } else if (arg == QLatin1String("-project")) { ++i; if (i == argc) { - printErr(u"The option -project requires a parameter.\n"_qs); + printErr(u"The option -project requires a parameter.\n"_s); return 1; } if (!projectDescriptionFile.isEmpty()) { - printErr(u"The option -project must appear only once.\n"_qs); + printErr(u"The option -project must appear only once.\n"_s); return 1; } projectDescriptionFile = args[i]; @@ -761,7 +792,7 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-target-language")) { ++i; if (i == argc) { - printErr(u"The option -target-language requires a parameter.\n"_qs); + printErr(u"The option -target-language requires a parameter.\n"_s); return 1; } targetLanguage = args[i]; @@ -769,7 +800,7 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-source-language")) { ++i; if (i == argc) { - printErr(u"The option -source-language requires a parameter.\n"_qs); + printErr(u"The option -source-language requires a parameter.\n"_s); return 1; } sourceLanguage = args[i]; @@ -777,7 +808,7 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-disable-heuristic")) { ++i; if (i == argc) { - printErr(u"The option -disable-heuristic requires a parameter.\n"_qs); + printErr(u"The option -disable-heuristic requires a parameter.\n"_s); return 1; } arg = args[i]; @@ -785,17 +816,15 @@ int main(int argc, char **argv) options &= ~HeuristicSameText; } else if (arg == QLatin1String("similartext")) { options &= ~HeuristicSimilarText; - } else if (arg == QLatin1String("number")) { - options &= ~HeuristicNumber; } else { - printErr(u"Invalid heuristic name passed to -disable-heuristic.\n"_qs); + printErr(u"Invalid heuristic name passed to -disable-heuristic.\n"_s); return 1; } continue; } else if (arg == QLatin1String("-locations")) { ++i; if (i == argc) { - printErr(u"The option -locations requires a parameter.\n"_qs); + printErr(u"The option -locations requires a parameter.\n"_s); return 1; } if (args[i] == QLatin1String("none")) { @@ -805,7 +834,7 @@ int main(int argc, char **argv) } else if (args[i] == QLatin1String("absolute")) { options |= AbsoluteLocations; } else { - printErr(u"Invalid parameter passed to -locations.\n"_qs); + printErr(u"Invalid parameter passed to -locations.\n"_s); return 1; } continue; @@ -839,7 +868,7 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-extensions")) { ++i; if (i == argc) { - printErr(u"The -extensions option should be followed by an extension list.\n"_qs); + printErr(u"The -extensions option should be followed by an extension list.\n"_s); return 1; } extensions = args[i]; @@ -847,7 +876,7 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-tr-function-alias")) { ++i; if (i == argc) { - printErr(u"The -tr-function-alias option should be followed by a list of function=alias mappings.\n"_qs); + printErr(u"The -tr-function-alias option should be followed by a list of function=alias mappings.\n"_s); return 1; } if (!handleTrFunctionAliases(args[i])) @@ -856,7 +885,7 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-pro")) { ++i; if (i == argc) { - printErr(u"The -pro option should be followed by a filename of .pro file.\n"_qs); + printErr(u"The -pro option should be followed by a filename of .pro file.\n"_s); return 1; } QString file = QDir::cleanPath(QFileInfo(args[i]).absoluteFilePath()); @@ -866,16 +895,16 @@ int main(int argc, char **argv) } else if (arg == QLatin1String("-pro-out")) { ++i; if (i == argc) { - printErr(u"The -pro-out option should be followed by a directory name.\n"_qs); + printErr(u"The -pro-out option should be followed by a directory name.\n"_s); return 1; } outDir = QDir::cleanPath(QFileInfo(args[i]).absoluteFilePath()); continue; } else if (arg.startsWith(QLatin1String("-I"))) { - if (arg.length() == 2) { + if (arg.size() == 2) { ++i; if (i == argc) { - printErr(u"The -I option should be followed by a path.\n"_qs); + printErr(u"The -I option should be followed by a path.\n"_s); return 1; } includePath += args[i]; @@ -894,6 +923,14 @@ int main(int argc, char **argv) } continue; } + else if (arg == QLatin1String("-project-roots")) { + while ((i + 1) != argc && !args[i + 1].startsWith(QLatin1String("-"))) { + i++; + rootDirs << args[i]; + } + rootDirs.removeDuplicates(); + continue; + } #endif else if (arg.startsWith(QLatin1String("-")) && arg != QLatin1String("-")) { printErr(QStringLiteral("Unrecognized option '%1'.\n").arg(arg)); @@ -912,8 +949,8 @@ int main(int argc, char **argv) QString lineContent = QString::fromLocal8Bit(lstFile.readLine().trimmed()); if (lineContent.startsWith(QLatin1String("-I"))) { - if (lineContent.length() == 2) { - printErr(u"The -I option should be followed by a path.\n"_qs); + if (lineContent.size() == 2) { + printErr(u"The -I option should be followed by a path.\n"_s); return 1; } includePath += lineContent.mid(2); @@ -925,9 +962,9 @@ int main(int argc, char **argv) files << arg; } if (metTsFlag) { - for (const QString &file : qAsConst(files)) { + for (const QString &file : std::as_const(files)) { bool found = false; - for (const Translator::FileFormat &fmt : qAsConst(Translator::registeredFileFormats())) { + for (const Translator::FileFormat &fmt : std::as_const(Translator::registeredFileFormats())) { if (file.endsWith(QLatin1Char('.') + fmt.extension, Qt::CaseInsensitive)) { QFileInfo fi(file); if (!fi.exists() || fi.isWritable()) { @@ -950,7 +987,7 @@ int main(int argc, char **argv) } else if (metXTsFlag) { alienFiles += files; } else { - for (const QString &file : qAsConst(files)) { + for (const QString &file : std::as_const(files)) { QFileInfo fi(file); if (!fi.exists()) { printErr(QStringLiteral("lupdate error: File '%1' does not exist.\n").arg(file)); @@ -977,8 +1014,8 @@ int main(int argc, char **argv) filters |= QDir::AllDirs | QDir::NoDotAndDotDot; QFileInfoList fileinfolist; recursiveFileInfoList(dir, extensionsNameFilters, filters, &fileinfolist); - int scanRootLen = dir.absolutePath().length(); - for (const QFileInfo &fi : qAsConst(fileinfolist)) { + int scanRootLen = dir.absolutePath().size(); + for (const QFileInfo &fi : std::as_const(fileinfolist)) { QString fn = QDir::cleanPath(fi.absoluteFilePath()); if (fn.endsWith(QLatin1String(".qrc"), Qt::CaseInsensitive)) { resourceFiles << fn; @@ -1019,13 +1056,21 @@ int main(int argc, char **argv) return 1; } - if (!targetLanguage.isEmpty() && tsFileNames.count() != 1) + if (!targetLanguage.isEmpty() && tsFileNames.size() != 1) printErr(u"lupdate warning: -target-language usually only" - " makes sense with exactly one TS file.\n"_qs); + " makes sense with exactly one TS file.\n"_s); + + if (proFiles.isEmpty() && resourceFiles.isEmpty() && sourceFiles.size() == 1 + && QFileInfo(sourceFiles.first()).fileName() == u"CMakeLists.txt"_s) { + printErr(u"lupdate error: Passing a CMakeLists.txt as project file is not supported.\n"_s + u"Please use the 'qt_add_lupdate' CMake command and build the "_s + u"'update_translations' target.\n"_s); + return 1; + } QString errorString; if (!proFiles.isEmpty()) { - runInternalQtTool(u"lupdate-pro"_qs, app.arguments().mid(1)); + runInternalQtTool(u"lupdate-pro"_s, app.arguments().mid(1)); return 0; } @@ -1042,13 +1087,16 @@ int main(int argc, char **argv) .arg(projectDescriptionFile)); return 1; } + removeExcludedSources(projectDescription); + for (Project &project : projectDescription) + expandQrcFiles(project); } bool fail = false; if (projectDescription.empty()) { if (tsFileNames.isEmpty()) printErr(u"lupdate warning:" - " no TS files specified. Only diagnostics will be produced.\n"_qs); + " no TS files specified. Only diagnostics will be produced.\n"_s); Translator fetchedTor; ConversionData cd; @@ -1058,7 +1106,8 @@ int main(int argc, char **argv) cd.m_includePath = includePath; cd.m_allCSources = allCSources; cd.m_compilationDatabaseDir = commandLineCompilationDatabaseDir; - for (const QString &resource : qAsConst(resourceFiles)) + cd.m_rootDirs = rootDirs; + for (const QString &resource : std::as_const(resourceFiles)) sourceFiles << getResources(resource); processSources(fetchedTor, sourceFiles, cd, &fail); updateTsFiles(fetchedTor, tsFileNames, alienFiles, diff --git a/src/linguist/lupdate/merge.cpp b/src/linguist/lupdate/merge.cpp index 57f77601b..f204fc73d 100644 --- a/src/linguist/lupdate/merge.cpp +++ b/src/linguist/lupdate/merge.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "lupdate.h" @@ -39,210 +14,6 @@ QT_BEGIN_NAMESPACE -static bool isDigitFriendly(QChar c) -{ - return c.isPunct() || c.isSpace(); -} - -static int numberLength(const QString &s, int i) -{ - if (i >= s.size() || !s.at(i).isDigit()) - return 0; - - int pos = i; - do { - ++i; - } while (i < s.size() - && (s.at(i).isDigit() - || (isDigitFriendly(s[i]) - && i + 1 < s.size() - && (s[i + 1].isDigit() - || (isDigitFriendly(s[i + 1]) - && i + 2 < s.size() - && s[i + 2].isDigit()))))); - return i - pos; -} - - -/* - Returns a version of 'key' where all numbers have been replaced by zeroes. If - there were none, returns "". -*/ -static QString zeroKey(const QString &key) -{ - QString zeroed; - bool metSomething = false; - - for (int i = 0; i < key.size(); ++i) { - int len = numberLength(key, i); - if (len > 0) { - i += len; - zeroed.append(QLatin1Char('0')); - metSomething = true; - } else { - zeroed.append(key.at(i)); - } - } - return metSomething ? zeroed : QString(); -} - -static QString translationAttempt(const QString &oldTranslation, - const QString &oldSource, const QString &newSource) -{ - int p = zeroKey(oldSource).count(QLatin1Char('0')); - QString attempt; - QStringList oldNumbers; - QStringList newNumbers; - QList<bool> met(p); - QList<int> matchedYet(p); - int i, j; - int k = 0, ell, best; - int m, n; - int pass; - - /* - This algorithm is hard to follow, so we'll consider an example - all along: oldTranslation is "XeT 3.0", oldSource is "TeX 3.0" - and newSource is "XeT 3.1". - - First, we set up two tables: oldNumbers and newNumbers. In our - example, oldNumber[0] is "3.0" and newNumber[0] is "3.1". - */ - for (i = 0, j = 0; i < oldSource.size(); i++, j++) { - m = numberLength(oldSource, i); - n = numberLength(newSource, j); - if (m > 0) { - oldNumbers.append(oldSource.mid(i, m + 1)); - newNumbers.append(newSource.mid(j, n + 1)); - i += m; - j += n; - met[k] = false; - matchedYet[k] = 0; - k++; - } - } - - /* - We now go over the old translation, "XeT 3.0", one letter at a - time, looking for numbers found in oldNumbers. Whenever such a - number is met, it is replaced with its newNumber equivalent. In - our example, the "3.0" of "XeT 3.0" becomes "3.1". - */ - for (i = 0; i < oldTranslation.length(); i++) { - attempt += oldTranslation[i]; - for (k = 0; k < p; k++) { - if (oldTranslation[i] == oldNumbers[k][matchedYet[k]]) - matchedYet[k]++; - else - matchedYet[k] = 0; - } - - /* - Let's find out if the last character ended a match. We make - two passes over the data. In the first pass, we try to - match only numbers that weren't matched yet; if that fails, - the second pass does the trick. This is useful in some - suspicious cases, flagged below. - */ - for (pass = 0; pass < 2; pass++) { - best = p; // an impossible value - for (k = 0; k < p; k++) { - if ((!met[k] || pass > 0) && - matchedYet[k] == oldNumbers[k].length() && - numberLength(oldTranslation, i + 1 - matchedYet[k]) == matchedYet[k]) { - // the longer the better - if (best == p || matchedYet[k] > matchedYet[best]) - best = k; - } - } - if (best != p) { - attempt.truncate(attempt.length() - matchedYet[best]); - attempt += newNumbers[best]; - met[best] = true; - for (k = 0; k < p; k++) - matchedYet[k] = 0; - break; - } - } - } - - /* - We flag two kinds of suspicious cases. They are identified as - such with comments such as "{2000?}" at the end. - - Example of the first kind: old source text "TeX 3.0" translated - as "XeT 2.0" is flagged "TeX 2.0 {3.0?}", no matter what the - new text is. - */ - for (k = 0; k < p; k++) { - if (!met[k]) - attempt += QLatin1String(" {") + newNumbers[k] + QLatin1String("?}"); - } - - /* - Example of the second kind: "1 of 1" translated as "1 af 1", - with new source text "1 of 2", generates "1 af 2 {1 or 2?}" - because it's not clear which of "1 af 2" and "2 af 1" is right. - */ - for (k = 0; k < p; k++) { - for (ell = 0; ell < p; ell++) { - if (k != ell && oldNumbers[k] == oldNumbers[ell] && - newNumbers[k] < newNumbers[ell]) - attempt += QLatin1String(" {") + newNumbers[k] + QLatin1String(" or ") + - newNumbers[ell] + QLatin1String("?}"); - } - } - return attempt; -} - - -/* - Augments a Translator with translations easily derived from - similar existing (probably obsolete) translations. - - For example, if "TeX 3.0" is translated as "XeT 3.0" and "TeX 3.1" - has no translation, "XeT 3.1" is added to the translator and is - marked Unfinished. - - Returns the number of additional messages that this heuristic translated. -*/ -int applyNumberHeuristic(Translator &tor) -{ - QMap<QString, QPair<QString, QString> > translated; - QList<bool> untranslated(tor.messageCount()); - int inserted = 0; - - for (int i = 0; i < tor.messageCount(); ++i) { - const TranslatorMessage &msg = tor.message(i); - bool hasTranslation = msg.isTranslated(); - if (msg.type() == TranslatorMessage::Unfinished) { - if (!hasTranslation) - untranslated[i] = true; - } else if (hasTranslation && msg.translations().count() == 1) { - const QString &key = zeroKey(msg.sourceText()); - if (!key.isEmpty()) - translated.insert(key, qMakePair(msg.sourceText(), msg.translation())); - } - } - - for (int i = 0; i < tor.messageCount(); ++i) { - if (untranslated[i]) { - TranslatorMessage &msg = tor.message(i); - const QString &key = zeroKey(msg.sourceText()); - if (!key.isEmpty()) { - const auto t = translated.constFind(key); - if (t != translated.constEnd() && t->first != msg.sourceText()) { - msg.setTranslation(translationAttempt(t->second, t->first, - msg.sourceText())); - inserted++; - } - } - } - } - return inserted; -} - - /* Augments a Translator with trivially derived translations. @@ -516,13 +287,6 @@ Translator merge( */ int sameTextHeuristicCount = (options & HeuristicSameText) ? applySameTextHeuristic(outTor) : 0; - /* - The number heuristic handles cases where a message has an - obsolete counterpart with mostly numbers differing in the - source text. - */ - int sameNumberHeuristicCount = (options & HeuristicNumber) ? applyNumberHeuristic(outTor) : 0; - if (options & Verbose) { int totalFound = neww + known; err += QStringLiteral(" Found %1 source text(s) (%2 new and %3 already existing)\n") @@ -536,9 +300,6 @@ Translator merge( } } - if (sameNumberHeuristicCount) - err += QStringLiteral(" Number heuristic provided %1 translation(s)\n") - .arg(sameNumberHeuristicCount); if (sameTextHeuristicCount) err += QStringLiteral(" Same-text heuristic provided %1 translation(s)\n") .arg(sameTextHeuristicCount); diff --git a/src/linguist/lupdate/python.cpp b/src/linguist/lupdate/python.cpp index 9ed3457e6..0bc3bf5e8 100644 --- a/src/linguist/lupdate/python.cpp +++ b/src/linguist/lupdate/python.cpp @@ -1,31 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2002-2007 Detlev Offenbach <detlev@die-offenbachs.de> -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2002-2007 Detlev Offenbach <detlev@die-offenbachs.de> +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include <translator.h> #include "lupdate.h" @@ -42,19 +17,27 @@ QT_BEGIN_NAMESPACE -static const char MagicComment[] = "TRANSLATOR "; +static const char PythonMagicComment[] = "TRANSLATOR "; /* The first part of this source file is the Python tokenizer. We skip most of Python; the only tokens that interest us are defined here. */ -enum Token { Tok_Eof, Tok_class, Tok_return, Tok_tr, +enum Token { Tok_Eof, Tok_class, Tok_def, Tok_return, Tok_tr, Tok_trUtf8, Tok_translate, Tok_Ident, Tok_Comment, Tok_Dot, Tok_String, Tok_LeftParen, Tok_RightParen, Tok_Comma, Tok_None, Tok_Integer}; +enum class StringType +{ + NoString, + String, + FormatString, + RawString +}; + /* The tokenizer maintains the following global variables. The names should be self-explanatory. @@ -76,6 +59,7 @@ static QByteArray id; QHash<QByteArray, Token> tokens = { {"None", Tok_None}, {"class", Tok_class}, + {"def", Tok_def}, {"return", Tok_return}, {"__tr", Tok_tr}, // Legacy? {"__trUtf8", Tok_trUtf8} @@ -101,8 +85,6 @@ using ContextPair = QPair<QByteArray, int>; using ContextStack = QStack<ContextPair>; static ContextStack yyContextStack; -static int yyContextPops; - static int getCharFromFile() { int c; @@ -120,17 +102,6 @@ static int getCharFromFile() } else if (yyCountingIndentation && (c == 32 || c == 9)) { yyContinuousSpaceCount++; } else { - if (yyIndentationSize == 1 && yyContinuousSpaceCount > yyIndentationSize) - yyIndentationSize = yyContinuousSpaceCount; - if (yyCountingIndentation && yyContextStack.count() > 1) { - ContextPair& top = yyContextStack.top(); - if (top.second == 0 && yyContinuousSpaceCount > 0) { - top.second = yyContinuousSpaceCount; - yyContinuousSpaceCount = 0; - } else if (yyContinuousSpaceCount < top.second) { - yyContextPops = (top.second - yyContinuousSpaceCount) / yyIndentationSize; - } - } yyCountingIndentation = false; } return c; @@ -156,18 +127,82 @@ static void startTokenizer(const QString &fileName, int (*getCharFunc)(), yyParenDepth = 0; yyCurLineNo = 1; - yyIndentationSize = 1; + yyIndentationSize = -1; yyContinuousSpaceCount = 0; yyCountingIndentation = false; yyContextStack.clear(); - yyContextPops = 0; } -static Token parseString() +static bool parseStringEscape(int quoteChar, StringType stringType) { static const char tab[] = "abfnrtv"; static const char backTab[] = "\a\b\f\n\r\t\v"; + yyCh = getChar(); + if (yyCh == EOF) + return false; + + if (stringType == StringType::RawString) { + if (yyCh != quoteChar) // Only quotes can be escaped in raw strings + yyString[yyStringLen++] = '\\'; + yyString[yyStringLen++] = yyCh; + yyCh = getChar(); + return true; + } + + if (yyCh == 'x') { + QByteArray hex = "0"; + yyCh = getChar(); + if (yyCh == EOF) + return false; + while (std::isxdigit(yyCh)) { + hex += char(yyCh); + yyCh = getChar(); + if (yyCh == EOF) + return false; + } + uint n; +#ifdef Q_CC_MSVC + sscanf_s(hex, "%x", &n); +#else + std::sscanf(hex, "%x", &n); +#endif + if (yyStringLen < sizeof(yyString) - 1) + yyString[yyStringLen++] = char(n); + return true; + } + + if (yyCh >= '0' && yyCh < '8') { + QByteArray oct; + int n = 0; + do { + oct += char(yyCh); + ++n; + yyCh = getChar(); + if (yyCh == EOF) + return false; + } while (yyCh >= '0' && yyCh < '8' && n < 3); +#ifdef Q_CC_MSVC + sscanf_s(oct, "%o", &n); +#else + std::sscanf(oct, "%o", &n); +#endif + if (yyStringLen < sizeof(yyString) - 1) + yyString[yyStringLen++] = char(n); + return true; + } + + const char *p = std::strchr(tab, yyCh); + if (yyStringLen < sizeof(yyString) - 1) { + yyString[yyStringLen++] = p == nullptr + ? char(yyCh) : backTab[p - tab]; + } + yyCh = getChar(); + return true; +} + +static Token parseString(StringType stringType = StringType::NoString) +{ int quoteChar = yyCh; bool tripleQuote = false; bool singleQuote = true; @@ -207,48 +242,8 @@ static Token parseString() } if (yyCh == '\\') { - yyCh = getChar(); - - if (yyCh == 'x') { - QByteArray hex = "0"; - - yyCh = getChar(); - while (std::isxdigit(yyCh)) { - hex += char(yyCh); - yyCh = getChar(); - } - uint n; -#ifdef Q_CC_MSVC - sscanf_s(hex, "%x", &n); -#else - std::sscanf(hex, "%x", &n); -#endif - if (yyStringLen < sizeof(yyString) - 1) - yyString[yyStringLen++] = char(n); - } else if (yyCh >= '0' && yyCh < '8') { - QByteArray oct; - int n = 0; - - do { - oct += char(yyCh); - ++n; - yyCh = getChar(); - } while (yyCh >= '0' && yyCh < '8' && n < 3); -#ifdef Q_CC_MSVC - sscanf_s(oct, "%o", &n); -#else - std::sscanf(oct, "%o", &n); -#endif - if (yyStringLen < sizeof(yyString) - 1) - yyString[yyStringLen++] = char(n); - } else { - const char *p = std::strchr(tab, yyCh); - if (yyStringLen < sizeof(yyString) - 1) { - yyString[yyStringLen++] = (p == nullptr) - ? char(yyCh) : backTab[p - tab]; - } - yyCh = getChar(); - } + if (!parseStringEscape(quoteChar, stringType)) + return Tok_Eof; } else { char *yStart = yyString + yyStringLen; char *yp = yStart; @@ -287,7 +282,7 @@ static QByteArray readLine() return result; } -static Token getToken() +static Token getToken(StringType stringType = StringType::NoString) { yyIdent.clear(); yyCommentLen = 0; @@ -313,6 +308,7 @@ static Token getToken() id = readLine().trimmed(); break; case EOF: + return Tok_Eof; case '\n': break; default: @@ -324,7 +320,7 @@ static Token getToken() break; case '"': case '\'': - return parseString(); + return parseString(stringType); case '(': yyParenDepth++; yyCh = getChar(); @@ -396,15 +392,34 @@ static bool match(Token t) return matches; } +static bool matchStringStart() +{ + if (yyTok == Tok_String) + return true; + // Check for f"bla{var}" and raw strings r"bla". + if (yyTok == Tok_Ident && yyIdent.size() == 1) { + switch (yyIdent.at(0)) { + case 'r': + yyTok = getToken(StringType::RawString); + return yyTok == Tok_String; + case 'f': + yyTok = getToken(StringType::FormatString); + return yyTok == Tok_String; + } + } + return false; +} + static bool matchString(QByteArray *s) { - const bool matches = (yyTok == Tok_String); s->clear(); - while (yyTok == Tok_String) { + bool ok = false; + while (matchStringStart()) { *s += yyString; yyTok = getToken(); + ok = true; } - return matches; + return ok; } static bool matchEncoding(bool *utf8) @@ -515,33 +530,57 @@ static bool parseTranslate(QByteArray *text, QByteArray *context, QByteArray *co if (match(Tok_RightParen)) return true; - // look for comment - if (!match(Tok_Comma) || !matchStringOrNone(comment)) + // not a comma or a right paren, illegal syntax + if (!match(Tok_Comma)) return false; + // python accepts trailing commas within parenthesis, so allow a comma with nothing after + if (match(Tok_RightParen)) + return true; + + // check for comment + if (!matchStringOrNone(comment)) + return false; // not a comment, or a trailing comma... something is wrong + if (match(Tok_RightParen)) return true; - // look for encoding + // not a comma or a right paren, illegal syntax if (!match(Tok_Comma)) return false; - if (matchEncoding(utf8)) { - if (!match(Tok_RightParen)) { - // look for the plural quantifier, - // this can be a number, an identifier or a function call, - // so for simplicity we mark it as plural if we know we have a comma instead of an - // right parentheses. - *plural = match(Tok_Comma); - } + // python accepts trailing commas within parenthesis, so allow a comma with nothing after + if (match(Tok_RightParen)) return true; + + // look for optional encoding information + if (matchEncoding(utf8)) { + if (match(Tok_RightParen)) + return true; + + // not a comma or a right paren, illegal syntax + if (!match(Tok_Comma)) + return false; + + // python accepts trailing commas within parenthesis, so allow a comma with nothing after + if (match(Tok_RightParen)) + return true; } - // This can be a QTranslator::translate("context", "source", "comment", n) plural translation - if (!matchExpression() || !match(Tok_RightParen)) + // Must be a plural expression + if (!matchExpression()) return false; + *plural = true; - return true; + + // Ignore any trailing comma here + match(Tok_Comma); + + // This must be the end, or there are too many parameters + if (match(Tok_RightParen)) + return true; + + return false; } static inline void setMessageParameters(TranslatorMessage *message) @@ -566,22 +605,33 @@ static void parse(Translator &tor, ConversionData &cd, QByteArray prefix; bool utf8 = false; - yyContextStack.push({initialContext, 0}); - yyTok = getToken(); while (yyTok != Tok_Eof) { - if (yyContextPops > 0) { - for ( int i = 0; i < yyContextPops; i++) - yyContextStack.pop(); - yyContextPops = 0; - } - switch (yyTok) { - case Tok_class: + case Tok_class: { + if (yyIndentationSize < 0 && yyContinuousSpaceCount > 0) + yyIndentationSize = yyContinuousSpaceCount; // First indented "class" + const int indent = yyIndentationSize > 0 + ? yyContinuousSpaceCount / yyIndentationSize : 0; + while (!yyContextStack.isEmpty() && yyContextStack.top().second >= indent) + yyContextStack.pop(); + yyTok = getToken(); + yyContextStack.push({yyIdent, indent}); yyTok = getToken(); - yyContextStack.push({yyIdent, 0}); - yyContinuousSpaceCount = 0; + } + break; + case Tok_def: + if (yyIndentationSize < 0 && yyContinuousSpaceCount > 0) + yyIndentationSize = yyContinuousSpaceCount; // First indented "def" + if (!yyContextStack.isEmpty()) { + // Pop classes if the function is further outdented than the class on the top + // (end of a nested class). + const int classIndent = yyIndentationSize > 0 + ? yyContinuousSpaceCount / yyIndentationSize - 1 : 0; + while (!yyContextStack.isEmpty() && yyContextStack.top().second > classIndent) + yyContextStack.pop(); + } yyTok = getToken(); break; case Tok_tr: @@ -607,7 +657,8 @@ static void parse(Translator &tor, ConversionData &cd, if (prefix.isEmpty()) context = defaultContext; else if (prefix == "self") - context = yyContextStack.top().first; + context = yyContextStack.isEmpty() + ? initialContext : yyContextStack.top().first; else context = prefix; @@ -649,8 +700,8 @@ static void parse(Translator &tor, ConversionData &cd, case Tok_Comment: comment = yyComment; comment = comment.simplified(); - if (comment.left(sizeof(MagicComment) - 1) == MagicComment) { - comment.remove(0, sizeof(MagicComment) - 1); + if (comment.left(sizeof(PythonMagicComment) - 1) == PythonMagicComment) { + comment.remove(0, sizeof(PythonMagicComment) - 1); int k = comment.indexOf(' '); if (k == -1) { context = comment; diff --git a/src/linguist/lupdate/qdeclarative.cpp b/src/linguist/lupdate/qdeclarative.cpp index 2f8d9111a..e62cfec15 100644 --- a/src/linguist/lupdate/qdeclarative.cpp +++ b/src/linguist/lupdate/qdeclarative.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "lupdate.h" @@ -33,6 +8,7 @@ #include <QtCore/QDebug> #include <QtCore/QFile> #include <QtCore/QString> +#include <QtCore/QTextStream> #include <private/qqmljsengine_p.h> #include <private/qqmljsparser_p.h> @@ -54,7 +30,7 @@ QT_BEGIN_NAMESPACE using namespace QQmlJS; -static QString MagicComment(QLatin1String("TRANSLATOR")); +using namespace Qt::StringLiterals; class FindTrCalls: protected AST::Visitor { @@ -82,7 +58,17 @@ protected: using AST::Visitor::endVisit; void accept(AST::Node *node) - { AST::Node::acceptChild(node, this); } + { AST::Node::accept(node, this); } + + bool visit(AST::UiPragma *node) override + { + if (!node->name.isNull()) { + if (node->name == "Translator"_L1) { + m_component = node->values->value.toString(); + } + } + return false; + } void endVisit(AST::CallExpression *node) override { @@ -110,12 +96,6 @@ protected: .arg(name)); return; } - if (AST::cast<AST::TemplateLiteral *>(node->arguments->expression)) { - yyMsg(identLineNo) - << qPrintable(QStringLiteral("%1() cannot be used with template literals. " - "Ignoring\n").arg(name)); - return; - } QString source; if (!createString(node->arguments->expression, &source)) @@ -246,6 +226,9 @@ private: if (createString(binop->right, out)) return true; } + } else if (AST::TemplateLiteral *templit = AST::cast<AST::TemplateLiteral *>(ast)) { + out->append(templit->value); + return true; } return false; @@ -287,7 +270,7 @@ QString createErrorString(const QString &filename, const QString &code, Parser & const QString textLine = lines.at(line > 0 ? line - 1 : 0); error += textLine + QLatin1Char('\n'); - for (int i = 0, end = qMin(column > 0 ? column - 1 : 0, textLine.length()); i < end; ++i) { + for (int i = 0, end = qMin(column > 0 ? column - 1 : 0, textLine.size()); i < end; ++i) { const QChar ch = textLine.at(i); if (ch.isSpace()) error += ch; @@ -339,7 +322,7 @@ void FindTrCalls::processComment(const SourceLocation &loc) const QStringView commentStr = engine->midRef(loc.begin(), loc.length); const QChar *chars = commentStr.constData(); - const int length = commentStr.length(); + const int length = commentStr.size(); // Try to match the logic of the C++ parser. if (*chars == QLatin1Char(':') && chars[1].isSpace()) { @@ -351,11 +334,17 @@ void FindTrCalls::processComment(const SourceLocation &loc) } else if (*chars == QLatin1Char('~') && chars[1].isSpace()) { QString text = QString(chars+2, length-2).trimmed(); int k = text.indexOf(QLatin1Char(' ')); - if (k > -1) - extra.insert(text.left(k), text.mid(k + 1).trimmed()); + if (k > -1) { + QString commentvalue = text.mid(k + 1).trimmed(); + if (commentvalue.startsWith(QLatin1Char('"')) && commentvalue.endsWith(QLatin1Char('"')) + && commentvalue.size() != 1) { + commentvalue = commentvalue.sliced(1, commentvalue.size() - 2); + } + extra.insert(text.left(k), commentvalue); + } } else if (*chars == QLatin1Char('%') && chars[1].isSpace()) { - sourcetext.reserve(sourcetext.length() + length-2); - ushort *ptr = (ushort *)sourcetext.data() + sourcetext.length(); + sourcetext.reserve(sourcetext.size() + length-2); + ushort *ptr = (ushort *)sourcetext.data() + sourcetext.size(); int p = 2, c; forever { if (p >= length) @@ -393,29 +382,6 @@ void FindTrCalls::processComment(const SourceLocation &loc) ushort c; while ((c = chars[idx].unicode()) == ' ' || c == '\t' || c == '\r' || c == '\n') ++idx; - if (!memcmp(chars + idx, MagicComment.unicode(), MagicComment.length() * 2)) { - idx += MagicComment.length(); - QString comment = QString(chars + idx, length - idx).simplified(); - int k = comment.indexOf(QLatin1Char(' ')); - if (k == -1) { - trcontext = comment; - } else { - trcontext = comment.left(k); - comment.remove(0, k + 1); - TranslatorMessage msg( - trcontext, QString(), - comment, QString(), - m_fileName, loc.startLine, QStringList(), - TranslatorMessage::Finished, /*plural=*/false); - msg.setExtraComment(extracomment.simplified()); - extracomment.clear(); - m_translator->append(msg); - m_translator->setExtras(extra); - extra.clear(); - } - - m_component = trcontext; - } } } diff --git a/src/linguist/lupdate/synchronized.h b/src/linguist/lupdate/synchronized.h index 46cbca57c..65bef5463 100644 --- a/src/linguist/lupdate/synchronized.h +++ b/src/linguist/lupdate/synchronized.h @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #ifndef SYNCHRONIZED_H #define SYNCHRONIZED_H diff --git a/src/linguist/lupdate/ui.cpp b/src/linguist/lupdate/ui.cpp index 1737b5342..1092035ae 100644 --- a/src/linguist/lupdate/ui.cpp +++ b/src/linguist/lupdate/ui.cpp @@ -1,30 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Linguist of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include "lupdate.h" @@ -39,6 +14,8 @@ QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; + class UiReader : public XmlParser { public: @@ -192,7 +169,7 @@ bool loadUI(Translator &translator, const QString &filename, ConversionData &cd) UiReader uiReader(translator, cd, reader); bool result = uiReader.parse(); if (!result) - cd.appendError(u"Parse error in UI file"_qs); + cd.appendError(u"Parse error in UI file"_s); return result; } |