From 4c3cdee2a5047d05201424bf9f80b6664619550d Mon Sep 17 00:00:00 2001 From: "Duncan P. N. Exon Smith" Date: Fri, 17 Mar 2017 22:55:13 +0000 Subject: Modules: Cache PCMs in memory and avoid a use-after-free Clang's internal build system for implicit modules uses lock files to ensure that after a process writes a PCM it will read the same one back in (without contention from other -cc1 commands). Since PCMs are read from disk repeatedly while invalidating, building, and importing, the lock is not released quickly. Furthermore, the LockFileManager is not robust in every environment. Other -cc1 commands can stall until timeout (after about eight minutes). This commit changes the lock file from being necessary for correctness to a (possibly dubious) performance hack. The remaining benefit is to reduce duplicate work in competing -cc1 commands which depend on the same module. Follow-up commits will change the internal build system to continue after a timeout, and reduce the timeout. Perhaps we should reconsider blocking at all. This also fixes a use-after-free, when one part of a compilation validates a PCM and starts using it, and another tries to swap out the PCM for something new. The PCMCache is a new type called MemoryBufferCache, which saves memory buffers based on their filename. Its ownership is shared by the CompilerInstance and ModuleManager. - The ModuleManager stores PCMs there that it loads from disk, never touching the disk if the cache is hot. - When modules fail to validate, they're removed from the cache. - When a CompilerInstance is spawned to build a new module, each already-loaded PCM is assumed to be valid, and is frozen to avoid the use-after-free. - Any newly-built module is written directly to the cache to avoid the round-trip to the filesystem, making lock files unnecessary for correctness. Original patch by Manman Ren; most testcases by Adrian Prantl! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@298165 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 91319bedd6..babef5dcc7 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -70,15 +70,15 @@ ExternalPreprocessorSource::~ExternalPreprocessorSource() { } Preprocessor::Preprocessor(std::shared_ptr PPOpts, DiagnosticsEngine &diags, LangOptions &opts, - SourceManager &SM, HeaderSearch &Headers, - ModuleLoader &TheModuleLoader, + SourceManager &SM, MemoryBufferCache &PCMCache, + HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup, bool OwnsHeaders, TranslationUnitKind TUKind) : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr), AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM), - ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), - TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), - Identifiers(opts, IILookup), + PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)), + HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), + ExternalSource(nullptr), Identifiers(opts, IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr), CodeCompletionFile(nullptr), CodeCompletionOffset(0), -- cgit v1.2.3 From cf26db83d56c4dcdec646533cbc1d871b92ead23 Mon Sep 17 00:00:00 2001 From: Renato Golin Date: Sat, 18 Mar 2017 12:31:32 +0000 Subject: Revert "Modules: Cache PCMs in memory and avoid a use-after-free" This reverts commit r298165, as it broke the ARM builds. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@298185 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index babef5dcc7..91319bedd6 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -70,15 +70,15 @@ ExternalPreprocessorSource::~ExternalPreprocessorSource() { } Preprocessor::Preprocessor(std::shared_ptr PPOpts, DiagnosticsEngine &diags, LangOptions &opts, - SourceManager &SM, MemoryBufferCache &PCMCache, - HeaderSearch &Headers, ModuleLoader &TheModuleLoader, + SourceManager &SM, HeaderSearch &Headers, + ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup, bool OwnsHeaders, TranslationUnitKind TUKind) : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr), AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM), - PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)), - HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), - ExternalSource(nullptr), Identifiers(opts, IILookup), + ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), + TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), + Identifiers(opts, IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr), CodeCompletionFile(nullptr), CodeCompletionOffset(0), -- cgit v1.2.3 From 2bf5f686e2ea17e971fb6b5a1739282d70dc3536 Mon Sep 17 00:00:00 2001 From: "Duncan P. N. Exon Smith" Date: Mon, 20 Mar 2017 17:58:26 +0000 Subject: Reapply "Modules: Cache PCMs in memory and avoid a use-after-free" This reverts commit r298185, effectively reapplying r298165, after fixing the new unit tests (PR32338). The memory buffer generator doesn't null-terminate the MemoryBuffer it creates; this version of the commit informs getMemBuffer about that to avoid the assert. Original commit message follows: ---- Clang's internal build system for implicit modules uses lock files to ensure that after a process writes a PCM it will read the same one back in (without contention from other -cc1 commands). Since PCMs are read from disk repeatedly while invalidating, building, and importing, the lock is not released quickly. Furthermore, the LockFileManager is not robust in every environment. Other -cc1 commands can stall until timeout (after about eight minutes). This commit changes the lock file from being necessary for correctness to a (possibly dubious) performance hack. The remaining benefit is to reduce duplicate work in competing -cc1 commands which depend on the same module. Follow-up commits will change the internal build system to continue after a timeout, and reduce the timeout. Perhaps we should reconsider blocking at all. This also fixes a use-after-free, when one part of a compilation validates a PCM and starts using it, and another tries to swap out the PCM for something new. The PCMCache is a new type called MemoryBufferCache, which saves memory buffers based on their filename. Its ownership is shared by the CompilerInstance and ModuleManager. - The ModuleManager stores PCMs there that it loads from disk, never touching the disk if the cache is hot. - When modules fail to validate, they're removed from the cache. - When a CompilerInstance is spawned to build a new module, each already-loaded PCM is assumed to be valid, and is frozen to avoid the use-after-free. - Any newly-built module is written directly to the cache to avoid the round-trip to the filesystem, making lock files unnecessary for correctness. Original patch by Manman Ren; most testcases by Adrian Prantl! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@298278 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 91319bedd6..babef5dcc7 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -70,15 +70,15 @@ ExternalPreprocessorSource::~ExternalPreprocessorSource() { } Preprocessor::Preprocessor(std::shared_ptr PPOpts, DiagnosticsEngine &diags, LangOptions &opts, - SourceManager &SM, HeaderSearch &Headers, - ModuleLoader &TheModuleLoader, + SourceManager &SM, MemoryBufferCache &PCMCache, + HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup, bool OwnsHeaders, TranslationUnitKind TUKind) : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr), AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM), - ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), - TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), - Identifiers(opts, IILookup), + PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)), + HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), + ExternalSource(nullptr), Identifiers(opts, IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr), CodeCompletionFile(nullptr), CodeCompletionOffset(0), -- cgit v1.2.3 From d49e8364d3107404aeea2d4ffae3bc8cd8830366 Mon Sep 17 00:00:00 2001 From: Bruno Cardoso Lopes Date: Tue, 21 Mar 2017 16:43:51 +0000 Subject: [Modules] Find PrivateHeaders when looking into subframeworks Fix the current parsing of subframeworks in modulemaps to lookup for headers based on whether they are frameworks. rdar://problem/30563982 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@298391 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 384e5707dc..61ad5948cd 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -1841,7 +1841,7 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, Module::UnresolvedHeaderDirective Header; Header.FileName = Tok.getString(); Header.FileNameLoc = consumeToken(); - + // Check whether we already have an umbrella. if (LeadingToken == MMToken::UmbrellaKeyword && ActiveModule->Umbrella) { Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash) @@ -1861,19 +1861,25 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, // Search for the header file within the search directory. SmallString<128> FullPathName(Directory->getName()); unsigned FullPathLength = FullPathName.size(); - + if (ActiveModule->isPartOfFramework()) { appendSubframeworkPaths(ActiveModule, RelativePathName); - + unsigned RelativePathLength = RelativePathName.size(); + // Check whether this file is in the public headers. llvm::sys::path::append(RelativePathName, "Headers", Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); File = SourceMgr.getFileManager().getFile(FullPathName); - + + // Check whether this file is in the private headers. if (!File) { - // Check whether this file is in the private headers. - // FIXME: Should we retain the subframework paths here? - RelativePathName.clear(); + // Ideally, private modules in the form 'FrameworkName.Private' should + // be defined as 'module FrameworkName.Private', and not as + // 'framework module FrameworkName.Private', since a 'Private.Framework' + // does not usually exist. However, since both are currently widely used + // for private modules, make sure we find the right path in both cases. + RelativePathName.resize(ActiveModule->IsFramework ? 0 + : RelativePathLength); FullPathName.resize(FullPathLength); llvm::sys::path::append(RelativePathName, "PrivateHeaders", Header.FileName); -- cgit v1.2.3 From 26d41bffe2da8282ce4116b7251568380c8b2ffb Mon Sep 17 00:00:00 2001 From: Sanne Wouda Date: Fri, 7 Apr 2017 10:13:00 +0000 Subject: Skip Unicode character expansion in assembly files Summary: When using the C preprocessor with assembly files, either with a capital `S` file extension, or with `-xassembler-with-cpp`, the Unicode escape sequence `\u` is ignored. The `\u` pattern can be used for expanding a macro argument that starts with `u`. Author: Salman Arif Reviewers: rengolin, olista01 Reviewed By: olista01 Subscribers: cfe-commits Differential Revision: https://reviews.llvm.org/D31765 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@299754 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 6025a66751..4c05193947 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -3603,17 +3603,19 @@ LexNextToken: // UCNs (C99 6.4.3, C++11 [lex.charset]p2) case '\\': - if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { - if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { - if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) - return true; // KeepWhitespaceMode + if (!LangOpts.AsmPreprocessor) { + if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { + if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { + if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) + return true; // KeepWhitespaceMode + + // We only saw whitespace, so just try again with this lexer. + // (We manually eliminate the tail call to avoid recursion.) + goto LexNextToken; + } - // We only saw whitespace, so just try again with this lexer. - // (We manually eliminate the tail call to avoid recursion.) - goto LexNextToken; + return LexUnicode(Result, CodePoint, CurPtr); } - - return LexUnicode(Result, CodePoint, CurPtr); } Kind = tok::unknown; -- cgit v1.2.3 From b3196fddb0623c60de3cfee3f85c2e518c7f8e4d Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Wed, 12 Apr 2017 11:03:25 +0000 Subject: Add support for __builtin_available to __has_builtin rdar://31576715 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300049 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPMacroExpansion.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/Lex') diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index de166c75e2..358c96a783 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1746,6 +1746,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { return llvm::StringSwitch(II->getName()) .Case("__make_integer_seq", LangOpts.CPlusPlus) .Case("__type_pack_element", LangOpts.CPlusPlus) + .Case("__builtin_available", true) .Default(false); } }); -- cgit v1.2.3 From 88cdb963dec3e6887528b4475f51381950891d4a Mon Sep 17 00:00:00 2001 From: David Blaikie Date: Wed, 12 Apr 2017 20:58:33 +0000 Subject: Modular Codegen: Separate flags for function and debug info support This allows using and testing these two features separately. (noteably, debug info is, so far as I know, always a win (basically). But function modular codegen is currently a loss for highly optimized code - where most of the linkonce_odr definitions are optimized away, so providing weak_odr definitions is only overhead) git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300104 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 61ad5948cd..4f3db8dd64 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -91,7 +91,6 @@ ModuleMap::ModuleMap(SourceManager &SourceMgr, DiagnosticsEngine &Diags, HeaderInfo(HeaderInfo), BuiltinIncludeDir(nullptr), SourceModule(nullptr), NumCreatedModules(0) { MMapLangOpts.LineComment = true; - MMapLangOpts.ModularCodegen = LangOpts.ModularCodegen; } ModuleMap::~ModuleMap() { -- cgit v1.2.3 From 380d805e6e18df6a276cd0f1dc67b573477d0cdd Mon Sep 17 00:00:00 2001 From: Yaron Keren Date: Sun, 16 Apr 2017 15:53:19 +0000 Subject: Use setUsedForHeaderGuard() accessor function instead of direcly accessing UsedForHeaderGuard. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300423 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 4db17c344b..cf0c953b61 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -303,9 +303,8 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { if (const FileEntry *FE = CurPPLexer->getFileEntry()) { HeaderInfo.SetFileControllingMacro(FE, ControllingMacro); if (MacroInfo *MI = - getMacroInfo(const_cast(ControllingMacro))) { - MI->UsedForHeaderGuard = true; - } + getMacroInfo(const_cast(ControllingMacro))) + MI->setUsedForHeaderGuard(true); if (const IdentifierInfo *DefinedMacro = CurPPLexer->MIOpt.GetDefinedMacro()) { if (!isMacroDefined(ControllingMacro) && -- cgit v1.2.3 From 9ea711133f6895cba2c401353964cc8718f12a73 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 17 Apr 2017 23:44:51 +0000 Subject: Fix mishandling of escaped newlines followed by newlines or nuls. Previously, if an escaped newline was followed by a newline or a nul, we'd lex the escaped newline as a bogus space character. This led to a bunch of different broken corner cases: For the pattern "\\\n\0#", we would then have a (horizontal) space whose spelling ends in a newline, and would decide that the '#' is at the start of a line, and incorrectly start preprocessing a directive in the middle of a logical source line. If we were already in the middle of a directive, this would result in our attempting to process multiple directives at the same time! This resulted in crashes, asserts, and hangs on invalid input, as discovered by fuzz-testing. For the pattern "\\\n" at EOF (with an implicit following nul byte), we would produce a bogus trailing space character with spelling "\\\n". This was mostly harmless, but would lead to clang-format getting confused and misformatting in rare cases. We now produce a trailing EOF token with spelling "\\\n", consistent with our handling for other similar cases -- an escaped newline is always part of the token containing the next character, if any. For the pattern "\\\n\n", this was somewhat more benign, but would produce an extraneous whitespace token to clients who care about preserving whitespace. However, it turns out that our lexing for line comments was relying on this bug due to an off-by-one error in its computation of the end of the comment, on the slow path where the comment might contain escaped newlines. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300515 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 4c05193947..5d0fe42def 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -1282,12 +1282,6 @@ Slash: Size += EscapedNewLineSize; Ptr += EscapedNewLineSize; - // If the char that we finally got was a \n, then we must have had - // something like \. We don't want to consume the - // second newline. - if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') - return ' '; - // Use slow version to accumulate a correct size field. return getCharAndSizeSlow(Ptr, Size, Tok); } @@ -1338,12 +1332,6 @@ Slash: Size += EscapedNewLineSize; Ptr += EscapedNewLineSize; - // If the char that we finally got was a \n, then we must have had - // something like \. We don't want to consume the - // second newline. - if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') - return ' '; - // Use slow version to accumulate a correct size field. return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); } @@ -2070,8 +2058,11 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, // Scan over the body of the comment. The common case, when scanning, is that // the comment contains normal ascii characters with nothing interesting in // them. As such, optimize for this case with the inner loop. + // + // This loop terminates with CurPtr pointing at the newline (or end of buffer) + // character that ends the line comment. char C; - do { + while (true) { C = *CurPtr; // Skip over characters in the fast loop. while (C != 0 && // Potentially EOF. @@ -2097,6 +2088,8 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, break; // This is a newline, we're done. // If there was space between the backslash and newline, warn about it. + // FIXME: This warning is bogus if trigraphs are disabled and the line + // ended with '?' '?' '\\' '\n'. if (HasSpace && !isLexingRawMode()) Diag(EscapePtr, diag::backslash_newline_space); } @@ -2140,9 +2133,9 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, } } - if (CurPtr == BufferEnd+1) { - --CurPtr; - break; + if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) { + --CurPtr; + break; } if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { @@ -2150,8 +2143,7 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, cutOffLexing(); return false; } - - } while (C != '\n' && C != '\r'); + } // Found but did not consume the newline. Notify comment handlers about the // comment unless we're in a #if 0 block. -- cgit v1.2.3 From 69c89b048e1a96066ddc9b4def3b1f27f988f44a Mon Sep 17 00:00:00 2001 From: Vassil Vassilev Date: Tue, 18 Apr 2017 20:57:29 +0000 Subject: PR30508: Downgrade error to warning if the umbrella folder doesn't exist. Patch by Yuka Takahashi (D32119)! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300594 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 4f3db8dd64..5b60ed3f81 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -2002,9 +2002,8 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) { } if (!Dir) { - Diags.Report(DirNameLoc, diag::err_mmap_umbrella_dir_not_found) + Diags.Report(DirNameLoc, diag::warn_mmap_umbrella_dir_not_found) << DirName; - HadError = true; return; } -- cgit v1.2.3 From 961ed8002bce57c3ab6d138853983c629bf058e4 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Tue, 18 Apr 2017 21:45:04 +0000 Subject: Do not warn about whitespace between ??/ trigraph and newline in line comments if trigraphs are disabled in the current language. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300609 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 5d0fe42def..dc911ef91b 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -1171,6 +1171,8 @@ const char *Lexer::SkipEscapedNewLines(const char *P) { // If not a trigraph for escape, bail out. if (P[1] != '?' || P[2] != '/') return P; + // FIXME: Take LangOpts into account; the language might not + // support trigraphs. AfterEscape = P+3; } else { return P; @@ -2079,17 +2081,17 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, HasSpace = true; } - if (*EscapePtr == '\\') // Escaped newline. + if (*EscapePtr == '\\') + // Escaped newline. CurPtr = EscapePtr; else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && - EscapePtr[-2] == '?') // Trigraph-escaped newline. + EscapePtr[-2] == '?' && LangOpts.Trigraphs) + // Trigraph-escaped newline. CurPtr = EscapePtr-2; else break; // This is a newline, we're done. // If there was space between the backslash and newline, warn about it. - // FIXME: This warning is bogus if trigraphs are disabled and the line - // ended with '?' '?' '\\' '\n'. if (HasSpace && !isLexingRawMode()) Diag(EscapePtr, diag::backslash_newline_space); } -- cgit v1.2.3 From aac41bcdb19f21fb20a2efdc19494b622ba29171 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Wed, 19 Apr 2017 08:58:56 +0000 Subject: Add support for editor placeholders to Clang This commit teaches Clang to recognize editor placeholders that are produced when an IDE like Xcode inserts a code-completion result that includes a placeholder. Now when the lexer sees a placeholder token, it emits an 'editor placeholder in source file' error and creates an identifier token that represents the placeholder. The parser/sema can now recognize the placeholders and can suppress the diagnostics related to the placeholders. This ensures that live issues in an IDE like Xcode won't get spurious diagnostics related to placeholders. This commit also adds a new compiler option named '-fallow-editor-placeholders' that silences the 'editor placeholder in source file' error. This is useful for an IDE like Xcode as we don't want to display those errors in live issues. rdar://31581400 Differential Revision: https://reviews.llvm.org/D32081 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@300667 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index dc911ef91b..003c9b5eed 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -2716,6 +2716,37 @@ bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { return false; } +static const char *findPlaceholderEnd(const char *CurPtr, + const char *BufferEnd) { + if (CurPtr == BufferEnd) + return nullptr; + BufferEnd -= 1; // Scan until the second last character. + for (; CurPtr != BufferEnd; ++CurPtr) { + if (CurPtr[0] == '#' && CurPtr[1] == '>') + return CurPtr + 2; + } + return nullptr; +} + +bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) { + assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!"); + if (!PP || LexingRawMode) + return false; + const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd); + if (!End) + return false; + const char *Start = CurPtr - 1; + if (!LangOpts.AllowEditorPlaceholders) + Diag(Start, diag::err_placeholder_in_source); + Result.startToken(); + FormTokenWithChars(Result, End, tok::raw_identifier); + Result.setRawIdentifierData(Start); + PP->LookUpIdentifierInfo(Result); + Result.setFlag(Token::IsEditorPlaceholder); + BufferPtr = End; + return true; +} + bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { if (PP && PP->isCodeCompletionEnabled()) { SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); @@ -3473,6 +3504,8 @@ LexNextToken: } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); Kind = tok::l_brace; + } else if (Char == '#' && lexEditorPlaceholder(Result, CurPtr)) { + return true; } else { Kind = tok::less; } -- cgit v1.2.3 From 2987ebbf305f2209d4a40e3f0b2c9fe2e014df3a Mon Sep 17 00:00:00 2001 From: Matthias Braun Date: Mon, 24 Apr 2017 18:41:00 +0000 Subject: Pragma: Fix DebugOverflowStack() resulting in endless loop. Drive-by fix (noticed while working on https://reviews.llvm.org/D32205): DebugOverflowStack() is supposed to provoke a stack overflow, however LLVM was smart enough to use the red-zone and fold the load into a tail jump on x86_64 optimizing this to an endless loop instead of a stack overflow. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301218 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Pragma.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index f81eaa31e9..87e105d1d0 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -989,9 +989,9 @@ struct PragmaDebugHandler : public PragmaHandler { #ifdef _MSC_VER #pragma warning(disable : 4717) #endif - static void DebugOverflowStack() { - void (*volatile Self)() = DebugOverflowStack; - Self(); + static void DebugOverflowStack(void (*P)() = nullptr) { + void (*volatile Self)(void(*P)()) = DebugOverflowStack; + Self(reinterpret_cast(Self)); } #ifdef _MSC_VER #pragma warning(default : 4717) -- cgit v1.2.3 From c2ca61f3d077e1eb1acf5d996991cd6522403a49 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 24 Apr 2017 23:12:30 +0000 Subject: [modules ts] Diagnose 'export' declarations outside of a module interface. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301271 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 5b60ed3f81..512d7dc5de 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -581,6 +581,7 @@ Module *ModuleMap::createModuleForInterfaceUnit(SourceLocation Loc, auto *Result = new Module(Name, Loc, nullptr, /*IsFramework*/ false, /*IsExplicit*/ false, NumCreatedModules++); + Result->Kind = Module::ModuleInterfaceUnit; Modules[Name] = SourceModule = Result; // Mark the main source file as being within the newly-created module so that -- cgit v1.2.3 From 9bc521065374e53923c6992332b77bed653ad22f Mon Sep 17 00:00:00 2001 From: Frederich Munch Date: Wed, 26 Apr 2017 19:47:31 +0000 Subject: PPCallbacks::MacroUndefined, change signature and add test. Summary: The PPCallbacks::MacroUndefined callback is currently insufficient for clients that need to track the MacroDirectives. This patch adds an additional argument to PPCallbacks::MacroUndefined that is the undef MacroDirective. Reviewers: bruno, manmanren Reviewed By: bruno Subscribers: nemanjai, cfe-commits Differential Revision: https://reviews.llvm.org/D29923 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301449 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 29 +++++++++++++++-------------- lib/Lex/PreprocessingRecord.cpp | 3 ++- 2 files changed, 17 insertions(+), 15 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 8a56ddf236..fd4e6d30de 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2592,25 +2592,26 @@ void Preprocessor::HandleUndefDirective() { // Okay, we have a valid identifier to undef. auto *II = MacroNameTok.getIdentifierInfo(); auto MD = getMacroDefinition(II); + UndefMacroDirective *Undef = nullptr; + + // If the macro is not defined, this is a noop undef. + if (const MacroInfo *MI = MD.getMacroInfo()) { + if (!MI->isUsed() && MI->isWarnIfUnused()) + Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used); + + if (MI->isWarnIfUnused()) + WarnUnusedMacroLocs.erase(MI->getDefinitionLoc()); + + Undef = AllocateUndefMacroDirective(MacroNameTok.getLocation()); + } // If the callbacks want to know, tell them about the macro #undef. // Note: no matter if the macro was defined or not. if (Callbacks) - Callbacks->MacroUndefined(MacroNameTok, MD); - - // If the macro is not defined, this is a noop undef, just return. - const MacroInfo *MI = MD.getMacroInfo(); - if (!MI) - return; - - if (!MI->isUsed() && MI->isWarnIfUnused()) - Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used); - - if (MI->isWarnIfUnused()) - WarnUnusedMacroLocs.erase(MI->getDefinitionLoc()); + Callbacks->MacroUndefined(MacroNameTok, MD, Undef); - appendMacroDirective(MacroNameTok.getIdentifierInfo(), - AllocateUndefMacroDirective(MacroNameTok.getLocation())); + if (Undef) + appendMacroDirective(II, Undef); } //===----------------------------------------------------------------------===// diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp index 13e15f3c94..03c4cbe589 100644 --- a/lib/Lex/PreprocessingRecord.cpp +++ b/lib/Lex/PreprocessingRecord.cpp @@ -422,7 +422,8 @@ void PreprocessingRecord::MacroDefined(const Token &Id, } void PreprocessingRecord::MacroUndefined(const Token &Id, - const MacroDefinition &MD) { + const MacroDefinition &MD, + const MacroDirective *Undef) { MD.forAllDefinitions([&](MacroInfo *MI) { MacroDefinitions.erase(MI); }); } -- cgit v1.2.3 From 313bd3bb2cfec5729b5da4b3a27e0f96c6ab0d06 Mon Sep 17 00:00:00 2001 From: Vedant Kumar Date: Wed, 26 Apr 2017 20:58:19 +0000 Subject: Revert "PPCallbacks::MacroUndefined, change signature and add test." This reverts commit r301449. It breaks the build with: MacroPPCallbacks.h:114:50: error: non-virtual member function marked 'override' hides virtual member function git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301469 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 29 ++++++++++++++--------------- lib/Lex/PreprocessingRecord.cpp | 3 +-- 2 files changed, 15 insertions(+), 17 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index fd4e6d30de..8a56ddf236 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2592,26 +2592,25 @@ void Preprocessor::HandleUndefDirective() { // Okay, we have a valid identifier to undef. auto *II = MacroNameTok.getIdentifierInfo(); auto MD = getMacroDefinition(II); - UndefMacroDirective *Undef = nullptr; - - // If the macro is not defined, this is a noop undef. - if (const MacroInfo *MI = MD.getMacroInfo()) { - if (!MI->isUsed() && MI->isWarnIfUnused()) - Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used); - - if (MI->isWarnIfUnused()) - WarnUnusedMacroLocs.erase(MI->getDefinitionLoc()); - - Undef = AllocateUndefMacroDirective(MacroNameTok.getLocation()); - } // If the callbacks want to know, tell them about the macro #undef. // Note: no matter if the macro was defined or not. if (Callbacks) - Callbacks->MacroUndefined(MacroNameTok, MD, Undef); + Callbacks->MacroUndefined(MacroNameTok, MD); + + // If the macro is not defined, this is a noop undef, just return. + const MacroInfo *MI = MD.getMacroInfo(); + if (!MI) + return; + + if (!MI->isUsed() && MI->isWarnIfUnused()) + Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used); + + if (MI->isWarnIfUnused()) + WarnUnusedMacroLocs.erase(MI->getDefinitionLoc()); - if (Undef) - appendMacroDirective(II, Undef); + appendMacroDirective(MacroNameTok.getIdentifierInfo(), + AllocateUndefMacroDirective(MacroNameTok.getLocation())); } //===----------------------------------------------------------------------===// diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp index 03c4cbe589..13e15f3c94 100644 --- a/lib/Lex/PreprocessingRecord.cpp +++ b/lib/Lex/PreprocessingRecord.cpp @@ -422,8 +422,7 @@ void PreprocessingRecord::MacroDefined(const Token &Id, } void PreprocessingRecord::MacroUndefined(const Token &Id, - const MacroDefinition &MD, - const MacroDirective *Undef) { + const MacroDefinition &MD) { MD.forAllDefinitions([&](MacroInfo *MI) { MacroDefinitions.erase(MI); }); } -- cgit v1.2.3 From e05b249e25bdf5805fa53e9a4f20c06a02823b51 Mon Sep 17 00:00:00 2001 From: Vedant Kumar Date: Wed, 26 Apr 2017 21:05:44 +0000 Subject: Revert "Revert "PPCallbacks::MacroUndefined, change signature and add test."" This reverts commit r301469. It isn't needed with r301470, which fixes the API break introduced in the original commit. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301472 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 29 +++++++++++++++-------------- lib/Lex/PreprocessingRecord.cpp | 3 ++- 2 files changed, 17 insertions(+), 15 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 8a56ddf236..fd4e6d30de 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2592,25 +2592,26 @@ void Preprocessor::HandleUndefDirective() { // Okay, we have a valid identifier to undef. auto *II = MacroNameTok.getIdentifierInfo(); auto MD = getMacroDefinition(II); + UndefMacroDirective *Undef = nullptr; + + // If the macro is not defined, this is a noop undef. + if (const MacroInfo *MI = MD.getMacroInfo()) { + if (!MI->isUsed() && MI->isWarnIfUnused()) + Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used); + + if (MI->isWarnIfUnused()) + WarnUnusedMacroLocs.erase(MI->getDefinitionLoc()); + + Undef = AllocateUndefMacroDirective(MacroNameTok.getLocation()); + } // If the callbacks want to know, tell them about the macro #undef. // Note: no matter if the macro was defined or not. if (Callbacks) - Callbacks->MacroUndefined(MacroNameTok, MD); - - // If the macro is not defined, this is a noop undef, just return. - const MacroInfo *MI = MD.getMacroInfo(); - if (!MI) - return; - - if (!MI->isUsed() && MI->isWarnIfUnused()) - Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used); - - if (MI->isWarnIfUnused()) - WarnUnusedMacroLocs.erase(MI->getDefinitionLoc()); + Callbacks->MacroUndefined(MacroNameTok, MD, Undef); - appendMacroDirective(MacroNameTok.getIdentifierInfo(), - AllocateUndefMacroDirective(MacroNameTok.getLocation())); + if (Undef) + appendMacroDirective(II, Undef); } //===----------------------------------------------------------------------===// diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp index 13e15f3c94..03c4cbe589 100644 --- a/lib/Lex/PreprocessingRecord.cpp +++ b/lib/Lex/PreprocessingRecord.cpp @@ -422,7 +422,8 @@ void PreprocessingRecord::MacroDefined(const Token &Id, } void PreprocessingRecord::MacroUndefined(const Token &Id, - const MacroDefinition &MD) { + const MacroDefinition &MD, + const MacroDirective *Undef) { MD.forAllDefinitions([&](MacroInfo *MI) { MacroDefinitions.erase(MI); }); } -- cgit v1.2.3 From 46755a96f6909bd941235bebc090498ccdfd66ff Mon Sep 17 00:00:00 2001 From: Yaron Keren Date: Thu, 27 Apr 2017 09:56:39 +0000 Subject: Constify SourceManager input to MacroInfo::getDefinitionLengthSlow, NFC. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301526 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroInfo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp index 924613dcb8..bec434085e 100644 --- a/lib/Lex/MacroInfo.cpp +++ b/lib/Lex/MacroInfo.cpp @@ -33,7 +33,7 @@ MacroInfo::MacroInfo(SourceLocation DefLoc) UsedForHeaderGuard(false) { } -unsigned MacroInfo::getDefinitionLengthSlow(SourceManager &SM) const { +unsigned MacroInfo::getDefinitionLengthSlow(const SourceManager &SM) const { assert(!IsDefinitionLengthCached); IsDefinitionLengthCached = true; -- cgit v1.2.3 From fc4c78381af3a65eb64567c94bac097b6b0246e4 Mon Sep 17 00:00:00 2001 From: "Duncan P. N. Exon Smith" Date: Thu, 27 Apr 2017 21:41:51 +0000 Subject: Preprocessor: Suppress -Wnonportable-include-path for header maps If a file search involves a header map, suppress -Wnonportable-include-path. It's firing lots of false positives for framework authors internally, and it's not trivial to fix. Consider a framework called "Foo" with a main (installed) framework header "Foo/Foo.h". It's atypical for "Foo.h" to actually live inside a directory called "Foo" in the source repository. Instead, the build system generates a header map while building the framework. If Foo.h lives at the top-level of the source repository (common), and the git repo is called ssh://some.url/foo.git, then the header map will have something like: Foo/Foo.h -> /Users/myname/code/foo/Foo.h where "/Users/myname/code/foo" is the clone of ssh://some.url/foo.git. After #import , the current implementation of -Wnonportable-include-path will falsely assume that Foo.h was found in a nonportable way, because of the name of the git clone (.../foo/Foo.h). However, that directory name was not involved in the header search at all. This commit adds an extra parameter to Preprocessor::LookupFile and HeaderSearch::LookupFile to track if the search used a header map, making it easy to suppress the warning. Longer term, once we find a way to avoid the false positive, we should turn the warning back on. rdar://problem/28863903 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301592 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 14 +++++++++++--- lib/Lex/PPDirectives.cpp | 27 +++++++++++---------------- lib/Lex/PPMacroExpansion.cpp | 2 +- lib/Lex/Pragma.cpp | 2 +- 4 files changed, 24 insertions(+), 21 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index 4ee3871928..bd425a07c3 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -624,7 +624,10 @@ const FileEntry *HeaderSearch::LookupFile( ArrayRef> Includers, SmallVectorImpl *SearchPath, SmallVectorImpl *RelativePath, Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, - bool SkipCache, bool BuildSystemModule) { + bool *IsMapped, bool SkipCache, bool BuildSystemModule) { + if (IsMapped) + *IsMapped = false; + if (SuggestedModule) *SuggestedModule = ModuleMap::KnownHeader(); @@ -754,8 +757,11 @@ const FileEntry *HeaderSearch::LookupFile( if (!SkipCache && CacheLookup.StartIdx == i+1) { // Skip querying potentially lots of directories for this lookup. i = CacheLookup.HitIdx; - if (CacheLookup.MappedName) + if (CacheLookup.MappedName) { Filename = CacheLookup.MappedName; + if (IsMapped) + *IsMapped = true; + } } else { // Otherwise, this is the first query, or the previous query didn't match // our search start. We will fill in our found location below, so prime the @@ -776,6 +782,8 @@ const FileEntry *HeaderSearch::LookupFile( if (HasBeenMapped) { CacheLookup.MappedName = copyString(Filename, LookupFileCache.getAllocator()); + if (IsMapped) + *IsMapped = true; } if (!FE) continue; @@ -839,7 +847,7 @@ const FileEntry *HeaderSearch::LookupFile( const FileEntry *FE = LookupFile(ScratchFilename, IncludeLoc, /*isAngled=*/true, FromDir, CurDir, Includers.front(), SearchPath, RelativePath, - RequestingModule, SuggestedModule); + RequestingModule, SuggestedModule, IsMapped); if (checkMSVCHeaderSearch(Diags, MSFE, FE, IncludeLoc)) { if (SuggestedModule) diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index fd4e6d30de..54578e8fb9 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -752,16 +752,11 @@ Preprocessor::getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, } const FileEntry *Preprocessor::LookupFile( - SourceLocation FilenameLoc, - StringRef Filename, - bool isAngled, - const DirectoryLookup *FromDir, - const FileEntry *FromFile, - const DirectoryLookup *&CurDir, - SmallVectorImpl *SearchPath, + SourceLocation FilenameLoc, StringRef Filename, bool isAngled, + const DirectoryLookup *FromDir, const FileEntry *FromFile, + const DirectoryLookup *&CurDir, SmallVectorImpl *SearchPath, SmallVectorImpl *RelativePath, - ModuleMap::KnownHeader *SuggestedModule, - bool SkipCache) { + ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache) { Module *RequestingModule = getModuleForLocation(FilenameLoc); bool RequestingModuleIsModuleInterface = !SourceMgr.isInMainFile(FilenameLoc); @@ -819,7 +814,7 @@ const FileEntry *Preprocessor::LookupFile( while (const FileEntry *FE = HeaderInfo.LookupFile( Filename, FilenameLoc, isAngled, TmpFromDir, TmpCurDir, Includers, SearchPath, RelativePath, RequestingModule, - SuggestedModule, SkipCache)) { + SuggestedModule, /*IsMapped=*/nullptr, SkipCache)) { // Keep looking as if this file did a #include_next. TmpFromDir = TmpCurDir; ++TmpFromDir; @@ -835,7 +830,7 @@ const FileEntry *Preprocessor::LookupFile( // Do a standard file entry lookup. const FileEntry *FE = HeaderInfo.LookupFile( Filename, FilenameLoc, isAngled, FromDir, CurDir, Includers, SearchPath, - RelativePath, RequestingModule, SuggestedModule, SkipCache, + RelativePath, RequestingModule, SuggestedModule, IsMapped, SkipCache, BuildSystemModule); if (FE) { if (SuggestedModule && !LangOpts.AsmPreprocessor) @@ -1783,6 +1778,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, } // Search include directories. + bool IsMapped = false; const DirectoryLookup *CurDir; SmallString<1024> SearchPath; SmallString<1024> RelativePath; @@ -1801,7 +1797,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, FilenameLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, isAngled, LookupFrom, LookupFromFile, CurDir, Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr, - &SuggestedModule); + &SuggestedModule, &IsMapped); if (!File) { if (Callbacks) { @@ -1818,7 +1814,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, FilenameLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, isAngled, LookupFrom, LookupFromFile, CurDir, nullptr, nullptr, - &SuggestedModule, /*SkipCache*/ true); + &SuggestedModule, &IsMapped, /*SkipCache*/ true); } } } @@ -1833,8 +1829,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, false, LookupFrom, LookupFromFile, CurDir, Callbacks ? &SearchPath : nullptr, - Callbacks ? &RelativePath : nullptr, - &SuggestedModule); + Callbacks ? &RelativePath : nullptr, &SuggestedModule, &IsMapped); if (File) { SourceRange Range(FilenameTok.getLocation(), CharEnd); Diag(FilenameTok, diag::err_pp_file_not_found_not_fatal) << @@ -1964,7 +1959,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, // Issue a diagnostic if the name of the file on disk has a different case // than the one we're about to open. const bool CheckIncludePathPortability = - File && !File->tryGetRealPathName().empty(); + !IsMapped && File && !File->tryGetRealPathName().empty(); if (CheckIncludePathPortability) { StringRef Name = LangOpts.MSVCCompat ? NormalizedPath.str() : Filename; diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 358c96a783..196223981d 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1422,7 +1422,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok, const DirectoryLookup *CurDir; const FileEntry *File = PP.LookupFile(FilenameLoc, Filename, isAngled, LookupFrom, LookupFromFile, - CurDir, nullptr, nullptr, nullptr); + CurDir, nullptr, nullptr, nullptr, nullptr); // Get the result value. A result of true means the file exists. return File != nullptr; diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index 87e105d1d0..fcaaa2d416 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -508,7 +508,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) { const DirectoryLookup *CurDir; const FileEntry *File = LookupFile(FilenameTok.getLocation(), Filename, isAngled, nullptr, - nullptr, CurDir, nullptr, nullptr, nullptr); + nullptr, CurDir, nullptr, nullptr, nullptr, nullptr); if (!File) { if (!SuppressIncludeNotFoundError) Diag(FilenameTok, diag::err_pp_file_not_found) << Filename; -- cgit v1.2.3 From c5a1b98af41696441c247686a80357f478fb000c Mon Sep 17 00:00:00 2001 From: Bruno Cardoso Lopes Date: Thu, 27 Apr 2017 22:29:10 +0000 Subject: [Modules] Refactor logic for incomplete umbrella warnings. NFC git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301596 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 73 +++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 37 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index cf0c953b61..9f68f3bb89 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -287,6 +287,40 @@ const char *Preprocessor::getCurLexerEndPos() { return EndPos; } +void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) { + assert(Mod.getUmbrellaHeader() && "Module must use umbrella header"); + SourceLocation StartLoc = + SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()); + if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc)) + return; + + ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap(); + const DirectoryEntry *Dir = Mod.getUmbrellaDir().Entry; + vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem(); + std::error_code EC; + for (vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), End; + Entry != End && !EC; Entry.increment(EC)) { + using llvm::StringSwitch; + + // Check whether this entry has an extension typically associated with + // headers. + if (!StringSwitch(llvm::sys::path::extension(Entry->getName())) + .Cases(".h", ".H", ".hh", ".hpp", true) + .Default(false)) + continue; + + if (const FileEntry *Header = getFileManager().getFile(Entry->getName())) + if (!getSourceManager().hasFileInfo(Header)) { + if (!ModMap.isHeaderInUnavailableModule(Header)) { + // Find the relative path that would access this header. + SmallString<128> RelativePath; + computeRelativePath(FileMgr, Dir, Header, RelativePath); + Diag(StartLoc, diag::warn_uncovered_module_header) + << Mod.getFullModuleName() << RelativePath; + } + } + } +} /// HandleEndOfFile - This callback is invoked when the lexer hits the end of /// the current file. This either returns the EOF token or pops a level off @@ -475,43 +509,8 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { // If we are building a module that has an umbrella header, make sure that // each of the headers within the directory covered by the umbrella header // was actually included by the umbrella header. - if (Module *Mod = getCurrentModule()) { - if (Mod->getUmbrellaHeader()) { - SourceLocation StartLoc - = SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()); - - if (!getDiagnostics().isIgnored(diag::warn_uncovered_module_header, - StartLoc)) { - ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap(); - const DirectoryEntry *Dir = Mod->getUmbrellaDir().Entry; - vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem(); - std::error_code EC; - for (vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), End; - Entry != End && !EC; Entry.increment(EC)) { - using llvm::StringSwitch; - - // Check whether this entry has an extension typically associated with - // headers. - if (!StringSwitch(llvm::sys::path::extension(Entry->getName())) - .Cases(".h", ".H", ".hh", ".hpp", true) - .Default(false)) - continue; - - if (const FileEntry *Header = - getFileManager().getFile(Entry->getName())) - if (!getSourceManager().hasFileInfo(Header)) { - if (!ModMap.isHeaderInUnavailableModule(Header)) { - // Find the relative path that would access this header. - SmallString<128> RelativePath; - computeRelativePath(FileMgr, Dir, Header, RelativePath); - Diag(StartLoc, diag::warn_uncovered_module_header) - << Mod->getFullModuleName() << RelativePath; - } - } - } - } - } - } + if (Module *Mod = getCurrentModule()) + diagnoseMissingHeaderInUmbrellaDir(*Mod); return true; } -- cgit v1.2.3 From 4ef94de34993f694aced084f55c7d2263ca7c367 Mon Sep 17 00:00:00 2001 From: Bruno Cardoso Lopes Date: Thu, 27 Apr 2017 22:29:14 +0000 Subject: [Modules] Improve diagnostics for incomplete umbrella One of the -Wincomplete-umbrella warnings diagnoses when a header is present in the directory but it's not present in the umbrella header. Currently, this warning only happens on top level modules; any submodule using an umbrella header does not get this warning. Fix that by also considering the submodules. Differential Revision: https://reviews.llvm.org/D32576 rdar://problem/22623686 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301597 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 9f68f3bb89..fcc49b3870 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -287,6 +287,14 @@ const char *Preprocessor::getCurLexerEndPos() { return EndPos; } +static void collectAllSubModulesWithUmbrellaHeader( + const Module &Mod, SmallVectorImpl &SubMods) { + if (Mod.getUmbrellaHeader()) + SubMods.push_back(&Mod); + for (auto *M : Mod.submodules()) + collectAllSubModulesWithUmbrellaHeader(*M, SubMods); +} + void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) { assert(Mod.getUmbrellaHeader() && "Module must use umbrella header"); SourceLocation StartLoc = @@ -507,10 +515,15 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { } // If we are building a module that has an umbrella header, make sure that - // each of the headers within the directory covered by the umbrella header - // was actually included by the umbrella header. - if (Module *Mod = getCurrentModule()) - diagnoseMissingHeaderInUmbrellaDir(*Mod); + // each of the headers within the directory, including all submodules, is + // covered by the umbrella header was actually included by the umbrella + // header. + if (Module *Mod = getCurrentModule()) { + llvm::SmallVector AllMods; + collectAllSubModulesWithUmbrellaHeader(*Mod, AllMods); + for (auto *M : AllMods) + diagnoseMissingHeaderInUmbrellaDir(*M); + } return true; } -- cgit v1.2.3 From ee60c74826a0ec615fcc7fef362686f4aae95c1a Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Sat, 29 Apr 2017 00:34:47 +0000 Subject: Add pragma to perform module import and use it in -E output. Many of our supported configurations support modules but do not have any first-class syntax to perform a module import. This leaves us with a problem: there is no way to represent the expansion of a #include that imports a module in the -E output for such languages. (We don't want to just leave it as a #include because that requires the consumer of the preprocessed source to have the same file system layout and include paths as the creator.) This patch adds a new pragma: #pragma clang module import MODULE.NAME.HERE that imports a module, and changes -E and -frewrite-includes to use it when rewriting a #include that maps to a module import. We don't make any attempt to use a native language syntax import if one exists, to get more consistent output. (If in the future, @import and #include have different semantics in some way, the pragma will track the #include semantics.) git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@301725 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 17 +++++++------- lib/Lex/Pragma.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 54578e8fb9..4826e399af 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -1588,18 +1588,18 @@ bool Preprocessor::ConcatenateIncludeName(SmallString<128> &FilenameBuffer, } /// \brief Push a token onto the token stream containing an annotation. -static void EnterAnnotationToken(Preprocessor &PP, - SourceLocation Begin, SourceLocation End, - tok::TokenKind Kind, void *AnnotationVal) { +void Preprocessor::EnterAnnotationToken(SourceRange Range, + tok::TokenKind Kind, + void *AnnotationVal) { // FIXME: Produce this as the current token directly, rather than // allocating a new token for it. auto Tok = llvm::make_unique(1); Tok[0].startToken(); Tok[0].setKind(Kind); - Tok[0].setLocation(Begin); - Tok[0].setAnnotationEndLoc(End); + Tok[0].setLocation(Range.getBegin()); + Tok[0].setAnnotationEndLoc(Range.getEnd()); Tok[0].setAnnotationValue(AnnotationVal); - PP.EnterTokenStream(std::move(Tok), 1, true); + EnterTokenStream(std::move(Tok), 1, true); } /// \brief Produce a diagnostic informing the user that a #include or similar @@ -2021,7 +2021,8 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, if (IncludeTok.getIdentifierInfo()->getPPKeywordID() != tok::pp___include_macros) - EnterAnnotationToken(*this, HashLoc, End, tok::annot_module_include, M); + EnterAnnotationToken(SourceRange(HashLoc, End), + tok::annot_module_include, M); } return; } @@ -2059,7 +2060,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, // submodule. // FIXME: There's no point doing this if we're handling a #__include_macros // directive. - EnterAnnotationToken(*this, HashLoc, End, tok::annot_module_begin, M); + EnterAnnotationToken(SourceRange(HashLoc, End), tok::annot_module_begin, M); } } diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index fcaaa2d416..576151a98b 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -534,6 +534,47 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) { } } +void Preprocessor::HandlePragmaModuleImport(Token &ImportTok) { + SourceLocation ImportLoc = ImportTok.getLocation(); + + Token Tok; + + llvm::SmallVector, 8> ModuleName; + while (true) { + LexUnexpandedToken(Tok); + if (Tok.isNot(tok::identifier)) { + Diag(Tok.getLocation(), + diag::err_pragma_module_import_expected_module_name) << 0; + return; + } + + ModuleName.emplace_back(Tok.getIdentifierInfo(), Tok.getLocation()); + + LexUnexpandedToken(Tok); + assert(Tok.isNot(tok::eof)); + if (Tok.is(tok::eod)) + break; + if (Tok.isNot(tok::period)) { + Diag(Tok.getLocation(), + diag::err_pragma_module_import_expected_module_name) << 1; + return; + } + } + + // If we have a non-empty module path, load the named module. + Module *Imported = + TheModuleLoader.loadModule(ImportLoc, ModuleName, Module::Hidden, + /*IsIncludeDirective=*/false); + if (!Imported) + return; + + makeModuleVisible(Imported, ImportLoc); + EnterAnnotationToken(SourceRange(ImportLoc, Tok.getLocation()), + tok::annot_module_include, Imported); + if (Callbacks) + Callbacks->moduleImport(ImportLoc, ModuleName, Imported); +} + /// ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro. /// Return the IdentifierInfo* associated with the macro to push or pop. IdentifierInfo *Preprocessor::ParsePragmaPushOrPopMacro(Token &Tok) { @@ -1301,6 +1342,19 @@ public: } }; +/// Handle the clang \#pragma module import extension. The syntax is: +/// \code +/// #pragma clang module import some.module.name +/// \endcode +struct PragmaModuleImportHandler : public PragmaHandler { + PragmaModuleImportHandler() : PragmaHandler("import") {} + + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &ImportTok) override { + PP.HandlePragmaModuleImport(ImportTok); + } +}; + /// PragmaPushMacroHandler - "\#pragma push_macro" saves the value of the /// macro on the top of the stack. struct PragmaPushMacroHandler : public PragmaHandler { @@ -1524,6 +1578,11 @@ void Preprocessor::RegisterBuiltinPragmas() { AddPragmaHandler("clang", new PragmaARCCFCodeAuditedHandler()); AddPragmaHandler("clang", new PragmaAssumeNonNullHandler()); + // #pragma clang module ... + auto *ModuleHandler = new PragmaNamespace("module"); + AddPragmaHandler("clang", ModuleHandler); + ModuleHandler->AddPragma(new PragmaModuleImportHandler()); + AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler()); AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler()); AddPragmaHandler("STDC", new PragmaSTDC_UnknownHandler()); -- cgit v1.2.3 From f195b64c780cc36265ea7d814e5d8c783638c08f Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Thu, 4 May 2017 00:29:54 +0000 Subject: Add #pragma clang module begin/end pragmas and generate them when preprocessing a module. These pragmas are intended to simulate the effect of entering or leaving a file with an associated module. This is not completely implemented yet: declarations between the pragmas will not be attributed to the correct module, but macro visibility is already functional. Modules named by #pragma clang module begin must already be known to clang (in some module map that's either loaded or on the search path). git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302098 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 8 +-- lib/Lex/PPLexerChange.cpp | 57 +++++++++++---- lib/Lex/PPMacroExpansion.cpp | 2 +- lib/Lex/Pragma.cpp | 167 ++++++++++++++++++++++++++++++++----------- lib/Lex/Preprocessor.cpp | 8 +-- 5 files changed, 176 insertions(+), 66 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 4826e399af..06fee8e5b0 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2049,12 +2049,12 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, M->getTopLevelModuleName() == getLangOpts().CurrentModule) return; - assert(!CurSubmodule && "should not have marked this as a module yet"); - CurSubmodule = M; + assert(!CurLexerSubmodule && "should not have marked this as a module yet"); + CurLexerSubmodule = M; // Let the macro handling code know that any future macros are within // the new submodule. - EnterSubmodule(M, HashLoc); + EnterSubmodule(M, HashLoc, /*ForPragma*/false); // Let the parser know that any future declarations are within the new // submodule. @@ -2082,7 +2082,7 @@ void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc, } else if (isInPrimaryFile()) { Lookup = nullptr; Diag(IncludeNextTok, diag::pp_include_next_in_primary); - } else if (CurSubmodule) { + } else if (CurLexerSubmodule) { // Start looking up in the directory *after* the one in which the current // file would be found, if any. assert(CurPPLexer && "#include_next directive in macro?"); diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index fcc49b3870..1938328c90 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -117,7 +117,7 @@ void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer, CurLexer.reset(TheLexer); CurPPLexer = TheLexer; CurDirLookup = CurDir; - CurSubmodule = nullptr; + CurLexerSubmodule = nullptr; if (CurLexerKind != CLK_LexAfterModuleImport) CurLexerKind = CLK_Lexer; @@ -142,7 +142,7 @@ void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL, CurDirLookup = CurDir; CurPTHLexer.reset(PL); CurPPLexer = CurPTHLexer.get(); - CurSubmodule = nullptr; + CurLexerSubmodule = nullptr; if (CurLexerKind != CLK_LexAfterModuleImport) CurLexerKind = CLK_PTHLexer; @@ -337,6 +337,26 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { assert(!CurTokenLexer && "Ending a file when currently in a macro!"); + // If we have an unclosed module region from a pragma at the end of a + // module, complain and close it now. + // FIXME: This is not correct if we are building a module from PTH. + const bool LeavingSubmodule = CurLexer && CurLexerSubmodule; + if ((LeavingSubmodule || IncludeMacroStack.empty()) && + !BuildingSubmoduleStack.empty() && + BuildingSubmoduleStack.back().IsPragma) { + Diag(BuildingSubmoduleStack.back().ImportLoc, + diag::err_pp_module_begin_without_module_end); + Module *M = LeaveSubmodule(/*ForPragma*/true); + + Result.startToken(); + const char *EndPos = getCurLexerEndPos(); + CurLexer->BufferPtr = EndPos; + CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end); + Result.setAnnotationEndLoc(Result.getLocation()); + Result.setAnnotationValue(M); + return true; + } + // See if this file had a controlling macro. if (CurPPLexer) { // Not ending a macro, ignore it. if (const IdentifierInfo *ControllingMacro = @@ -442,18 +462,17 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { if (Callbacks && !isEndOfMacro && CurPPLexer) ExitedFID = CurPPLexer->getFileID(); - bool LeavingSubmodule = CurSubmodule && CurLexer; if (LeavingSubmodule) { + // We're done with this submodule. + Module *M = LeaveSubmodule(/*ForPragma*/false); + // Notify the parser that we've left the module. const char *EndPos = getCurLexerEndPos(); Result.startToken(); CurLexer->BufferPtr = EndPos; CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end); Result.setAnnotationEndLoc(Result.getLocation()); - Result.setAnnotationValue(CurSubmodule); - - // We're done with this submodule. - LeaveSubmodule(); + Result.setAnnotationValue(M); } // We're done with the #included file. @@ -628,11 +647,13 @@ void Preprocessor::HandleMicrosoftCommentPaste(Token &Tok) { assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode"); } -void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc) { +void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) { if (!getLangOpts().ModulesLocalVisibility) { // Just track that we entered this submodule. - BuildingSubmoduleStack.push_back(BuildingSubmoduleInfo( - M, ImportLoc, CurSubmoduleState, PendingModuleMacroNames.size())); + BuildingSubmoduleStack.push_back( + BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState, + PendingModuleMacroNames.size())); return; } @@ -673,8 +694,9 @@ void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc) { } // Track that we entered this module. - BuildingSubmoduleStack.push_back(BuildingSubmoduleInfo( - M, ImportLoc, CurSubmoduleState, PendingModuleMacroNames.size())); + BuildingSubmoduleStack.push_back( + BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState, + PendingModuleMacroNames.size())); // Switch to this submodule as the current submodule. CurSubmoduleState = &State; @@ -697,7 +719,13 @@ bool Preprocessor::needModuleMacros() const { return getLangOpts().isCompilingModule(); } -void Preprocessor::LeaveSubmodule() { +Module *Preprocessor::LeaveSubmodule(bool ForPragma) { + if (BuildingSubmoduleStack.empty() || + BuildingSubmoduleStack.back().IsPragma != ForPragma) { + assert(ForPragma && "non-pragma module enter/leave mismatch"); + return nullptr; + } + auto &Info = BuildingSubmoduleStack.back(); Module *LeavingMod = Info.M; @@ -711,7 +739,7 @@ void Preprocessor::LeaveSubmodule() { // of pending names for the surrounding submodule. BuildingSubmoduleStack.pop_back(); makeModuleVisible(LeavingMod, ImportLoc); - return; + return LeavingMod; } // Create ModuleMacros for any macros defined in this submodule. @@ -800,4 +828,5 @@ void Preprocessor::LeaveSubmodule() { // A nested #include makes the included submodule visible. makeModuleVisible(LeavingMod, ImportLoc); + return LeavingMod; } diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 196223981d..6c7663994a 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1453,7 +1453,7 @@ static bool EvaluateHasIncludeNext(Token &Tok, } else if (PP.isInPrimaryFile()) { Lookup = nullptr; PP.Diag(Tok, diag::pp_include_next_in_primary); - } else if (PP.getCurrentSubmodule()) { + } else if (PP.getCurrentLexerSubmodule()) { // Start looking up in the directory *after* the one in which the current // file would be found, if any. assert(PP.getCurrentLexer() && "#include_next directive in macro?"); diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index 576151a98b..51da2baac9 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -534,47 +534,6 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) { } } -void Preprocessor::HandlePragmaModuleImport(Token &ImportTok) { - SourceLocation ImportLoc = ImportTok.getLocation(); - - Token Tok; - - llvm::SmallVector, 8> ModuleName; - while (true) { - LexUnexpandedToken(Tok); - if (Tok.isNot(tok::identifier)) { - Diag(Tok.getLocation(), - diag::err_pragma_module_import_expected_module_name) << 0; - return; - } - - ModuleName.emplace_back(Tok.getIdentifierInfo(), Tok.getLocation()); - - LexUnexpandedToken(Tok); - assert(Tok.isNot(tok::eof)); - if (Tok.is(tok::eod)) - break; - if (Tok.isNot(tok::period)) { - Diag(Tok.getLocation(), - diag::err_pragma_module_import_expected_module_name) << 1; - return; - } - } - - // If we have a non-empty module path, load the named module. - Module *Imported = - TheModuleLoader.loadModule(ImportLoc, ModuleName, Module::Hidden, - /*IsIncludeDirective=*/false); - if (!Imported) - return; - - makeModuleVisible(Imported, ImportLoc); - EnterAnnotationToken(SourceRange(ImportLoc, Tok.getLocation()), - tok::annot_module_include, Imported); - if (Callbacks) - Callbacks->moduleImport(ImportLoc, ModuleName, Imported); -} - /// ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro. /// Return the IdentifierInfo* associated with the macro to push or pop. IdentifierInfo *Preprocessor::ParsePragmaPushOrPopMacro(Token &Tok) { @@ -1342,6 +1301,26 @@ public: } }; +static bool LexModuleName( + Preprocessor &PP, Token &Tok, + llvm::SmallVectorImpl> + &ModuleName) { + while (true) { + PP.LexUnexpandedToken(Tok); + if (Tok.isNot(tok::identifier)) { + PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) + << ModuleName.empty(); + return true; + } + + ModuleName.emplace_back(Tok.getIdentifierInfo(), Tok.getLocation()); + + PP.LexUnexpandedToken(Tok); + if (Tok.isNot(tok::period)) + return false; + } +} + /// Handle the clang \#pragma module import extension. The syntax is: /// \code /// #pragma clang module import some.module.name @@ -1350,8 +1329,108 @@ struct PragmaModuleImportHandler : public PragmaHandler { PragmaModuleImportHandler() : PragmaHandler("import") {} void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, - Token &ImportTok) override { - PP.HandlePragmaModuleImport(ImportTok); + Token &Tok) override { + SourceLocation ImportLoc = Tok.getLocation(); + + // Read the module name. + llvm::SmallVector, 8> + ModuleName; + if (LexModuleName(PP, Tok, ModuleName)) + return; + + if (Tok.isNot(tok::eod)) + PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma"; + + // If we have a non-empty module path, load the named module. + Module *Imported = + PP.getModuleLoader().loadModule(ImportLoc, ModuleName, Module::Hidden, + /*IsIncludeDirective=*/false); + if (!Imported) + return; + + PP.makeModuleVisible(Imported, ImportLoc); + PP.EnterAnnotationToken(SourceRange(ImportLoc, ModuleName.back().second), + tok::annot_module_include, Imported); + if (auto *CB = PP.getPPCallbacks()) + CB->moduleImport(ImportLoc, ModuleName, Imported); + } +}; + +/// Handle the clang \#pragma module begin extension. The syntax is: +/// \code +/// #pragma clang module begin some.module.name +/// ... +/// #pragma clang module end +/// \endcode +struct PragmaModuleBeginHandler : public PragmaHandler { + PragmaModuleBeginHandler() : PragmaHandler("begin") {} + + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &Tok) override { + SourceLocation BeginLoc = Tok.getLocation(); + + // Read the module name. + llvm::SmallVector, 8> + ModuleName; + if (LexModuleName(PP, Tok, ModuleName)) + return; + + if (Tok.isNot(tok::eod)) + PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma"; + + // We can only enter submodules of the current module. + StringRef Current = PP.getLangOpts().CurrentModule; + if (ModuleName.front().first->getName() != Current) { + PP.Diag(ModuleName.front().second, diag::err_pp_module_begin_wrong_module) + << ModuleName.front().first << (ModuleName.size() > 1) + << Current.empty() << Current; + return; + } + + // Find the module we're entering. We require that a module map for it + // be loaded or implicitly loadable. + // FIXME: We could create the submodule here. We'd need to know whether + // it's supposed to be explicit, but not much else. + Module *M = PP.getHeaderSearchInfo().getModuleMap().findModule(Current); + if (!M) { + PP.Diag(ModuleName.front().second, + diag::err_pp_module_begin_no_module_map) << Current; + return; + } + for (unsigned I = 1; I != ModuleName.size(); ++I) { + auto *NewM = M->findSubmodule(ModuleName[I].first->getName()); + if (!NewM) { + PP.Diag(ModuleName[I].second, diag::err_pp_module_begin_no_submodule) + << M->getFullModuleName() << ModuleName[I].first; + return; + } + M = NewM; + } + + // Enter the scope of the submodule. + PP.EnterSubmodule(M, BeginLoc, /*ForPragma*/true); + PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().second), + tok::annot_module_begin, M); + } +}; + +/// Handle the clang \#pragma module end extension. +struct PragmaModuleEndHandler : public PragmaHandler { + PragmaModuleEndHandler() : PragmaHandler("end") {} + + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &Tok) override { + SourceLocation Loc = Tok.getLocation(); + + PP.LexUnexpandedToken(Tok); + if (Tok.isNot(tok::eod)) + PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma"; + + Module *M = PP.LeaveSubmodule(/*ForPragma*/true); + if (M) + PP.EnterAnnotationToken(SourceRange(Loc), tok::annot_module_end, M); + else + PP.Diag(Loc, diag::err_pp_module_end_without_module_begin); } }; @@ -1582,6 +1661,8 @@ void Preprocessor::RegisterBuiltinPragmas() { auto *ModuleHandler = new PragmaNamespace("module"); AddPragmaHandler("clang", ModuleHandler); ModuleHandler->AddPragma(new PragmaModuleImportHandler()); + ModuleHandler->AddPragma(new PragmaModuleBeginHandler()); + ModuleHandler->AddPragma(new PragmaModuleEndHandler()); AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler()); AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler()); diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index babef5dcc7..e409ab0365 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -85,10 +85,10 @@ Preprocessor::Preprocessor(std::shared_ptr PPOpts, LastTokenWasAt(false), ModuleImportExpectsIdentifier(false), CodeCompletionReached(false), CodeCompletionII(nullptr), MainFileDir(nullptr), SkipMainFilePreamble(0, true), CurPPLexer(nullptr), - CurDirLookup(nullptr), CurLexerKind(CLK_Lexer), CurSubmodule(nullptr), - Callbacks(nullptr), CurSubmoduleState(&NullSubmoduleState), - MacroArgCache(nullptr), Record(nullptr), MIChainHead(nullptr), - DeserialMIChainHead(nullptr) { + CurDirLookup(nullptr), CurLexerKind(CLK_Lexer), + CurLexerSubmodule(nullptr), Callbacks(nullptr), + CurSubmoduleState(&NullSubmoduleState), MacroArgCache(nullptr), + Record(nullptr), MIChainHead(nullptr), DeserialMIChainHead(nullptr) { OwnsHeaderSearch = OwnsHeaders; CounterValue = 0; // __COUNTER__ starts at 0. -- cgit v1.2.3 From ea3a0ef86d3f4d8f859f9eca4ef97134484c2610 Mon Sep 17 00:00:00 2001 From: James Y Knight Date: Thu, 4 May 2017 21:31:17 +0000 Subject: Fix whitespace before token-paste of an argument. The whitespace should come from the argument name in the macro expansion, rather than from the token passed to the macro (same as it does when not pasting). Added a new test case for the change in behavior to stringize_space.c. FileCheck'ized macro_paste_commaext.c, tweaked the test case, and added a comment; no behavioral change to this test. Differential Revision: https://reviews.llvm.org/D30427 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302195 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index a53c8014eb..049e046cec 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -183,6 +183,12 @@ void TokenLexer::ExpandFunctionArguments() { // preprocessor already verified that the following token is a macro name // when the #define was parsed. const Token &CurTok = Tokens[i]; + // We don't want a space for the next token after a paste + // operator. In valid code, the token will get smooshed onto the + // preceding one anyway. In assembler-with-cpp mode, invalid + // pastes are allowed through: in this case, we do not want the + // extra whitespace to be added. For example, we want ". ## foo" + // -> ".foo" not ". foo". if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) NextTokGetsSpace = true; @@ -317,6 +323,7 @@ void TokenLexer::ExpandFunctionArguments() { const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); unsigned NumToks = MacroArgs::getArgLength(ArgToks); if (NumToks) { // Not an empty argument? + bool VaArgsPseudoPaste = false; // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when // the expander trys to paste ',' with the first token of the __VA_ARGS__ @@ -325,6 +332,7 @@ void TokenLexer::ExpandFunctionArguments() { ResultToks[ResultToks.size()-2].is(tok::comma) && (unsigned)ArgNo == Macro->getNumArgs()-1 && Macro->isVariadic()) { + VaArgsPseudoPaste = true; // Remove the paste operator, report use of the extension. PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma); } @@ -344,18 +352,16 @@ void TokenLexer::ExpandFunctionArguments() { ResultToks.end()-NumToks, ResultToks.end()); } - // If this token (the macro argument) was supposed to get leading - // whitespace, transfer this information onto the first token of the - // expansion. - // - // Do not do this if the paste operator occurs before the macro argument, - // as in "A ## MACROARG". In valid code, the first token will get - // smooshed onto the preceding one anyway (forming AMACROARG). In - // assembler-with-cpp mode, invalid pastes are allowed through: in this - // case, we do not want the extra whitespace to be added. For example, - // we want ". ## foo" -> ".foo" not ". foo". - if (NextTokGetsSpace) - ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace); + // Transfer the leading whitespace information from the token + // (the macro argument) onto the first token of the + // expansion. Note that we don't do this for the GNU + // pseudo-paste extension ", ## __VA_ARGS__". + if (!VaArgsPseudoPaste) { + ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine, + false); + ResultToks[ResultToks.size() - NumToks].setFlagValue( + Token::LeadingSpace, NextTokGetsSpace); + } NextTokGetsSpace = false; continue; -- cgit v1.2.3 From 90500c1350ce925c24aa3da49b42657140999764 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Fri, 5 May 2017 16:42:44 +0000 Subject: Add a fix-it for -Wunguarded-availability This patch adds a fix-it for the -Wunguarded-availability warning. This fix-it is similar to the Swift one: it suggests that you wrap the statement in an `if (@available)` check. The produced fixits are indented (just like the Swift ones) to make them look nice in Xcode's fix-it preview. rdar://31680358 Differential Revision: https://reviews.llvm.org/D32424 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302253 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 66 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 17 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 003c9b5eed..3d6fe91115 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -452,6 +452,29 @@ bool Lexer::getRawToken(SourceLocation Loc, Token &Result, return false; } +/// Returns the pointer that points to the beginning of line that contains +/// the given offset, or null if the offset if invalid. +static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) { + const char *BufStart = Buffer.data(); + if (Offset >= Buffer.size()) + return nullptr; + const char *StrData = BufStart + Offset; + + if (StrData[0] == '\n' || StrData[0] == '\r') + return StrData; + + const char *LexStart = StrData; + while (LexStart != BufStart) { + if (LexStart[0] == '\n' || LexStart[0] == '\r') { + ++LexStart; + break; + } + + --LexStart; + } + return LexStart; +} + static SourceLocation getBeginningOfFileToken(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) { @@ -467,27 +490,15 @@ static SourceLocation getBeginningOfFileToken(SourceLocation Loc, // Back up from the current location until we hit the beginning of a line // (or the buffer). We'll relex from that point. - const char *BufStart = Buffer.data(); - if (LocInfo.second >= Buffer.size()) - return Loc; - - const char *StrData = BufStart+LocInfo.second; - if (StrData[0] == '\n' || StrData[0] == '\r') + const char *StrData = Buffer.data() + LocInfo.second; + const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second); + if (!LexStart || LexStart == StrData) return Loc; - - const char *LexStart = StrData; - while (LexStart != BufStart) { - if (LexStart[0] == '\n' || LexStart[0] == '\r') { - ++LexStart; - break; - } - - --LexStart; - } // Create a lexer starting at the beginning of this token. SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); - Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); + Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart, + Buffer.end()); TheLexer.SetCommentRetentionState(true); // Lex tokens until we find the token that contains the source location. @@ -1038,6 +1049,27 @@ bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { return isIdentifierBody(c, LangOpts.DollarIdents); } +StringRef Lexer::getIndentationForLine(SourceLocation Loc, + const SourceManager &SM) { + if (Loc.isInvalid() || Loc.isMacroID()) + return ""; + std::pair LocInfo = SM.getDecomposedLoc(Loc); + if (LocInfo.first.isInvalid()) + return ""; + bool Invalid = false; + StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); + if (Invalid) + return ""; + const char *Line = findBeginningOfLine(Buffer, LocInfo.second); + if (!Line) + return ""; + StringRef Rest = Buffer.substr(Line - Buffer.data()); + size_t NumWhitespaceChars = Rest.find_first_not_of(" \t"); + return NumWhitespaceChars == StringRef::npos + ? "" + : Rest.take_front(NumWhitespaceChars); +} + //===----------------------------------------------------------------------===// // Diagnostics forwarding code. //===----------------------------------------------------------------------===// -- cgit v1.2.3 From bc4ab6d64d9314f4391752e17068eea2b9377e53 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 5 May 2017 22:18:51 +0000 Subject: Add support for building modules from preprocessed source. To support this, an optional marker "#pragma clang module contents" is recognized in module map files, and the rest of the module map file from that point onwards is treated as the source of the module. Preprocessing a module map produces the input module followed by the marker and then the preprocessed contents of the module. Ignoring line markers, a preprocessed module might look like this: module A { header "a.h" } #pragma clang module contents #pragma clang module begin A // ... a.h ... #pragma clang module end The preprocessed output generates line markers, which are not accepted by the module map parser, so -x c++-module-map-cpp-output should be used to compile such outputs. A couple of major parts do not work yet: 1) The files that are listed in the module map must exist on disk, in order to build the on-disk header -> module lookup table in the PCM file. To fix this, we need the preprocessed output to track the file size and other stat information we might use to build the lookup table. 2) Declaration ownership semantics don't work properly yet, since mapping from a source location to a module relies on mapping from FileIDs to modules, which we can't do if module transitions can occur in the middle of a file. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302309 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 10 +++++---- lib/Lex/ModuleMap.cpp | 58 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 13 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index bd425a07c3..f5b7c59e44 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -1325,7 +1325,8 @@ static const FileEntry *getPrivateModuleMap(const FileEntry *File, return FileMgr.getFile(PrivateFilename); } -bool HeaderSearch::loadModuleMapFile(const FileEntry *File, bool IsSystem) { +bool HeaderSearch::loadModuleMapFile(const FileEntry *File, bool IsSystem, + FileID ID, unsigned *Offset) { // Find the directory for the module. For frameworks, that may require going // up from the 'Modules' directory. const DirectoryEntry *Dir = nullptr; @@ -1344,7 +1345,7 @@ bool HeaderSearch::loadModuleMapFile(const FileEntry *File, bool IsSystem) { } } - switch (loadModuleMapFileImpl(File, IsSystem, Dir)) { + switch (loadModuleMapFileImpl(File, IsSystem, Dir, ID, Offset)) { case LMM_AlreadyLoaded: case LMM_NewlyLoaded: return false; @@ -1357,7 +1358,8 @@ bool HeaderSearch::loadModuleMapFile(const FileEntry *File, bool IsSystem) { HeaderSearch::LoadModuleMapResult HeaderSearch::loadModuleMapFileImpl(const FileEntry *File, bool IsSystem, - const DirectoryEntry *Dir) { + const DirectoryEntry *Dir, FileID ID, + unsigned *Offset) { assert(File && "expected FileEntry"); // Check whether we've already loaded this module map, and mark it as being @@ -1366,7 +1368,7 @@ HeaderSearch::loadModuleMapFileImpl(const FileEntry *File, bool IsSystem, if (!AddResult.second) return AddResult.first->second ? LMM_AlreadyLoaded : LMM_InvalidModuleMap; - if (ModMap.parseModuleMapFile(File, IsSystem, Dir)) { + if (ModMap.parseModuleMapFile(File, IsSystem, Dir, ID, Offset)) { LoadedModuleMaps[File] = false; return LMM_InvalidModuleMap; } diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 512d7dc5de..70d37d3d70 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -1132,14 +1132,17 @@ namespace clang { } bool parseModuleMapFile(); + + bool terminatedByDirective() { return false; } + SourceLocation getLocation() { return Tok.getLocation(); } }; } SourceLocation ModuleMapParser::consumeToken() { -retry: SourceLocation Result = Tok.getLocation(); + +retry: Tok.clear(); - Token LToken; L.LexFromRawLexer(LToken); Tok.Location = LToken.getLocation().getRawEncoding(); @@ -1232,9 +1235,28 @@ retry: case tok::comment: goto retry; - + + case tok::hash: + // A module map can be terminated prematurely by + // #pragma clang module contents + // When building the module, we'll treat the rest of the file as the + // contents of the module. + { + auto NextIsIdent = [&](StringRef Str) -> bool { + L.LexFromRawLexer(LToken); + return !LToken.isAtStartOfLine() && LToken.is(tok::raw_identifier) && + LToken.getRawIdentifier() == Str; + }; + if (NextIsIdent("pragma") && NextIsIdent("clang") && + NextIsIdent("module") && NextIsIdent("contents")) { + Tok.Kind = MMToken::EndOfFile; + break; + } + } + LLVM_FALLTHROUGH; + default: - Diags.Report(LToken.getLocation(), diag::err_mmap_unknown_token); + Diags.Report(Tok.getLocation(), diag::err_mmap_unknown_token); HadError = true; goto retry; } @@ -1682,7 +1704,8 @@ void ModuleMapParser::parseExternModuleDecl() { File, /*IsSystem=*/false, Map.HeaderInfo.getHeaderSearchOpts().ModuleMapFileHomeIsCwd ? Directory - : File->getDir(), ExternLoc); + : File->getDir(), + FileID(), nullptr, ExternLoc); } /// Whether to add the requirement \p Feature to the module \p M. @@ -2522,28 +2545,45 @@ bool ModuleMapParser::parseModuleMapFile() { } bool ModuleMap::parseModuleMapFile(const FileEntry *File, bool IsSystem, - const DirectoryEntry *Dir, + const DirectoryEntry *Dir, FileID ID, + unsigned *Offset, SourceLocation ExternModuleLoc) { + assert(Target && "Missing target information"); llvm::DenseMap::iterator Known = ParsedModuleMap.find(File); if (Known != ParsedModuleMap.end()) return Known->second; + // If the module map file wasn't already entered, do so now. + if (ID.isInvalid()) { + auto FileCharacter = IsSystem ? SrcMgr::C_System : SrcMgr::C_User; + ID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter); + } + assert(Target && "Missing target information"); - auto FileCharacter = IsSystem ? SrcMgr::C_System : SrcMgr::C_User; - FileID ID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter); const llvm::MemoryBuffer *Buffer = SourceMgr.getBuffer(ID); if (!Buffer) return ParsedModuleMap[File] = true; + assert((!Offset || *Offset <= Buffer->getBufferSize()) && + "invalid buffer offset"); // Parse this module map file. - Lexer L(ID, SourceMgr.getBuffer(ID), SourceMgr, MMapLangOpts); + Lexer L(SourceMgr.getLocForStartOfFile(ID), MMapLangOpts, + Buffer->getBufferStart(), + Buffer->getBufferStart() + (Offset ? *Offset : 0), + Buffer->getBufferEnd()); SourceLocation Start = L.getSourceLocation(); ModuleMapParser Parser(L, SourceMgr, Target, Diags, *this, File, Dir, BuiltinIncludeDir, IsSystem); bool Result = Parser.parseModuleMapFile(); ParsedModuleMap[File] = Result; + if (Offset) { + auto Loc = SourceMgr.getDecomposedLoc(Parser.getLocation()); + assert(Loc.first == ID && "stopped in a different file?"); + *Offset = Loc.second; + } + // Notify callbacks that we parsed it. for (const auto &Cb : Callbacks) Cb->moduleMapFileRead(Start, *File, IsSystem); -- cgit v1.2.3 From 4d1775b4eab190b2916bd2e5b19d1d55731e2cd9 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 5 May 2017 22:34:07 +0000 Subject: Permit keywords in module names in #pragma clang module *. This is necessary to be able to build a libc++ module from preprocessed source (due to the submodule std.new). git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302312 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Pragma.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index 51da2baac9..99d56182c1 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -1307,7 +1307,7 @@ static bool LexModuleName( &ModuleName) { while (true) { PP.LexUnexpandedToken(Tok); - if (Tok.isNot(tok::identifier)) { + if (Tok.isAnnotation() || !Tok.getIdentifierInfo()) { PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << ModuleName.empty(); return true; -- cgit v1.2.3 From 977625bd6af0feaa1324925acb7c0689f10fa0b9 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 8 May 2017 20:30:47 +0000 Subject: If we are building a module, and we read a second description of the same module from a different module map, ignore it. This happens during builds of preprocessed modules (where it is harmless). git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302463 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 70d37d3d70..568894c32b 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -1485,7 +1485,19 @@ void ModuleMapParser::parseModuleDecl() { // Determine whether this (sub)module has already been defined. if (Module *Existing = Map.lookupModuleQualified(ModuleName, ActiveModule)) { - if (Existing->DefinitionLoc.isInvalid() && !ActiveModule) { + // We might see a (re)definition of a module that we already have a + // definition for in two cases: + // - If we loaded one definition from an AST file and we've just found a + // corresponding definition in a module map file, or + bool LoadedFromASTFile = Existing->DefinitionLoc.isInvalid(); + // - If we're building a (preprocessed) module and we've just loaded the + // module map file from which it was created. + bool ParsedAsMainInput = + Map.LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap && + Map.LangOpts.CurrentModule == ModuleName && + SourceMgr.getDecomposedLoc(ModuleNameLoc).first != + SourceMgr.getDecomposedLoc(Existing->DefinitionLoc).first; + if (!ActiveModule && (LoadedFromASTFile || ParsedAsMainInput)) { // Skip the module definition. skipUntil(MMToken::RBrace); if (Tok.is(MMToken::RBrace)) -- cgit v1.2.3 From a4e6a18c0f27fa88ab403ba530f45f4a780a6601 Mon Sep 17 00:00:00 2001 From: Bruno Cardoso Lopes Date: Tue, 9 May 2017 00:41:38 +0000 Subject: [Modules] Allow umbrella frameworks to define private submodules for subframeworks In r298391 we fixed the umbrella framework model to work when submodules named "Private" are used. This complements the work by allowing the umbrella framework model to work in general. rdar://problem/31790067 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302491 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 568894c32b..6f44dc757e 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -1913,8 +1913,10 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, // 'framework module FrameworkName.Private', since a 'Private.Framework' // does not usually exist. However, since both are currently widely used // for private modules, make sure we find the right path in both cases. - RelativePathName.resize(ActiveModule->IsFramework ? 0 - : RelativePathLength); + if (ActiveModule->IsFramework && ActiveModule->Name == "Private") + RelativePathName.clear(); + else + RelativePathName.resize(RelativePathLength); FullPathName.resize(FullPathLength); llvm::sys::path::append(RelativePathName, "PrivateHeaders", Header.FileName); -- cgit v1.2.3 From c156232e13edf1d0deab270634b6fea861aa0e5d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 12 May 2017 18:56:03 +0000 Subject: [modules] Simplify module macro handling in non-local-submodule-visibility mode. When reaching the end of a module, we used to convert its macros to ModuleMacros but also leave them in the MacroDirective chain for the identifier. This meant that every lookup of such a macro would find two (identical) definitions. It also made it difficult to determine the correct owner for a macro when reaching the end of a module: the most recent MacroDirective in the chain could be from an #included submodule rather than the current module. Simplify this: whenever we convert a MacroDirective to a ModuleMacro when leaving a module, clear out the MacroDirective chain for that identifier, and just rely on the ModuleMacro to provide the macro definition information. (We don't want to do this for local submodule visibility mode, because in that mode we maintain a distinct MacroDirective chain for each submodule, and we need to keep around the prior MacroDirective in case we re-enter the submodule -- for instance, if its header is #included more than once in a module build, we need the include guard directive to stick around. But the problem doesn't arise in this case for the same reason: each submodule has its own MacroDirective chain, so the macros don't leak out of submodules in the first place.) git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302932 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 1938328c90..a3c537a43b 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -731,7 +731,7 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { Module *LeavingMod = Info.M; SourceLocation ImportLoc = Info.ImportLoc; - if (!needModuleMacros() || + if (!needModuleMacros() || (!getLangOpts().ModulesLocalVisibility && LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) { // If we don't need module macros, or this is not a module for which we @@ -777,17 +777,6 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) { assert(MD && "broken macro directive chain"); - // Stop on macros defined in other submodules of this module that we - // #included along the way. There's no point doing this if we're - // tracking local submodule visibility, since there can be no such - // directives in our list. - if (!getLangOpts().ModulesLocalVisibility) { - Module *Mod = getModuleContainingLocation(MD->getLocation()); - if (Mod != LeavingMod && - Mod->getTopLevelModule() == LeavingMod->getTopLevelModule()) - break; - } - if (auto *VisMD = dyn_cast(MD)) { // The latest visibility directive for a name in a submodule affects // all the directives that come before it. @@ -809,6 +798,12 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { if (Def || !Macro.getOverriddenMacros().empty()) addModuleMacro(LeavingMod, II, Def, Macro.getOverriddenMacros(), IsNew); + + if (!getLangOpts().ModulesLocalVisibility) { + // This macro is exposed to the rest of this compilation as a + // ModuleMacro; we don't need to track its MacroDirective any more. + Macro.setLatest(nullptr); + } break; } } -- cgit v1.2.3 From 2dac30a6d7668a1de94f2509b6e533e16b3117a7 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 12 May 2017 20:42:54 +0000 Subject: Revert r302932, as it appears to be breaking stage2 for some of our modules-enabled buildbots. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302947 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index a3c537a43b..1938328c90 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -731,7 +731,7 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { Module *LeavingMod = Info.M; SourceLocation ImportLoc = Info.ImportLoc; - if (!needModuleMacros() || + if (!needModuleMacros() || (!getLangOpts().ModulesLocalVisibility && LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) { // If we don't need module macros, or this is not a module for which we @@ -777,6 +777,17 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) { assert(MD && "broken macro directive chain"); + // Stop on macros defined in other submodules of this module that we + // #included along the way. There's no point doing this if we're + // tracking local submodule visibility, since there can be no such + // directives in our list. + if (!getLangOpts().ModulesLocalVisibility) { + Module *Mod = getModuleContainingLocation(MD->getLocation()); + if (Mod != LeavingMod && + Mod->getTopLevelModule() == LeavingMod->getTopLevelModule()) + break; + } + if (auto *VisMD = dyn_cast(MD)) { // The latest visibility directive for a name in a submodule affects // all the directives that come before it. @@ -798,12 +809,6 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { if (Def || !Macro.getOverriddenMacros().empty()) addModuleMacro(LeavingMod, II, Def, Macro.getOverriddenMacros(), IsNew); - - if (!getLangOpts().ModulesLocalVisibility) { - // This macro is exposed to the rest of this compilation as a - // ModuleMacro; we don't need to track its MacroDirective any more. - Macro.setLatest(nullptr); - } break; } } -- cgit v1.2.3 From c9d8bd8ed2a5aa8c7e0934e12fc0bf38c78646da Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 12 May 2017 23:40:52 +0000 Subject: Remove unused tracking of owning module for MacroInfo objects. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@302966 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroInfo.cpp | 2 -- lib/Lex/PPDirectives.cpp | 27 ++------------------------- lib/Lex/Preprocessor.cpp | 7 +------ 3 files changed, 3 insertions(+), 33 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp index bec434085e..1e5deeb191 100644 --- a/lib/Lex/MacroInfo.cpp +++ b/lib/Lex/MacroInfo.cpp @@ -29,7 +29,6 @@ MacroInfo::MacroInfo(SourceLocation DefLoc) IsUsed(false), IsAllowRedefinitionsWithoutWarning(false), IsWarnIfUnused(false), - FromASTFile(false), UsedForHeaderGuard(false) { } @@ -137,7 +136,6 @@ LLVM_DUMP_METHOD void MacroInfo::dump() const { if (IsAllowRedefinitionsWithoutWarning) Out << " allow_redefinitions_without_warning"; if (IsWarnIfUnused) Out << " warn_if_unused"; - if (FromASTFile) Out << " imported"; if (UsedForHeaderGuard) Out << " header_guard"; Out << "\n #define "; diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 06fee8e5b0..faf8809e4e 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -54,35 +54,12 @@ using namespace clang; // Utility Methods for Preprocessor Directive Handling. //===----------------------------------------------------------------------===// -MacroInfo *Preprocessor::AllocateMacroInfo() { - MacroInfoChain *MIChain = BP.Allocate(); - MIChain->Next = MIChainHead; +MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) { + auto *MIChain = new (BP) MacroInfoChain{L, MIChainHead}; MIChainHead = MIChain; return &MIChain->MI; } -MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) { - MacroInfo *MI = AllocateMacroInfo(); - new (MI) MacroInfo(L); - return MI; -} - -MacroInfo *Preprocessor::AllocateDeserializedMacroInfo(SourceLocation L, - unsigned SubModuleID) { - static_assert(alignof(MacroInfo) >= sizeof(SubModuleID), - "alignment for MacroInfo is less than the ID"); - DeserializedMacroInfoChain *MIChain = - BP.Allocate(); - MIChain->Next = DeserialMIChainHead; - DeserialMIChainHead = MIChain; - - MacroInfo *MI = &MIChain->MI; - new (MI) MacroInfo(L); - MI->FromASTFile = true; - MI->setOwningModuleID(SubModuleID); - return MI; -} - DefMacroDirective *Preprocessor::AllocateDefMacroDirective(MacroInfo *MI, SourceLocation Loc) { return new (BP) DefMacroDirective(MI, Loc); diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index e409ab0365..dce8c1efda 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -88,7 +88,7 @@ Preprocessor::Preprocessor(std::shared_ptr PPOpts, CurDirLookup(nullptr), CurLexerKind(CLK_Lexer), CurLexerSubmodule(nullptr), Callbacks(nullptr), CurSubmoduleState(&NullSubmoduleState), MacroArgCache(nullptr), - Record(nullptr), MIChainHead(nullptr), DeserialMIChainHead(nullptr) { + Record(nullptr), MIChainHead(nullptr) { OwnsHeaderSearch = OwnsHeaders; CounterValue = 0; // __COUNTER__ starts at 0. @@ -169,11 +169,6 @@ Preprocessor::~Preprocessor() { std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr); CurTokenLexer.reset(); - while (DeserializedMacroInfoChain *I = DeserialMIChainHead) { - DeserialMIChainHead = I->Next; - I->~DeserializedMacroInfoChain(); - } - // Free any cached MacroArgs. for (MacroArgs *ArgList = MacroArgCache; ArgList;) ArgList = ArgList->deallocate(); -- cgit v1.2.3 From bd77dc6b8f04ac9afcb5487e91d7a5034521e2cc Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Wed, 17 May 2017 11:08:36 +0000 Subject: [Lexer] Ensure that the token is not an annotation token when retrieving the identifer info for an Objective-C keyword This commit fixes an assertion that's triggered in getIdentifier when the token is an annotation token. rdar://32225463 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303246 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 3d6fe91115..92942fd09a 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -43,6 +43,8 @@ using namespace clang; /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { + if (isAnnotation()) + return false; if (IdentifierInfo *II = getIdentifierInfo()) return II->getObjCKeywordID() == objcKey; return false; @@ -50,6 +52,8 @@ bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { /// getObjCKeywordID - Return the ObjC keyword kind. tok::ObjCKeywordKind Token::getObjCKeywordID() const { + if (isAnnotation()) + return tok::objc_not_keyword; IdentifierInfo *specId = getIdentifierInfo(); return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; } -- cgit v1.2.3 From 838eab83af5727fd23b3d052368e2c3bbe3be2a6 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 19 May 2017 23:32:38 +0000 Subject: [modules] Simplify module macro handling in non-local-submodule-visibility mode. When reaching the end of a module, we used to convert its macros to ModuleMacros but also leave them in the MacroDirective chain for the identifier. This meant that every lookup of such a macro would find two (identical) definitions. It also made it difficult to determine the correct owner for a macro when reaching the end of a module: the most recent MacroDirective in the chain could be from an #included submodule rather than the current module. Simplify this: whenever we convert a MacroDirective to a ModuleMacro when leaving a module, clear out the MacroDirective chain for that identifier, and just rely on the ModuleMacro to provide the macro definition information. (We don't want to do this for local submodule visibility mode, because in that mode we maintain a distinct MacroDirective chain for each submodule, and we need to keep around the prior MacroDirective in case we re-enter the submodule -- for instance, if its header is #included more than once in a module build, we need the include guard directive to stick around. But the problem doesn't arise in this case for the same reason: each submodule has its own MacroDirective chain, so the macros don't leak out of submodules in the first place.) This reinstates r302932, reverted in r302947, with a fix for a bug that resulted in us sometimes losing macro definitions due to failing to clear out the overridden module macro list when promoting a directive to a module macro. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303468 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 1938328c90..5a589d6a17 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -731,7 +731,7 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { Module *LeavingMod = Info.M; SourceLocation ImportLoc = Info.ImportLoc; - if (!needModuleMacros() || + if (!needModuleMacros() || (!getLangOpts().ModulesLocalVisibility && LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) { // If we don't need module macros, or this is not a module for which we @@ -777,17 +777,6 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) { assert(MD && "broken macro directive chain"); - // Stop on macros defined in other submodules of this module that we - // #included along the way. There's no point doing this if we're - // tracking local submodule visibility, since there can be no such - // directives in our list. - if (!getLangOpts().ModulesLocalVisibility) { - Module *Mod = getModuleContainingLocation(MD->getLocation()); - if (Mod != LeavingMod && - Mod->getTopLevelModule() == LeavingMod->getTopLevelModule()) - break; - } - if (auto *VisMD = dyn_cast(MD)) { // The latest visibility directive for a name in a submodule affects // all the directives that come before it. @@ -809,6 +798,13 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) { if (Def || !Macro.getOverriddenMacros().empty()) addModuleMacro(LeavingMod, II, Def, Macro.getOverriddenMacros(), IsNew); + + if (!getLangOpts().ModulesLocalVisibility) { + // This macro is exposed to the rest of this compilation as a + // ModuleMacro; we don't need to track its MacroDirective any more. + Macro.setLatest(nullptr); + Macro.setOverriddenMacros(*this, {}); + } break; } } -- cgit v1.2.3 From a41b9ecc818758cf022e4cd153c3e1cc65e6f43b Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 19 May 2017 23:49:00 +0000 Subject: Remove last (unnecessary) use of mapping from SourceLocation to Module and remove the mechanism for doing so. This mechanism was incorrect in the presence of preprocessed modules (and #pragma clang module begin/end). git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303469 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 33 --------------------------------- lib/Lex/PPDirectives.cpp | 13 +++---------- 2 files changed, 3 insertions(+), 43 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 6f44dc757e..1f7003a2a4 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -950,39 +950,6 @@ bool ModuleMap::resolveConflicts(Module *Mod, bool Complain) { return !Mod->UnresolvedConflicts.empty(); } -Module *ModuleMap::inferModuleFromLocation(FullSourceLoc Loc) { - if (Loc.isInvalid()) - return nullptr; - - if (UmbrellaDirs.empty() && Headers.empty()) - return nullptr; - - // Use the expansion location to determine which module we're in. - FullSourceLoc ExpansionLoc = Loc.getExpansionLoc(); - if (!ExpansionLoc.isFileID()) - return nullptr; - - const SourceManager &SrcMgr = Loc.getManager(); - FileID ExpansionFileID = ExpansionLoc.getFileID(); - - while (const FileEntry *ExpansionFile - = SrcMgr.getFileEntryForID(ExpansionFileID)) { - // Find the module that owns this header (if any). - if (Module *Mod = findModuleForHeader(ExpansionFile).getModule()) - return Mod; - - // No module owns this header, so look up the inclusion chain to see if - // any included header has an associated module. - SourceLocation IncludeLoc = SrcMgr.getIncludeLoc(ExpansionFileID); - if (IncludeLoc.isInvalid()) - return nullptr; - - ExpansionFileID = SrcMgr.getFileID(IncludeLoc); - } - - return nullptr; -} - //----------------------------------------------------------------------------// // Module map file parser //----------------------------------------------------------------------------// diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index faf8809e4e..0534aeb10c 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -667,24 +667,17 @@ Module *Preprocessor::getModuleForLocation(SourceLocation Loc) { : HeaderInfo.lookupModule(getLangOpts().CurrentModule); } -Module *Preprocessor::getModuleContainingLocation(SourceLocation Loc) { - return HeaderInfo.getModuleMap().inferModuleFromLocation( - FullSourceLoc(Loc, SourceMgr)); -} - const FileEntry * Preprocessor::getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, + Module *M, SourceLocation Loc) { + assert(M && "no module to include"); + // If we have a module import syntax, we shouldn't include a header to // make a particular module visible. if (getLangOpts().ObjC2) return nullptr; - // Figure out which module we'd want to import. - Module *M = getModuleContainingLocation(Loc); - if (!M) - return nullptr; - Module *TopM = M->getTopLevelModule(); Module *IncM = getModuleForLocation(IncLoc); -- cgit v1.2.3 From ecac99cbb4fc1411a821718d6c254333641c1198 Mon Sep 17 00:00:00 2001 From: Reid Kleckner Date: Mon, 22 May 2017 21:42:58 +0000 Subject: Give files from #line the characteristics of the current file This allows #line directives to appear in system headers that have code that clang would normally warn on. This is compatible with GCC, which is easy to test by running `gcc -E`. Fixes PR30752 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303582 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 42 ++++++++++++++++++++++-------------------- lib/Lex/Pragma.cpp | 4 ++-- 2 files changed, 24 insertions(+), 22 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 0534aeb10c..030717b8bd 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -1171,18 +1171,26 @@ void Preprocessor::HandleLineDirective() { CheckEndOfDirective("line", true); } - SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID); + // Take the file kind of the file containing the #line directive. #line + // directives are often used for generated sources from the same codebase, so + // the new file should generally be classified the same way as the current + // file. This is visible in GCC's pre-processed output, which rewrites #line + // to GNU line markers. + SrcMgr::CharacteristicKind FileKind = + SourceMgr.getFileCharacteristic(DigitTok.getLocation()); + + SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID, false, + false, FileKind); if (Callbacks) Callbacks->FileChanged(CurPPLexer->getSourceLocation(), - PPCallbacks::RenameFile, - SrcMgr::C_User); + PPCallbacks::RenameFile, FileKind); } /// ReadLineMarkerFlags - Parse and validate any flags at the end of a GNU line /// marker directive. static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit, - bool &IsSystemHeader, bool &IsExternCHeader, + SrcMgr::CharacteristicKind &FileKind, Preprocessor &PP) { unsigned FlagVal; Token FlagTok; @@ -1233,7 +1241,7 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit, return true; } - IsSystemHeader = true; + FileKind = SrcMgr::C_System; PP.Lex(FlagTok); if (FlagTok.is(tok::eod)) return false; @@ -1247,7 +1255,7 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit, return true; } - IsExternCHeader = true; + FileKind = SrcMgr::C_ExternCSystem; PP.Lex(FlagTok); if (FlagTok.is(tok::eod)) return false; @@ -1277,14 +1285,15 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) { Lex(StrTok); bool IsFileEntry = false, IsFileExit = false; - bool IsSystemHeader = false, IsExternCHeader = false; int FilenameID = -1; + SrcMgr::CharacteristicKind FileKind = SrcMgr::C_User; // If the StrTok is "eod", then it wasn't present. Otherwise, it must be a // string followed by eod. - if (StrTok.is(tok::eod)) - ; // ok - else if (StrTok.isNot(tok::string_literal)) { + if (StrTok.is(tok::eod)) { + // Treat this like "#line NN", which doesn't change file characteristics. + FileKind = SourceMgr.getFileCharacteristic(DigitTok.getLocation()); + } else if (StrTok.isNot(tok::string_literal)) { Diag(StrTok, diag::err_pp_linemarker_invalid_filename); return DiscardUntilEndOfDirective(); } else if (StrTok.hasUDSuffix()) { @@ -1303,15 +1312,13 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) { FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString()); // If a filename was present, read any flags that are present. - if (ReadLineMarkerFlags(IsFileEntry, IsFileExit, - IsSystemHeader, IsExternCHeader, *this)) + if (ReadLineMarkerFlags(IsFileEntry, IsFileExit, FileKind, *this)) return; } // Create a line note with this information. - SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID, - IsFileEntry, IsFileExit, - IsSystemHeader, IsExternCHeader); + SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID, IsFileEntry, + IsFileExit, FileKind); // If the preprocessor has callbacks installed, notify them of the #line // change. This is used so that the line marker comes out in -E mode for @@ -1322,11 +1329,6 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) { Reason = PPCallbacks::EnterFile; else if (IsFileExit) Reason = PPCallbacks::ExitFile; - SrcMgr::CharacteristicKind FileKind = SrcMgr::C_User; - if (IsExternCHeader) - FileKind = SrcMgr::C_ExternCSystem; - else if (IsSystemHeader) - FileKind = SrcMgr::C_System; Callbacks->FileChanged(CurPPLexer->getSourceLocation(), Reason, FileKind); } diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index 99d56182c1..2d078a4e76 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -475,9 +475,9 @@ void Preprocessor::HandlePragmaSystemHeader(Token &SysHeaderTok) { // Emit a line marker. This will change any source locations from this point // forward to realize they are in a system header. // Create a line note with this information. - SourceMgr.AddLineNote(SysHeaderTok.getLocation(), PLoc.getLine()+1, + SourceMgr.AddLineNote(SysHeaderTok.getLocation(), PLoc.getLine() + 1, FilenameID, /*IsEntry=*/false, /*IsExit=*/false, - /*IsSystem=*/true, /*IsExternC=*/false); + SrcMgr::C_System); } /// HandlePragmaDependency - Handle \#pragma GCC dependency "foo" blah. -- cgit v1.2.3 From ed911ada8c59bf5af2d5a1476e8a5b48893ffc22 Mon Sep 17 00:00:00 2001 From: Tim Northover Date: Tue, 23 May 2017 21:41:49 +0000 Subject: Sema: allow imaginary constants via GNU extension if UDL overloads not present. C++14 added user-defined literal support for complex numbers so that you can write something like "complex val = 2i". However, there is an existing GNU extension supporting this syntax and interpreting the result as a _Complex type. This changes parsing so that such literals are interpreted in terms of C++14's operators if an overload is present but otherwise falls back to the original GNU extension. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303694 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index fbfd3fe5cc..91993b1e41 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -652,9 +652,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, break; } } - // "i", "if", and "il" are user-defined suffixes in C++1y. - if (*s == 'i' && PP.getLangOpts().CPlusPlus14) - break; // fall through. case 'j': case 'J': @@ -667,36 +664,34 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, break; } - if (s != ThisTokEnd) { + // "i", "if", and "il" are user-defined suffixes in C++1y. + if (s != ThisTokEnd || isImaginary) { // FIXME: Don't bother expanding UCNs if !tok.hasUCN(). expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)); if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) { - // Any suffix pieces we might have parsed are actually part of the - // ud-suffix. - isLong = false; - isUnsigned = false; - isLongLong = false; - isFloat = false; - isHalf = false; - isImaginary = false; - MicrosoftInteger = 0; + if (!isImaginary) { + // Any suffix pieces we might have parsed are actually part of the + // ud-suffix. + isLong = false; + isUnsigned = false; + isLongLong = false; + isFloat = false; + isHalf = false; + isImaginary = false; + MicrosoftInteger = 0; + } saw_ud_suffix = true; return; } - // Report an error if there are any. - PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), - diag::err_invalid_suffix_constant) - << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin) << isFPConstant; - hadError = true; - return; - } - - if (isImaginary) { - PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, - ImaginarySuffixLoc - ThisTokBegin), - diag::ext_imaginary_constant); + if (s != ThisTokEnd) { + // Report an error if there are any. + PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), + diag::err_invalid_suffix_constant) + << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin) << isFPConstant; + hadError = true; + } } } -- cgit v1.2.3 From b4d1da0e78733b71e791726473e28d513a7e5ece Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Wed, 24 May 2017 00:46:27 +0000 Subject: Enhance the 'diagnose_if' attribute so that we can apply it for ObjC methods and properties as well This is an initial commit to allow using it with constant expressions, a follow-up commit will enable full support for it in ObjC methods. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303712 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPMacroExpansion.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/Lex') diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 6c7663994a..2f141a1b20 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1166,6 +1166,7 @@ static bool HasFeature(const Preprocessor &PP, StringRef Feature) { .Case("objc_generics", LangOpts.ObjC2) .Case("objc_generics_variance", LangOpts.ObjC2) .Case("objc_class_property", LangOpts.ObjC2) + .Case("objc_diagnose_if_attr", LangOpts.ObjC2) // C11 features .Case("c_alignas", LangOpts.C11) .Case("c_alignof", LangOpts.C11) -- cgit v1.2.3 From f54f06399b3670bbdce87b2ded7719f6633229a2 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Wed, 24 May 2017 01:38:00 +0000 Subject: Change __has_feature(objc_diagnose_if_attr) to __has_feature(attribute_diagnose_if_objc) for consistency with rest of attribute checks. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303713 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPMacroExpansion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 2f141a1b20..a6bfc32e22 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1125,6 +1125,7 @@ static bool HasFeature(const Preprocessor &PP, StringRef Feature) { .Case("attribute_overloadable", true) .Case("attribute_unavailable_with_message", true) .Case("attribute_unused_on_fields", true) + .Case("attribute_diagnose_if_objc", true) .Case("blocks", LangOpts.Blocks) .Case("c_thread_safety_attributes", true) .Case("cxx_exceptions", LangOpts.CXXExceptions) @@ -1166,7 +1167,6 @@ static bool HasFeature(const Preprocessor &PP, StringRef Feature) { .Case("objc_generics", LangOpts.ObjC2) .Case("objc_generics_variance", LangOpts.ObjC2) .Case("objc_class_property", LangOpts.ObjC2) - .Case("objc_diagnose_if_attr", LangOpts.ObjC2) // C11 features .Case("c_alignas", LangOpts.C11) .Case("c_alignof", LangOpts.C11) -- cgit v1.2.3 From 6c360dcc838e90d6fad8a2501b916c7744967c29 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 24 May 2017 10:38:09 +0000 Subject: Fix 'set but not used' [-Wunused-but-set-variable] warning git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303734 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index 91993b1e41..1fead55e80 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -563,7 +563,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, // Parse the suffix. At this point we can classify whether we have an FP or // integer constant. bool isFPConstant = isFloatingLiteral(); - const char *ImaginarySuffixLoc = nullptr; // Loop over all of the characters of the suffix. If we see something bad, // we break out of the loop. @@ -657,7 +656,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, case 'J': if (isImaginary) break; // Cannot be repeated. isImaginary = true; - ImaginarySuffixLoc = s; continue; // Success. } // If we reached here, there was an error or a ud-suffix. -- cgit v1.2.3 From 03e95d2ffb90719219513cabb351bbd88b2ee6d2 Mon Sep 17 00:00:00 2001 From: Tim Northover Date: Wed, 24 May 2017 22:18:35 +0000 Subject: Revert "Sema: allow imaginary constants via GNU extension if UDL overloads not present." This reverts commit r303697. It broke libc++ tests that were specifically checking incompatibility in C++14 mode. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303813 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index 1fead55e80..1e2cbde825 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -651,6 +651,9 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, break; } } + // "i", "if", and "il" are user-defined suffixes in C++1y. + if (*s == 'i' && PP.getLangOpts().CPlusPlus14) + break; // fall through. case 'j': case 'J': @@ -662,34 +665,35 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, break; } - // "i", "if", and "il" are user-defined suffixes in C++1y. - if (s != ThisTokEnd || isImaginary) { + if (s != ThisTokEnd) { // FIXME: Don't bother expanding UCNs if !tok.hasUCN(). expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)); if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) { - if (!isImaginary) { - // Any suffix pieces we might have parsed are actually part of the - // ud-suffix. - isLong = false; - isUnsigned = false; - isLongLong = false; - isFloat = false; - isHalf = false; - isImaginary = false; - MicrosoftInteger = 0; - } + // Any suffix pieces we might have parsed are actually part of the + // ud-suffix. + isLong = false; + isUnsigned = false; + isLongLong = false; + isFloat = false; + isHalf = false; + isImaginary = false; + MicrosoftInteger = 0; saw_ud_suffix = true; return; } - if (s != ThisTokEnd) { - // Report an error if there are any. - PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), - diag::err_invalid_suffix_constant) - << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin) << isFPConstant; - hadError = true; - } + // Report an error if there are any. + PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), + diag::err_invalid_suffix_constant) + << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin) << isFPConstant; + hadError = true; + return; + } + + if (isImaginary) { + PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), + diag::ext_imaginary_constant); } } -- cgit v1.2.3 From bc7fbfa106a523b15654c40e21fca1475339fad8 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 26 May 2017 00:01:53 +0000 Subject: Factor resolving of header directives -> files out of module map parser. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@303945 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 201 ++++++++++++++++++++++++++------------------------ 1 file changed, 104 insertions(+), 97 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 1f7003a2a4..8c57931e47 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -84,6 +84,90 @@ Module *ModuleMap::resolveModuleId(const ModuleId &Id, Module *Mod, return Context; } +/// \brief Append to \p Paths the set of paths needed to get to the +/// subframework in which the given module lives. +static void appendSubframeworkPaths(Module *Mod, + SmallVectorImpl &Path) { + // Collect the framework names from the given module to the top-level module. + SmallVector Paths; + for (; Mod; Mod = Mod->Parent) { + if (Mod->IsFramework) + Paths.push_back(Mod->Name); + } + + if (Paths.empty()) + return; + + // Add Frameworks/Name.framework for each subframework. + for (unsigned I = Paths.size() - 1; I != 0; --I) + llvm::sys::path::append(Path, "Frameworks", Paths[I-1] + ".framework"); +} + +const FileEntry * +ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header, + SmallVectorImpl &RelativePathName) { + if (llvm::sys::path::is_absolute(Header.FileName)) { + RelativePathName.clear(); + RelativePathName.append(Header.FileName.begin(), Header.FileName.end()); + return SourceMgr.getFileManager().getFile(Header.FileName); + } + + // Search for the header file within the module's home directory. + auto *Directory = M->Directory; + SmallString<128> FullPathName(Directory->getName()); + unsigned FullPathLength = FullPathName.size(); + + if (M->isPartOfFramework()) { + appendSubframeworkPaths(M, RelativePathName); + unsigned RelativePathLength = RelativePathName.size(); + + // Check whether this file is in the public headers. + llvm::sys::path::append(RelativePathName, "Headers", Header.FileName); + llvm::sys::path::append(FullPathName, RelativePathName); + if (auto *File = SourceMgr.getFileManager().getFile(FullPathName)) + return File; + + // Check whether this file is in the private headers. + // Ideally, private modules in the form 'FrameworkName.Private' should + // be defined as 'module FrameworkName.Private', and not as + // 'framework module FrameworkName.Private', since a 'Private.Framework' + // does not usually exist. However, since both are currently widely used + // for private modules, make sure we find the right path in both cases. + if (M->IsFramework && M->Name == "Private") + RelativePathName.clear(); + else + RelativePathName.resize(RelativePathLength); + FullPathName.resize(FullPathLength); + llvm::sys::path::append(RelativePathName, "PrivateHeaders", + Header.FileName); + llvm::sys::path::append(FullPathName, RelativePathName); + return SourceMgr.getFileManager().getFile(FullPathName); + } + + // Lookup for normal headers. + llvm::sys::path::append(RelativePathName, Header.FileName); + llvm::sys::path::append(FullPathName, RelativePathName); + return SourceMgr.getFileManager().getFile(FullPathName); +} + +const FileEntry * +ModuleMap::resolveAsBuiltinHeader(Module *M, + Module::UnresolvedHeaderDirective Header, + SmallVectorImpl &BuiltinPathName) { + if (llvm::sys::path::is_absolute(Header.FileName) || M->isPartOfFramework() || + !M->IsSystem || Header.IsUmbrella || !BuiltinIncludeDir || + BuiltinIncludeDir == M->Directory || !isBuiltinHeader(Header.FileName)) + return nullptr; + + // This is a system module with a top-level header. This header + // may have a counterpart (or replacement) in the set of headers + // supplied by Clang. Find that builtin header. + llvm::sys::path::append(BuiltinPathName, BuiltinIncludeDir->getName(), + Header.FileName); + return SourceMgr.getFileManager().getFile( + StringRef(BuiltinPathName.data(), BuiltinPathName.size())); +} + ModuleMap::ModuleMap(SourceManager &SourceMgr, DiagnosticsEngine &Diags, const LangOptions &LangOpts, const TargetInfo *Target, HeaderSearch &HeaderInfo) @@ -1026,9 +1110,6 @@ namespace clang { /// be resolved relative to. const DirectoryEntry *Directory; - /// \brief The directory containing Clang-supplied headers. - const DirectoryEntry *BuiltinIncludeDir; - /// \brief Whether this module map is in a system header directory. bool IsSystem; @@ -1087,12 +1168,10 @@ namespace clang { ModuleMap &Map, const FileEntry *ModuleMapFile, const DirectoryEntry *Directory, - const DirectoryEntry *BuiltinIncludeDir, bool IsSystem) : L(L), SourceMgr(SourceMgr), Target(Target), Diags(Diags), Map(Map), ModuleMapFile(ModuleMapFile), Directory(Directory), - BuiltinIncludeDir(BuiltinIncludeDir), IsSystem(IsSystem), - HadError(false), ActiveModule(nullptr) + IsSystem(IsSystem), HadError(false), ActiveModule(nullptr) { Tok.clear(); consumeToken(); @@ -1772,25 +1851,6 @@ void ModuleMapParser::parseRequiresDecl() { } while (true); } -/// \brief Append to \p Paths the set of paths needed to get to the -/// subframework in which the given module lives. -static void appendSubframeworkPaths(Module *Mod, - SmallVectorImpl &Path) { - // Collect the framework names from the given module to the top-level module. - SmallVector Paths; - for (; Mod; Mod = Mod->Parent) { - if (Mod->IsFramework) - Paths.push_back(Mod->Name); - } - - if (Paths.empty()) - return; - - // Add Frameworks/Name.framework for each subframework. - for (unsigned I = Paths.size() - 1; I != 0; --I) - llvm::sys::path::append(Path, "Frameworks", Paths[I-1] + ".framework"); -} - /// \brief Parse a header declaration. /// /// header-declaration: @@ -1843,85 +1903,36 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, Module::UnresolvedHeaderDirective Header; Header.FileName = Tok.getString(); Header.FileNameLoc = consumeToken(); + Header.IsUmbrella = LeadingToken == MMToken::UmbrellaKeyword; // Check whether we already have an umbrella. - if (LeadingToken == MMToken::UmbrellaKeyword && ActiveModule->Umbrella) { + if (Header.IsUmbrella && ActiveModule->Umbrella) { Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash) << ActiveModule->getFullModuleName(); HadError = true; return; } - // Look for this file. - const FileEntry *File = nullptr; - const FileEntry *BuiltinFile = nullptr; - SmallString<128> RelativePathName; - if (llvm::sys::path::is_absolute(Header.FileName)) { - RelativePathName = Header.FileName; - File = SourceMgr.getFileManager().getFile(RelativePathName); - } else { - // Search for the header file within the search directory. - SmallString<128> FullPathName(Directory->getName()); - unsigned FullPathLength = FullPathName.size(); - - if (ActiveModule->isPartOfFramework()) { - appendSubframeworkPaths(ActiveModule, RelativePathName); - unsigned RelativePathLength = RelativePathName.size(); - - // Check whether this file is in the public headers. - llvm::sys::path::append(RelativePathName, "Headers", Header.FileName); - llvm::sys::path::append(FullPathName, RelativePathName); - File = SourceMgr.getFileManager().getFile(FullPathName); - - // Check whether this file is in the private headers. - if (!File) { - // Ideally, private modules in the form 'FrameworkName.Private' should - // be defined as 'module FrameworkName.Private', and not as - // 'framework module FrameworkName.Private', since a 'Private.Framework' - // does not usually exist. However, since both are currently widely used - // for private modules, make sure we find the right path in both cases. - if (ActiveModule->IsFramework && ActiveModule->Name == "Private") - RelativePathName.clear(); - else - RelativePathName.resize(RelativePathLength); - FullPathName.resize(FullPathLength); - llvm::sys::path::append(RelativePathName, "PrivateHeaders", - Header.FileName); - llvm::sys::path::append(FullPathName, RelativePathName); - File = SourceMgr.getFileManager().getFile(FullPathName); - } - } else { - // Lookup for normal headers. - llvm::sys::path::append(RelativePathName, Header.FileName); - llvm::sys::path::append(FullPathName, RelativePathName); - File = SourceMgr.getFileManager().getFile(FullPathName); - - // If this is a system module with a top-level header, this header - // may have a counterpart (or replacement) in the set of headers - // supplied by Clang. Find that builtin header. - if (ActiveModule->IsSystem && LeadingToken != MMToken::UmbrellaKeyword && - BuiltinIncludeDir && BuiltinIncludeDir != Directory && - ModuleMap::isBuiltinHeader(Header.FileName)) { - SmallString<128> BuiltinPathName(BuiltinIncludeDir->getName()); - llvm::sys::path::append(BuiltinPathName, Header.FileName); - BuiltinFile = SourceMgr.getFileManager().getFile(BuiltinPathName); - - // If Clang supplies this header but the underlying system does not, - // just silently swap in our builtin version. Otherwise, we'll end - // up adding both (later). - if (BuiltinFile && !File) { - File = BuiltinFile; - RelativePathName = BuiltinPathName; - BuiltinFile = nullptr; - } - } - } + // Look for this file by name if we don't have any stat information. + SmallString<128> RelativePathName, BuiltinPathName; + const FileEntry *File = + Map.resolveHeader(ActiveModule, Header, RelativePathName); + const FileEntry *BuiltinFile = + Map.resolveAsBuiltinHeader(ActiveModule, Header, BuiltinPathName); + + // If Clang supplies this header but the underlying system does not, + // just silently swap in our builtin version. Otherwise, we'll end + // up adding both (later). + if (BuiltinFile && !File) { + RelativePathName = BuiltinPathName; + File = BuiltinFile; + BuiltinFile = nullptr; } // FIXME: We shouldn't be eagerly stat'ing every file named in a module map. // Come up with a lazy way to do this. if (File) { - if (LeadingToken == MMToken::UmbrellaKeyword) { + if (Header.IsUmbrella) { const DirectoryEntry *UmbrellaDir = File->getDir(); if (Module *UmbrellaModule = Map.UmbrellaDirs[UmbrellaDir]) { Diags.Report(LeadingLoc, diag::err_mmap_umbrella_clash) @@ -1938,10 +1949,7 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, // If there is a builtin counterpart to this file, add it now so it can // wrap the system header. if (BuiltinFile) { - // FIXME: Taking the name from the FileEntry is unstable and can give - // different results depending on how we've previously named that file - // in this build. - Module::Header H = { BuiltinFile->getName(), BuiltinFile }; + Module::Header H = { BuiltinPathName.str(), BuiltinFile }; Map.addHeader(ActiveModule, H, Role); // If we have both a builtin and system version of the file, the @@ -1960,7 +1968,6 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, // If we find a module that has a missing header, we mark this module as // unavailable and store the header directive for displaying diagnostics. - Header.IsUmbrella = LeadingToken == MMToken::UmbrellaKeyword; ActiveModule->markUnavailable(); ActiveModule->MissingHeaders.push_back(Header); } @@ -2555,7 +2562,7 @@ bool ModuleMap::parseModuleMapFile(const FileEntry *File, bool IsSystem, Buffer->getBufferEnd()); SourceLocation Start = L.getSourceLocation(); ModuleMapParser Parser(L, SourceMgr, Target, Diags, *this, File, Dir, - BuiltinIncludeDir, IsSystem); + IsSystem); bool Result = Parser.parseModuleMapFile(); ParsedModuleMap[File] = Result; -- cgit v1.2.3 From 86d77d167e74736ccd991ddae2933d54589a9473 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Tue, 30 May 2017 02:03:19 +0000 Subject: [modules] When we #include a local submodule header that we've already built, and it has an include guard, produce callbacks for a module import, not for a skipped non-modular header. Fixes -E output when preprocessing a module to list these cases as a module import, rather than suppressing the #include and losing the import side effect. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304183 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 030717b8bd..8b5877934f 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -1906,6 +1906,25 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, } } + // The #included file will be considered to be a system header if either it is + // in a system include directory, or if the #includer is a system include + // header. + SrcMgr::CharacteristicKind FileCharacter = + SourceMgr.getFileCharacteristic(FilenameTok.getLocation()); + if (File) + FileCharacter = std::max(HeaderInfo.getFileDirFlavor(File), FileCharacter); + + // Ask HeaderInfo if we should enter this #include file. If not, #including + // this file will have no effect. + bool SkipHeader = false; + if (ShouldEnter && File && + !HeaderInfo.ShouldEnterIncludeFile(*this, File, isImport, + getLangOpts().Modules, + SuggestedModule.getModule())) { + ShouldEnter = false; + SkipHeader = true; + } + if (Callbacks) { // Notify the callback object that we've seen an inclusion directive. Callbacks->InclusionDirective( @@ -1913,18 +1932,13 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, isAngled, FilenameRange, File, SearchPath, RelativePath, ShouldEnter ? nullptr : SuggestedModule.getModule()); + if (SkipHeader && !SuggestedModule.getModule()) + Callbacks->FileSkipped(*File, FilenameTok, FileCharacter); } if (!File) return; - // The #included file will be considered to be a system header if either it is - // in a system include directory, or if the #includer is a system include - // header. - SrcMgr::CharacteristicKind FileCharacter = - std::max(HeaderInfo.getFileDirFlavor(File), - SourceMgr.getFileCharacteristic(FilenameTok.getLocation())); - // FIXME: If we have a suggested module, and we've already visited this file, // don't bother entering it again. We know it has no further effect. @@ -1964,19 +1978,6 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, } } - // Ask HeaderInfo if we should enter this #include file. If not, #including - // this file will have no effect. - bool SkipHeader = false; - if (ShouldEnter && - !HeaderInfo.ShouldEnterIncludeFile(*this, File, isImport, - getLangOpts().Modules, - SuggestedModule.getModule())) { - ShouldEnter = false; - SkipHeader = true; - if (Callbacks) - Callbacks->FileSkipped(*File, FilenameTok, FileCharacter); - } - // If we don't need to enter the file, stop now. if (!ShouldEnter) { // If this is a module import, make it visible if needed. -- cgit v1.2.3 From bb482b389e541428d0fa93c244d54d22eef3a384 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Tue, 30 May 2017 05:22:59 +0000 Subject: Diagnose attempts to build a preprocessed module that defines an unavailable submodule. The errors we would otherwise get are incomprehensible, as we would enter the module but not make its contents visible to itself. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304190 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Pragma.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index 2d078a4e76..e1d981527b 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -1407,6 +1407,24 @@ struct PragmaModuleBeginHandler : public PragmaHandler { M = NewM; } + // If the module isn't available, it doesn't make sense to enter it. + if (!M->isAvailable()) { + Module::Requirement Requirement; + Module::UnresolvedHeaderDirective MissingHeader; + (void)M->isAvailable(PP.getLangOpts(), PP.getTargetInfo(), + Requirement, MissingHeader); + if (MissingHeader.FileNameLoc.isValid()) { + PP.Diag(MissingHeader.FileNameLoc, diag::err_module_header_missing) + << MissingHeader.IsUmbrella << MissingHeader.FileName; + } else { + PP.Diag(M->DefinitionLoc, diag::err_module_unavailable) + << M->getFullModuleName() << Requirement.second << Requirement.first; + } + PP.Diag(BeginLoc, diag::note_pp_module_begin_here) + << M->getTopLevelModuleName(); + return; + } + // Enter the scope of the submodule. PP.EnterSubmodule(M, BeginLoc, /*ForPragma*/true); PP.EnterAnnotationToken(SourceRange(BeginLoc, ModuleName.back().second), -- cgit v1.2.3 From 0289cea07c4cf50b36f6f9c9e60ac2d35c624946 Mon Sep 17 00:00:00 2001 From: Erik Verbruggen Date: Tue, 30 May 2017 11:54:55 +0000 Subject: Allow for unfinished #if blocks in preambles Previously, a preamble only included #if blocks (and friends like ifdef) if there was a corresponding #endif before any declaration or definition. The problem is that any header file that uses include guards will not have a preamble generated, which can make code-completion very slow. To prevent errors about unbalanced preprocessor conditionals in the preamble, and unbalanced preprocessor conditionals after a preamble containing unfinished conditionals, the conditional stack is stored in the pch file. This fixes PR26045. Differential Revision: http://reviews.llvm.org/D15994 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304207 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 39 +++++++++++---------------------------- lib/Lex/PPLexerChange.cpp | 6 ++++++ lib/Lex/Preprocessor.cpp | 9 +++++++++ 3 files changed, 26 insertions(+), 28 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 92942fd09a..f5a35e97d6 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -550,8 +550,6 @@ namespace { enum PreambleDirectiveKind { PDK_Skipped, - PDK_StartIf, - PDK_EndIf, PDK_Unknown }; @@ -574,8 +572,6 @@ std::pair Lexer::ComputePreamble(StringRef Buffer, bool InPreprocessorDirective = false; Token TheTok; - Token IfStartTok; - unsigned IfCount = 0; SourceLocation ActiveCommentLoc; unsigned MaxLineOffset = 0; @@ -658,33 +654,18 @@ std::pair Lexer::ComputePreamble(StringRef Buffer, .Case("sccs", PDK_Skipped) .Case("assert", PDK_Skipped) .Case("unassert", PDK_Skipped) - .Case("if", PDK_StartIf) - .Case("ifdef", PDK_StartIf) - .Case("ifndef", PDK_StartIf) + .Case("if", PDK_Skipped) + .Case("ifdef", PDK_Skipped) + .Case("ifndef", PDK_Skipped) .Case("elif", PDK_Skipped) .Case("else", PDK_Skipped) - .Case("endif", PDK_EndIf) + .Case("endif", PDK_Skipped) .Default(PDK_Unknown); switch (PDK) { case PDK_Skipped: continue; - case PDK_StartIf: - if (IfCount == 0) - IfStartTok = HashTok; - - ++IfCount; - continue; - - case PDK_EndIf: - // Mismatched #endif. The preamble ends here. - if (IfCount == 0) - break; - - --IfCount; - continue; - case PDK_Unknown: // We don't know what this directive is; stop at the '#'. break; @@ -705,16 +686,13 @@ std::pair Lexer::ComputePreamble(StringRef Buffer, } while (true); SourceLocation End; - if (IfCount) - End = IfStartTok.getLocation(); - else if (ActiveCommentLoc.isValid()) + if (ActiveCommentLoc.isValid()) End = ActiveCommentLoc; // don't truncate a decl comment. else End = TheTok.getLocation(); return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), - IfCount? IfStartTok.isAtStartOfLine() - : TheTok.isAtStartOfLine()); + TheTok.isAtStartOfLine()); } /// AdvanceToTokenCharacter - Given a location that specifies the start of a @@ -2570,6 +2548,11 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { return true; } + if (PP->isRecordingPreamble() && !PP->isInMainFile()) { + PP->setRecordedPreambleConditionalStack(ConditionalStack); + ConditionalStack.clear(); + } + // Issue diagnostics for unterminated #if and missing newline. // If we are in a #if directive, emit an error. diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 5a589d6a17..1c0cd56368 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -46,6 +46,12 @@ bool Preprocessor::isInPrimaryFile() const { }); } +bool Preprocessor::isInMainFile() const { + if (IsFileLexer()) + return IncludeMacroStack.size() == 0; + return true; +} + /// getCurrentLexer - Return the current file lexer being lexed from. Note /// that this ignores any potentially active macro expansions and _Pragma /// expansions going on at the time. diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index dce8c1efda..3596337c24 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -150,6 +150,9 @@ Preprocessor::Preprocessor(std::shared_ptr PPOpts, Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr; Ident_AbnormalTermination = nullptr; } + + if (this->PPOpts->GeneratePreamble) + PreambleConditionalStack.startRecording(); } Preprocessor::~Preprocessor() { @@ -532,6 +535,12 @@ void Preprocessor::EnterMainSourceFile() { // Start parsing the predefines. EnterSourceFile(FID, nullptr, SourceLocation()); + + // Restore the conditional stack from the preamble, if there is one. + if (PreambleConditionalStack.isReplaying()) { + CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); + PreambleConditionalStack.doneReplaying(); + } } void Preprocessor::EndSourceFile() { -- cgit v1.2.3 From 62955aa44343371b203bc2103cad2f75fd91622c Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Wed, 31 May 2017 20:56:55 +0000 Subject: [modules] When compiling a preprocessed module map, look for headers relative to the original module map. Also use the path and name of the original module map when emitting that information into the .pcm file. The upshot of this is that the produced .pcm file will track information for headers in their original locations (where the module was preprocessed), not relative to whatever directory the preprocessed module map was in when it was built. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304346 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index f5b7c59e44..9084bc352f 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -1326,14 +1326,27 @@ static const FileEntry *getPrivateModuleMap(const FileEntry *File, } bool HeaderSearch::loadModuleMapFile(const FileEntry *File, bool IsSystem, - FileID ID, unsigned *Offset) { + FileID ID, unsigned *Offset, + StringRef OriginalModuleMapFile) { // Find the directory for the module. For frameworks, that may require going // up from the 'Modules' directory. const DirectoryEntry *Dir = nullptr; if (getHeaderSearchOpts().ModuleMapFileHomeIsCwd) Dir = FileMgr.getDirectory("."); else { - Dir = File->getDir(); + if (!OriginalModuleMapFile.empty()) { + // We're building a preprocessed module map. Find or invent the directory + // that it originally occupied. + Dir = FileMgr.getDirectory( + llvm::sys::path::parent_path(OriginalModuleMapFile)); + if (!Dir) { + auto *FakeFile = FileMgr.getVirtualFile(OriginalModuleMapFile, 0, 0); + Dir = FakeFile->getDir(); + } + } else { + Dir = File->getDir(); + } + StringRef DirName(Dir->getName()); if (llvm::sys::path::filename(DirName) == "Modules") { DirName = llvm::sys::path::parent_path(DirName); -- cgit v1.2.3 From 146ecad762cb83f8e00d8027188b622c6a8ce15d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 2 Jun 2017 01:55:39 +0000 Subject: Support lazy stat'ing of files referenced by module maps. This patch adds support for a `header` declaration in a module map to specify certain `stat` information (currently, size and mtime) about that header file. This has two purposes: - It removes the need to eagerly `stat` every file referenced by a module map. Instead, we track a list of unresolved header files with each size / mtime (actually, for simplicity, we track submodules with such headers), and when attempting to look up a header file based on a `FileEntry`, we check if there are any unresolved header directives with that `FileEntry`'s size / mtime and perform deferred `stat`s if so. - It permits a preprocessed module to be compiled without the original files being present on disk. The only reason we used to need those files was to get the `stat` information in order to do header -> module lookups when using the module. If we're provided with the `stat` information in the preprocessed module, we can avoid requiring the files to exist. Unlike most `header` directives, if a `header` directive with `stat` information has no corresponding on-disk file the enclosing module is *not* marked unavailable (so that behavior is consistent regardless of whether we've resolved a header directive, and so that preprocessed modules don't get marked unavailable). We could actually do this for all `header` directives: the only reason we mark the module unavailable if headers are missing is to give a diagnostic slightly earlier (rather than waiting until we actually try to build the module / load and validate its .pcm file). Differential Revision: https://reviews.llvm.org/D33703 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304515 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 2 + lib/Lex/ModuleMap.cpp | 334 +++++++++++++++++++++++++++++++++++------------ lib/Lex/PPDirectives.cpp | 2 + 3 files changed, 254 insertions(+), 84 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index 9084bc352f..1ebcc0a1c6 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -1114,6 +1114,8 @@ bool HeaderSearch::ShouldEnterIncludeFile(Preprocessor &PP, auto TryEnterImported = [&](void) -> bool { if (!ModulesEnabled) return false; + // Ensure FileInfo bits are up to date. + ModMap.resolveHeaderDirectives(File); // Modules with builtins are special; multiple modules use builtins as // modular headers, example: // diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 8c57931e47..018d59e5e8 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -36,6 +36,37 @@ #endif using namespace clang; +Module::HeaderKind ModuleMap::headerRoleToKind(ModuleHeaderRole Role) { + switch ((int)Role) { + default: llvm_unreachable("unknown header role"); + case NormalHeader: + return Module::HK_Normal; + case PrivateHeader: + return Module::HK_Private; + case TextualHeader: + return Module::HK_Textual; + case PrivateHeader | TextualHeader: + return Module::HK_PrivateTextual; + } +} + +ModuleMap::ModuleHeaderRole +ModuleMap::headerKindToRole(Module::HeaderKind Kind) { + switch ((int)Kind) { + case Module::HK_Normal: + return NormalHeader; + case Module::HK_Private: + return PrivateHeader; + case Module::HK_Textual: + return TextualHeader; + case Module::HK_PrivateTextual: + return ModuleHeaderRole(PrivateHeader | TextualHeader); + case Module::HK_Excluded: + llvm_unreachable("unexpected header kind"); + } + llvm_unreachable("unknown header kind"); +} + Module::ExportDecl ModuleMap::resolveExport(Module *Mod, const Module::UnresolvedExportDecl &Unresolved, @@ -104,12 +135,22 @@ static void appendSubframeworkPaths(Module *Mod, } const FileEntry * -ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header, - SmallVectorImpl &RelativePathName) { +ModuleMap::findHeader(Module *M, + const Module::UnresolvedHeaderDirective &Header, + SmallVectorImpl &RelativePathName) { + auto GetFile = [&](StringRef Filename) -> const FileEntry * { + auto *File = SourceMgr.getFileManager().getFile(Filename); + if (!File || + (Header.Size && File->getSize() != *Header.Size) || + (Header.ModTime && File->getModificationTime() != *Header.ModTime)) + return nullptr; + return File; + }; + if (llvm::sys::path::is_absolute(Header.FileName)) { RelativePathName.clear(); RelativePathName.append(Header.FileName.begin(), Header.FileName.end()); - return SourceMgr.getFileManager().getFile(Header.FileName); + return GetFile(Header.FileName); } // Search for the header file within the module's home directory. @@ -124,7 +165,7 @@ ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header, // Check whether this file is in the public headers. llvm::sys::path::append(RelativePathName, "Headers", Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); - if (auto *File = SourceMgr.getFileManager().getFile(FullPathName)) + if (auto *File = GetFile(FullPathName)) return File; // Check whether this file is in the private headers. @@ -141,31 +182,74 @@ ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header, llvm::sys::path::append(RelativePathName, "PrivateHeaders", Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); - return SourceMgr.getFileManager().getFile(FullPathName); + return GetFile(FullPathName); } // Lookup for normal headers. llvm::sys::path::append(RelativePathName, Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); - return SourceMgr.getFileManager().getFile(FullPathName); + return GetFile(FullPathName); } -const FileEntry * -ModuleMap::resolveAsBuiltinHeader(Module *M, - Module::UnresolvedHeaderDirective Header, - SmallVectorImpl &BuiltinPathName) { - if (llvm::sys::path::is_absolute(Header.FileName) || M->isPartOfFramework() || - !M->IsSystem || Header.IsUmbrella || !BuiltinIncludeDir || - BuiltinIncludeDir == M->Directory || !isBuiltinHeader(Header.FileName)) - return nullptr; +void ModuleMap::resolveHeader(Module *Mod, + const Module::UnresolvedHeaderDirective &Header) { + SmallString<128> RelativePathName; + if (const FileEntry *File = findHeader(Mod, Header, RelativePathName)) { + if (Header.IsUmbrella) { + const DirectoryEntry *UmbrellaDir = File->getDir(); + if (Module *UmbrellaMod = UmbrellaDirs[UmbrellaDir]) + Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash) + << UmbrellaMod->getFullModuleName(); + else + // Record this umbrella header. + setUmbrellaHeader(Mod, File, RelativePathName.str()); + } else { + Module::Header H = {RelativePathName.str(), File}; + if (Header.Kind == Module::HK_Excluded) + excludeHeader(Mod, H); + else + addHeader(Mod, H, headerKindToRole(Header.Kind)); + } + } else if (Header.HasBuiltinHeader && !Header.Size && !Header.ModTime) { + // There's a builtin header but no corresponding on-disk header. Assume + // this was supposed to modularize the builtin header alone. + } else if (Header.Kind == Module::HK_Excluded) { + // Ignore missing excluded header files. They're optional anyway. + } else { + // If we find a module that has a missing header, we mark this module as + // unavailable and store the header directive for displaying diagnostics. + Mod->MissingHeaders.push_back(Header); + // A missing header with stat information doesn't make the module + // unavailable; this keeps our behavior consistent as headers are lazily + // resolved. (Such a module still can't be built though, except from + // preprocessed source.) + if (!Header.Size && !Header.ModTime) + Mod->markUnavailable(); + } +} + +bool ModuleMap::resolveAsBuiltinHeader( + Module *Mod, const Module::UnresolvedHeaderDirective &Header) { + if (Header.Kind == Module::HK_Excluded || + llvm::sys::path::is_absolute(Header.FileName) || + Mod->isPartOfFramework() || !Mod->IsSystem || Header.IsUmbrella || + !BuiltinIncludeDir || BuiltinIncludeDir == Mod->Directory || + !isBuiltinHeader(Header.FileName)) + return false; // This is a system module with a top-level header. This header // may have a counterpart (or replacement) in the set of headers // supplied by Clang. Find that builtin header. - llvm::sys::path::append(BuiltinPathName, BuiltinIncludeDir->getName(), - Header.FileName); - return SourceMgr.getFileManager().getFile( - StringRef(BuiltinPathName.data(), BuiltinPathName.size())); + SmallString<128> Path; + llvm::sys::path::append(Path, BuiltinIncludeDir->getName(), Header.FileName); + auto *File = SourceMgr.getFileManager().getFile(Path); + if (!File) + return false; + + auto Role = headerKindToRole(Header.Kind); + Module::Header H = {Path.str(), File}; + addHeader(Mod, H, Role); + return true; } ModuleMap::ModuleMap(SourceManager &SourceMgr, DiagnosticsEngine &Diags, @@ -246,6 +330,7 @@ bool ModuleMap::isBuiltinHeader(StringRef FileName) { ModuleMap::HeadersMap::iterator ModuleMap::findKnownHeader(const FileEntry *File) { + resolveHeaderDirectives(File); HeadersMap::iterator Known = Headers.find(File); if (HeaderInfo.getHeaderSearchOpts().ImplicitModuleMaps && Known == Headers.end() && File->getDir() == BuiltinIncludeDir && @@ -328,8 +413,10 @@ void ModuleMap::diagnoseHeaderInclusion(Module *RequestingModule, if (getTopLevelOrNull(RequestingModule) != getTopLevelOrNull(SourceModule)) return; - if (RequestingModule) + if (RequestingModule) { resolveUses(RequestingModule, /*Complain=*/false); + resolveHeaderDirectives(RequestingModule); + } bool Excluded = false; Module *Private = nullptr; @@ -511,6 +598,7 @@ ModuleMap::findOrCreateModuleForHeaderInUmbrellaDir(const FileEntry *File) { ArrayRef ModuleMap::findAllModulesForHeader(const FileEntry *File) const { + resolveHeaderDirectives(File); auto It = Headers.find(File); if (It == Headers.end()) return None; @@ -524,6 +612,7 @@ bool ModuleMap::isHeaderInUnavailableModule(const FileEntry *Header) const { bool ModuleMap::isHeaderUnavailableInModule(const FileEntry *Header, const Module *RequestingModule) const { + resolveHeaderDirectives(Header); HeadersMap::const_iterator Known = Headers.find(Header); if (Known != Headers.end()) { for (SmallVectorImpl::const_iterator @@ -896,20 +985,65 @@ void ModuleMap::setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir, UmbrellaDirs[UmbrellaDir] = Mod; } -static Module::HeaderKind headerRoleToKind(ModuleMap::ModuleHeaderRole Role) { - switch ((int)Role) { - default: llvm_unreachable("unknown header role"); - case ModuleMap::NormalHeader: - return Module::HK_Normal; - case ModuleMap::PrivateHeader: - return Module::HK_Private; - case ModuleMap::TextualHeader: - return Module::HK_Textual; - case ModuleMap::PrivateHeader | ModuleMap::TextualHeader: - return Module::HK_PrivateTextual; +void ModuleMap::addUnresolvedHeader(Module *Mod, + Module::UnresolvedHeaderDirective Header) { + // If there is a builtin counterpart to this file, add it now so it can + // wrap the system header. + if (resolveAsBuiltinHeader(Mod, Header)) { + // If we have both a builtin and system version of the file, the + // builtin version may want to inject macros into the system header, so + // force the system header to be treated as a textual header in this + // case. + Header.Kind = headerRoleToKind(ModuleMap::ModuleHeaderRole( + headerKindToRole(Header.Kind) | ModuleMap::TextualHeader)); + Header.HasBuiltinHeader = true; + } + + // If possible, don't stat the header until we need to. This requires the + // user to have provided us with some stat information about the file. + // FIXME: Add support for lazily stat'ing umbrella headers and excluded + // headers. + if ((Header.Size || Header.ModTime) && !Header.IsUmbrella && + Header.Kind != Module::HK_Excluded) { + // We expect more variation in mtime than size, so if we're given both, + // use the mtime as the key. + if (Header.ModTime) + LazyHeadersByModTime[*Header.ModTime].push_back(Mod); + else + LazyHeadersBySize[*Header.Size].push_back(Mod); + Mod->UnresolvedHeaders.push_back(Header); + return; + } + + // We don't have stat information or can't defer looking this file up. + // Perform the lookup now. + resolveHeader(Mod, Header); +} + +void ModuleMap::resolveHeaderDirectives(const FileEntry *File) const { + auto BySize = LazyHeadersBySize.find(File->getSize()); + if (BySize != LazyHeadersBySize.end()) { + for (auto *M : BySize->second) + resolveHeaderDirectives(M); + LazyHeadersBySize.erase(BySize); + } + + auto ByModTime = LazyHeadersByModTime.find(File->getModificationTime()); + if (ByModTime != LazyHeadersByModTime.end()) { + for (auto *M : ByModTime->second) + resolveHeaderDirectives(M); + LazyHeadersByModTime.erase(ByModTime); } } +void ModuleMap::resolveHeaderDirectives(Module *Mod) const { + for (auto &Header : Mod->UnresolvedHeaders) + // This operation is logically const; we're just changing how we represent + // the header information for this file. + const_cast(this)->resolveHeader(Mod, Header); + Mod->UnresolvedHeaders.clear(); +} + void ModuleMap::addHeader(Module *Mod, Module::Header Header, ModuleHeaderRole Role, bool Imported) { KnownHeader KH(Mod, Role); @@ -1063,6 +1197,7 @@ namespace clang { RequiresKeyword, Star, StringLiteral, + IntegerLiteral, TextualKeyword, LBrace, RBrace, @@ -1072,7 +1207,12 @@ namespace clang { unsigned Location; unsigned StringLength; - const char *StringData; + union { + // If Kind != IntegerLiteral. + const char *StringData; + // If Kind == IntegerLiteral. + uint64_t IntegerValue; + }; void clear() { Kind = EndOfFile; @@ -1086,9 +1226,14 @@ namespace clang { SourceLocation getLocation() const { return SourceLocation::getFromRawEncoding(Location); } + + uint64_t getInteger() const { + return Kind == IntegerLiteral ? IntegerValue : 0; + } StringRef getString() const { - return StringRef(StringData, StringLength); + return Kind == IntegerLiteral ? StringRef() + : StringRef(StringData, StringLength); } }; @@ -1278,6 +1423,25 @@ retry: Tok.StringLength = Length; break; } + + case tok::numeric_constant: { + // We don't support any suffixes or other complications. + SmallString<32> SpellingBuffer; + SpellingBuffer.resize(LToken.getLength() + 1); + const char *Start = SpellingBuffer.data(); + unsigned Length = + Lexer::getSpelling(LToken, Start, SourceMgr, L.getLangOpts()); + uint64_t Value; + if (StringRef(Start, Length).getAsInteger(0, Value)) { + Diags.Report(Tok.getLocation(), diag::err_mmap_unknown_token); + HadError = true; + goto retry; + } + + Tok.Kind = MMToken::IntegerLiteral; + Tok.IntegerValue = Value; + break; + } case tok::comment: goto retry; @@ -1904,6 +2068,9 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, Header.FileName = Tok.getString(); Header.FileNameLoc = consumeToken(); Header.IsUmbrella = LeadingToken == MMToken::UmbrellaKeyword; + Header.Kind = + (LeadingToken == MMToken::ExcludeKeyword ? Module::HK_Excluded + : Map.headerRoleToKind(Role)); // Check whether we already have an umbrella. if (Header.IsUmbrella && ActiveModule->Umbrella) { @@ -1913,64 +2080,62 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken, return; } - // Look for this file by name if we don't have any stat information. - SmallString<128> RelativePathName, BuiltinPathName; - const FileEntry *File = - Map.resolveHeader(ActiveModule, Header, RelativePathName); - const FileEntry *BuiltinFile = - Map.resolveAsBuiltinHeader(ActiveModule, Header, BuiltinPathName); + // If we were given stat information, parse it so we can skip looking for + // the file. + if (Tok.is(MMToken::LBrace)) { + SourceLocation LBraceLoc = consumeToken(); + + while (!Tok.is(MMToken::RBrace) && !Tok.is(MMToken::EndOfFile)) { + enum Attribute { Size, ModTime, Unknown }; + StringRef Str = Tok.getString(); + SourceLocation Loc = consumeToken(); + switch (llvm::StringSwitch(Str) + .Case("size", Size) + .Case("mtime", ModTime) + .Default(Unknown)) { + case Size: + if (Header.Size) + Diags.Report(Loc, diag::err_mmap_duplicate_header_attribute) << Str; + if (!Tok.is(MMToken::IntegerLiteral)) { + Diags.Report(Tok.getLocation(), + diag::err_mmap_invalid_header_attribute_value) << Str; + skipUntil(MMToken::RBrace); + break; + } + Header.Size = Tok.getInteger(); + consumeToken(); + break; - // If Clang supplies this header but the underlying system does not, - // just silently swap in our builtin version. Otherwise, we'll end - // up adding both (later). - if (BuiltinFile && !File) { - RelativePathName = BuiltinPathName; - File = BuiltinFile; - BuiltinFile = nullptr; - } + case ModTime: + if (Header.ModTime) + Diags.Report(Loc, diag::err_mmap_duplicate_header_attribute) << Str; + if (!Tok.is(MMToken::IntegerLiteral)) { + Diags.Report(Tok.getLocation(), + diag::err_mmap_invalid_header_attribute_value) << Str; + skipUntil(MMToken::RBrace); + break; + } + Header.ModTime = Tok.getInteger(); + consumeToken(); + break; - // FIXME: We shouldn't be eagerly stat'ing every file named in a module map. - // Come up with a lazy way to do this. - if (File) { - if (Header.IsUmbrella) { - const DirectoryEntry *UmbrellaDir = File->getDir(); - if (Module *UmbrellaModule = Map.UmbrellaDirs[UmbrellaDir]) { - Diags.Report(LeadingLoc, diag::err_mmap_umbrella_clash) - << UmbrellaModule->getFullModuleName(); - HadError = true; - } else { - // Record this umbrella header. - Map.setUmbrellaHeader(ActiveModule, File, RelativePathName.str()); - } - } else if (LeadingToken == MMToken::ExcludeKeyword) { - Module::Header H = {RelativePathName.str(), File}; - Map.excludeHeader(ActiveModule, H); - } else { - // If there is a builtin counterpart to this file, add it now so it can - // wrap the system header. - if (BuiltinFile) { - Module::Header H = { BuiltinPathName.str(), BuiltinFile }; - Map.addHeader(ActiveModule, H, Role); - - // If we have both a builtin and system version of the file, the - // builtin version may want to inject macros into the system header, so - // force the system header to be treated as a textual header in this - // case. - Role = ModuleMap::ModuleHeaderRole(Role | ModuleMap::TextualHeader); + case Unknown: + Diags.Report(Loc, diag::err_mmap_expected_header_attribute); + skipUntil(MMToken::RBrace); + break; } - - // Record this header. - Module::Header H = { RelativePathName.str(), File }; - Map.addHeader(ActiveModule, H, Role); } - } else if (LeadingToken != MMToken::ExcludeKeyword) { - // Ignore excluded header files. They're optional anyway. - // If we find a module that has a missing header, we mark this module as - // unavailable and store the header directive for displaying diagnostics. - ActiveModule->markUnavailable(); - ActiveModule->MissingHeaders.push_back(Header); + if (Tok.is(MMToken::RBrace)) + consumeToken(); + else { + Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace); + Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match); + HadError = true; + } } + + Map.addUnresolvedHeader(ActiveModule, std::move(Header)); } static int compareModuleHeaders(const Module::Header *A, @@ -2521,6 +2686,7 @@ bool ModuleMapParser::parseModuleMapFile() { case MMToken::RequiresKeyword: case MMToken::Star: case MMToken::StringLiteral: + case MMToken::IntegerLiteral: case MMToken::TextualKeyword: case MMToken::UmbrellaKeyword: case MMToken::UseKeyword: diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 8b5877934f..2d3ad69098 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -689,6 +689,8 @@ Preprocessor::getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, while (!Loc.isInvalid() && !SM.isInMainFile(Loc)) { auto ID = SM.getFileID(SM.getExpansionLoc(Loc)); auto *FE = SM.getFileEntryForID(ID); + if (!FE) + break; bool InTextualHeader = false; for (auto Header : HeaderInfo.getModuleMap().findAllModulesForHeader(FE)) { -- cgit v1.2.3 From 3fb1501e7d602b1014424b62adae4dc13ea292e1 Mon Sep 17 00:00:00 2001 From: Galina Kistanova Date: Sat, 3 Jun 2017 06:25:29 +0000 Subject: Added LLVM_FALLTHROUGH to address warning: this statement may fall through + formatted. NFC. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304642 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index 1e2cbde825..a598a46781 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -456,10 +456,17 @@ static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, // Finally, we write the bytes into ResultBuf. ResultBuf += bytesToWrite; switch (bytesToWrite) { // note: everything falls through. - case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; - case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; - case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; - case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]); + case 4: + *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; + LLVM_FALLTHROUGH; + case 3: + *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; + LLVM_FALLTHROUGH; + case 2: + *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; + LLVM_FALLTHROUGH; + case 1: + *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]); } // Update the buffer. ResultBuf += bytesToWrite; -- cgit v1.2.3 From 6129830cbea46c1cf81a5a9b399276ea8184ff27 Mon Sep 17 00:00:00 2001 From: Galina Kistanova Date: Sat, 3 Jun 2017 06:25:47 +0000 Subject: Added LLVM_FALLTHROUGH to address warning: this statement may fall through. NFC. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304643 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index f5a35e97d6..447ff212f0 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -2498,6 +2498,7 @@ void Lexer::ReadToEndOfLine(SmallVectorImpl *Result) { break; } // FALL THROUGH. + LLVM_FALLTHROUGH; case '\r': case '\n': // Okay, we found the end of the line. First, back up past the \0, \r, \n. @@ -3247,6 +3248,7 @@ LexNextToken: return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), tok::wide_char_constant); // FALL THROUGH, treating L like the start of an identifier. + LLVM_FALLTHROUGH; // C99 6.4.2: Identifiers. case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': -- cgit v1.2.3 From 747fcbb8908f48825386c6c9eb8fb2fd4bf9444d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 5 Jun 2017 18:57:56 +0000 Subject: Factor out and unify emission of "module is unavailable" diagnostics. Inspired by post-commit review of r304190. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304728 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 38 +++++++++++++++++++++++--------------- lib/Lex/Pragma.cpp | 14 ++------------ 2 files changed, 25 insertions(+), 27 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 2d3ad69098..b2c3c2e707 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -1654,6 +1654,26 @@ static bool trySimplifyPath(SmallVectorImpl &Components, return SuggestReplacement; } +bool Preprocessor::checkModuleIsAvailable(const LangOptions &LangOpts, + const TargetInfo &TargetInfo, + DiagnosticsEngine &Diags, Module *M) { + Module::Requirement Requirement; + Module::UnresolvedHeaderDirective MissingHeader; + if (M->isAvailable(LangOpts, TargetInfo, Requirement, MissingHeader)) + return false; + + if (MissingHeader.FileNameLoc.isValid()) { + Diags.Report(MissingHeader.FileNameLoc, diag::err_module_header_missing) + << MissingHeader.IsUmbrella << MissingHeader.FileName; + } else { + // FIXME: Track the location at which the requirement was specified, and + // use it here. + Diags.Report(M->DefinitionLoc, diag::err_module_unavailable) + << M->getFullModuleName() << Requirement.second << Requirement.first; + } + return true; +} + /// HandleIncludeDirective - The "\#include" tokens have just been read, read /// the file to be included from the lexer, then include it! This is a common /// routine with functionality shared between \#include, \#include_next and @@ -1835,23 +1855,11 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, // unavailable, diagnose the situation and bail out. // FIXME: Remove this; loadModule does the same check (but produces // slightly worse diagnostics). - if (!SuggestedModule.getModule()->isAvailable()) { - Module::Requirement Requirement; - Module::UnresolvedHeaderDirective MissingHeader; - Module *M = SuggestedModule.getModule(); - // Identify the cause. - (void)M->isAvailable(getLangOpts(), getTargetInfo(), Requirement, - MissingHeader); - if (MissingHeader.FileNameLoc.isValid()) { - Diag(MissingHeader.FileNameLoc, diag::err_module_header_missing) - << MissingHeader.IsUmbrella << MissingHeader.FileName; - } else { - Diag(M->DefinitionLoc, diag::err_module_unavailable) - << M->getFullModuleName() << Requirement.second << Requirement.first; - } + if (checkModuleIsAvailable(getLangOpts(), getTargetInfo(), getDiagnostics(), + SuggestedModule.getModule())) { Diag(FilenameTok.getLocation(), diag::note_implicit_top_level_module_import_here) - << M->getTopLevelModuleName(); + << SuggestedModule.getModule()->getTopLevelModuleName(); return; } diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index e1d981527b..dd52abd74f 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -1408,18 +1408,8 @@ struct PragmaModuleBeginHandler : public PragmaHandler { } // If the module isn't available, it doesn't make sense to enter it. - if (!M->isAvailable()) { - Module::Requirement Requirement; - Module::UnresolvedHeaderDirective MissingHeader; - (void)M->isAvailable(PP.getLangOpts(), PP.getTargetInfo(), - Requirement, MissingHeader); - if (MissingHeader.FileNameLoc.isValid()) { - PP.Diag(MissingHeader.FileNameLoc, diag::err_module_header_missing) - << MissingHeader.IsUmbrella << MissingHeader.FileName; - } else { - PP.Diag(M->DefinitionLoc, diag::err_module_unavailable) - << M->getFullModuleName() << Requirement.second << Requirement.first; - } + if (Preprocessor::checkModuleIsAvailable( + PP.getLangOpts(), PP.getTargetInfo(), PP.getDiagnostics(), M)) { PP.Diag(BeginLoc, diag::note_pp_module_begin_here) << M->getTopLevelModuleName(); return; -- cgit v1.2.3 From 4ed69c25ab95ef923b9917a3c3599665ca7bbda7 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Fri, 9 Jun 2017 01:20:48 +0000 Subject: [libclang] Introduce a new parsing option 'CXTranslationUnit_SingleFileParse' that puts preprocessor in a mode for parsing a single file only. This is useful for parsing a single file, as a fast/inaccurate 'mode' that can still provide declarations from the file, like the classes and their methods. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305044 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index b2c3c2e707..89c2ebd00a 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -30,6 +30,7 @@ #include "clang/Lex/PPCallbacks.h" #include "clang/Lex/Pragma.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" #include "clang/Lex/PTHLexer.h" #include "clang/Lex/Token.h" #include "llvm/ADT/ArrayRef.h" @@ -1845,10 +1846,13 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, // we've imported or already built. bool ShouldEnter = true; + if (PPOpts->SingleFileParseMode) + ShouldEnter = false; + // Determine whether we should try to import the module for this #include, if // there is one. Don't do so if precompiled module support is disabled or we // are processing this module textually (because we're building the module). - if (File && SuggestedModule && getLangOpts().Modules && + if (ShouldEnter && File && SuggestedModule && getLangOpts().Modules && SuggestedModule.getModule()->getTopLevelModuleName() != getLangOpts().CurrentModule) { // If this include corresponds to a module but that module is -- cgit v1.2.3 From 3148514e3baca9b32f9da60915df952cc8b227b9 Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Fri, 9 Jun 2017 16:29:35 +0000 Subject: support operator keywords used in Windows SDK to support operator keywords used in Windows SDK, alter token type when seen in system headers Hello, I submitted D33505 to address this problem, but the proposal was rejected as too big a hammer. This change will allow clang to parse the WindowsSDK header which uses the operator name "or" as a field name. Treat cpp operator keywords as ordinary identifiers inside the Microsoft headers, but treat them as usual in the user's program. Original Submitter: Melanie Blower (mibintc) Differential Revision: https://reviews.llvm.org/D33782 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305087 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 3596337c24..77b46ae5cc 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -580,7 +580,11 @@ IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { // Update the token info (identifier info and appropriate token kind). Identifier.setIdentifierInfo(II); - Identifier.setKind(II->getTokenID()); + if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() && + getSourceManager().isInSystemHeader(Identifier.getLocation())) + Identifier.setKind(clang::tok::identifier); + else + Identifier.setKind(II->getTokenID()); return II; } -- cgit v1.2.3 From ae2de79ce245a2bac0c19e905a78ba7fa923820c Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 9 Jun 2017 19:22:32 +0000 Subject: Add #pragma clang module build/endbuild pragmas for performing a module build as part of a compilation. This is intended for two purposes: 1) Writing self-contained test cases for modules: we can now write a single source file test that builds some number of module files on the side and imports them. 2) Debugging / test case reduction. A single-source testcase is much more amenable to reduction, compared to a VFS tarball or .pcm files. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305101 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Pragma.cpp | 117 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index dd52abd74f..c16478dd2b 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -754,6 +754,88 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) { getHeaderSearchInfo().AddIncludeAlias(OriginalSource, ReplaceFileName); } +void Preprocessor::HandlePragmaModuleBuild(Token &Tok) { + SourceLocation Loc = Tok.getLocation(); + + LexUnexpandedToken(Tok); + if (Tok.isAnnotation() || !Tok.getIdentifierInfo()) { + Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << true; + return; + } + IdentifierInfo *ModuleName = Tok.getIdentifierInfo(); + + LexUnexpandedToken(Tok); + if (Tok.isNot(tok::eod)) { + Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma"; + DiscardUntilEndOfDirective(); + } + + if (CurPTHLexer) { + // FIXME: Support this somehow? + Diag(Loc, diag::err_pp_module_build_pth); + return; + } + + CurLexer->LexingRawMode = true; + + auto TryConsumeIdentifier = [&](StringRef Ident) -> bool { + if (Tok.getKind() != tok::raw_identifier || + Tok.getRawIdentifier() != Ident) + return false; + CurLexer->Lex(Tok); + return true; + }; + + // Scan forward looking for the end of the module. + const char *Start = CurLexer->getBufferLocation(); + const char *End = nullptr; + unsigned NestingLevel = 1; + while (true) { + End = CurLexer->getBufferLocation(); + CurLexer->Lex(Tok); + + if (Tok.is(tok::eof)) { + Diag(Loc, diag::err_pp_module_build_missing_end); + break; + } + + if (Tok.isNot(tok::hash) || !Tok.isAtStartOfLine()) { + // Token was part of module; keep going. + continue; + } + + // We hit something directive-shaped; check to see if this is the end + // of the module build. + CurLexer->ParsingPreprocessorDirective = true; + CurLexer->Lex(Tok); + if (TryConsumeIdentifier("pragma") && TryConsumeIdentifier("clang") && + TryConsumeIdentifier("module")) { + if (TryConsumeIdentifier("build")) + // #pragma clang module build -> entering a nested module build. + ++NestingLevel; + else if (TryConsumeIdentifier("endbuild")) { + // #pragma clang module endbuild -> leaving a module build. + if (--NestingLevel == 0) + break; + } + // We should either be looking at the EOD or more of the current directive + // preceding the EOD. Either way we can ignore this token and keep going. + assert(Tok.getKind() != tok::eof && "missing EOD before EOF"); + } + } + + CurLexer->LexingRawMode = false; + + // Load the extracted text as a preprocessed module. + assert(CurLexer->getBuffer().begin() <= Start && + Start <= CurLexer->getBuffer().end() && + CurLexer->getBuffer().begin() <= End && + End <= CurLexer->getBuffer().end() && + "module source range not contained within same file buffer"); + TheModuleLoader.loadModuleFromSource(Loc, ModuleName->getName(), + StringRef(Start, End - Start)); +} + /// AddPragmaHandler - Add the specified pragma handler to the preprocessor. /// If 'Namespace' is non-null, then it is a token required to exist on the /// pragma line before the pragma string starts, e.g. "STDC" or "GCC". @@ -1442,6 +1524,39 @@ struct PragmaModuleEndHandler : public PragmaHandler { } }; +/// Handle the clang \#pragma module build extension. +struct PragmaModuleBuildHandler : public PragmaHandler { + PragmaModuleBuildHandler() : PragmaHandler("build") {} + + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &Tok) override { + PP.HandlePragmaModuleBuild(Tok); + } +}; + +/// Handle the clang \#pragma module load extension. +struct PragmaModuleLoadHandler : public PragmaHandler { + PragmaModuleLoadHandler() : PragmaHandler("load") {} + + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &Tok) override { + SourceLocation Loc = Tok.getLocation(); + + // Read the module name. + llvm::SmallVector, 8> + ModuleName; + if (LexModuleName(PP, Tok, ModuleName)) + return; + + if (Tok.isNot(tok::eod)) + PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma"; + + // Load the module, don't make it visible. + PP.getModuleLoader().loadModule(Loc, ModuleName, Module::Hidden, + /*IsIncludeDirective=*/false); + } +}; + /// PragmaPushMacroHandler - "\#pragma push_macro" saves the value of the /// macro on the top of the stack. struct PragmaPushMacroHandler : public PragmaHandler { @@ -1671,6 +1786,8 @@ void Preprocessor::RegisterBuiltinPragmas() { ModuleHandler->AddPragma(new PragmaModuleImportHandler()); ModuleHandler->AddPragma(new PragmaModuleBeginHandler()); ModuleHandler->AddPragma(new PragmaModuleEndHandler()); + ModuleHandler->AddPragma(new PragmaModuleBuildHandler()); + ModuleHandler->AddPragma(new PragmaModuleLoadHandler()); AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler()); AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler()); -- cgit v1.2.3 From 6feeb00acd06ed4183dc89e1264124d5daff6f94 Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Fri, 9 Jun 2017 22:50:02 +0000 Subject: Support operator keywords used in Windows SDK(fix ubsan) UBSan found an issue with a nullptr being assigned to a reference. This was because a following function went back and checked the identifier in the CPPOperatorName case. This patch corrects that location with the original logic as well. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305128 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 77b46ae5cc..f9a399cd7f 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -713,7 +713,9 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) { // C++ 2.11p2: If this is an alternative representation of a C++ operator, // then we act as if it is the actual operator and not the textual // representation of it. - if (II.isCPlusPlusOperatorKeyword()) + if (II.isCPlusPlusOperatorKeyword() && + !(getLangOpts().MSVCCompat && + getSourceManager().isInSystemHeader(Identifier.getLocation()))) Identifier.setIdentifierInfo(nullptr); // If this is an extension token, diagnose its use. -- cgit v1.2.3 From bdfd67ac5ada2f9aa15f61293a861f2aa4e69c84 Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Wed, 14 Jun 2017 23:09:01 +0000 Subject: [Preprocessor]Correct Macro-Arg allocation of StringifiedArguments, correct getNumArguments StringifiedArguments is allocated (resized) based on the size the getNumArguments function. However, this function ACTUALLY currently returns the amount of total UnexpArgTokens which is minimum the same as the new implementation of getNumMacroArguments, since empty/omitted arguments result in 1 UnexpArgToken, and included ones at minimum include 2 (1 for the arg itself, 1 for eof). This patch renames the otherwise unused getNumArguments to be more clear that it is the number of arguments that the Macro expects, and thus the maximum number that can be stringified. This patch also replaces the explicit memset (which results in value instantiation of the new tokens, PLUS clearing the memory) with brace initialization. Differential Revision: https://reviews.llvm.org/D32046 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305425 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index 1c1979d8e8..a201d16590 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -44,20 +44,22 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, // Otherwise, use the best fit. ClosestMatch = (*Entry)->NumUnexpArgTokens; } - + MacroArgs *Result; if (!ResultEnt) { // Allocate memory for a MacroArgs object with the lexer tokens at the end. - Result = (MacroArgs*)malloc(sizeof(MacroArgs) + - UnexpArgTokens.size() * sizeof(Token)); + Result = (MacroArgs *)malloc(sizeof(MacroArgs) + + UnexpArgTokens.size() * sizeof(Token)); // Construct the MacroArgs object. - new (Result) MacroArgs(UnexpArgTokens.size(), VarargsElided); + new (Result) + MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumArgs()); } else { Result = *ResultEnt; // Unlink this node from the preprocessors singly linked list. *ResultEnt = Result->ArgCache; Result->NumUnexpArgTokens = UnexpArgTokens.size(); Result->VarargsElided = VarargsElided; + Result->NumMacroArgs = MI->getNumArgs(); } // Copy the actual unexpanded tokens to immediately after the result ptr. @@ -298,12 +300,10 @@ const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd) { - assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!"); - if (StringifiedArgs.empty()) { - StringifiedArgs.resize(getNumArguments()); - memset((void*)&StringifiedArgs[0], 0, - sizeof(StringifiedArgs[0])*getNumArguments()); - } + assert(ArgNo < getNumMacroArguments() && "Invalid argument number!"); + if (StringifiedArgs.empty()) + StringifiedArgs.resize(getNumMacroArguments(), {}); + if (StringifiedArgs[ArgNo].isNot(tok::string_literal)) StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP, /*Charify=*/false, -- cgit v1.2.3 From fb40328fb184e12d521a3d0179779bf29390b4b7 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Fri, 16 Jun 2017 20:13:39 +0000 Subject: [PR33394] Avoid lexing editor placeholders when Clang is used only for preprocessing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit r300667 added support for editor placeholder to Clang. That commit didn’t take into account that users who use Clang for preprocessing only (-E) will get the "editor placeholder in source file" error when preprocessing their source (PR33394). This commit ensures that Clang doesn't lex editor placeholders when running a preprocessor only action. rdar://32718000 Differential Revision: https://reviews.llvm.org/D34256 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305576 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 447ff212f0..012189aa6f 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -19,6 +19,7 @@ #include "clang/Lex/LexDiagnostic.h" #include "clang/Lex/LiteralSupport.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/Compiler.h" @@ -2750,7 +2751,7 @@ static const char *findPlaceholderEnd(const char *CurPtr, bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) { assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!"); - if (!PP || LexingRawMode) + if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode) return false; const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd); if (!End) -- cgit v1.2.3 From 0be2e497aa26778cd044e98df261c1c7e1e23571 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 19 Jun 2017 23:09:36 +0000 Subject: Support non-identifier module names when preprocessing modules. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305758 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Pragma.cpp | 70 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 26 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index c16478dd2b..bf2363a0a6 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -20,6 +20,7 @@ #include "clang/Basic/TokenKinds.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/LexDiagnostic.h" +#include "clang/Lex/LiteralSupport.h" #include "clang/Lex/MacroInfo.h" #include "clang/Lex/PPCallbacks.h" #include "clang/Lex/Preprocessor.h" @@ -754,15 +755,52 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) { getHeaderSearchInfo().AddIncludeAlias(OriginalSource, ReplaceFileName); } +// Lex a component of a module name: either an identifier or a string literal; +// for components that can be expressed both ways, the two forms are equivalent. +static bool LexModuleNameComponent( + Preprocessor &PP, Token &Tok, + std::pair &ModuleNameComponent, + bool First) { + PP.LexUnexpandedToken(Tok); + if (Tok.is(tok::string_literal) && !Tok.hasUDSuffix()) { + StringLiteralParser Literal(Tok, PP); + if (Literal.hadError) + return true; + ModuleNameComponent = std::make_pair( + PP.getIdentifierInfo(Literal.GetString()), Tok.getLocation()); + } else if (!Tok.isAnnotation() && Tok.getIdentifierInfo()) { + ModuleNameComponent = + std::make_pair(Tok.getIdentifierInfo(), Tok.getLocation()); + } else { + PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << First; + return true; + } + return false; +} + +static bool LexModuleName( + Preprocessor &PP, Token &Tok, + llvm::SmallVectorImpl> + &ModuleName) { + while (true) { + std::pair NameComponent; + if (LexModuleNameComponent(PP, Tok, NameComponent, ModuleName.empty())) + return true; + ModuleName.push_back(NameComponent); + + PP.LexUnexpandedToken(Tok); + if (Tok.isNot(tok::period)) + return false; + } +} + void Preprocessor::HandlePragmaModuleBuild(Token &Tok) { SourceLocation Loc = Tok.getLocation(); - LexUnexpandedToken(Tok); - if (Tok.isAnnotation() || !Tok.getIdentifierInfo()) { - Diag(Tok.getLocation(), diag::err_pp_expected_module_name) << true; + std::pair ModuleNameLoc; + if (LexModuleNameComponent(*this, Tok, ModuleNameLoc, true)) return; - } - IdentifierInfo *ModuleName = Tok.getIdentifierInfo(); + IdentifierInfo *ModuleName = ModuleNameLoc.first; LexUnexpandedToken(Tok); if (Tok.isNot(tok::eod)) { @@ -1383,26 +1421,6 @@ public: } }; -static bool LexModuleName( - Preprocessor &PP, Token &Tok, - llvm::SmallVectorImpl> - &ModuleName) { - while (true) { - PP.LexUnexpandedToken(Tok); - if (Tok.isAnnotation() || !Tok.getIdentifierInfo()) { - PP.Diag(Tok.getLocation(), diag::err_pp_expected_module_name) - << ModuleName.empty(); - return true; - } - - ModuleName.emplace_back(Tok.getIdentifierInfo(), Tok.getLocation()); - - PP.LexUnexpandedToken(Tok); - if (Tok.isNot(tok::period)) - return false; - } -} - /// Handle the clang \#pragma module import extension. The syntax is: /// \code /// #pragma clang module import some.module.name @@ -1473,7 +1491,7 @@ struct PragmaModuleBeginHandler : public PragmaHandler { // be loaded or implicitly loadable. // FIXME: We could create the submodule here. We'd need to know whether // it's supposed to be explicit, but not much else. - Module *M = PP.getHeaderSearchInfo().getModuleMap().findModule(Current); + Module *M = PP.getHeaderSearchInfo().lookupModule(Current); if (!M) { PP.Diag(ModuleName.front().second, diag::err_pp_module_begin_no_module_map) << Current; -- cgit v1.2.3 From 7a2305ca4dfb7a94e5e524ed034aeb3768d8bf9a Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Tue, 20 Jun 2017 14:36:58 +0000 Subject: [preprocessor] When preprocessor option 'SingleFileParseMode' is enabled, parse all directive blocks if the condition uses undefined macros This is useful for being able to parse the preprocessor directive blocks even if the header, that defined the macro that is checked, hasn't been included. Differential Revision: https://reviews.llvm.org/D34263 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305797 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 38 +++++++++++++++++++++++++++++++++----- lib/Lex/PPExpressions.cpp | 29 ++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 14 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 89c2ebd00a..422bfc8968 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -538,7 +538,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc, assert(CurPPLexer->LexingRawMode && "We have to be skipping here!"); CurPPLexer->LexingRawMode = false; IdentifierInfo *IfNDefMacro = nullptr; - const bool CondValue = EvaluateDirectiveExpression(IfNDefMacro); + const bool CondValue = EvaluateDirectiveExpression(IfNDefMacro).Conditional; CurPPLexer->LexingRawMode = true; if (Callbacks) { const SourceLocation CondEnd = CurPPLexer->getSourceLocation(); @@ -635,7 +635,7 @@ void Preprocessor::PTHSkipExcludedConditionalBlock() { // Evaluate the condition of the #elif. IdentifierInfo *IfNDefMacro = nullptr; CurPTHLexer->ParsingPreprocessorDirective = true; - bool ShouldEnter = EvaluateDirectiveExpression(IfNDefMacro); + bool ShouldEnter = EvaluateDirectiveExpression(IfNDefMacro).Conditional; CurPTHLexer->ParsingPreprocessorDirective = false; // If this condition is true, enter it! @@ -2654,7 +2654,13 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef, } // Should we include the stuff contained by this directive? - if (!MI == isIfndef) { + if (PPOpts->SingleFileParseMode && !MI) { + // In 'single-file-parse mode' undefined identifiers trigger parsing of all + // the directive blocks. + CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(), + /*wasskip*/true, /*foundnonskip*/false, + /*foundelse*/false); + } else if (!MI == isIfndef) { // Yes, remember that we are inside a conditional, then lex the next token. CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(), /*wasskip*/false, /*foundnonskip*/true, @@ -2676,7 +2682,8 @@ void Preprocessor::HandleIfDirective(Token &IfToken, // Parse and evaluate the conditional expression. IdentifierInfo *IfNDefMacro = nullptr; const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation(); - const bool ConditionalTrue = EvaluateDirectiveExpression(IfNDefMacro); + const DirectiveEvalResult DER = EvaluateDirectiveExpression(IfNDefMacro); + const bool ConditionalTrue = DER.Conditional; const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation(); // If this condition is equivalent to #ifndef X, and if this is the first @@ -2695,7 +2702,12 @@ void Preprocessor::HandleIfDirective(Token &IfToken, (ConditionalTrue ? PPCallbacks::CVK_True : PPCallbacks::CVK_False)); // Should we include the stuff contained by this directive? - if (ConditionalTrue) { + if (PPOpts->SingleFileParseMode && DER.IncludedUndefinedIds) { + // In 'single-file-parse mode' undefined identifiers trigger parsing of all + // the directive blocks. + CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/true, + /*foundnonskip*/false, /*foundelse*/false); + } else if (ConditionalTrue) { // Yes, remember that we are inside a conditional, then lex the next token. CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/false, /*foundnonskip*/true, /*foundelse*/false); @@ -2756,6 +2768,14 @@ void Preprocessor::HandleElseDirective(Token &Result) { if (Callbacks) Callbacks->Else(Result.getLocation(), CI.IfLoc); + if (PPOpts->SingleFileParseMode && CI.WasSkipping) { + // In 'single-file-parse mode' undefined identifiers trigger parsing of all + // the directive blocks. + CurPPLexer->pushConditionalLevel(CI.IfLoc, /*wasskip*/false, + /*foundnonskip*/true, /*foundelse*/true); + return; + } + // Finally, skip the rest of the contents of this block. SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true, /*FoundElse*/true, Result.getLocation()); @@ -2791,6 +2811,14 @@ void Preprocessor::HandleElifDirective(Token &ElifToken) { SourceRange(ConditionalBegin, ConditionalEnd), PPCallbacks::CVK_NotEvaluated, CI.IfLoc); + if (PPOpts->SingleFileParseMode && CI.WasSkipping) { + // In 'single-file-parse mode' undefined identifiers trigger parsing of all + // the directive blocks. + CurPPLexer->pushConditionalLevel(ElifToken.getLocation(), /*wasskip*/true, + /*foundnonskip*/false, /*foundelse*/false); + return; + } + // Finally, skip the rest of the contents of this block. SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true, /*FoundElse*/CI.FoundElse, diff --git a/lib/Lex/PPExpressions.cpp b/lib/Lex/PPExpressions.cpp index 862a4713e4..12f5084298 100644 --- a/lib/Lex/PPExpressions.cpp +++ b/lib/Lex/PPExpressions.cpp @@ -73,6 +73,7 @@ public: static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, Token &PeekTok, bool ValueLive, + bool &IncludedUndefinedIds, Preprocessor &PP); /// DefinedTracker - This struct is used while parsing expressions to keep track @@ -93,6 +94,7 @@ struct DefinedTracker { /// TheMacro - When the state is DefinedMacro or NotDefinedMacro, this /// indicates the macro that was checked. IdentifierInfo *TheMacro; + bool IncludedUndefinedIds = false; }; /// EvaluateDefined - Process a 'defined(sym)' expression. @@ -128,6 +130,7 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT, MacroDefinition Macro = PP.getMacroDefinition(II); Result.Val = !!Macro; Result.Val.setIsUnsigned(false); // Result is signed intmax_t. + DT.IncludedUndefinedIds = !Macro; // If there is a macro, mark it used. if (Result.Val != 0 && ValueLive) @@ -255,6 +258,8 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT, Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0. Result.setIdentifier(II); Result.setRange(PeekTok.getLocation()); + DT.IncludedUndefinedIds = (II->getTokenID() != tok::kw_true && + II->getTokenID() != tok::kw_false); PP.LexNonComment(PeekTok); return false; } @@ -400,7 +405,8 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT, // Just use DT unmodified as our result. } else { // Otherwise, we have something like (x+y), and we consumed '(x'. - if (EvaluateDirectiveSubExpr(Result, 1, PeekTok, ValueLive, PP)) + if (EvaluateDirectiveSubExpr(Result, 1, PeekTok, ValueLive, + DT.IncludedUndefinedIds, PP)) return true; if (PeekTok.isNot(tok::r_paren)) { @@ -532,6 +538,7 @@ static void diagnoseUnexpectedOperator(Preprocessor &PP, PPValue &LHS, /// evaluation, such as division by zero warnings. static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, Token &PeekTok, bool ValueLive, + bool &IncludedUndefinedIds, Preprocessor &PP) { unsigned PeekPrec = getPrecedence(PeekTok.getKind()); // If this token isn't valid, report the error. @@ -571,6 +578,7 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, // Parse the RHS of the operator. DefinedTracker DT; if (EvaluateValue(RHS, PeekTok, DT, RHSIsLive, PP)) return true; + IncludedUndefinedIds = DT.IncludedUndefinedIds; // Remember the precedence of this operator and get the precedence of the // operator immediately to the right of the RHS. @@ -601,7 +609,8 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, RHSPrec = ThisPrec+1; if (PeekPrec >= RHSPrec) { - if (EvaluateDirectiveSubExpr(RHS, RHSPrec, PeekTok, RHSIsLive, PP)) + if (EvaluateDirectiveSubExpr(RHS, RHSPrec, PeekTok, RHSIsLive, + IncludedUndefinedIds, PP)) return true; PeekPrec = getPrecedence(PeekTok.getKind()); } @@ -769,7 +778,8 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, // Parse anything after the : with the same precedence as ?. We allow // things of equal precedence because ?: is right associative. if (EvaluateDirectiveSubExpr(AfterColonVal, ThisPrec, - PeekTok, AfterColonLive, PP)) + PeekTok, AfterColonLive, + IncludedUndefinedIds, PP)) return true; // Now that we have the condition, the LHS and the RHS of the :, evaluate. @@ -806,7 +816,8 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec, /// EvaluateDirectiveExpression - Evaluate an integer constant expression that /// may occur after a #if or #elif directive. If the expression is equivalent /// to "!defined(X)" return X in IfNDefMacro. -bool Preprocessor::EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) { +Preprocessor::DirectiveEvalResult +Preprocessor::EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) { SaveAndRestore PPDir(ParsingIfOrElifDirective, true); // Save the current state of 'DisableMacroExpansion' and reset it to false. If // 'DisableMacroExpansion' is true, then we must be in a macro argument list @@ -833,7 +844,7 @@ bool Preprocessor::EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) { // Restore 'DisableMacroExpansion'. DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective; - return false; + return {false, DT.IncludedUndefinedIds}; } // If we are at the end of the expression after just parsing a value, there @@ -847,20 +858,20 @@ bool Preprocessor::EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) { // Restore 'DisableMacroExpansion'. DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective; - return ResVal.Val != 0; + return {ResVal.Val != 0, DT.IncludedUndefinedIds}; } // Otherwise, we must have a binary operator (e.g. "#if 1 < 2"), so parse the // operator and the stuff after it. if (EvaluateDirectiveSubExpr(ResVal, getPrecedence(tok::question), - Tok, true, *this)) { + Tok, true, DT.IncludedUndefinedIds, *this)) { // Parse error, skip the rest of the macro line. if (Tok.isNot(tok::eod)) DiscardUntilEndOfDirective(); // Restore 'DisableMacroExpansion'. DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective; - return false; + return {false, DT.IncludedUndefinedIds}; } // If we aren't at the tok::eod token, something bad happened, like an extra @@ -872,5 +883,5 @@ bool Preprocessor::EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) { // Restore 'DisableMacroExpansion'. DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective; - return ResVal.Val != 0; + return {ResVal.Val != 0, DT.IncludedUndefinedIds}; } -- cgit v1.2.3 From b5b472a91f6b26ce2aba5a81d13b36cdefbb3dc1 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Wed, 21 Jun 2017 18:52:44 +0000 Subject: [preprocessor] Fix assertion hit when 'SingleFileParseMode' option is enabled and #if with an undefined identifier and without #else 'HandleEndifDirective' asserts that 'WasSkipping' is false, so switch to using 'FoundNonSkip' as the hint for 'SingleFileParseMode' to keep going with parsing. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305940 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 422bfc8968..8c79e50176 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2658,7 +2658,7 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef, // In 'single-file-parse mode' undefined identifiers trigger parsing of all // the directive blocks. CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(), - /*wasskip*/true, /*foundnonskip*/false, + /*wasskip*/false, /*foundnonskip*/false, /*foundelse*/false); } else if (!MI == isIfndef) { // Yes, remember that we are inside a conditional, then lex the next token. @@ -2705,7 +2705,7 @@ void Preprocessor::HandleIfDirective(Token &IfToken, if (PPOpts->SingleFileParseMode && DER.IncludedUndefinedIds) { // In 'single-file-parse mode' undefined identifiers trigger parsing of all // the directive blocks. - CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/true, + CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/false, /*foundnonskip*/false, /*foundelse*/false); } else if (ConditionalTrue) { // Yes, remember that we are inside a conditional, then lex the next token. @@ -2768,11 +2768,11 @@ void Preprocessor::HandleElseDirective(Token &Result) { if (Callbacks) Callbacks->Else(Result.getLocation(), CI.IfLoc); - if (PPOpts->SingleFileParseMode && CI.WasSkipping) { + if (PPOpts->SingleFileParseMode && !CI.FoundNonSkip) { // In 'single-file-parse mode' undefined identifiers trigger parsing of all // the directive blocks. CurPPLexer->pushConditionalLevel(CI.IfLoc, /*wasskip*/false, - /*foundnonskip*/true, /*foundelse*/true); + /*foundnonskip*/false, /*foundelse*/true); return; } @@ -2811,10 +2811,10 @@ void Preprocessor::HandleElifDirective(Token &ElifToken) { SourceRange(ConditionalBegin, ConditionalEnd), PPCallbacks::CVK_NotEvaluated, CI.IfLoc); - if (PPOpts->SingleFileParseMode && CI.WasSkipping) { + if (PPOpts->SingleFileParseMode && !CI.FoundNonSkip) { // In 'single-file-parse mode' undefined identifiers trigger parsing of all // the directive blocks. - CurPPLexer->pushConditionalLevel(ElifToken.getLocation(), /*wasskip*/true, + CurPPLexer->pushConditionalLevel(ElifToken.getLocation(), /*wasskip*/false, /*foundnonskip*/false, /*foundelse*/false); return; } -- cgit v1.2.3 From 563c0ec65e052e93085d4e3b7fede21da5ea768f Mon Sep 17 00:00:00 2001 From: George Burgess IV Date: Tue, 27 Jun 2017 21:31:31 +0000 Subject: [Sema] Allow unmarked overloadable functions. This patch extends the `overloadable` attribute to allow for one function with a given name to not be marked with the `overloadable` attribute. The overload without the `overloadable` attribute will not have its name mangled. So, the following code is now legal: void foo(void) __attribute__((overloadable)); void foo(int); void foo(float) __attribute__((overloadable)); In addition, this patch fixes a bug where we'd accept code with `__attribute__((overloadable))` inconsistently applied. In other words, we used to accept: void foo(void); void foo(void) __attribute__((overloadable)); But we will do this no longer, since it defeats the original purpose of requiring `__attribute__((overloadable))` on all redeclarations of a function. This breakage seems to not be an issue in practice, since the only code I could find that had this pattern often looked like: void foo(void); void foo(void) __attribute__((overloadable)) __asm__("foo"); void foo(int) __attribute__((overloadable)); ...Which can now be simplified by simply removing the asm label and overloadable attribute from the redeclaration of `void foo(void);` Differential Revision: https://reviews.llvm.org/D32332 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@306467 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPMacroExpansion.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index a6bfc32e22..8af9a50cc2 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1315,6 +1315,8 @@ static bool HasExtension(const Preprocessor &PP, StringRef Extension) { .Case("cxx_binary_literals", true) .Case("cxx_init_captures", LangOpts.CPlusPlus11) .Case("cxx_variable_templates", LangOpts.CPlusPlus) + // Miscellaneous language extensions + .Case("overloadable_unmarked", true) .Default(false); } -- cgit v1.2.3 From 2d7d5c241bce687a4c7018a023efb7b22769ca9d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Thu, 29 Jun 2017 02:19:42 +0000 Subject: Track the set of module maps read while building a .pcm file and reload those when preprocessing from that .pcm file. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@306628 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 018d59e5e8..40f78ce25c 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -2710,7 +2710,8 @@ bool ModuleMap::parseModuleMapFile(const FileEntry *File, bool IsSystem, // If the module map file wasn't already entered, do so now. if (ID.isInvalid()) { - auto FileCharacter = IsSystem ? SrcMgr::C_System : SrcMgr::C_User; + auto FileCharacter = + IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap; ID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter); } -- cgit v1.2.3 From 0fc70de55d536d0366cb6ee19fbfd23c7d3578a4 Mon Sep 17 00:00:00 2001 From: Erik Verbruggen Date: Wed, 5 Jul 2017 09:44:07 +0000 Subject: Fix invalid warnings for header guards in preambles Fixes https://bugs.llvm.org/show_bug.cgi?id=33574 Differential Revision: https://reviews.llvm.org/D34882 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@307134 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 2 +- lib/Lex/PPLexerChange.cpp | 6 ------ lib/Lex/Preprocessor.cpp | 2 ++ 3 files changed, 3 insertions(+), 7 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 012189aa6f..61bcef8cb7 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -2550,7 +2550,7 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { return true; } - if (PP->isRecordingPreamble() && !PP->isInMainFile()) { + if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) { PP->setRecordedPreambleConditionalStack(ConditionalStack); ConditionalStack.clear(); } diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 1c0cd56368..5a589d6a17 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -46,12 +46,6 @@ bool Preprocessor::isInPrimaryFile() const { }); } -bool Preprocessor::isInMainFile() const { - if (IsFileLexer()) - return IncludeMacroStack.size() == 0; - return true; -} - /// getCurrentLexer - Return the current file lexer being lexed from. Note /// that this ignores any potentially active macro expansions and _Pragma /// expansions going on at the time. diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index f9a399cd7f..63f39524d1 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -535,7 +535,9 @@ void Preprocessor::EnterMainSourceFile() { // Start parsing the predefines. EnterSourceFile(FID, nullptr, SourceLocation()); +} +void Preprocessor::replayPreambleConditionalStack() { // Restore the conditional stack from the preamble, if there is one. if (PreambleConditionalStack.isReplaying()) { CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); -- cgit v1.2.3 From baa150782312a70256b17d5383d7a5cd6c8dec32 Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Fri, 14 Jul 2017 09:23:40 +0000 Subject: Keep the IdentifierInfo in the Token for alternative operator keyword The goal of this commit is to fix clang-format so it does not merge tokens when using the alternative spelling keywords. (eg: "not foo" should not become "notfoo") The problem is that Preprocessor::HandleIdentifier used to drop the identifier info from the token for these keyword. This means the first condition of TokenAnnotator::spaceRequiredBefore is not met. We could add explicit check for the spelling in that condition, but I think it is better to keep the IdentifierInfo and handle the operator keyword explicitly when needed. That actually leads to simpler code, and probably slightly more efficient as well. Another side effect of this change is that __identifier(and) will now work as one would expect, removing a FIXME from the MicrosoftExtensions.cpp test Differential Revision: https://reviews.llvm.org/D35172 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@308008 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 14 +++-------- lib/Lex/PPExpressions.cpp | 59 +++++++++++++++++++++++++---------------------- lib/Lex/Preprocessor.cpp | 8 ------- 3 files changed, 35 insertions(+), 46 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 8c79e50176..4440cf2094 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -220,26 +220,18 @@ bool Preprocessor::CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, return Diag(MacroNameTok, diag::err_pp_missing_macro_name); IdentifierInfo *II = MacroNameTok.getIdentifierInfo(); - if (!II) { - bool Invalid = false; - std::string Spelling = getSpelling(MacroNameTok, &Invalid); - if (Invalid) - return Diag(MacroNameTok, diag::err_pp_macro_not_identifier); - II = getIdentifierInfo(Spelling); - - if (!II->isCPlusPlusOperatorKeyword()) - return Diag(MacroNameTok, diag::err_pp_macro_not_identifier); + if (!II) + return Diag(MacroNameTok, diag::err_pp_macro_not_identifier); + if (II->isCPlusPlusOperatorKeyword()) { // C++ 2.5p2: Alternative tokens behave the same as its primary token // except for their spellings. Diag(MacroNameTok, getLangOpts().MicrosoftExt ? diag::ext_pp_operator_used_as_macro_name : diag::err_pp_operator_used_as_macro_name) << II << MacroNameTok.getKind(); - // Allow #defining |and| and friends for Microsoft compatibility or // recovery when legacy C headers are included in C++. - MacroNameTok.setIdentifierInfo(II); } if ((isDefineUndef != MU_Other) && II->getPPKeywordID() == tok::pp_defined) { diff --git a/lib/Lex/PPExpressions.cpp b/lib/Lex/PPExpressions.cpp index 12f5084298..d8431827e9 100644 --- a/lib/Lex/PPExpressions.cpp +++ b/lib/Lex/PPExpressions.cpp @@ -237,35 +237,32 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT, PP.setCodeCompletionReached(); PP.LexNonComment(PeekTok); } - - // If this token's spelling is a pp-identifier, check to see if it is - // 'defined' or if it is a macro. Note that we check here because many - // keywords are pp-identifiers, so we can't check the kind. - if (IdentifierInfo *II = PeekTok.getIdentifierInfo()) { - // Handle "defined X" and "defined(X)". - if (II->isStr("defined")) - return EvaluateDefined(Result, PeekTok, DT, ValueLive, PP); - - // If this identifier isn't 'defined' or one of the special - // preprocessor keywords and it wasn't macro expanded, it turns - // into a simple 0, unless it is the C++ keyword "true", in which case it - // turns into "1". - if (ValueLive && - II->getTokenID() != tok::kw_true && - II->getTokenID() != tok::kw_false) - PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II; - Result.Val = II->getTokenID() == tok::kw_true; - Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0. - Result.setIdentifier(II); - Result.setRange(PeekTok.getLocation()); - DT.IncludedUndefinedIds = (II->getTokenID() != tok::kw_true && - II->getTokenID() != tok::kw_false); - PP.LexNonComment(PeekTok); - return false; - } switch (PeekTok.getKind()) { - default: // Non-value token. + default: + // If this token's spelling is a pp-identifier, check to see if it is + // 'defined' or if it is a macro. Note that we check here because many + // keywords are pp-identifiers, so we can't check the kind. + if (IdentifierInfo *II = PeekTok.getIdentifierInfo()) { + // Handle "defined X" and "defined(X)". + if (II->isStr("defined")) + return EvaluateDefined(Result, PeekTok, DT, ValueLive, PP); + + if (!II->isCPlusPlusOperatorKeyword()) { + // If this identifier isn't 'defined' or one of the special + // preprocessor keywords and it wasn't macro expanded, it turns + // into a simple 0 + if (ValueLive) + PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II; + Result.Val = 0; + Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0. + Result.setIdentifier(II); + Result.setRange(PeekTok.getLocation()); + DT.IncludedUndefinedIds = true; + PP.LexNonComment(PeekTok); + return false; + } + } PP.Diag(PeekTok, diag::err_pp_expr_bad_token_start_expr); return true; case tok::eod: @@ -481,6 +478,14 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT, DT.State = DefinedTracker::DefinedMacro; return false; } + case tok::kw_true: + case tok::kw_false: + Result.Val = PeekTok.getKind() == tok::kw_true; + Result.Val.setIsUnsigned(false); // "0" is signed intmax_t 0. + Result.setIdentifier(PeekTok.getIdentifierInfo()); + Result.setRange(PeekTok.getLocation()); + PP.LexNonComment(PeekTok); + return false; // FIXME: Handle #assert } diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 63f39524d1..d1dc8e1c00 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -712,14 +712,6 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) { II.setIsFutureCompatKeyword(false); } - // C++ 2.11p2: If this is an alternative representation of a C++ operator, - // then we act as if it is the actual operator and not the textual - // representation of it. - if (II.isCPlusPlusOperatorKeyword() && - !(getLangOpts().MSVCCompat && - getSourceManager().isInSystemHeader(Identifier.getLocation()))) - Identifier.setIdentifierInfo(nullptr); - // If this is an extension token, diagnose its use. // We avoid diagnosing tokens that originate from macro definitions. // FIXME: This warning is disabled in cases where it shouldn't be, -- cgit v1.2.3 From fc60afa41c22bbd3cde2075d5ade4f8698e99243 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Mon, 17 Jul 2017 01:27:53 +0000 Subject: [NFC] Refactor the Preprocessor function that handles Macro definitions and rename Arguments to Parameters in Macro Definitions. - Extracted the reading of the tokens out into a separate function. - Replace 'Argument' with 'Parameter' when referring to the identifiers of the macro definition (as opposed to the supplied arguments - MacroArgs - during the macro invocation). This is in preparation for submitting patches for review to implement __VA_OPT__ which will otherwise just keep lengthening the HandleDefineDirective function and making it less comprehensible. Thanks! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@308157 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 10 +++--- lib/Lex/MacroInfo.cpp | 19 +++++----- lib/Lex/PPDirectives.cpp | 82 ++++++++++++++++++++++++++------------------ lib/Lex/PPMacroExpansion.cpp | 10 +++--- lib/Lex/TokenLexer.cpp | 12 +++---- 5 files changed, 75 insertions(+), 58 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index a201d16590..f791d8d4ba 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -52,14 +52,14 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, UnexpArgTokens.size() * sizeof(Token)); // Construct the MacroArgs object. new (Result) - MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumArgs()); + MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumParams()); } else { Result = *ResultEnt; // Unlink this node from the preprocessors singly linked list. *ResultEnt = Result->ArgCache; Result->NumUnexpArgTokens = UnexpArgTokens.size(); Result->VarargsElided = VarargsElided; - Result->NumMacroArgs = MI->getNumArgs(); + Result->NumMacroArgs = MI->getNumParams(); } // Copy the actual unexpanded tokens to immediately after the result ptr. @@ -148,11 +148,11 @@ bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok, const std::vector & MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP) { - assert(Arg < MI->getNumArgs() && "Invalid argument number!"); + assert(Arg < MI->getNumParams() && "Invalid argument number!"); // If we have already computed this, return it. - if (PreExpArgTokens.size() < MI->getNumArgs()) - PreExpArgTokens.resize(MI->getNumArgs()); + if (PreExpArgTokens.size() < MI->getNumParams()) + PreExpArgTokens.resize(MI->getNumParams()); std::vector &Result = PreExpArgTokens[Arg]; if (!Result.empty()) return Result; diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp index 1e5deeb191..6dc7841bc1 100644 --- a/lib/Lex/MacroInfo.cpp +++ b/lib/Lex/MacroInfo.cpp @@ -17,8 +17,8 @@ using namespace clang; MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc), - ArgumentList(nullptr), - NumArguments(0), + ParameterList(nullptr), + NumParameters(0), IsDefinitionLengthCached(false), IsFunctionLike(false), IsC99Varargs(false), @@ -74,7 +74,7 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, // Check # tokens in replacement, number of args, and various flags all match. if (ReplacementTokens.size() != Other.ReplacementTokens.size() || - getNumArgs() != Other.getNumArgs() || + getNumParams() != Other.getNumParams() || isFunctionLike() != Other.isFunctionLike() || isC99Varargs() != Other.isC99Varargs() || isGNUVarargs() != Other.isGNUVarargs()) @@ -82,7 +82,8 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, if (Lexically) { // Check arguments. - for (arg_iterator I = arg_begin(), OI = Other.arg_begin(), E = arg_end(); + for (param_iterator I = param_begin(), OI = Other.param_begin(), + E = param_end(); I != E; ++I, ++OI) if (*I != *OI) return false; } @@ -109,10 +110,10 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, return false; // With syntactic equivalence the parameter names can be different as long // as they are used in the same place. - int AArgNum = getArgumentNum(A.getIdentifierInfo()); + int AArgNum = getParameterNum(A.getIdentifierInfo()); if (AArgNum == -1) return false; - if (AArgNum != Other.getArgumentNum(B.getIdentifierInfo())) + if (AArgNum != Other.getParameterNum(B.getIdentifierInfo())) return false; continue; } @@ -141,12 +142,12 @@ LLVM_DUMP_METHOD void MacroInfo::dump() const { Out << "\n #define "; if (IsFunctionLike) { Out << "("; - for (unsigned I = 0; I != NumArguments; ++I) { + for (unsigned I = 0; I != NumParameters; ++I) { if (I) Out << ", "; - Out << ArgumentList[I]->getName(); + Out << ParameterList[I]->getName(); } if (IsC99Varargs || IsGNUVarargs) { - if (NumArguments && IsC99Varargs) Out << ", "; + if (NumParameters && IsC99Varargs) Out << ", "; Out << "..."; } Out << ")"; diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 4440cf2094..b2450f516b 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2135,11 +2135,11 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, // Preprocessor Macro Directive Handling. //===----------------------------------------------------------------------===// -/// ReadMacroDefinitionArgList - The ( starting an argument list of a macro +/// ReadMacroParameterList - The ( starting an argument list of a macro /// definition has just been read. Lex the rest of the arguments and the /// closing ), updating MI with what we learn. Return true if an error occurs /// parsing the arg list. -bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { +bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { SmallVector Arguments; while (true) { @@ -2173,7 +2173,7 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { // Add the __VA_ARGS__ identifier as an argument. Arguments.push_back(Ident__VA_ARGS__); MI->setIsC99Varargs(); - MI->setArgumentList(Arguments, BP); + MI->setParameterList(Arguments, BP); return false; case tok::eod: // #define X( Diag(Tok, diag::err_pp_missing_rparen_in_macro_def); @@ -2207,7 +2207,7 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { Diag(Tok, diag::err_pp_expected_comma_in_arg_list); return true; case tok::r_paren: // #define X(A) - MI->setArgumentList(Arguments, BP); + MI->setParameterList(Arguments, BP); return false; case tok::comma: // #define X(A, break; @@ -2223,7 +2223,7 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { } MI->setIsGNUVarargs(); - MI->setArgumentList(Arguments, BP); + MI->setParameterList(Arguments, BP); return false; } } @@ -2272,28 +2272,20 @@ static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI, MI->getNumTokens() == 0; } -/// HandleDefineDirective - Implements \#define. This consumes the entire macro -/// line then lets the caller lex the next real token. -void Preprocessor::HandleDefineDirective(Token &DefineTok, - bool ImmediatelyAfterHeaderGuard) { - ++NumDefined; - - Token MacroNameTok; - bool MacroShadowsKeyword; - ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword); +// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the +// entire line) of the macro's tokens and adds them to MacroInfo, and while +// doing so performs certain validity checks including (but not limited to): +// - # (stringization) is followed by a macro parameter +// +// Returns a nullptr if an invalid sequence of tokens is encountered or returns +// a pointer to a MacroInfo object. - // Error reading macro name? If so, diagnostic already issued. - if (MacroNameTok.is(tok::eod)) - return; +MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( + const Token &MacroNameTok, const bool ImmediatelyAfterHeaderGuard) { Token LastTok = MacroNameTok; - - // If we are supposed to keep comments in #defines, reenable comment saving - // mode. - if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments); - // Create the new macro. - MacroInfo *MI = AllocateMacroInfo(MacroNameTok.getLocation()); + MacroInfo *const MI = AllocateMacroInfo(MacroNameTok.getLocation()); Token Tok; LexUnexpandedToken(Tok); @@ -2315,11 +2307,11 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, } else if (Tok.is(tok::l_paren)) { // This is a function-like macro definition. Read the argument list. MI->setIsFunctionLike(); - if (ReadMacroDefinitionArgList(MI, LastTok)) { + if (ReadMacroParameterList(MI, LastTok)) { // Throw away the rest of the line. if (CurPPLexer->ParsingPreprocessorDirective) DiscardUntilEndOfDirective(); - return; + return nullptr; } // If this is a definition of a variadic C99 function-like macro, not using @@ -2426,7 +2418,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, // Check for a valid macro arg identifier. if (Tok.getIdentifierInfo() == nullptr || - MI->getArgumentNum(Tok.getIdentifierInfo()) == -1) { + MI->getParameterNum(Tok.getIdentifierInfo()) == -1) { // If this is assembler-with-cpp mode, we accept random gibberish after // the '#' because '#' is often a comment character. However, change @@ -2442,7 +2434,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, // Disable __VA_ARGS__ again. Ident__VA_ARGS__->setIsPoisoned(true); - return; + return nullptr; } } @@ -2455,15 +2447,39 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, LexUnexpandedToken(Tok); } } + MI->setDefinitionEndLoc(LastTok.getLocation()); + // Disable __VA_ARGS__ again. + Ident__VA_ARGS__->setIsPoisoned(true); + + return MI; +} +/// HandleDefineDirective - Implements \#define. This consumes the entire macro +/// line then lets the caller lex the next real token. +void Preprocessor::HandleDefineDirective( + Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) { + ++NumDefined; + + Token MacroNameTok; + bool MacroShadowsKeyword; + ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword); + + // Error reading macro name? If so, diagnostic already issued. + if (MacroNameTok.is(tok::eod)) + return; + + // If we are supposed to keep comments in #defines, reenable comment saving + // mode. + if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments); + + MacroInfo *const MI = ReadOptionalMacroParameterListAndBody( + MacroNameTok, ImmediatelyAfterHeaderGuard); + + if (!MI) return; if (MacroShadowsKeyword && !isConfigurationPattern(MacroNameTok, MI, getLangOpts())) { Diag(MacroNameTok, diag::warn_pp_macro_hides_keyword); - } - - // Disable __VA_ARGS__ again. - Ident__VA_ARGS__->setIsPoisoned(true); - + } // Check that there is no paste (##) operator at the beginning or end of the // replacement list. unsigned NumTokens = MI->getNumTokens(); @@ -2478,7 +2494,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, } } - MI->setDefinitionEndLoc(LastTok.getLocation()); + // Finally, if this identifier already had a macro defined for it, verify that // the macro bodies are identical, and issue diagnostics if they are not. diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 8af9a50cc2..3f8ede23da 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -412,7 +412,7 @@ static bool isTrivialSingleTokenExpansion(const MacroInfo *MI, // If this is a function-like macro invocation, it's safe to trivially expand // as long as the identifier is not a macro argument. - return std::find(MI->arg_begin(), MI->arg_end(), II) == MI->arg_end(); + return std::find(MI->param_begin(), MI->param_end(), II) == MI->param_end(); } /// isNextPPTokenLParen - Determine whether the next preprocessor token to be @@ -492,7 +492,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, // Preprocessor directives used inside macro arguments are not portable, and // this enables the warning. InMacroArgs = true; - Args = ReadFunctionLikeMacroArgs(Identifier, MI, ExpansionEnd); + Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEnd); // Finished parsing args. InMacroArgs = false; @@ -745,11 +745,11 @@ static bool GenerateNewArgTokens(Preprocessor &PP, /// token is the '(' of the macro, this method is invoked to read all of the /// actual arguments specified for the macro invocation. This returns null on /// error. -MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName, +MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI, SourceLocation &MacroEnd) { // The number of fixed arguments to parse. - unsigned NumFixedArgsLeft = MI->getNumArgs(); + unsigned NumFixedArgsLeft = MI->getNumParams(); bool isVariadic = MI->isVariadic(); // Outer loop, while there are more arguments, keep reading them. @@ -889,7 +889,7 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName, // Okay, we either found the r_paren. Check to see if we parsed too few // arguments. - unsigned MinArgsExpected = MI->getNumArgs(); + unsigned MinArgsExpected = MI->getNumParams(); // If this is not a variadic macro, and too many args were specified, emit // an error. diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 049e046cec..c2e49ba919 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -67,7 +67,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, // If this is a function-like macro, expand the arguments and change // Tokens to point to the expanded tokens. - if (Macro->isFunctionLike() && Macro->getNumArgs()) + if (Macro->isFunctionLike() && Macro->getNumParams()) ExpandFunctionArguments(); // Mark the macro as currently disabled, so that it is not recursively @@ -122,7 +122,7 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( SmallVectorImpl &ResultToks, bool HasPasteOperator, MacroInfo *Macro, unsigned MacroArgNo, Preprocessor &PP) { // Is the macro argument __VA_ARGS__? - if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) + if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1) return false; // In Microsoft-compatibility mode, a comma is removed in the expansion @@ -137,7 +137,7 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( // with GNU extensions, it is removed regardless of named arguments. // Microsoft also appears to support this extension, unofficially. if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode - && Macro->getNumArgs() < 2) + && Macro->getNumParams() < 2) return false; // Is a comma available to be removed? @@ -193,7 +193,7 @@ void TokenLexer::ExpandFunctionArguments() { NextTokGetsSpace = true; if (CurTok.isOneOf(tok::hash, tok::hashat)) { - int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); + int ArgNo = Macro->getParameterNum(Tokens[i+1].getIdentifierInfo()); assert(ArgNo != -1 && "Token following # is not an argument?"); SourceLocation ExpansionLocStart = @@ -237,7 +237,7 @@ void TokenLexer::ExpandFunctionArguments() { // Otherwise, if this is not an argument token, just add the token to the // output buffer. IdentifierInfo *II = CurTok.getIdentifierInfo(); - int ArgNo = II ? Macro->getArgumentNum(II) : -1; + int ArgNo = II ? Macro->getParameterNum(II) : -1; if (ArgNo == -1) { // This isn't an argument, just add it. ResultToks.push_back(CurTok); @@ -330,7 +330,7 @@ void TokenLexer::ExpandFunctionArguments() { // expansion. if (NonEmptyPasteBefore && ResultToks.size() >= 2 && ResultToks[ResultToks.size()-2].is(tok::comma) && - (unsigned)ArgNo == Macro->getNumArgs()-1 && + (unsigned)ArgNo == Macro->getNumParams()-1 && Macro->isVariadic()) { VaArgsPseudoPaste = true; // Remove the paste operator, report use of the extension. -- cgit v1.2.3 From fa8c4ac40846278e478dfe472e74063fca3d3f1e Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Mon, 17 Jul 2017 02:03:21 +0000 Subject: Revert changes from my previous refactoring - will need to fix dependencies in clang's extra tooling (such as clang-tidy etc.). Sorry about that. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@308158 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 10 +++--- lib/Lex/MacroInfo.cpp | 19 +++++----- lib/Lex/PPDirectives.cpp | 82 ++++++++++++++++++-------------------------- lib/Lex/PPMacroExpansion.cpp | 10 +++--- lib/Lex/TokenLexer.cpp | 12 +++---- 5 files changed, 58 insertions(+), 75 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index f791d8d4ba..a201d16590 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -52,14 +52,14 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, UnexpArgTokens.size() * sizeof(Token)); // Construct the MacroArgs object. new (Result) - MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumParams()); + MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumArgs()); } else { Result = *ResultEnt; // Unlink this node from the preprocessors singly linked list. *ResultEnt = Result->ArgCache; Result->NumUnexpArgTokens = UnexpArgTokens.size(); Result->VarargsElided = VarargsElided; - Result->NumMacroArgs = MI->getNumParams(); + Result->NumMacroArgs = MI->getNumArgs(); } // Copy the actual unexpanded tokens to immediately after the result ptr. @@ -148,11 +148,11 @@ bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok, const std::vector & MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP) { - assert(Arg < MI->getNumParams() && "Invalid argument number!"); + assert(Arg < MI->getNumArgs() && "Invalid argument number!"); // If we have already computed this, return it. - if (PreExpArgTokens.size() < MI->getNumParams()) - PreExpArgTokens.resize(MI->getNumParams()); + if (PreExpArgTokens.size() < MI->getNumArgs()) + PreExpArgTokens.resize(MI->getNumArgs()); std::vector &Result = PreExpArgTokens[Arg]; if (!Result.empty()) return Result; diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp index 6dc7841bc1..1e5deeb191 100644 --- a/lib/Lex/MacroInfo.cpp +++ b/lib/Lex/MacroInfo.cpp @@ -17,8 +17,8 @@ using namespace clang; MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc), - ParameterList(nullptr), - NumParameters(0), + ArgumentList(nullptr), + NumArguments(0), IsDefinitionLengthCached(false), IsFunctionLike(false), IsC99Varargs(false), @@ -74,7 +74,7 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, // Check # tokens in replacement, number of args, and various flags all match. if (ReplacementTokens.size() != Other.ReplacementTokens.size() || - getNumParams() != Other.getNumParams() || + getNumArgs() != Other.getNumArgs() || isFunctionLike() != Other.isFunctionLike() || isC99Varargs() != Other.isC99Varargs() || isGNUVarargs() != Other.isGNUVarargs()) @@ -82,8 +82,7 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, if (Lexically) { // Check arguments. - for (param_iterator I = param_begin(), OI = Other.param_begin(), - E = param_end(); + for (arg_iterator I = arg_begin(), OI = Other.arg_begin(), E = arg_end(); I != E; ++I, ++OI) if (*I != *OI) return false; } @@ -110,10 +109,10 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, return false; // With syntactic equivalence the parameter names can be different as long // as they are used in the same place. - int AArgNum = getParameterNum(A.getIdentifierInfo()); + int AArgNum = getArgumentNum(A.getIdentifierInfo()); if (AArgNum == -1) return false; - if (AArgNum != Other.getParameterNum(B.getIdentifierInfo())) + if (AArgNum != Other.getArgumentNum(B.getIdentifierInfo())) return false; continue; } @@ -142,12 +141,12 @@ LLVM_DUMP_METHOD void MacroInfo::dump() const { Out << "\n #define "; if (IsFunctionLike) { Out << "("; - for (unsigned I = 0; I != NumParameters; ++I) { + for (unsigned I = 0; I != NumArguments; ++I) { if (I) Out << ", "; - Out << ParameterList[I]->getName(); + Out << ArgumentList[I]->getName(); } if (IsC99Varargs || IsGNUVarargs) { - if (NumParameters && IsC99Varargs) Out << ", "; + if (NumArguments && IsC99Varargs) Out << ", "; Out << "..."; } Out << ")"; diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index b2450f516b..4440cf2094 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2135,11 +2135,11 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, // Preprocessor Macro Directive Handling. //===----------------------------------------------------------------------===// -/// ReadMacroParameterList - The ( starting an argument list of a macro +/// ReadMacroDefinitionArgList - The ( starting an argument list of a macro /// definition has just been read. Lex the rest of the arguments and the /// closing ), updating MI with what we learn. Return true if an error occurs /// parsing the arg list. -bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { +bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { SmallVector Arguments; while (true) { @@ -2173,7 +2173,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { // Add the __VA_ARGS__ identifier as an argument. Arguments.push_back(Ident__VA_ARGS__); MI->setIsC99Varargs(); - MI->setParameterList(Arguments, BP); + MI->setArgumentList(Arguments, BP); return false; case tok::eod: // #define X( Diag(Tok, diag::err_pp_missing_rparen_in_macro_def); @@ -2207,7 +2207,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { Diag(Tok, diag::err_pp_expected_comma_in_arg_list); return true; case tok::r_paren: // #define X(A) - MI->setParameterList(Arguments, BP); + MI->setArgumentList(Arguments, BP); return false; case tok::comma: // #define X(A, break; @@ -2223,7 +2223,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { } MI->setIsGNUVarargs(); - MI->setParameterList(Arguments, BP); + MI->setArgumentList(Arguments, BP); return false; } } @@ -2272,20 +2272,28 @@ static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI, MI->getNumTokens() == 0; } -// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the -// entire line) of the macro's tokens and adds them to MacroInfo, and while -// doing so performs certain validity checks including (but not limited to): -// - # (stringization) is followed by a macro parameter -// -// Returns a nullptr if an invalid sequence of tokens is encountered or returns -// a pointer to a MacroInfo object. +/// HandleDefineDirective - Implements \#define. This consumes the entire macro +/// line then lets the caller lex the next real token. +void Preprocessor::HandleDefineDirective(Token &DefineTok, + bool ImmediatelyAfterHeaderGuard) { + ++NumDefined; + + Token MacroNameTok; + bool MacroShadowsKeyword; + ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword); -MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( - const Token &MacroNameTok, const bool ImmediatelyAfterHeaderGuard) { + // Error reading macro name? If so, diagnostic already issued. + if (MacroNameTok.is(tok::eod)) + return; Token LastTok = MacroNameTok; + + // If we are supposed to keep comments in #defines, reenable comment saving + // mode. + if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments); + // Create the new macro. - MacroInfo *const MI = AllocateMacroInfo(MacroNameTok.getLocation()); + MacroInfo *MI = AllocateMacroInfo(MacroNameTok.getLocation()); Token Tok; LexUnexpandedToken(Tok); @@ -2307,11 +2315,11 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( } else if (Tok.is(tok::l_paren)) { // This is a function-like macro definition. Read the argument list. MI->setIsFunctionLike(); - if (ReadMacroParameterList(MI, LastTok)) { + if (ReadMacroDefinitionArgList(MI, LastTok)) { // Throw away the rest of the line. if (CurPPLexer->ParsingPreprocessorDirective) DiscardUntilEndOfDirective(); - return nullptr; + return; } // If this is a definition of a variadic C99 function-like macro, not using @@ -2418,7 +2426,7 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( // Check for a valid macro arg identifier. if (Tok.getIdentifierInfo() == nullptr || - MI->getParameterNum(Tok.getIdentifierInfo()) == -1) { + MI->getArgumentNum(Tok.getIdentifierInfo()) == -1) { // If this is assembler-with-cpp mode, we accept random gibberish after // the '#' because '#' is often a comment character. However, change @@ -2434,7 +2442,7 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( // Disable __VA_ARGS__ again. Ident__VA_ARGS__->setIsPoisoned(true); - return nullptr; + return; } } @@ -2447,39 +2455,15 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( LexUnexpandedToken(Tok); } } - MI->setDefinitionEndLoc(LastTok.getLocation()); - // Disable __VA_ARGS__ again. - Ident__VA_ARGS__->setIsPoisoned(true); - - return MI; -} -/// HandleDefineDirective - Implements \#define. This consumes the entire macro -/// line then lets the caller lex the next real token. -void Preprocessor::HandleDefineDirective( - Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) { - ++NumDefined; - - Token MacroNameTok; - bool MacroShadowsKeyword; - ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword); - - // Error reading macro name? If so, diagnostic already issued. - if (MacroNameTok.is(tok::eod)) - return; - - // If we are supposed to keep comments in #defines, reenable comment saving - // mode. - if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments); - - MacroInfo *const MI = ReadOptionalMacroParameterListAndBody( - MacroNameTok, ImmediatelyAfterHeaderGuard); - - if (!MI) return; if (MacroShadowsKeyword && !isConfigurationPattern(MacroNameTok, MI, getLangOpts())) { Diag(MacroNameTok, diag::warn_pp_macro_hides_keyword); - } + } + + // Disable __VA_ARGS__ again. + Ident__VA_ARGS__->setIsPoisoned(true); + // Check that there is no paste (##) operator at the beginning or end of the // replacement list. unsigned NumTokens = MI->getNumTokens(); @@ -2494,7 +2478,7 @@ void Preprocessor::HandleDefineDirective( } } - + MI->setDefinitionEndLoc(LastTok.getLocation()); // Finally, if this identifier already had a macro defined for it, verify that // the macro bodies are identical, and issue diagnostics if they are not. diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 3f8ede23da..8af9a50cc2 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -412,7 +412,7 @@ static bool isTrivialSingleTokenExpansion(const MacroInfo *MI, // If this is a function-like macro invocation, it's safe to trivially expand // as long as the identifier is not a macro argument. - return std::find(MI->param_begin(), MI->param_end(), II) == MI->param_end(); + return std::find(MI->arg_begin(), MI->arg_end(), II) == MI->arg_end(); } /// isNextPPTokenLParen - Determine whether the next preprocessor token to be @@ -492,7 +492,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, // Preprocessor directives used inside macro arguments are not portable, and // this enables the warning. InMacroArgs = true; - Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEnd); + Args = ReadFunctionLikeMacroArgs(Identifier, MI, ExpansionEnd); // Finished parsing args. InMacroArgs = false; @@ -745,11 +745,11 @@ static bool GenerateNewArgTokens(Preprocessor &PP, /// token is the '(' of the macro, this method is invoked to read all of the /// actual arguments specified for the macro invocation. This returns null on /// error. -MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, +MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName, MacroInfo *MI, SourceLocation &MacroEnd) { // The number of fixed arguments to parse. - unsigned NumFixedArgsLeft = MI->getNumParams(); + unsigned NumFixedArgsLeft = MI->getNumArgs(); bool isVariadic = MI->isVariadic(); // Outer loop, while there are more arguments, keep reading them. @@ -889,7 +889,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, // Okay, we either found the r_paren. Check to see if we parsed too few // arguments. - unsigned MinArgsExpected = MI->getNumParams(); + unsigned MinArgsExpected = MI->getNumArgs(); // If this is not a variadic macro, and too many args were specified, emit // an error. diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index c2e49ba919..049e046cec 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -67,7 +67,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, // If this is a function-like macro, expand the arguments and change // Tokens to point to the expanded tokens. - if (Macro->isFunctionLike() && Macro->getNumParams()) + if (Macro->isFunctionLike() && Macro->getNumArgs()) ExpandFunctionArguments(); // Mark the macro as currently disabled, so that it is not recursively @@ -122,7 +122,7 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( SmallVectorImpl &ResultToks, bool HasPasteOperator, MacroInfo *Macro, unsigned MacroArgNo, Preprocessor &PP) { // Is the macro argument __VA_ARGS__? - if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1) + if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) return false; // In Microsoft-compatibility mode, a comma is removed in the expansion @@ -137,7 +137,7 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( // with GNU extensions, it is removed regardless of named arguments. // Microsoft also appears to support this extension, unofficially. if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode - && Macro->getNumParams() < 2) + && Macro->getNumArgs() < 2) return false; // Is a comma available to be removed? @@ -193,7 +193,7 @@ void TokenLexer::ExpandFunctionArguments() { NextTokGetsSpace = true; if (CurTok.isOneOf(tok::hash, tok::hashat)) { - int ArgNo = Macro->getParameterNum(Tokens[i+1].getIdentifierInfo()); + int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); assert(ArgNo != -1 && "Token following # is not an argument?"); SourceLocation ExpansionLocStart = @@ -237,7 +237,7 @@ void TokenLexer::ExpandFunctionArguments() { // Otherwise, if this is not an argument token, just add the token to the // output buffer. IdentifierInfo *II = CurTok.getIdentifierInfo(); - int ArgNo = II ? Macro->getParameterNum(II) : -1; + int ArgNo = II ? Macro->getArgumentNum(II) : -1; if (ArgNo == -1) { // This isn't an argument, just add it. ResultToks.push_back(CurTok); @@ -330,7 +330,7 @@ void TokenLexer::ExpandFunctionArguments() { // expansion. if (NonEmptyPasteBefore && ResultToks.size() >= 2 && ResultToks[ResultToks.size()-2].is(tok::comma) && - (unsigned)ArgNo == Macro->getNumParams()-1 && + (unsigned)ArgNo == Macro->getNumArgs()-1 && Macro->isVariadic()) { VaArgsPseudoPaste = true; // Remove the paste operator, report use of the extension. -- cgit v1.2.3 From d1a84831f513795b7768e725aee35e476169c973 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Mon, 17 Jul 2017 17:18:43 +0000 Subject: [NFC] Refactor the Preprocessor function that handles Macro definitions and rename Arguments to Parameters in Macro Definitions. - Extracted the reading of the tokens out into a separate function. - Replace 'Argument' with 'Parameter' when referring to the identifiers of the macro definition (as opposed to the supplied arguments - MacroArgs - during the macro invocation). This is in preparation for submitting patches for review to implement __VA_OPT__ which will otherwise just keep lengthening the HandleDefineDirective function and making it less comprehensible. I will also directly update some extra clang tooling that is broken by the change from Argument to Parameter. Hopefully the bots will stay appeased. Thanks! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@308190 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 10 +++--- lib/Lex/MacroInfo.cpp | 19 +++++----- lib/Lex/PPDirectives.cpp | 82 ++++++++++++++++++++++++++------------------ lib/Lex/PPMacroExpansion.cpp | 10 +++--- lib/Lex/TokenLexer.cpp | 12 +++---- 5 files changed, 75 insertions(+), 58 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index a201d16590..f791d8d4ba 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -52,14 +52,14 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, UnexpArgTokens.size() * sizeof(Token)); // Construct the MacroArgs object. new (Result) - MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumArgs()); + MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumParams()); } else { Result = *ResultEnt; // Unlink this node from the preprocessors singly linked list. *ResultEnt = Result->ArgCache; Result->NumUnexpArgTokens = UnexpArgTokens.size(); Result->VarargsElided = VarargsElided; - Result->NumMacroArgs = MI->getNumArgs(); + Result->NumMacroArgs = MI->getNumParams(); } // Copy the actual unexpanded tokens to immediately after the result ptr. @@ -148,11 +148,11 @@ bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok, const std::vector & MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP) { - assert(Arg < MI->getNumArgs() && "Invalid argument number!"); + assert(Arg < MI->getNumParams() && "Invalid argument number!"); // If we have already computed this, return it. - if (PreExpArgTokens.size() < MI->getNumArgs()) - PreExpArgTokens.resize(MI->getNumArgs()); + if (PreExpArgTokens.size() < MI->getNumParams()) + PreExpArgTokens.resize(MI->getNumParams()); std::vector &Result = PreExpArgTokens[Arg]; if (!Result.empty()) return Result; diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp index 1e5deeb191..6dc7841bc1 100644 --- a/lib/Lex/MacroInfo.cpp +++ b/lib/Lex/MacroInfo.cpp @@ -17,8 +17,8 @@ using namespace clang; MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc), - ArgumentList(nullptr), - NumArguments(0), + ParameterList(nullptr), + NumParameters(0), IsDefinitionLengthCached(false), IsFunctionLike(false), IsC99Varargs(false), @@ -74,7 +74,7 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, // Check # tokens in replacement, number of args, and various flags all match. if (ReplacementTokens.size() != Other.ReplacementTokens.size() || - getNumArgs() != Other.getNumArgs() || + getNumParams() != Other.getNumParams() || isFunctionLike() != Other.isFunctionLike() || isC99Varargs() != Other.isC99Varargs() || isGNUVarargs() != Other.isGNUVarargs()) @@ -82,7 +82,8 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, if (Lexically) { // Check arguments. - for (arg_iterator I = arg_begin(), OI = Other.arg_begin(), E = arg_end(); + for (param_iterator I = param_begin(), OI = Other.param_begin(), + E = param_end(); I != E; ++I, ++OI) if (*I != *OI) return false; } @@ -109,10 +110,10 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP, return false; // With syntactic equivalence the parameter names can be different as long // as they are used in the same place. - int AArgNum = getArgumentNum(A.getIdentifierInfo()); + int AArgNum = getParameterNum(A.getIdentifierInfo()); if (AArgNum == -1) return false; - if (AArgNum != Other.getArgumentNum(B.getIdentifierInfo())) + if (AArgNum != Other.getParameterNum(B.getIdentifierInfo())) return false; continue; } @@ -141,12 +142,12 @@ LLVM_DUMP_METHOD void MacroInfo::dump() const { Out << "\n #define "; if (IsFunctionLike) { Out << "("; - for (unsigned I = 0; I != NumArguments; ++I) { + for (unsigned I = 0; I != NumParameters; ++I) { if (I) Out << ", "; - Out << ArgumentList[I]->getName(); + Out << ParameterList[I]->getName(); } if (IsC99Varargs || IsGNUVarargs) { - if (NumArguments && IsC99Varargs) Out << ", "; + if (NumParameters && IsC99Varargs) Out << ", "; Out << "..."; } Out << ")"; diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 4440cf2094..b2450f516b 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2135,11 +2135,11 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, // Preprocessor Macro Directive Handling. //===----------------------------------------------------------------------===// -/// ReadMacroDefinitionArgList - The ( starting an argument list of a macro +/// ReadMacroParameterList - The ( starting an argument list of a macro /// definition has just been read. Lex the rest of the arguments and the /// closing ), updating MI with what we learn. Return true if an error occurs /// parsing the arg list. -bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { +bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { SmallVector Arguments; while (true) { @@ -2173,7 +2173,7 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { // Add the __VA_ARGS__ identifier as an argument. Arguments.push_back(Ident__VA_ARGS__); MI->setIsC99Varargs(); - MI->setArgumentList(Arguments, BP); + MI->setParameterList(Arguments, BP); return false; case tok::eod: // #define X( Diag(Tok, diag::err_pp_missing_rparen_in_macro_def); @@ -2207,7 +2207,7 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { Diag(Tok, diag::err_pp_expected_comma_in_arg_list); return true; case tok::r_paren: // #define X(A) - MI->setArgumentList(Arguments, BP); + MI->setParameterList(Arguments, BP); return false; case tok::comma: // #define X(A, break; @@ -2223,7 +2223,7 @@ bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) { } MI->setIsGNUVarargs(); - MI->setArgumentList(Arguments, BP); + MI->setParameterList(Arguments, BP); return false; } } @@ -2272,28 +2272,20 @@ static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI, MI->getNumTokens() == 0; } -/// HandleDefineDirective - Implements \#define. This consumes the entire macro -/// line then lets the caller lex the next real token. -void Preprocessor::HandleDefineDirective(Token &DefineTok, - bool ImmediatelyAfterHeaderGuard) { - ++NumDefined; - - Token MacroNameTok; - bool MacroShadowsKeyword; - ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword); +// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the +// entire line) of the macro's tokens and adds them to MacroInfo, and while +// doing so performs certain validity checks including (but not limited to): +// - # (stringization) is followed by a macro parameter +// +// Returns a nullptr if an invalid sequence of tokens is encountered or returns +// a pointer to a MacroInfo object. - // Error reading macro name? If so, diagnostic already issued. - if (MacroNameTok.is(tok::eod)) - return; +MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( + const Token &MacroNameTok, const bool ImmediatelyAfterHeaderGuard) { Token LastTok = MacroNameTok; - - // If we are supposed to keep comments in #defines, reenable comment saving - // mode. - if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments); - // Create the new macro. - MacroInfo *MI = AllocateMacroInfo(MacroNameTok.getLocation()); + MacroInfo *const MI = AllocateMacroInfo(MacroNameTok.getLocation()); Token Tok; LexUnexpandedToken(Tok); @@ -2315,11 +2307,11 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, } else if (Tok.is(tok::l_paren)) { // This is a function-like macro definition. Read the argument list. MI->setIsFunctionLike(); - if (ReadMacroDefinitionArgList(MI, LastTok)) { + if (ReadMacroParameterList(MI, LastTok)) { // Throw away the rest of the line. if (CurPPLexer->ParsingPreprocessorDirective) DiscardUntilEndOfDirective(); - return; + return nullptr; } // If this is a definition of a variadic C99 function-like macro, not using @@ -2426,7 +2418,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, // Check for a valid macro arg identifier. if (Tok.getIdentifierInfo() == nullptr || - MI->getArgumentNum(Tok.getIdentifierInfo()) == -1) { + MI->getParameterNum(Tok.getIdentifierInfo()) == -1) { // If this is assembler-with-cpp mode, we accept random gibberish after // the '#' because '#' is often a comment character. However, change @@ -2442,7 +2434,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, // Disable __VA_ARGS__ again. Ident__VA_ARGS__->setIsPoisoned(true); - return; + return nullptr; } } @@ -2455,15 +2447,39 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, LexUnexpandedToken(Tok); } } + MI->setDefinitionEndLoc(LastTok.getLocation()); + // Disable __VA_ARGS__ again. + Ident__VA_ARGS__->setIsPoisoned(true); + + return MI; +} +/// HandleDefineDirective - Implements \#define. This consumes the entire macro +/// line then lets the caller lex the next real token. +void Preprocessor::HandleDefineDirective( + Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) { + ++NumDefined; + + Token MacroNameTok; + bool MacroShadowsKeyword; + ReadMacroName(MacroNameTok, MU_Define, &MacroShadowsKeyword); + + // Error reading macro name? If so, diagnostic already issued. + if (MacroNameTok.is(tok::eod)) + return; + + // If we are supposed to keep comments in #defines, reenable comment saving + // mode. + if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments); + + MacroInfo *const MI = ReadOptionalMacroParameterListAndBody( + MacroNameTok, ImmediatelyAfterHeaderGuard); + + if (!MI) return; if (MacroShadowsKeyword && !isConfigurationPattern(MacroNameTok, MI, getLangOpts())) { Diag(MacroNameTok, diag::warn_pp_macro_hides_keyword); - } - - // Disable __VA_ARGS__ again. - Ident__VA_ARGS__->setIsPoisoned(true); - + } // Check that there is no paste (##) operator at the beginning or end of the // replacement list. unsigned NumTokens = MI->getNumTokens(); @@ -2478,7 +2494,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok, } } - MI->setDefinitionEndLoc(LastTok.getLocation()); + // Finally, if this identifier already had a macro defined for it, verify that // the macro bodies are identical, and issue diagnostics if they are not. diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 8af9a50cc2..3f8ede23da 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -412,7 +412,7 @@ static bool isTrivialSingleTokenExpansion(const MacroInfo *MI, // If this is a function-like macro invocation, it's safe to trivially expand // as long as the identifier is not a macro argument. - return std::find(MI->arg_begin(), MI->arg_end(), II) == MI->arg_end(); + return std::find(MI->param_begin(), MI->param_end(), II) == MI->param_end(); } /// isNextPPTokenLParen - Determine whether the next preprocessor token to be @@ -492,7 +492,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, // Preprocessor directives used inside macro arguments are not portable, and // this enables the warning. InMacroArgs = true; - Args = ReadFunctionLikeMacroArgs(Identifier, MI, ExpansionEnd); + Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEnd); // Finished parsing args. InMacroArgs = false; @@ -745,11 +745,11 @@ static bool GenerateNewArgTokens(Preprocessor &PP, /// token is the '(' of the macro, this method is invoked to read all of the /// actual arguments specified for the macro invocation. This returns null on /// error. -MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName, +MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI, SourceLocation &MacroEnd) { // The number of fixed arguments to parse. - unsigned NumFixedArgsLeft = MI->getNumArgs(); + unsigned NumFixedArgsLeft = MI->getNumParams(); bool isVariadic = MI->isVariadic(); // Outer loop, while there are more arguments, keep reading them. @@ -889,7 +889,7 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName, // Okay, we either found the r_paren. Check to see if we parsed too few // arguments. - unsigned MinArgsExpected = MI->getNumArgs(); + unsigned MinArgsExpected = MI->getNumParams(); // If this is not a variadic macro, and too many args were specified, emit // an error. diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 049e046cec..c2e49ba919 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -67,7 +67,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, // If this is a function-like macro, expand the arguments and change // Tokens to point to the expanded tokens. - if (Macro->isFunctionLike() && Macro->getNumArgs()) + if (Macro->isFunctionLike() && Macro->getNumParams()) ExpandFunctionArguments(); // Mark the macro as currently disabled, so that it is not recursively @@ -122,7 +122,7 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( SmallVectorImpl &ResultToks, bool HasPasteOperator, MacroInfo *Macro, unsigned MacroArgNo, Preprocessor &PP) { // Is the macro argument __VA_ARGS__? - if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) + if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1) return false; // In Microsoft-compatibility mode, a comma is removed in the expansion @@ -137,7 +137,7 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( // with GNU extensions, it is removed regardless of named arguments. // Microsoft also appears to support this extension, unofficially. if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode - && Macro->getNumArgs() < 2) + && Macro->getNumParams() < 2) return false; // Is a comma available to be removed? @@ -193,7 +193,7 @@ void TokenLexer::ExpandFunctionArguments() { NextTokGetsSpace = true; if (CurTok.isOneOf(tok::hash, tok::hashat)) { - int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); + int ArgNo = Macro->getParameterNum(Tokens[i+1].getIdentifierInfo()); assert(ArgNo != -1 && "Token following # is not an argument?"); SourceLocation ExpansionLocStart = @@ -237,7 +237,7 @@ void TokenLexer::ExpandFunctionArguments() { // Otherwise, if this is not an argument token, just add the token to the // output buffer. IdentifierInfo *II = CurTok.getIdentifierInfo(); - int ArgNo = II ? Macro->getArgumentNum(II) : -1; + int ArgNo = II ? Macro->getParameterNum(II) : -1; if (ArgNo == -1) { // This isn't an argument, just add it. ResultToks.push_back(CurTok); @@ -330,7 +330,7 @@ void TokenLexer::ExpandFunctionArguments() { // expansion. if (NonEmptyPasteBefore && ResultToks.size() >= 2 && ResultToks[ResultToks.size()-2].is(tok::comma) && - (unsigned)ArgNo == Macro->getNumArgs()-1 && + (unsigned)ArgNo == Macro->getNumParams()-1 && Macro->isVariadic()) { VaArgsPseudoPaste = true; // Remove the paste operator, report use of the extension. -- cgit v1.2.3 From 0dfbcb01b8b699ac0f4eea0274ab1bb04397b701 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Thu, 20 Jul 2017 01:10:56 +0000 Subject: [NFC] Update local variable names to upper-case as per LLVM Coding Standards. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@308574 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index c2e49ba919..10e21b3b04 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -178,28 +178,28 @@ void TokenLexer::ExpandFunctionArguments() { // we install the newly expanded sequence as the new 'Tokens' list. bool MadeChange = false; - for (unsigned i = 0, e = NumTokens; i != e; ++i) { + for (unsigned I = 0, E = NumTokens; I != E; ++I) { // If we found the stringify operator, get the argument stringified. The // preprocessor already verified that the following token is a macro name // when the #define was parsed. - const Token &CurTok = Tokens[i]; + const Token &CurTok = Tokens[I]; // We don't want a space for the next token after a paste // operator. In valid code, the token will get smooshed onto the // preceding one anyway. In assembler-with-cpp mode, invalid // pastes are allowed through: in this case, we do not want the // extra whitespace to be added. For example, we want ". ## foo" // -> ".foo" not ". foo". - if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) + if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) NextTokGetsSpace = true; if (CurTok.isOneOf(tok::hash, tok::hashat)) { - int ArgNo = Macro->getParameterNum(Tokens[i+1].getIdentifierInfo()); + int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo()); assert(ArgNo != -1 && "Token following # is not an argument?"); SourceLocation ExpansionLocStart = getExpansionLocForMacroDefLoc(CurTok.getLocation()); SourceLocation ExpansionLocEnd = - getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation()); + getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation()); Token Res; if (CurTok.is(tok::hash)) // Stringify @@ -222,7 +222,7 @@ void TokenLexer::ExpandFunctionArguments() { ResultToks.push_back(Res); MadeChange = true; - ++i; // Skip arg name. + ++I; // Skip arg name. NextTokGetsSpace = false; continue; } @@ -230,8 +230,8 @@ void TokenLexer::ExpandFunctionArguments() { // Find out if there is a paste (##) operator before or after the token. bool NonEmptyPasteBefore = !ResultToks.empty() && ResultToks.back().is(tok::hashhash); - bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash); - bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash); + bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash); + bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash); assert(!NonEmptyPasteBefore || PasteBefore); // Otherwise, if this is not an argument token, just add the token to the @@ -374,7 +374,7 @@ void TokenLexer::ExpandFunctionArguments() { if (PasteAfter) { // Discard the argument token and skip (don't copy to the expansion // buffer) the paste operator after it. - ++i; + ++I; continue; } -- cgit v1.2.3 From d939eff44caebbfed6b954bd418e7b8ab90ce87d Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Tue, 25 Jul 2017 03:15:36 +0000 Subject: [NFC] Use RAII to un-poison and then re-poison __VA_ARGS__ - This will also be used for the forthcoming __VA_OPT__ feature approved for C++2a. - recommended by rsmith during his review of the __VA_OPT__ patch (https://reviews.llvm.org/D35782) git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@308948 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index b2450f516b..6b09398e46 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -33,6 +33,7 @@ #include "clang/Lex/PreprocessorOptions.h" #include "clang/Lex/PTHLexer.h" #include "clang/Lex/Token.h" +#include "clang/Lex/VariadicMacroSupport.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" @@ -2290,6 +2291,10 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( Token Tok; LexUnexpandedToken(Tok); + // Used to un-poison and then re-poison identifiers of the __VA_ARGS__ ilk + // within their appropriate context. + VariadicMacroScopeGuard VariadicMacroScopeGuard(*this); + // If this is a function-like macro definition, parse the argument list, // marking each of the identifiers as being used as macro arguments. Also, // check other constraints on the first token of the macro body. @@ -2314,14 +2319,14 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( return nullptr; } - // If this is a definition of a variadic C99 function-like macro, not using - // the GNU named varargs extension, enabled __VA_ARGS__. + // If this is a definition of an ISO C/C++ variadic function-like macro (not + // using the GNU named varargs extension) inform our variadic scope guard + // which un-poisons and re-poisons certain identifiers (e.g. __VA_ARGS__) + // allowed only within the definition of a variadic macro. - // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. - // This gets unpoisoned where it is allowed. - assert(Ident__VA_ARGS__->isPoisoned() && "__VA_ARGS__ should be poisoned!"); - if (MI->isC99Varargs()) - Ident__VA_ARGS__->setIsPoisoned(false); + if (MI->isC99Varargs()) { + VariadicMacroScopeGuard.enterScope(); + } // Read the first token after the arg list for down below. LexUnexpandedToken(Tok); @@ -2431,9 +2436,6 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( } else { Diag(Tok, diag::err_pp_stringize_not_parameter) << LastTok.is(tok::hashat); - - // Disable __VA_ARGS__ again. - Ident__VA_ARGS__->setIsPoisoned(true); return nullptr; } } @@ -2448,9 +2450,6 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( } } MI->setDefinitionEndLoc(LastTok.getLocation()); - // Disable __VA_ARGS__ again. - Ident__VA_ARGS__->setIsPoisoned(true); - return MI; } /// HandleDefineDirective - Implements \#define. This consumes the entire macro -- cgit v1.2.3 From f29d44a306f1ad20d1b2bf82ccbd6b7848786909 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Sun, 30 Jul 2017 06:31:29 +0000 Subject: PR33902: Invalidate line number cache when adding more text to existing buffer. This led to crashes as the line number cache would report a bogus line number for a line of code, and we'd try to find a nonexistent column within the line when printing diagnostics. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@309503 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ScratchBuffer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/ScratchBuffer.cpp b/lib/Lex/ScratchBuffer.cpp index cd8a27e76c..e0f3966fce 100644 --- a/lib/Lex/ScratchBuffer.cpp +++ b/lib/Lex/ScratchBuffer.cpp @@ -35,6 +35,14 @@ SourceLocation ScratchBuffer::getToken(const char *Buf, unsigned Len, const char *&DestPtr) { if (BytesUsed+Len+2 > ScratchBufSize) AllocScratchBuffer(Len+2); + else { + // Clear out the source line cache if it's already been computed. + // FIXME: Allow this to be incrementally extended. + auto *ContentCache = const_cast( + SourceMgr.getSLocEntry(SourceMgr.getFileID(BufferStartLoc)) + .getFile().getContentCache()); + ContentCache->SourceLineCache = nullptr; + } // Prefix the token with a \n, so that it looks like it is the first thing on // its own virtual line in caret diagnostics. -- cgit v1.2.3 From 7e859c0fd66b5d9e54d113974ed9bfd8098980ba Mon Sep 17 00:00:00 2001 From: Tim Northover Date: Tue, 8 Aug 2017 22:03:54 +0000 Subject: Lexer: always allow imaginary constants in GNU mode. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310423 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index a598a46781..52b259edce 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -659,7 +659,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, } } // "i", "if", and "il" are user-defined suffixes in C++1y. - if (*s == 'i' && PP.getLangOpts().CPlusPlus14) + if (*s == 'i' && !PP.getLangOpts().GNUMode) break; // fall through. case 'j': -- cgit v1.2.3 From a7bbae30542dc157f953d16d4acf3d61e1cc661c Mon Sep 17 00:00:00 2001 From: Tim Northover Date: Tue, 8 Aug 2017 23:17:51 +0000 Subject: Revert "Lexer: always allow imaginary constants in GNU mode." This reverts r310423. It was committed by mistake, I intended to commit the improved diagnostics for implicit conversions instead. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310426 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index 52b259edce..a598a46781 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -659,7 +659,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, } } // "i", "if", and "il" are user-defined suffixes in C++1y. - if (*s == 'i' && !PP.getLangOpts().GNUMode) + if (*s == 'i' && PP.getLangOpts().CPlusPlus14) break; // fall through. case 'j': -- cgit v1.2.3 From 442707be84f5e5b20bf3d489a2d1423c8f06b4eb Mon Sep 17 00:00:00 2001 From: Tim Northover Date: Wed, 9 Aug 2017 14:56:48 +0000 Subject: Reapply Sema: allow imaginary constants via GNU extension if UDL overloads not present. C++14 added user-defined literal support for complex numbers so that you can write something like "complex val = 2i". However, there is an existing GNU extension supporting this syntax and interpreting the result as a _Complex type. This changes parsing so that such literals are interpreted in terms of C++14's operators if an overload is present but otherwise falls back to the original GNU extension. (We now have more robust diagnostics for implicit conversions so the libc++ test that caused the original revert still passes). git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310478 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index a598a46781..3cc45cea97 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -658,9 +658,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, break; } } - // "i", "if", and "il" are user-defined suffixes in C++1y. - if (*s == 'i' && PP.getLangOpts().CPlusPlus14) - break; // fall through. case 'j': case 'J': @@ -672,35 +669,34 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, break; } - if (s != ThisTokEnd) { + // "i", "if", and "il" are user-defined suffixes in C++1y. + if (s != ThisTokEnd || isImaginary) { // FIXME: Don't bother expanding UCNs if !tok.hasUCN(). expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)); if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) { - // Any suffix pieces we might have parsed are actually part of the - // ud-suffix. - isLong = false; - isUnsigned = false; - isLongLong = false; - isFloat = false; - isHalf = false; - isImaginary = false; - MicrosoftInteger = 0; + if (!isImaginary) { + // Any suffix pieces we might have parsed are actually part of the + // ud-suffix. + isLong = false; + isUnsigned = false; + isLongLong = false; + isFloat = false; + isHalf = false; + isImaginary = false; + MicrosoftInteger = 0; + } saw_ud_suffix = true; return; } - // Report an error if there are any. - PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), - diag::err_invalid_suffix_constant) - << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin) << isFPConstant; - hadError = true; - return; - } - - if (isImaginary) { - PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), - diag::ext_imaginary_constant); + if (s != ThisTokEnd) { + // Report an error if there are any. + PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), + diag::err_invalid_suffix_constant) + << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin) << isFPConstant; + hadError = true; + } } } -- cgit v1.2.3 From a2fc7dbc981deab96d1633331da793e2333c7521 Mon Sep 17 00:00:00 2001 From: Alexander Kornienko Date: Thu, 10 Aug 2017 10:06:16 +0000 Subject: [Lexer] Finding beginning of token with escaped new line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Lexer::GetBeginningOfToken produced invalid location when backtracking across escaped new lines. This fixes PR26228 Reviewers: akyrtzi, alexfh, rsmith, doug.gregor Reviewed By: alexfh Subscribers: alexfh, cfe-commits Patch by Paweł Żukowski! Differential Revision: https://reviews.llvm.org/D30748 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310576 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 72 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 28 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 61bcef8cb7..79472961c0 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -463,19 +463,15 @@ static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) { const char *BufStart = Buffer.data(); if (Offset >= Buffer.size()) return nullptr; - const char *StrData = BufStart + Offset; - if (StrData[0] == '\n' || StrData[0] == '\r') - return StrData; - - const char *LexStart = StrData; - while (LexStart != BufStart) { - if (LexStart[0] == '\n' || LexStart[0] == '\r') { + const char *LexStart = BufStart + Offset; + for (; LexStart != BufStart; --LexStart) { + if (isVerticalWhitespace(LexStart[0]) && + !Lexer::isNewLineEscaped(BufStart, LexStart)) { + // LexStart should point at first character of logical line. ++LexStart; break; } - - --LexStart; } return LexStart; } @@ -487,7 +483,7 @@ static SourceLocation getBeginningOfFileToken(SourceLocation Loc, std::pair LocInfo = SM.getDecomposedLoc(Loc); if (LocInfo.first.isInvalid()) return Loc; - + bool Invalid = false; StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); if (Invalid) @@ -499,31 +495,31 @@ static SourceLocation getBeginningOfFileToken(SourceLocation Loc, const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second); if (!LexStart || LexStart == StrData) return Loc; - + // Create a lexer starting at the beginning of this token. SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart, Buffer.end()); TheLexer.SetCommentRetentionState(true); - + // Lex tokens until we find the token that contains the source location. Token TheTok; do { TheLexer.LexFromRawLexer(TheTok); - + if (TheLexer.getBufferLocation() > StrData) { // Lexing this token has taken the lexer past the source location we're // looking for. If the current token encompasses our source location, // return the beginning of that token. if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) return TheTok.getLocation(); - + // We ended up skipping over the source location entirely, which means // that it points into whitespace. We're done here. break; } } while (TheTok.getKind() != tok::eof); - + // We've passed our source location; just return the original source location. return Loc; } @@ -531,20 +527,20 @@ static SourceLocation getBeginningOfFileToken(SourceLocation Loc, SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) { - if (Loc.isFileID()) - return getBeginningOfFileToken(Loc, SM, LangOpts); - - if (!SM.isMacroArgExpansion(Loc)) - return Loc; + if (Loc.isFileID()) + return getBeginningOfFileToken(Loc, SM, LangOpts); + + if (!SM.isMacroArgExpansion(Loc)) + return Loc; - SourceLocation FileLoc = SM.getSpellingLoc(Loc); - SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); - std::pair FileLocInfo = SM.getDecomposedLoc(FileLoc); - std::pair BeginFileLocInfo - = SM.getDecomposedLoc(BeginFileLoc); - assert(FileLocInfo.first == BeginFileLocInfo.first && - FileLocInfo.second >= BeginFileLocInfo.second); - return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); + SourceLocation FileLoc = SM.getSpellingLoc(Loc); + SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); + std::pair FileLocInfo = SM.getDecomposedLoc(FileLoc); + std::pair BeginFileLocInfo = + SM.getDecomposedLoc(BeginFileLoc); + assert(FileLocInfo.first == BeginFileLocInfo.first && + FileLocInfo.second >= BeginFileLocInfo.second); + return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); } namespace { @@ -1032,6 +1028,26 @@ bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { return isIdentifierBody(c, LangOpts.DollarIdents); } +bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) { + assert(isVerticalWhitespace(Str[0])); + if (Str - 1 < BufferStart) + return false; + + if ((Str[0] == '\n' && Str[-1] == '\r') || + (Str[0] == '\r' && Str[-1] == '\n')) { + if (Str - 2 < BufferStart) + return false; + --Str; + } + --Str; + + // Rewind to first non-space character: + while (Str > BufferStart && isHorizontalWhitespace(*Str)) + --Str; + + return *Str == '\\'; +} + StringRef Lexer::getIndentationForLine(SourceLocation Loc, const SourceManager &SM) { if (Loc.isInvalid() || Loc.isMacroID()) -- cgit v1.2.3 From 15f32a5cedf2ebf7f24cf76a91dbdd49d70a33de Mon Sep 17 00:00:00 2001 From: Bruno Cardoso Lopes Date: Thu, 10 Aug 2017 15:16:24 +0000 Subject: [Modules] Prevent #import to reenter header if not building a module. When non-modular headers are imported while not building a module but in -fmodules mode, be conservative and preserve the default #import semantic: do not reenter headers. rdar://problem/33745031 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310605 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index 1ebcc0a1c6..4b5df8c1cc 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -1143,7 +1143,7 @@ bool HeaderSearch::ShouldEnterIncludeFile(Preprocessor &PP, // headers find in the wild might rely only on #import and do not contain // controlling macros, be conservative and only try to enter textual headers // if such macro is present. - if (!FileInfo.isModuleHeader && + if (FileInfo.isCompilingModuleHeader && !FileInfo.isModuleHeader && FileInfo.getControllingMacro(ExternalLookup)) TryEnterHdr = true; return TryEnterHdr; -- cgit v1.2.3 From 1b599a5f36a0d4b26cf3c647712916de29b0d915 Mon Sep 17 00:00:00 2001 From: Bruno Cardoso Lopes Date: Sat, 12 Aug 2017 01:38:26 +0000 Subject: Revert "[Modules] Prevent #import to reenter header if not building a module." This reverts commit r310605. Richard pointed out a better way to achieve this, which I'll post a patch for soon. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310775 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index 4b5df8c1cc..1ebcc0a1c6 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -1143,7 +1143,7 @@ bool HeaderSearch::ShouldEnterIncludeFile(Preprocessor &PP, // headers find in the wild might rely only on #import and do not contain // controlling macros, be conservative and only try to enter textual headers // if such macro is present. - if (FileInfo.isCompilingModuleHeader && !FileInfo.isModuleHeader && + if (!FileInfo.isModuleHeader && FileInfo.getControllingMacro(ExternalLookup)) TryEnterHdr = true; return TryEnterHdr; -- cgit v1.2.3 From d1cebef95412e885e43d92beaf3d2233e5a79b7b Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Sun, 13 Aug 2017 21:32:33 +0000 Subject: [c++2a] Treat 'concept' and 'requires' as keywords, add compat warning for C++17 and before. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310803 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index d1dc8e1c00..158d0eca27 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -630,6 +630,8 @@ static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, return llvm::StringSwitch(II.getName()) #define CXX11_KEYWORD(NAME, FLAGS) \ .Case(#NAME, diag::warn_cxx11_keyword) +#define CXX2A_KEYWORD(NAME, FLAGS) \ + .Case(#NAME, diag::warn_cxx2a_keyword) #include "clang/Basic/TokenKinds.def" ; -- cgit v1.2.3 From 0018aec614ff892c8996c77e2345c3d7731d2539 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Sun, 13 Aug 2017 23:37:29 +0000 Subject: Rename cxx1z -> cxx17 across all diagnostic IDs. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@310805 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index 3cc45cea97..f1738483f0 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -847,7 +847,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { ? diag::ext_hex_literal_invalid : diag::ext_hex_constant_invalid); else if (PP.getLangOpts().CPlusPlus1z) - PP.Diag(TokLoc, diag::warn_cxx1z_hex_literal); + PP.Diag(TokLoc, diag::warn_cxx17_hex_literal); } else if (saw_period) { PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), diag::err_hex_constant_requires) -- cgit v1.2.3 From 0d2d302e5428d57d63f6d6da87433f4e2668cfdd Mon Sep 17 00:00:00 2001 From: Ilya Biryukov Date: Mon, 21 Aug 2017 12:03:08 +0000 Subject: Fixed a crash on replaying Preamble's PP conditional stack. Summary: The crash occurs when the first token after a preamble is a macro expansion. Fixed by moving replayPreambleConditionalStack from Parser into Preprocessor. It is now called right after the predefines file is processed. Reviewers: erikjv, bkramer, klimek, yvvan Reviewed By: bkramer Subscribers: cfe-commits Differential Revision: https://reviews.llvm.org/D36872 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@311330 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 13 ++++++++++++- lib/Lex/Preprocessor.cpp | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 5a589d6a17..36d7028da6 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -458,10 +458,16 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs); } + bool ExitedFromPredefinesFile = false; FileID ExitedFID; - if (Callbacks && !isEndOfMacro && CurPPLexer) + if (!isEndOfMacro && CurPPLexer) { ExitedFID = CurPPLexer->getFileID(); + assert(PredefinesFileID.isValid() && + "HandleEndOfFile is called before PredefinesFileId is set"); + ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID); + } + if (LeavingSubmodule) { // We're done with this submodule. Module *M = LeaveSubmodule(/*ForPragma*/false); @@ -489,6 +495,11 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { PPCallbacks::ExitFile, FileType, ExitedFID); } + // Restore conditional stack from the preamble right after exiting from the + // predefines file. + if (ExitedFromPredefinesFile) + replayPreambleConditionalStack(); + // Client should lex another token unless we generated an EOM. return LeavingSubmodule; } diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 158d0eca27..e1294994df 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -540,6 +540,8 @@ void Preprocessor::EnterMainSourceFile() { void Preprocessor::replayPreambleConditionalStack() { // Restore the conditional stack from the preamble, if there is one. if (PreambleConditionalStack.isReplaying()) { + assert(CurPPLexer && + "CurPPLexer is null when calling replayPreambleConditionalStack."); CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); PreambleConditionalStack.doneReplaying(); } -- cgit v1.2.3 From 6d8609db4d42605a8a0e2e2620e9e38bf1b29d3c Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Thu, 24 Aug 2017 18:36:07 +0000 Subject: [Preprocessor] Correct internal token parsing of newline characters in CRLF Discovered due to a goofy git setup, the test system-headerline-directive.c (and a few others) failed because the token-consumption will consume only the '\r' in CRLF, making the preprocessor's printed value give the wrong line number when returning from an include. For example: (line 1):#include \r\n The "file exit" code causes the printer to try to print the 'returned to the main file' line. It looks up what the current line number is. However, since the current 'token' is the '\n' (since only the \r was consumed), it will give the line number as '1", not '2'. This results in a few failed tests, but more importantly, results in error messages being incorrect when compiling a previously preprocessed file. Differential Revision: https://reviews.llvm.org/D37079 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@311683 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 79472961c0..976a0d2fbd 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -3073,6 +3073,8 @@ LexNextToken: case '\n': case '\r': + if (CurPtr[0] != Char && (CurPtr[0] == '\n' || CurPtr[0] == '\r')) + Char = getAndAdvanceChar(CurPtr, Result); // If we are inside a preprocessor directive and we see the end of line, // we know we are done with the directive, so return an EOD token. if (ParsingPreprocessorDirective) { -- cgit v1.2.3 From ca9ec867b62382f5174cb0fab6c178dd4dd43f58 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 30 Aug 2017 08:45:59 +0000 Subject: [modules] Add ability to specify module name to module file mapping Extend the -fmodule-file option to support the [=] value format. If the name is omitted, then the old semantics is preserved (the module file is loaded whether needed or not). If the name is specified, then the mapping is treated as just another prebuilt module search mechanism, similar to -fprebuilt-module-path, and the module file is only loaded if actually used (e.g., via import). With one exception: this mapping also overrides module file references embedded in other modules (which can be useful if module files are moved/renamed as often happens during remote compilation). This override semantics requires some extra work: we now store the module name in addition to the file name in the serialized AST representation. Reviewed By: rsmith Differential Revision: https://reviews.llvm.org/D35020 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312105 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index 1ebcc0a1c6..b18d27376a 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -128,21 +128,24 @@ void HeaderSearch::getHeaderMapFileNames( Names.push_back(HM.first->getName()); } -std::string HeaderSearch::getModuleFileName(Module *Module) { +std::string HeaderSearch::getCachedModuleFileName(Module *Module) { const FileEntry *ModuleMap = getModuleMap().getModuleMapFileForUniquing(Module); - return getModuleFileName(Module->Name, ModuleMap->getName(), - /*UsePrebuiltPath*/false); + return getCachedModuleFileName(Module->Name, ModuleMap->getName()); } -std::string HeaderSearch::getModuleFileName(StringRef ModuleName, - StringRef ModuleMapPath, - bool UsePrebuiltPath) { - if (UsePrebuiltPath) { - if (HSOpts->PrebuiltModulePaths.empty()) +std::string HeaderSearch::getPrebuiltModuleFileName(StringRef ModuleName, + bool FileMapOnly) { + // First check the module name to pcm file map. + auto i (HSOpts->PrebuiltModuleFiles.find(ModuleName)); + if (i != HSOpts->PrebuiltModuleFiles.end()) + return i->second; + + if (FileMapOnly || HSOpts->PrebuiltModulePaths.empty()) return std::string(); - // Go though each prebuilt module path and try to find the pcm file. + // Then go through each prebuilt module directory and try to find the pcm + // file. for (const std::string &Dir : HSOpts->PrebuiltModulePaths) { SmallString<256> Result(Dir); llvm::sys::fs::make_absolute(Result); @@ -154,6 +157,8 @@ std::string HeaderSearch::getModuleFileName(StringRef ModuleName, return std::string(); } +std::string HeaderSearch::getCachedModuleFileName(StringRef ModuleName, + StringRef ModuleMapPath) { // If we don't have a module cache path or aren't supposed to use one, we // can't do anything. if (getModuleCachePath().empty()) -- cgit v1.2.3 From 6f7f03ccb12e16b4a5bf79638276cb7df4271e68 Mon Sep 17 00:00:00 2001 From: Victor Leschuk Date: Wed, 30 Aug 2017 11:31:56 +0000 Subject: Revert r312105 [modules] Add ability to specify module name to module file mapping Looks like it breaks win10 builder. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312112 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index b18d27376a..1ebcc0a1c6 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -128,24 +128,21 @@ void HeaderSearch::getHeaderMapFileNames( Names.push_back(HM.first->getName()); } -std::string HeaderSearch::getCachedModuleFileName(Module *Module) { +std::string HeaderSearch::getModuleFileName(Module *Module) { const FileEntry *ModuleMap = getModuleMap().getModuleMapFileForUniquing(Module); - return getCachedModuleFileName(Module->Name, ModuleMap->getName()); + return getModuleFileName(Module->Name, ModuleMap->getName(), + /*UsePrebuiltPath*/false); } -std::string HeaderSearch::getPrebuiltModuleFileName(StringRef ModuleName, - bool FileMapOnly) { - // First check the module name to pcm file map. - auto i (HSOpts->PrebuiltModuleFiles.find(ModuleName)); - if (i != HSOpts->PrebuiltModuleFiles.end()) - return i->second; - - if (FileMapOnly || HSOpts->PrebuiltModulePaths.empty()) +std::string HeaderSearch::getModuleFileName(StringRef ModuleName, + StringRef ModuleMapPath, + bool UsePrebuiltPath) { + if (UsePrebuiltPath) { + if (HSOpts->PrebuiltModulePaths.empty()) return std::string(); - // Then go through each prebuilt module directory and try to find the pcm - // file. + // Go though each prebuilt module path and try to find the pcm file. for (const std::string &Dir : HSOpts->PrebuiltModulePaths) { SmallString<256> Result(Dir); llvm::sys::fs::make_absolute(Result); @@ -157,8 +154,6 @@ std::string HeaderSearch::getPrebuiltModuleFileName(StringRef ModuleName, return std::string(); } -std::string HeaderSearch::getCachedModuleFileName(StringRef ModuleName, - StringRef ModuleMapPath) { // If we don't have a module cache path or aren't supposed to use one, we // can't do anything. if (getModuleCachePath().empty()) -- cgit v1.2.3 From 90051f7676a0198fd55d8435eb27815493c7bf4e Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Thu, 31 Aug 2017 06:26:43 +0000 Subject: [modules] Add ability to specify module name to module file mapping (reapply) Extend the -fmodule-file option to support the [=] value format. If the name is omitted, then the old semantics is preserved (the module file is loaded whether needed or not). If the name is specified, then the mapping is treated as just another prebuilt module search mechanism, similar to -fprebuilt-module-path, and the module file is only loaded if actually used (e.g., via import). With one exception: this mapping also overrides module file references embedded in other modules (which can be useful if module files are moved/renamed as often happens during remote compilation). This override semantics requires some extra work: we now store the module name in addition to the file name in the serialized AST representation. Reviewed By: rsmith Differential Revision: https://reviews.llvm.org/D35020 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312220 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/HeaderSearch.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp index 1ebcc0a1c6..b18d27376a 100644 --- a/lib/Lex/HeaderSearch.cpp +++ b/lib/Lex/HeaderSearch.cpp @@ -128,21 +128,24 @@ void HeaderSearch::getHeaderMapFileNames( Names.push_back(HM.first->getName()); } -std::string HeaderSearch::getModuleFileName(Module *Module) { +std::string HeaderSearch::getCachedModuleFileName(Module *Module) { const FileEntry *ModuleMap = getModuleMap().getModuleMapFileForUniquing(Module); - return getModuleFileName(Module->Name, ModuleMap->getName(), - /*UsePrebuiltPath*/false); + return getCachedModuleFileName(Module->Name, ModuleMap->getName()); } -std::string HeaderSearch::getModuleFileName(StringRef ModuleName, - StringRef ModuleMapPath, - bool UsePrebuiltPath) { - if (UsePrebuiltPath) { - if (HSOpts->PrebuiltModulePaths.empty()) +std::string HeaderSearch::getPrebuiltModuleFileName(StringRef ModuleName, + bool FileMapOnly) { + // First check the module name to pcm file map. + auto i (HSOpts->PrebuiltModuleFiles.find(ModuleName)); + if (i != HSOpts->PrebuiltModuleFiles.end()) + return i->second; + + if (FileMapOnly || HSOpts->PrebuiltModulePaths.empty()) return std::string(); - // Go though each prebuilt module path and try to find the pcm file. + // Then go through each prebuilt module directory and try to find the pcm + // file. for (const std::string &Dir : HSOpts->PrebuiltModulePaths) { SmallString<256> Result(Dir); llvm::sys::fs::make_absolute(Result); @@ -154,6 +157,8 @@ std::string HeaderSearch::getModuleFileName(StringRef ModuleName, return std::string(); } +std::string HeaderSearch::getCachedModuleFileName(StringRef ModuleName, + StringRef ModuleMapPath) { // If we don't have a module cache path or aren't supposed to use one, we // can't do anything. if (getModuleCachePath().empty()) -- cgit v1.2.3 From 5f4efc24c888d4c43af20c2c5a8e028720b8644d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 4 Sep 2017 05:37:53 +0000 Subject: Implement Itanium name mangling support for C++ Modules TS. This follows the scheme agreed with Nathan Sidwell, which can be found here: https://gcc.gnu.org/wiki/cxx-modules?action=AttachFile This will be proposed to the itanium-cxx-abi list once we have some experience with how well it works; the ABI for this TS should be considered unstable until it is part of the Itanium C++ ABI. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312467 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 40f78ce25c..db2f952e3c 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -746,8 +746,16 @@ std::pair ModuleMap::findOrCreateModule(StringRef Name, return std::make_pair(Result, true); } +Module *ModuleMap::createGlobalModuleForInterfaceUnit(SourceLocation Loc) { + auto *Result = new Module("", Loc, nullptr, /*IsFramework*/ false, + /*IsExplicit*/ true, NumCreatedModules++); + Result->Kind = Module::GlobalModuleFragment; + return Result; +} + Module *ModuleMap::createModuleForInterfaceUnit(SourceLocation Loc, - StringRef Name) { + StringRef Name, + Module *GlobalModule) { assert(LangOpts.CurrentModule == Name && "module name mismatch"); assert(!Modules[Name] && "redefining existing module"); @@ -757,6 +765,9 @@ Module *ModuleMap::createModuleForInterfaceUnit(SourceLocation Loc, Result->Kind = Module::ModuleInterfaceUnit; Modules[Name] = SourceModule = Result; + // Reparent the current global module fragment as a submodule of this module. + GlobalModule->setParent(Result); + // Mark the main source file as being within the newly-created module so that // declarations and macros are properly visibility-restricted to it. auto *MainFile = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()); -- cgit v1.2.3 From 5de618f7e28a184fe29085adf8b1e042fbf6dbff Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Tue, 5 Sep 2017 17:32:36 +0000 Subject: [Preprocessor] Correct internal token parsing of newline characters in CRLF Correct implementation: Apparently I managed in r311683 to submit the wrong version of the patch for this, so I'm correcting it now. Differential Revision: https://reviews.llvm.org/D37079 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312542 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 976a0d2fbd..928c24d94c 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -3071,10 +3071,11 @@ LexNextToken: Kind = tok::unknown; break; - case '\n': case '\r': - if (CurPtr[0] != Char && (CurPtr[0] == '\n' || CurPtr[0] == '\r')) + if (CurPtr[0] == '\n') Char = getAndAdvanceChar(CurPtr, Result); + LLVM_FALLTHROUGH; + case '\n': // If we are inside a preprocessor directive and we see the end of line, // we know we are done with the directive, so return an EOD token. if (ParsingPreprocessorDirective) { -- cgit v1.2.3 From 76d554642795496a2936ac28b1b17e4adfa923c6 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Tue, 5 Sep 2017 21:46:22 +0000 Subject: Fix memory leak after r312467. The ModuleMap is the owner of the global module object until it's reparented under a real module. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312580 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index db2f952e3c..b01080e55a 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -256,8 +256,7 @@ ModuleMap::ModuleMap(SourceManager &SourceMgr, DiagnosticsEngine &Diags, const LangOptions &LangOpts, const TargetInfo *Target, HeaderSearch &HeaderInfo) : SourceMgr(SourceMgr), Diags(Diags), LangOpts(LangOpts), Target(Target), - HeaderInfo(HeaderInfo), BuiltinIncludeDir(nullptr), - SourceModule(nullptr), NumCreatedModules(0) { + HeaderInfo(HeaderInfo) { MMapLangOpts.LineComment = true; } @@ -747,10 +746,12 @@ std::pair ModuleMap::findOrCreateModule(StringRef Name, } Module *ModuleMap::createGlobalModuleForInterfaceUnit(SourceLocation Loc) { - auto *Result = new Module("", Loc, nullptr, /*IsFramework*/ false, - /*IsExplicit*/ true, NumCreatedModules++); - Result->Kind = Module::GlobalModuleFragment; - return Result; + assert(!PendingGlobalModule && "created multiple global modules"); + PendingGlobalModule.reset( + new Module("", Loc, nullptr, /*IsFramework*/ false, + /*IsExplicit*/ true, NumCreatedModules++)); + PendingGlobalModule->Kind = Module::GlobalModuleFragment; + return PendingGlobalModule.get(); } Module *ModuleMap::createModuleForInterfaceUnit(SourceLocation Loc, @@ -766,7 +767,10 @@ Module *ModuleMap::createModuleForInterfaceUnit(SourceLocation Loc, Modules[Name] = SourceModule = Result; // Reparent the current global module fragment as a submodule of this module. + assert(GlobalModule == PendingGlobalModule.get() && + "unexpected global module"); GlobalModule->setParent(Result); + PendingGlobalModule.release(); // now owned by parent // Mark the main source file as being within the newly-created module so that // declarations and macros are properly visibility-restricted to it. -- cgit v1.2.3 From b9fefa5ec7a31eebaa84c4938ec7efff89db9972 Mon Sep 17 00:00:00 2001 From: Sjoerd Meijer Date: Fri, 8 Sep 2017 09:42:32 +0000 Subject: Add _Float16 as a C/C++ source language type This adds _Float16 as a source language type, which is a 16-bit floating point type defined in C11 extension ISO/IEC TS 18661-3. In follow up patches documentation and more tests will be added. Differential Revision: https://reviews.llvm.org/D33719 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312781 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index f1738483f0..5f6e5efa23 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -544,6 +544,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, isHalf = false; isFloat = false; isImaginary = false; + isFloat16 = false; isFloat128 = false; MicrosoftInteger = 0; hadError = false; @@ -588,6 +589,13 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, if (!isFPConstant) break; // Error for integer constant. if (isHalf || isFloat || isLong || isFloat128) break; // HF, FF, LF, QF invalid. + + if (s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') { + s += 2; // success, eat up 2 characters. + isFloat16 = true; + continue; + } + isFloat = true; continue; // Success. case 'q': // FP Suffix for "__float128" @@ -681,6 +689,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, isUnsigned = false; isLongLong = false; isFloat = false; + isFloat16 = false; isHalf = false; isImaginary = false; MicrosoftInteger = 0; -- cgit v1.2.3 From 185b81b1f8cd4522708f8f90128320fd4985896c Mon Sep 17 00:00:00 2001 From: Sjoerd Meijer Date: Fri, 8 Sep 2017 10:20:52 +0000 Subject: Revert "Add _Float16 as a C/C++ source language type" The clang-with-lto-ubuntu bot didn't like the new regression test, revert while I investigate the issue. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312784 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 9 --------- 1 file changed, 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index 5f6e5efa23..f1738483f0 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -544,7 +544,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, isHalf = false; isFloat = false; isImaginary = false; - isFloat16 = false; isFloat128 = false; MicrosoftInteger = 0; hadError = false; @@ -589,13 +588,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, if (!isFPConstant) break; // Error for integer constant. if (isHalf || isFloat || isLong || isFloat128) break; // HF, FF, LF, QF invalid. - - if (s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') { - s += 2; // success, eat up 2 characters. - isFloat16 = true; - continue; - } - isFloat = true; continue; // Success. case 'q': // FP Suffix for "__float128" @@ -689,7 +681,6 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, isUnsigned = false; isLongLong = false; isFloat = false; - isFloat16 = false; isHalf = false; isImaginary = false; MicrosoftInteger = 0; -- cgit v1.2.3 From 87063256b97d572c2a6d108d509af2de5034fba3 Mon Sep 17 00:00:00 2001 From: Sjoerd Meijer Date: Fri, 8 Sep 2017 15:15:00 +0000 Subject: Recommit "Add _Float16 as a C/C++ source language type" This is a recommit of r312781; in some build configurations variable names are omitted, so changed the new regression test accordingly. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312794 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/LiteralSupport.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp index f1738483f0..5f6e5efa23 100644 --- a/lib/Lex/LiteralSupport.cpp +++ b/lib/Lex/LiteralSupport.cpp @@ -544,6 +544,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, isHalf = false; isFloat = false; isImaginary = false; + isFloat16 = false; isFloat128 = false; MicrosoftInteger = 0; hadError = false; @@ -588,6 +589,13 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, if (!isFPConstant) break; // Error for integer constant. if (isHalf || isFloat || isLong || isFloat128) break; // HF, FF, LF, QF invalid. + + if (s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') { + s += 2; // success, eat up 2 characters. + isFloat16 = true; + continue; + } + isFloat = true; continue; // Success. case 'q': // FP Suffix for "__float128" @@ -681,6 +689,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, isUnsigned = false; isLongLong = false; isFloat = false; + isFloat16 = false; isHalf = false; isImaginary = false; MicrosoftInteger = 0; -- cgit v1.2.3 From c9445f041caab06d4aa184cb0e0b15a5d27feeaf Mon Sep 17 00:00:00 2001 From: Vedant Kumar Date: Mon, 11 Sep 2017 20:47:42 +0000 Subject: [Lexer] Report more precise skipped regions (PR34166) This patch teaches the preprocessor to report more precise source ranges for code that is skipped due to conditional directives. The new behavior includes the '#' from the opening directive and the full text of the line containing the closing directive in the skipped area. This matches up clang's behavior (we don't IRGen the code between the closing "endif" and the end of a line). This also affects the code coverage implementation. See llvm.org/PR34166 (this also happens to be rdar://problem/23224058). The old behavior (report the end of the skipped range as the end location of the 'endif' token) is preserved for indexing clients. Differential Revision: https://reviews.llvm.org/D36642 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@312947 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 59 ++++++++++++++++++++++++----------------- lib/Lex/PreprocessingRecord.cpp | 5 ++-- 2 files changed, 37 insertions(+), 27 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 6b09398e46..556e76356d 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -79,7 +79,8 @@ Preprocessor::AllocateVisibilityMacroDirective(SourceLocation Loc, } /// \brief Read and discard all tokens remaining on the current line until -/// the tok::eod token is found. +/// the tok::eod token is found. If the discarded tokens are in a skipped range, +/// complete the range and pass it to the \c SourceRangeSkipped callback. void Preprocessor::DiscardUntilEndOfDirective() { Token Tmp; do { @@ -350,7 +351,8 @@ void Preprocessor::CheckEndOfDirective(const char *DirType, bool EnableMacros) { /// If ElseOk is true, then \#else directives are ok, if not, then we have /// already seen one so a \#else directive is a duplicate. When this returns, /// the caller can lex the first valid token. -void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc, +void Preprocessor::SkipExcludedConditionalBlock(const Token &HashToken, + SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc) { @@ -558,10 +560,10 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc, // the #if block. CurPPLexer->LexingRawMode = false; - if (Callbacks) { - SourceLocation BeginLoc = ElseLoc.isValid() ? ElseLoc : IfTokenLoc; - Callbacks->SourceRangeSkipped(SourceRange(BeginLoc, Tok.getLocation())); - } + if (Callbacks) + Callbacks->SourceRangeSkipped( + SourceRange(HashToken.getLocation(), CurPPLexer->getSourceLocation()), + Tok.getLocation()); } void Preprocessor::PTHSkipExcludedConditionalBlock() { @@ -949,15 +951,17 @@ void Preprocessor::HandleDirective(Token &Result) { default: break; // C99 6.10.1 - Conditional Inclusion. case tok::pp_if: - return HandleIfDirective(Result, ReadAnyTokensBeforeDirective); + return HandleIfDirective(Result, SavedHash, ReadAnyTokensBeforeDirective); case tok::pp_ifdef: - return HandleIfdefDirective(Result, false, true/*not valid for miopt*/); + return HandleIfdefDirective(Result, SavedHash, false, + true /*not valid for miopt*/); case tok::pp_ifndef: - return HandleIfdefDirective(Result, true, ReadAnyTokensBeforeDirective); + return HandleIfdefDirective(Result, SavedHash, true, + ReadAnyTokensBeforeDirective); case tok::pp_elif: - return HandleElifDirective(Result); + return HandleElifDirective(Result, SavedHash); case tok::pp_else: - return HandleElseDirective(Result); + return HandleElseDirective(Result, SavedHash); case tok::pp_endif: return HandleEndifDirective(Result); @@ -2613,7 +2617,9 @@ void Preprocessor::HandleUndefDirective() { /// true if any tokens have been returned or pp-directives activated before this /// \#ifndef has been lexed. /// -void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef, +void Preprocessor::HandleIfdefDirective(Token &Result, + const Token &HashToken, + bool isIfndef, bool ReadAnyTokensBeforeDirective) { ++NumIf; Token DirectiveTok = Result; @@ -2625,8 +2631,8 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef, if (MacroNameTok.is(tok::eod)) { // Skip code until we get to #endif. This helps with recovery by not // emitting an error when the #endif is reached. - SkipExcludedConditionalBlock(DirectiveTok.getLocation(), - /*Foundnonskip*/false, /*FoundElse*/false); + SkipExcludedConditionalBlock(HashToken, DirectiveTok.getLocation(), + /*Foundnonskip*/ false, /*FoundElse*/ false); return; } @@ -2674,15 +2680,16 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef, /*foundelse*/false); } else { // No, skip the contents of this block. - SkipExcludedConditionalBlock(DirectiveTok.getLocation(), - /*Foundnonskip*/false, - /*FoundElse*/false); + SkipExcludedConditionalBlock(HashToken, DirectiveTok.getLocation(), + /*Foundnonskip*/ false, + /*FoundElse*/ false); } } /// HandleIfDirective - Implements the \#if directive. /// void Preprocessor::HandleIfDirective(Token &IfToken, + const Token &HashToken, bool ReadAnyTokensBeforeDirective) { ++NumIf; @@ -2720,8 +2727,9 @@ void Preprocessor::HandleIfDirective(Token &IfToken, /*foundnonskip*/true, /*foundelse*/false); } else { // No, skip the contents of this block. - SkipExcludedConditionalBlock(IfToken.getLocation(), /*Foundnonskip*/false, - /*FoundElse*/false); + SkipExcludedConditionalBlock(HashToken, IfToken.getLocation(), + /*Foundnonskip*/ false, + /*FoundElse*/ false); } } @@ -2753,7 +2761,7 @@ void Preprocessor::HandleEndifDirective(Token &EndifToken) { /// HandleElseDirective - Implements the \#else directive. /// -void Preprocessor::HandleElseDirective(Token &Result) { +void Preprocessor::HandleElseDirective(Token &Result, const Token &HashToken) { ++NumElse; // #else directive in a non-skipping conditional... start skipping. @@ -2784,13 +2792,14 @@ void Preprocessor::HandleElseDirective(Token &Result) { } // Finally, skip the rest of the contents of this block. - SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true, - /*FoundElse*/true, Result.getLocation()); + SkipExcludedConditionalBlock(HashToken, CI.IfLoc, /*Foundnonskip*/ true, + /*FoundElse*/ true, Result.getLocation()); } /// HandleElifDirective - Implements the \#elif directive. /// -void Preprocessor::HandleElifDirective(Token &ElifToken) { +void Preprocessor::HandleElifDirective(Token &ElifToken, + const Token &HashToken) { ++NumElse; // #elif directive in a non-skipping conditional... start skipping. @@ -2827,7 +2836,7 @@ void Preprocessor::HandleElifDirective(Token &ElifToken) { } // Finally, skip the rest of the contents of this block. - SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true, - /*FoundElse*/CI.FoundElse, + SkipExcludedConditionalBlock(HashToken, CI.IfLoc, /*Foundnonskip*/ true, + /*FoundElse*/ CI.FoundElse, ElifToken.getLocation()); } diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp index 03c4cbe589..954b569bb0 100644 --- a/lib/Lex/PreprocessingRecord.cpp +++ b/lib/Lex/PreprocessingRecord.cpp @@ -400,8 +400,9 @@ void PreprocessingRecord::Defined(const Token &MacroNameTok, MacroNameTok.getLocation()); } -void PreprocessingRecord::SourceRangeSkipped(SourceRange Range) { - SkippedRanges.push_back(Range); +void PreprocessingRecord::SourceRangeSkipped(SourceRange Range, + SourceLocation EndifLoc) { + SkippedRanges.emplace_back(Range.getBegin(), EndifLoc); } void PreprocessingRecord::MacroExpands(const Token &Id, -- cgit v1.2.3 From 622d54d5a584280c680148ffa1a1ba6b8a3772f8 Mon Sep 17 00:00:00 2001 From: Ilya Biryukov Date: Tue, 12 Sep 2017 08:35:57 +0000 Subject: Fix recording preamble's conditional stack in skipped PP branches. Summary: This fixes PR34547. `Lexer::LexEndOfFile` handles recording of ConditionalStack for preamble and reporting errors about unmatched conditionalal PP directives. However, SkipExcludedConditionalBlock contianed duplicated logic for reporting errors and clearing ConditionalStack, but not for preamble recording. This fix removes error reporting logic from `SkipExcludedConditionalBlock`, unmatched PP conditionals are now reported inside `Lexer::LexEndOfFile`. Reviewers: erikjv, klimek, bkramer Reviewed By: erikjv Subscribers: nik, cfe-commits Differential Revision: https://reviews.llvm.org/D37700 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313014 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 556e76356d..eb9f11f3d4 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -383,15 +383,8 @@ void Preprocessor::SkipExcludedConditionalBlock(const Token &HashToken, // If this is the end of the buffer, we have an error. if (Tok.is(tok::eof)) { - // Emit errors for each unterminated conditional on the stack, including - // the current one. - while (!CurPPLexer->ConditionalStack.empty()) { - if (CurLexer->getFileLoc() != CodeCompletionFileLoc) - Diag(CurPPLexer->ConditionalStack.back().IfLoc, - diag::err_pp_unterminated_conditional); - CurPPLexer->ConditionalStack.pop_back(); - } - + // We don't emit errors for unterminated conditionals here, + // Lexer::LexEndOfFile can do that propertly. // Just return and let the caller lex after this #include. break; } -- cgit v1.2.3 From 763bd9136e59e77b6fbc68841855632547dd9882 Mon Sep 17 00:00:00 2001 From: Douglas Gregor Date: Thu, 14 Sep 2017 23:38:44 +0000 Subject: [Module map] Introduce a private module re-export directive. Introduce a new "export_as" directive for top-level modules, which indicates that the current module is a "private" module whose symbols will eventually be exported through the named "public" module. This is in support of a common pattern in the Darwin ecosystem where a single public framework is constructed of several private frameworks, with (currently) header duplication and some support from the linker. Addresses rdar://problem/34438420. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313316 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/ModuleMap.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index b01080e55a..fc7fe138c5 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -1201,6 +1201,7 @@ namespace clang { ExcludeKeyword, ExplicitKeyword, ExportKeyword, + ExportAsKeyword, ExternKeyword, FrameworkKeyword, LinkKeyword, @@ -1312,6 +1313,7 @@ namespace clang { SourceLocation LeadingLoc); void parseUmbrellaDirDecl(SourceLocation UmbrellaLoc); void parseExportDecl(); + void parseExportAsDecl(); void parseUseDecl(); void parseLinkDecl(); void parseConfigMacros(); @@ -1363,6 +1365,7 @@ retry: .Case("exclude", MMToken::ExcludeKeyword) .Case("explicit", MMToken::ExplicitKeyword) .Case("export", MMToken::ExportKeyword) + .Case("export_as", MMToken::ExportAsKeyword) .Case("extern", MMToken::ExternKeyword) .Case("framework", MMToken::FrameworkKeyword) .Case("header", MMToken::HeaderKeyword) @@ -1590,6 +1593,7 @@ namespace { /// header-declaration /// submodule-declaration /// export-declaration +/// export-as-declaration /// link-declaration /// /// submodule-declaration: @@ -1824,6 +1828,10 @@ void ModuleMapParser::parseModuleDecl() { parseExportDecl(); break; + case MMToken::ExportAsKeyword: + parseExportAsDecl(); + break; + case MMToken::UseKeyword: parseUseDecl(); break; @@ -2284,6 +2292,41 @@ void ModuleMapParser::parseExportDecl() { ActiveModule->UnresolvedExports.push_back(Unresolved); } +/// \brief Parse a module export_as declaration. +/// +/// export-as-declaration: +/// 'export_as' identifier +void ModuleMapParser::parseExportAsDecl() { + assert(Tok.is(MMToken::ExportAsKeyword)); + consumeToken(); + + if (!Tok.is(MMToken::Identifier)) { + Diags.Report(Tok.getLocation(), diag::err_mmap_module_id); + HadError = true; + return; + } + + if (ActiveModule->Parent) { + Diags.Report(Tok.getLocation(), diag::err_mmap_submodule_export_as); + consumeToken(); + return; + } + + if (!ActiveModule->ExportAsModule.empty()) { + if (ActiveModule->ExportAsModule == Tok.getString()) { + Diags.Report(Tok.getLocation(), diag::warn_mmap_redundant_export_as) + << ActiveModule->Name << Tok.getString(); + } else { + Diags.Report(Tok.getLocation(), diag::err_mmap_conflicting_export_as) + << ActiveModule->Name << ActiveModule->ExportAsModule + << Tok.getString(); + } + } + + ActiveModule->ExportAsModule = Tok.getString(); + consumeToken(); +} + /// \brief Parse a module use declaration. /// /// use-declaration: @@ -2689,6 +2732,7 @@ bool ModuleMapParser::parseModuleMapFile() { case MMToken::Exclaim: case MMToken::ExcludeKeyword: case MMToken::ExportKeyword: + case MMToken::ExportAsKeyword: case MMToken::HeaderKeyword: case MMToken::Identifier: case MMToken::LBrace: -- cgit v1.2.3 From ad56fa864310d75036da71597ca7381aada68c09 Mon Sep 17 00:00:00 2001 From: Vedant Kumar Date: Sat, 16 Sep 2017 06:26:51 +0000 Subject: Remove comment accidentally committed with D36642. NFC. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313446 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index eb9f11f3d4..1d8056eb75 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -79,8 +79,7 @@ Preprocessor::AllocateVisibilityMacroDirective(SourceLocation Loc, } /// \brief Read and discard all tokens remaining on the current line until -/// the tok::eod token is found. If the discarded tokens are in a skipped range, -/// complete the range and pass it to the \c SourceRangeSkipped callback. +/// the tok::eod token is found. void Preprocessor::DiscardUntilEndOfDirective() { Token Tmp; do { -- cgit v1.2.3 From fea155028f9b2ec46b7753f3079aa40f9d35288c Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Mon, 18 Sep 2017 04:55:31 +0000 Subject: Reformat. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313510 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 36d7028da6..5686871675 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -40,10 +40,9 @@ bool Preprocessor::isInPrimaryFile() const { // If there are any stacked lexers, we're in a #include. assert(IsFileLexer(IncludeMacroStack[0]) && "Top level include stack isn't our primary lexer?"); - return std::none_of(IncludeMacroStack.begin() + 1, IncludeMacroStack.end(), - [this](const IncludeStackInfo &ISI) -> bool { - return IsFileLexer(ISI); - }); + return std::none_of( + IncludeMacroStack.begin() + 1, IncludeMacroStack.end(), + [this](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); } /// getCurrentLexer - Return the current file lexer being lexed from. Note -- cgit v1.2.3 From faa91c43b5a571fde880e1539a461bae426e93e1 Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Mon, 18 Sep 2017 04:55:33 +0000 Subject: Fix a warning discovered by rL313487. [-Wunused-lambda-capture] git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313511 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 5686871675..aa7cf17f08 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -42,7 +42,7 @@ bool Preprocessor::isInPrimaryFile() const { "Top level include stack isn't our primary lexer?"); return std::none_of( IncludeMacroStack.begin() + 1, IncludeMacroStack.end(), - [this](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); + [](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); } /// getCurrentLexer - Return the current file lexer being lexed from. Note -- cgit v1.2.3 From 5a5f2707cbd5a5d3050e93aa2e54f0f60559edd2 Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Mon, 18 Sep 2017 05:52:57 +0000 Subject: Revert rL313511, "Fix a warning discovered by rL313487. [-Wunused-lambda-capture]" It was incompatible to msc. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313513 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index aa7cf17f08..5686871675 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -42,7 +42,7 @@ bool Preprocessor::isInPrimaryFile() const { "Top level include stack isn't our primary lexer?"); return std::none_of( IncludeMacroStack.begin() + 1, IncludeMacroStack.end(), - [](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); + [this](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); } /// getCurrentLexer - Return the current file lexer being lexed from. Note -- cgit v1.2.3 From 2af104bd92de0d171ffdd8f997725da87efeaa2a Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Mon, 18 Sep 2017 08:26:01 +0000 Subject: Another attempt to fix warning discovered by r313487. [-Wunused-lambda-capture] git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313521 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPLexerChange.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index 5686871675..e484e9c4c3 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -42,7 +42,7 @@ bool Preprocessor::isInPrimaryFile() const { "Top level include stack isn't our primary lexer?"); return std::none_of( IncludeMacroStack.begin() + 1, IncludeMacroStack.end(), - [this](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); + [&](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); } /// getCurrentLexer - Return the current file lexer being lexed from. Note -- cgit v1.2.3 From aacb7039877a344e6063dc8e1214f82ef3b69fca Mon Sep 17 00:00:00 2001 From: Cameron Desrochers Date: Wed, 20 Sep 2017 19:03:37 +0000 Subject: [PCH] Fixed preamble breaking with BOM presence (and particularly, fluctuating BOM presence) This patch fixes broken preamble-skipping when the preamble region includes a byte order mark (BOM). Previously, parsing would fail if preamble PCH generation was enabled and a BOM was present. This also fixes preamble invalidation when a BOM appears or disappears. This may seem to be an obscure edge case, but it happens regularly with IDEs that pass buffer overrides that never (or always) have a BOM, yet the underlying file from the initial parse that generated a PCH might (or might not) have a BOM. I've included a test case for these scenarios. Differential Revision: https://reviews.llvm.org/D37491 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313796 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 14 +++++++------- lib/Lex/Preprocessor.cpp | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 928c24d94c..b7f97a583d 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -552,9 +552,9 @@ namespace { } // end anonymous namespace -std::pair Lexer::ComputePreamble(StringRef Buffer, - const LangOptions &LangOpts, - unsigned MaxLines) { +PreambleBounds Lexer::ComputePreamble(StringRef Buffer, + const LangOptions &LangOpts, + unsigned MaxLines) { // Create a lexer starting at the beginning of the file. Note that we use a // "fake" file source location at offset 1 so that the lexer will track our // position within the file. @@ -688,7 +688,7 @@ std::pair Lexer::ComputePreamble(StringRef Buffer, else End = TheTok.getLocation(); - return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), + return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(), TheTok.isAtStartOfLine()); } @@ -1394,9 +1394,9 @@ Slash: // Helper methods for lexing. //===----------------------------------------------------------------------===// -/// \brief Routine that indiscriminately skips bytes in the source file. -void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { - BufferPtr += Bytes; +/// \brief Routine that indiscriminately sets the offset into the source file. +void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) { + BufferPtr = BufferStart + Offset; if (BufferPtr > BufferEnd) BufferPtr = BufferEnd; // FIXME: What exactly does the StartOfLine bit mean? There are two diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index e1294994df..1f9a469bc5 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -516,9 +516,9 @@ void Preprocessor::EnterMainSourceFile() { // If we've been asked to skip bytes in the main file (e.g., as part of a // precompiled preamble), do so now. if (SkipMainFilePreamble.first > 0) - CurLexer->SkipBytes(SkipMainFilePreamble.first, - SkipMainFilePreamble.second); - + CurLexer->SetByteOffset(SkipMainFilePreamble.first, + SkipMainFilePreamble.second); + // Tell the header info that the main file was entered. If the file is later // #imported, it won't be re-entered. if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) -- cgit v1.2.3 From abb16417bac8d1da0d6698741dd07dcc7912a320 Mon Sep 17 00:00:00 2001 From: Cameron Desrochers Date: Wed, 20 Sep 2017 19:37:37 +0000 Subject: Fixed unused variable warning introduced in r313796 causing build failure git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313802 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index b7f97a583d..9885ab9d3e 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -564,9 +564,6 @@ PreambleBounds Lexer::ComputePreamble(StringRef Buffer, Buffer.end()); TheLexer.SetCommentRetentionState(true); - // StartLoc will differ from FileLoc if there is a BOM that was skipped. - SourceLocation StartLoc = TheLexer.getSourceLocation(); - bool InPreprocessorDirective = false; Token TheTok; SourceLocation ActiveCommentLoc; -- cgit v1.2.3 From 07667dbf1f85bc9f495cb03f775ac181474ab6e3 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Tue, 26 Sep 2017 18:38:56 +0000 Subject: Delete trailing whitespace. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314232 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPMacroExpansion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 3f8ede23da..b0330f2881 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1023,7 +1023,7 @@ Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer, size_t newIndex = MacroExpandedTokens.size(); bool cacheNeedsToGrow = tokens.size() > - MacroExpandedTokens.capacity()-MacroExpandedTokens.size(); + MacroExpandedTokens.capacity()-MacroExpandedTokens.size(); MacroExpandedTokens.append(tokens.begin(), tokens.end()); if (cacheNeedsToGrow) { -- cgit v1.2.3 From ce3b2b06b6219dc72e976235e8c08f141e2e0ff6 Mon Sep 17 00:00:00 2001 From: Eli Friedman Date: Wed, 27 Sep 2017 23:29:37 +0000 Subject: [Preprocessor] Preserve #pragma clang assume_nonnull in preprocessed output Patch by Zbigniew Sarbinowski! Differential Revision: https://reviews.llvm.org/D37861 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314364 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Pragma.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index bf2363a0a6..b5d0f93954 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -1725,6 +1725,7 @@ struct PragmaAssumeNonNullHandler : public PragmaHandler { // The start location we want after processing this. SourceLocation NewLoc; + PPCallbacks *Callbacks = PP.getPPCallbacks(); if (IsBegin) { // Complain about attempts to re-enter an audit. @@ -1733,6 +1734,8 @@ struct PragmaAssumeNonNullHandler : public PragmaHandler { PP.Diag(BeginLoc, diag::note_pragma_entered_here); } NewLoc = Loc; + if (Callbacks) + Callbacks->PragmaAssumeNonNullBegin(NewLoc); } else { // Complain about attempts to leave an audit that doesn't exist. if (!BeginLoc.isValid()) { @@ -1740,6 +1743,8 @@ struct PragmaAssumeNonNullHandler : public PragmaHandler { return; } NewLoc = SourceLocation(); + if (Callbacks) + Callbacks->PragmaAssumeNonNullEnd(NewLoc); } PP.setPragmaAssumeNonNullLoc(NewLoc); -- cgit v1.2.3 From 35b54a9ed7493be94a126f1152caf86ff388d36a Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Thu, 28 Sep 2017 01:50:23 +0000 Subject: [NFC] Modernize MacroArgs using TrailingObjects Refactor MacroArgs to use TrailingObjects when creating a variably sized object on the heap to store the unexpanded tokens immediately after the MacroArgs object. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314372 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index f791d8d4ba..c816fee085 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -33,7 +33,7 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, // See if we have an entry with a big enough argument list to reuse on the // free list. If so, reuse it. for (MacroArgs **Entry = &PP.MacroArgCache; *Entry; - Entry = &(*Entry)->ArgCache) + Entry = &(*Entry)->ArgCache) { if ((*Entry)->NumUnexpArgTokens >= UnexpArgTokens.size() && (*Entry)->NumUnexpArgTokens < ClosestMatch) { ResultEnt = Entry; @@ -44,14 +44,12 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, // Otherwise, use the best fit. ClosestMatch = (*Entry)->NumUnexpArgTokens; } - + } MacroArgs *Result; if (!ResultEnt) { - // Allocate memory for a MacroArgs object with the lexer tokens at the end. - Result = (MacroArgs *)malloc(sizeof(MacroArgs) + - UnexpArgTokens.size() * sizeof(Token)); - // Construct the MacroArgs object. - new (Result) + // Allocate memory for a MacroArgs object with the lexer tokens at the end, + // and construct the MacroArgs object. + Result = new (std::malloc(totalSizeToAlloc(UnexpArgTokens.size()))) MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumParams()); } else { Result = *ResultEnt; @@ -63,9 +61,14 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, } // Copy the actual unexpanded tokens to immediately after the result ptr. - if (!UnexpArgTokens.empty()) + if (!UnexpArgTokens.empty()) { + static_assert(std::is_trivially_copyable_v, + "assume trivial copyability if copying into the " + "uninitialized array (as opposed to reusing a cached " + "MacroArgs)"); std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), - const_cast(Result->getUnexpArgument(0))); + Result->getTrailingObjects()); + } return Result; } @@ -93,6 +96,8 @@ MacroArgs *MacroArgs::deallocate() { // Run the dtor to deallocate the vectors. this->~MacroArgs(); // Release the memory for the object. + static_assert(std::is_trivially_destructible_v, + "assume trivially destructible and forego destructors"); free(this); return Next; @@ -115,7 +120,7 @@ unsigned MacroArgs::getArgLength(const Token *ArgPtr) { const Token *MacroArgs::getUnexpArgument(unsigned Arg) const { // The unexpanded argument tokens start immediately after the MacroArgs object // in memory. - const Token *Start = (const Token *)(this+1); + const Token *Start = getTrailingObjects(); const Token *Result = Start; // Scan to find Arg. for (; Arg; ++Result) { -- cgit v1.2.3 From 29487927c0f5d8cd6b23978a0216b17041161cc5 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Thu, 28 Sep 2017 02:00:40 +0000 Subject: [NFC] Don't use C++17 standard lib variable template helper traits, instead use ::value. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314373 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index c816fee085..b96b1a8995 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -62,7 +62,7 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, // Copy the actual unexpanded tokens to immediately after the result ptr. if (!UnexpArgTokens.empty()) { - static_assert(std::is_trivially_copyable_v, + static_assert(std::is_trivially_copyable::value, "assume trivial copyability if copying into the " "uninitialized array (as opposed to reusing a cached " "MacroArgs)"); @@ -96,7 +96,7 @@ MacroArgs *MacroArgs::deallocate() { // Run the dtor to deallocate the vectors. this->~MacroArgs(); // Release the memory for the object. - static_assert(std::is_trivially_destructible_v, + static_assert(std::is_trivially_destructible::value, "assume trivially destructible and forego destructors"); free(this); -- cgit v1.2.3 From 0e1f5c3820a5d54c0d3328fdf5fddc9dee2a590d Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 28 Sep 2017 08:50:30 +0000 Subject: Use std::is_trivial instead of is_trivially_copyable. The oldest versions of GCC we support (before 5) didn't support that trait. is_trivial is stronger superset that clang::Token fulfills, so just use that instead. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314391 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index b96b1a8995..37a7d5c1ff 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -62,11 +62,11 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI, // Copy the actual unexpanded tokens to immediately after the result ptr. if (!UnexpArgTokens.empty()) { - static_assert(std::is_trivially_copyable::value, + static_assert(std::is_trivial::value, "assume trivial copyability if copying into the " "uninitialized array (as opposed to reusing a cached " "MacroArgs)"); - std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), + std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), Result->getTrailingObjects()); } -- cgit v1.2.3 From cd3ff5f109745f792f9cdaa1cf615d0551c7c37c Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Fri, 29 Sep 2017 02:17:31 +0000 Subject: [NFC] Rename variable 'Arguments' to 'Parameters' when lexing the Macro Definition. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314483 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 1d8056eb75..9c2860dc9c 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2137,14 +2137,14 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, /// closing ), updating MI with what we learn. Return true if an error occurs /// parsing the arg list. bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { - SmallVector Arguments; + SmallVector Parameters; while (true) { LexUnexpandedToken(Tok); switch (Tok.getKind()) { case tok::r_paren: // Found the end of the argument list. - if (Arguments.empty()) // #define FOO() + if (Parameters.empty()) // #define FOO() return false; // Otherwise we have #define FOO(A,) Diag(Tok, diag::err_pp_expected_ident_in_arg_list); @@ -2168,9 +2168,9 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { return true; } // Add the __VA_ARGS__ identifier as an argument. - Arguments.push_back(Ident__VA_ARGS__); + Parameters.push_back(Ident__VA_ARGS__); MI->setIsC99Varargs(); - MI->setParameterList(Arguments, BP); + MI->setParameterList(Parameters, BP); return false; case tok::eod: // #define X( Diag(Tok, diag::err_pp_missing_rparen_in_macro_def); @@ -2187,14 +2187,14 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { // If this is already used as an argument, it is used multiple times (e.g. // #define X(A,A. - if (std::find(Arguments.begin(), Arguments.end(), II) != - Arguments.end()) { // C99 6.10.3p6 + if (std::find(Parameters.begin(), Parameters.end(), II) != + Parameters.end()) { // C99 6.10.3p6 Diag(Tok, diag::err_pp_duplicate_name_in_arg_list) << II; return true; } // Add the argument to the macro info. - Arguments.push_back(II); + Parameters.push_back(II); // Lex the token after the identifier. LexUnexpandedToken(Tok); @@ -2204,7 +2204,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { Diag(Tok, diag::err_pp_expected_comma_in_arg_list); return true; case tok::r_paren: // #define X(A) - MI->setParameterList(Arguments, BP); + MI->setParameterList(Parameters, BP); return false; case tok::comma: // #define X(A, break; @@ -2220,7 +2220,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { } MI->setIsGNUVarargs(); - MI->setParameterList(Arguments, BP); + MI->setParameterList(Parameters, BP); return false; } } -- cgit v1.2.3 From 8c6abdf1bad624a2838f083882d95ce95e9b9b4a Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Fri, 29 Sep 2017 02:43:22 +0000 Subject: [NFC] Replace 'arguments' with 'parameters' in comments relating to lexing a macro definition. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314484 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/PPDirectives.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 9c2860dc9c..318931663c 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2132,10 +2132,10 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, // Preprocessor Macro Directive Handling. //===----------------------------------------------------------------------===// -/// ReadMacroParameterList - The ( starting an argument list of a macro -/// definition has just been read. Lex the rest of the arguments and the +/// ReadMacroParameterList - The ( starting a parameter list of a macro +/// definition has just been read. Lex the rest of the parameters and the /// closing ), updating MI with what we learn. Return true if an error occurs -/// parsing the arg list. +/// parsing the param list. bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { SmallVector Parameters; @@ -2143,7 +2143,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { LexUnexpandedToken(Tok); switch (Tok.getKind()) { case tok::r_paren: - // Found the end of the argument list. + // Found the end of the parameter list. if (Parameters.empty()) // #define FOO() return false; // Otherwise we have #define FOO(A,) @@ -2167,7 +2167,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { Diag(Tok, diag::err_pp_missing_rparen_in_macro_def); return true; } - // Add the __VA_ARGS__ identifier as an argument. + // Add the __VA_ARGS__ identifier as a parameter. Parameters.push_back(Ident__VA_ARGS__); MI->setIsC99Varargs(); MI->setParameterList(Parameters, BP); @@ -2185,7 +2185,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { return true; } - // If this is already used as an argument, it is used multiple times (e.g. + // If this is already used as a parameter, it is used multiple times (e.g. // #define X(A,A. if (std::find(Parameters.begin(), Parameters.end(), II) != Parameters.end()) { // C99 6.10.3p6 @@ -2193,7 +2193,7 @@ bool Preprocessor::ReadMacroParameterList(MacroInfo *MI, Token &Tok) { return true; } - // Add the argument to the macro info. + // Add the parameter to the macro info. Parameters.push_back(II); // Lex the token after the identifier. -- cgit v1.2.3 From 51cc2d29165533c6c0c07ddd1ab02cc0dfa8cb79 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Sat, 30 Sep 2017 13:58:38 +0000 Subject: [NFC] Remove superfluous parameter - MacroArgs already knows the maximum number of arguments that can be supplied to the macro. No need to pass MacroInfo (information about the macro definition) to the call to getPreExpArgument (which by the way might benefit from being called getExpandedArgument() ?) for it to compute the number of arguments. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314593 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 11 +++++------ lib/Lex/TokenLexer.cpp | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index 37a7d5c1ff..de54f392a4 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -150,14 +150,13 @@ bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok, /// getPreExpArgument - Return the pre-expanded form of the specified /// argument. -const std::vector & -MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI, - Preprocessor &PP) { - assert(Arg < MI->getNumParams() && "Invalid argument number!"); +const std::vector &MacroArgs::getPreExpArgument(unsigned Arg, + Preprocessor &PP) { + assert(Arg < getNumMacroArguments() && "Invalid argument number!"); // If we have already computed this, return it. - if (PreExpArgTokens.size() < MI->getNumParams()) - PreExpArgTokens.resize(MI->getNumParams()); + if (PreExpArgTokens.size() < getNumMacroArguments()) + PreExpArgTokens.resize(getNumMacroArguments()); std::vector &Result = PreExpArgTokens[Arg]; if (!Result.empty()) return Result; diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 10e21b3b04..4341d56b4e 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -275,7 +275,7 @@ void TokenLexer::ExpandFunctionArguments() { // avoids some work in common cases. const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) - ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0]; + ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; else ResultArgToks = ArgTok; // Use non-preexpanded tokens. -- cgit v1.2.3 From 744bd75982e8bbcb7d74fa15e4d77c94c19040df Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Sat, 30 Sep 2017 19:34:27 +0000 Subject: [NFC] Add assertion that we assume a valid macro argument index. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314600 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index de54f392a4..d2525c0cd6 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -118,10 +118,13 @@ unsigned MacroArgs::getArgLength(const Token *ArgPtr) { /// getUnexpArgument - Return the unexpanded tokens for the specified formal. /// const Token *MacroArgs::getUnexpArgument(unsigned Arg) const { + + assert(Arg < getNumMacroArguments() && "Invalid arg #"); // The unexpanded argument tokens start immediately after the MacroArgs object // in memory. const Token *Start = getTrailingObjects(); const Token *Result = Start; + // Scan to find Arg. for (; Arg; ++Result) { assert(Result < Start+NumUnexpArgTokens && "Invalid arg #"); -- cgit v1.2.3 From 6d25f92413b65d3fc7f10f1baa7500daa8b48c0c Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Tue, 3 Oct 2017 00:52:14 +0000 Subject: [NFC] Refactor PasteTokens so that it can be passed the Token Stream and Index to start concatenating at. In passing: - change the name of the function to pasteTokens c/w coding standards - rename CurToken to CurTokenIdx (since it is not the token, but the index) - add doxygen comments to document some of pasteTokens' functionality - use parameter names different from the data member names. This will be useful for implementing __VA_OPT__ (https://reviews.llvm.org/D35782#inline-322587) git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314747 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 96 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 38 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 4341d56b4e..18211ff25b 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -31,7 +31,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, Macro = MI; ActualArgs = Actuals; - CurToken = 0; + CurTokenIdx = 0; ExpandLocStart = Tok.getLocation(); ExpandLocEnd = ELEnd; @@ -90,7 +90,7 @@ void TokenLexer::Init(const Token *TokArray, unsigned NumToks, OwnsTokens = ownsTokens; DisableMacroExpansion = disableMacroExpansion; NumTokens = NumToks; - CurToken = 0; + CurTokenIdx = 0; ExpandLocStart = ExpandLocEnd = SourceLocation(); AtStartOfLine = false; HasLeadingSpace = false; @@ -431,7 +431,7 @@ bool TokenLexer::Lex(Token &Tok) { Tok.startToken(); Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); - if (CurToken == 0) + if (CurTokenIdx == 0) Tok.setFlag(Token::LeadingEmptyMacro); return PP.HandleEndOfTokenLexer(Tok); } @@ -440,25 +440,25 @@ bool TokenLexer::Lex(Token &Tok) { // If this is the first token of the expanded result, we inherit spacing // properties later. - bool isFirstToken = CurToken == 0; + bool isFirstToken = CurTokenIdx == 0; // Get the next token to return. - Tok = Tokens[CurToken++]; + Tok = Tokens[CurTokenIdx++]; bool TokenIsFromPaste = false; // If this token is followed by a token paste (##) operator, paste the tokens! // Note that ## is a normal token when not expanding a macro. if (!isAtEnd() && Macro && - (Tokens[CurToken].is(tok::hashhash) || + (Tokens[CurTokenIdx].is(tok::hashhash) || // Special processing of L#x macros in -fms-compatibility mode. // Microsoft compiler is able to form a wide string literal from // 'L#macro_arg' construct in a function-like macro. (PP.getLangOpts().MSVCCompat && - isWideStringLiteralFromMacro(Tok, Tokens[CurToken])))) { + isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) { // When handling the microsoft /##/ extension, the final token is - // returned by PasteTokens, not the pasted token. - if (PasteTokens(Tok)) + // returned by pasteTokens, not the pasted token. + if (pasteTokens(Tok)) return true; TokenIsFromPaste = true; @@ -521,40 +521,60 @@ bool TokenLexer::Lex(Token &Tok) { return true; } -/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## +bool TokenLexer::pasteTokens(Token &Tok) { + return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx); +} +/// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ## /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there -/// are more ## after it, chomp them iteratively. Return the result as Tok. +/// are more ## after it, chomp them iteratively. Return the result as LHSTok. /// If this returns true, the caller should immediately return the token. -bool TokenLexer::PasteTokens(Token &Tok) { +bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef TokenStream, + unsigned int &CurIdx) { + assert(CurIdx > 0 && "## can not be the first token within tokens"); + assert(TokenStream[CurIdx].is(tok::hashhash) || + (PP.getLangOpts().MSVCCompat && + isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx])) && + "Token at this Index must be ## or part of the MSVC 'L " + "#macro-arg' pasting pair"); + + assert(std::is_trivial::value && + !std::memcmp(&LHSTok, &TokenStream[CurIdx - 1], sizeof(Token)) && + "LHSTok must equal the token preceding the hashhash"); + // MSVC: If previous token was pasted, this must be a recovery from an invalid // paste operation. Ignore spaces before this token to mimic MSVC output. // Required for generating valid UUID strings in some MS headers. - if (PP.getLangOpts().MicrosoftExt && (CurToken >= 2) && - Tokens[CurToken - 2].is(tok::hashhash)) - Tok.clearFlag(Token::LeadingSpace); + if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) && + TokenStream[CurIdx - 2].is(tok::hashhash)) + LHSTok.clearFlag(Token::LeadingSpace); SmallString<128> Buffer; const char *ResultTokStrPtr = nullptr; - SourceLocation StartLoc = Tok.getLocation(); + SourceLocation StartLoc = LHSTok.getLocation(); SourceLocation PasteOpLoc; + + auto IsAtEnd = [&TokenStream, &CurIdx] { + return TokenStream.size() == CurIdx; + }; + do { // Consume the ## operator if any. - PasteOpLoc = Tokens[CurToken].getLocation(); - if (Tokens[CurToken].is(tok::hashhash)) - ++CurToken; - assert(!isAtEnd() && "No token on the RHS of a paste operator!"); + PasteOpLoc = TokenStream[CurIdx].getLocation(); + if (TokenStream[CurIdx].is(tok::hashhash)) + ++CurIdx; + assert(!IsAtEnd() && "No token on the RHS of a paste operator!"); // Get the RHS token. - const Token &RHS = Tokens[CurToken]; + const Token &RHS = TokenStream[CurIdx]; // Allocate space for the result token. This is guaranteed to be enough for // the two tokens. - Buffer.resize(Tok.getLength() + RHS.getLength()); + Buffer.resize(LHSTok.getLength() + RHS.getLength()); // Get the spelling of the LHS token in Buffer. const char *BufPtr = &Buffer[0]; bool Invalid = false; - unsigned LHSLen = PP.getSpelling(Tok, BufPtr, &Invalid); + unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid); if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! memcpy(&Buffer[0], BufPtr, LHSLen); if (Invalid) @@ -586,7 +606,7 @@ bool TokenLexer::PasteTokens(Token &Tok) { // Lex the resultant pasted token into Result. Token Result; - if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) { + if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) { // Common paste case: identifier+identifier = identifier. Avoid creating // a lexer and other overhead. PP.IncrementPasteCounter(true); @@ -626,7 +646,7 @@ bool TokenLexer::PasteTokens(Token &Tok) { isInvalid |= Result.is(tok::eof); // If pasting the two tokens didn't form a full new token, this is an - // error. This occurs with "x ## +" and other stuff. Return with Tok + // error. This occurs with "x ## +" and other stuff. Return with LHSTok // unmodified and with RHS as the next token to lex. if (isInvalid) { // Explicitly convert the token location to have proper expansion @@ -637,9 +657,9 @@ bool TokenLexer::PasteTokens(Token &Tok) { // Test for the Microsoft extension of /##/ turning into // here on the // error path. - if (PP.getLangOpts().MicrosoftExt && Tok.is(tok::slash) && + if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) && RHS.is(tok::slash)) { - HandleMicrosoftCommentPaste(Tok, Loc); + HandleMicrosoftCommentPaste(LHSTok, Loc); return true; } @@ -664,15 +684,15 @@ bool TokenLexer::PasteTokens(Token &Tok) { } // Transfer properties of the LHS over the Result. - Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine()); - Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace()); + Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine()); + Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace()); // Finally, replace LHS with the result, consume the RHS, and iterate. - ++CurToken; - Tok = Result; - } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)); + ++CurIdx; + LHSTok = Result; + } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash)); - SourceLocation EndLoc = Tokens[CurToken - 1].getLocation(); + SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation(); // The token's current location indicate where the token was lexed from. We // need this information to compute the spelling of the token, but any @@ -690,16 +710,16 @@ bool TokenLexer::PasteTokens(Token &Tok) { while (SM.getFileID(EndLoc) != MacroFID) EndLoc = SM.getImmediateExpansionRange(EndLoc).second; - Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc, - Tok.getLength())); + LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc, + LHSTok.getLength())); // Now that we got the result token, it will be subject to expansion. Since // token pasting re-lexes the result token in raw mode, identifier information // isn't looked up. As such, if the result is an identifier, look up id info. - if (Tok.is(tok::raw_identifier)) { + if (LHSTok.is(tok::raw_identifier)) { // Look up the identifier info for the token. We disabled identifier lookup // by saying we're skipping contents, so we need to do this manually. - PP.LookUpIdentifierInfo(Tok); + PP.LookUpIdentifierInfo(LHSTok); } return false; } @@ -711,7 +731,7 @@ unsigned TokenLexer::isNextTokenLParen() const { // Out of tokens? if (isAtEnd()) return 2; - return Tokens[CurToken].is(tok::l_paren); + return Tokens[CurTokenIdx].is(tok::l_paren); } /// isParsingPreprocessorDirective - Return true if we are in the middle of a -- cgit v1.2.3 From 1d05c89b9e096cc8464448681fb05f5f9f705451 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Tue, 3 Oct 2017 01:20:40 +0000 Subject: Remove an assertion I added from the refactoring of pasteTokens (https://reviews.llvm.org/rL314747). - it made the bots v angry! I'm not exactly sure why the assertion doesn't hold - if anyone has any insight - would appreciate it. Thanks! git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314748 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 18211ff25b..f8f9400db9 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -537,10 +537,6 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef TokenStream, "Token at this Index must be ## or part of the MSVC 'L " "#macro-arg' pasting pair"); - assert(std::is_trivial::value && - !std::memcmp(&LHSTok, &TokenStream[CurIdx - 1], sizeof(Token)) && - "LHSTok must equal the token preceding the hashhash"); - // MSVC: If previous token was pasted, this must be a recovery from an invalid // paste operation. Ignore spaces before this token to mimic MSVC output. // Required for generating valid UUID strings in some MS headers. -- cgit v1.2.3 From 6a419e7edff49ad1ca5a4d7a80962c9d8bd1dc70 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Tue, 3 Oct 2017 01:33:36 +0000 Subject: Add parens around the boolean condition of one of the added asserts in r314747 ... ... in the hopes of teaching the bots the gift of silence ;) For quick reference: https://reviews.llvm.org/rL314747 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314753 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index f8f9400db9..df6ad5276a 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -531,9 +531,9 @@ bool TokenLexer::pasteTokens(Token &Tok) { bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef TokenStream, unsigned int &CurIdx) { assert(CurIdx > 0 && "## can not be the first token within tokens"); - assert(TokenStream[CurIdx].is(tok::hashhash) || + assert((TokenStream[CurIdx].is(tok::hashhash) || (PP.getLangOpts().MSVCCompat && - isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx])) && + isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) && "Token at this Index must be ## or part of the MSVC 'L " "#macro-arg' pasting pair"); -- cgit v1.2.3 From bd279d876518592ad5ddf5c9aee15bef8d551f0c Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Wed, 11 Oct 2017 00:41:20 +0000 Subject: A '<' with a trigraph '#' is not a valid editor placeholder Credit to OSS-Fuzz for discovery: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=3137#c5 rdar://34923985 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@315398 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 9885ab9d3e..0c179c0fb8 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -3542,7 +3542,8 @@ LexNextToken: } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); Kind = tok::l_brace; - } else if (Char == '#' && lexEditorPlaceholder(Result, CurPtr)) { + } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 && + lexEditorPlaceholder(Result, CurPtr)) { return true; } else { Kind = tok::less; -- cgit v1.2.3 From 9047763be73c6c687a868e265a439e77e2d0ca57 Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Thu, 12 Oct 2017 09:42:14 +0000 Subject: Fix warnings. [-Wdocumentation] git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@315573 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index df6ad5276a..6ec128efc1 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -847,9 +847,8 @@ static void updateConsecutiveMacroArgTokens(SourceManager &SM, /// \brief Creates SLocEntries and updates the locations of macro argument /// tokens to their new expanded locations. /// -/// \param ArgIdDefLoc the location of the macro argument id inside the macro +/// \param ArgIdSpellLoc the location of the macro argument id inside the macro /// definition. -/// \param Tokens the macro argument tokens to update. void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, Token *begin_tokens, Token *end_tokens) { -- cgit v1.2.3 From 4318ef1cb398b9b72aea287a815c09cfe68a2b27 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Sat, 14 Oct 2017 01:18:30 +0000 Subject: [Lex] Avoid out-of-bounds dereference in SkipLineComment Credit to OSS-Fuzz for discovery: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=3145 rdar://34526482 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@315785 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 0c179c0fb8..b85e0f03dc 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -2144,7 +2144,8 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, // If we read multiple characters, and one of those characters was a \r or // \n, then we had an escaped newline within the comment. Emit diagnostic // unless the next line is also a // comment. - if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { + if (CurPtr != OldPtr + 1 && C != '/' && + (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) { for (; OldPtr != CurPtr; ++OldPtr) if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { // Okay, we found a // comment that ends in a newline, if the next -- cgit v1.2.3 From 3a7dad24b1470dee2bad556d69990fd3e8d328f4 Mon Sep 17 00:00:00 2001 From: Faisal Vali Date: Sun, 15 Oct 2017 01:26:26 +0000 Subject: [c++2a] Implement P0306 __VA_OPT__ (Comma omission and comma deletion) This patch implements an extension to the preprocessor: __VA_OPT__(contents) --> which expands into its contents if variadic arguments are supplied to the parent macro, or behaves as an empty token if none. - Currently this feature is only enabled for C++2a (this could be enabled, with some careful tweaks, for other dialects with the appropriate extension or compatibility warnings) - The patch was reviewed here: https://reviews.llvm.org/D35782 and asides from the above (and moving some of the definition and expansion recognition logic into the corresponding state machines), I believe I incorporated all of Richard's suggestions. A few technicalities (most of which were clarified through private correspondence between rsmith, hubert and thomas) are worth mentioning. Given: #define F(a,...) a #__VA_OPT__(a ## a) a ## __VA_OPT__(__VA_ARGS__) - The call F(,) Does not supply any tokens for the variadic arguments and hence VA_OPT behaves as a placeholder. - When expanding VA_OPT (for e.g. F(,1) token pasting occurs eagerly within its contents if the contents need to be stringified. - A hash or a hashhash prior to VA_OPT does not inhibit expansion of arguments if they are the first token within VA_OPT. - When a variadic argument is supplied, argument substitution occurs within the contents as does stringification - and these resulting tokens are inserted back into the macro expansions token stream just prior to the entire stream being rescanned and concatenated. See wg21.link/P0306 for further details on the feature. Acknowledgment: This patch would have been poorer if not for Richard Smith's usual thoughtful analysis and feedback. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@315840 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/MacroArgs.cpp | 10 +++ lib/Lex/PPDirectives.cpp | 67 ++++++++++++++++-- lib/Lex/Preprocessor.cpp | 22 ++++-- lib/Lex/TokenLexer.cpp | 173 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 251 insertions(+), 21 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp index d2525c0cd6..5c0f0623c3 100644 --- a/lib/Lex/MacroArgs.cpp +++ b/lib/Lex/MacroArgs.cpp @@ -135,6 +135,16 @@ const Token *MacroArgs::getUnexpArgument(unsigned Arg) const { return Result; } +// This function assumes that the variadic arguments are the tokens +// corresponding to the last parameter (ellipsis) - and since tokens are +// separated by the 'eof' token, if that is the only token corresponding to that +// last parameter, we know no variadic arguments were supplied. +bool MacroArgs::invokedWithVariadicArgument(const MacroInfo *const MI) const { + if (!MI->isVariadic()) + return false; + const int VariadicArgIndex = getNumMacroArguments() - 1; + return getUnexpArgument(VariadicArgIndex)->isNot(tok::eof); +} /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected /// by pre-expansion, return false. Otherwise, conservatively return true. diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 318931663c..f9a97505bf 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -2368,12 +2368,50 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( // Otherwise, read the body of a function-like macro. While we are at it, // check C99 6.10.3.2p1: ensure that # operators are followed by macro // parameters in function-like macro expansions. + + VAOptDefinitionContext VAOCtx(*this); + while (Tok.isNot(tok::eod)) { LastTok = Tok; if (!Tok.isOneOf(tok::hash, tok::hashat, tok::hashhash)) { MI->AddTokenToBody(Tok); + if (VAOCtx.isVAOptToken(Tok)) { + // If we're already within a VAOPT, emit an error. + if (VAOCtx.isInVAOpt()) { + Diag(Tok, diag::err_pp_vaopt_nested_use); + return nullptr; + } + // Ensure VAOPT is followed by a '(' . + LexUnexpandedToken(Tok); + if (Tok.isNot(tok::l_paren)) { + Diag(Tok, diag::err_pp_missing_lparen_in_vaopt_use); + return nullptr; + } + MI->AddTokenToBody(Tok); + VAOCtx.sawVAOptFollowedByOpeningParens(Tok.getLocation()); + LexUnexpandedToken(Tok); + if (Tok.is(tok::hashhash)) { + Diag(Tok, diag::err_vaopt_paste_at_start); + return nullptr; + } + continue; + } else if (VAOCtx.isInVAOpt()) { + if (Tok.is(tok::r_paren)) { + if (VAOCtx.sawClosingParen()) { + const unsigned NumTokens = MI->getNumTokens(); + assert(NumTokens >= 3 && "Must have seen at least __VA_OPT__( " + "and a subsequent tok::r_paren"); + if (MI->getReplacementToken(NumTokens - 2).is(tok::hashhash)) { + Diag(Tok, diag::err_vaopt_paste_at_end); + return nullptr; + } + } + } else if (Tok.is(tok::l_paren)) { + VAOCtx.sawOpeningParen(Tok.getLocation()); + } + } // Get the next token of the macro. LexUnexpandedToken(Tok); continue; @@ -2414,12 +2452,14 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( continue; } + // Our Token is a stringization operator. // Get the next token of the macro. LexUnexpandedToken(Tok); - // Check for a valid macro arg identifier. - if (Tok.getIdentifierInfo() == nullptr || - MI->getParameterNum(Tok.getIdentifierInfo()) == -1) { + // Check for a valid macro arg identifier or __VA_OPT__. + if (!VAOCtx.isVAOptToken(Tok) && + (Tok.getIdentifierInfo() == nullptr || + MI->getParameterNum(Tok.getIdentifierInfo()) == -1)) { // If this is assembler-with-cpp mode, we accept random gibberish after // the '#' because '#' is often a comment character. However, change @@ -2438,11 +2478,24 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( // Things look ok, add the '#' and param name tokens to the macro. MI->AddTokenToBody(LastTok); - MI->AddTokenToBody(Tok); - LastTok = Tok; - // Get the next token of the macro. - LexUnexpandedToken(Tok); + // If the token following '#' is VAOPT, let the next iteration handle it + // and check it for correctness, otherwise add the token and prime the + // loop with the next one. + if (!VAOCtx.isVAOptToken(Tok)) { + MI->AddTokenToBody(Tok); + LastTok = Tok; + + // Get the next token of the macro. + LexUnexpandedToken(Tok); + } + } + if (VAOCtx.isInVAOpt()) { + assert(Tok.is(tok::eod) && "Must be at End Of preprocessing Directive"); + Diag(Tok, diag::err_pp_expected_after) + << LastTok.getKind() << tok::r_paren; + Diag(VAOCtx.getUnmatchedOpeningParenLoc(), diag::note_matching) << tok::l_paren; + return nullptr; } } MI->setDefinitionEndLoc(LastTok.getLocation()); diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 1f9a469bc5..65df6a57f1 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -121,12 +121,18 @@ Preprocessor::Preprocessor(std::shared_ptr PPOpts, // We haven't read anything from the external source. ReadMacrosFromExternalSource = false; - - // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. - // This gets unpoisoned where it is allowed. + + // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of + // a macro. They get unpoisoned where it is allowed. (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); - + if (getLangOpts().CPlusPlus2a) { + (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned(); + SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use); + } else { + Ident__VA_OPT__ = nullptr; + } + // Initialize the pragma handlers. RegisterBuiltinPragmas(); @@ -667,13 +673,15 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) { // unpoisoned it if we're defining a C99 macro. if (II.isOutOfDate()) { bool CurrentIsPoisoned = false; - if (&II == Ident__VA_ARGS__) - CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned(); + const bool IsSpecialVariadicMacro = + &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__; + if (IsSpecialVariadicMacro) + CurrentIsPoisoned = II.isPoisoned(); updateOutOfDateIdentifier(II); Identifier.setKind(II.getTokenID()); - if (&II == Ident__VA_ARGS__) + if (IsSpecialVariadicMacro) II.setIsPoisoned(CurrentIsPoisoned); } diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 6ec128efc1..c0883dd48e 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -17,6 +17,7 @@ #include "clang/Lex/MacroArgs.h" #include "clang/Lex/MacroInfo.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/VariadicMacroSupport.h" #include "llvm/ADT/SmallString.h" using namespace clang; @@ -168,6 +169,65 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( return true; } +void TokenLexer::stringifyVAOPTContents( + SmallVectorImpl &ResultToks, const VAOptExpansionContext &VCtx, + const SourceLocation VAOPTClosingParenLoc) { + const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt(); + const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt; + Token *const VAOPTTokens = + NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr; + + SmallVector ConcatenatedVAOPTResultToks; + // FIXME: Should we keep track within VCtx that we did or didnot + // encounter pasting - and only then perform this loop. + + // Perform token pasting (concatenation) prior to stringization. + for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens; + ++CurTokenIdx) { + const unsigned int PrevTokenIdx = CurTokenIdx; + + if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) { + assert(CurTokenIdx != 0 && + "Can not have __VAOPT__ contents begin with a ##"); + Token &LHS = VAOPTTokens[CurTokenIdx - 1]; + pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens), + CurTokenIdx); + // CurTokenIdx is either the same as NumTokens or one past the + // last token concatenated. + // PrevTokenIdx is the index of the hashhash + const unsigned NumTokensPastedTogether = CurTokenIdx - PrevTokenIdx + 1; + // Replace the token prior to the first ## in this iteration. + ConcatenatedVAOPTResultToks.back() = LHS; + if (CurTokenIdx == NumVAOptTokens) + break; + } + ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]); + } + + ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok()); + // Get the SourceLocation that represents the start location within + // the macro definition that marks where this string is substituted + // into: i.e. the __VA_OPT__ and the ')' within the spelling of the + // macro definition, and use it to indicate that the stringified token + // was generated from that location. + const SourceLocation ExpansionLocStartWithinMacro = + getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc()); + const SourceLocation ExpansionLocEndWithinMacro = + getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc); + + Token StringifiedVAOPT = MacroArgs::StringifyArgument( + &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/, + ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro); + + if (VCtx.getLeadingSpaceForStringifiedToken()) + StringifiedVAOPT.setFlag(Token::LeadingSpace); + + StringifiedVAOPT.setFlag(Token::StringifiedInMacro); + // Resize (shrink) the token stream to just capture this stringified token. + ResultToks.resize(NumToksPriorToVAOpt + 1); + ResultToks.back() = StringifiedVAOPT; +} + /// Expand the arguments of a function-like macro so that we can quickly /// return preexpanded tokens from Tokens. void TokenLexer::ExpandFunctionArguments() { @@ -178,10 +238,13 @@ void TokenLexer::ExpandFunctionArguments() { // we install the newly expanded sequence as the new 'Tokens' list. bool MadeChange = false; + const bool CalledWithVariadicArguments = + ActualArgs->invokedWithVariadicArgument(Macro); + + VAOptExpansionContext VCtx(PP); + for (unsigned I = 0, E = NumTokens; I != E; ++I) { - // If we found the stringify operator, get the argument stringified. The - // preprocessor already verified that the following token is a macro name - // when the #define was parsed. + const Token &CurTok = Tokens[I]; // We don't want a space for the next token after a paste // operator. In valid code, the token will get smooshed onto the @@ -192,10 +255,98 @@ void TokenLexer::ExpandFunctionArguments() { if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) NextTokGetsSpace = true; + if (VCtx.isVAOptToken(CurTok)) { + MadeChange = true; + assert(Tokens[I + 1].is(tok::l_paren) && + "__VA_OPT__ must be followed by '('"); + + ++I; // Skip the l_paren + VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(), + ResultToks.size()); + + continue; + } + + // We have entered into the __VA_OPT__ context, so handle tokens + // appropriately. + if (VCtx.isInVAOpt()) { + // If we are about to process a token that is either an argument to + // __VA_OPT__ or its closing rparen, then: + // 1) If the token is the closing rparen that exits us out of __VA_OPT__, + // perform any necessary stringification or placemarker processing, + // and/or skip to the next token. + // 2) else if macro was invoked without variadic arguments skip this + // token. + // 3) else (macro was invoked with variadic arguments) process the token + // normally. + + if (Tokens[I].is(tok::l_paren)) + VCtx.sawOpeningParen(Tokens[I].getLocation()); + // Continue skipping tokens within __VA_OPT__ if the macro was not + // called with variadic arguments, else let the rest of the loop handle + // this token. Note sawClosingParen() returns true only if the r_paren matches + // the closing r_paren of the __VA_OPT__. + if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) { + if (!CalledWithVariadicArguments) { + // Skip this token. + continue; + } + // ... else the macro was called with variadic arguments, and we do not + // have a closing rparen - so process this token normally. + + } else { + // Current token is the closing r_paren which marks the end of the + // __VA_OPT__ invocation, so handle any place-marker pasting (if + // empty) by removing hashhash either before (if exists) or after. And + // also stringify the entire contents if VAOPT was preceded by a hash, + // but do so only after any token concatenation that needs to occur + // within the contents of VAOPT. + + if (VCtx.hasStringifyOrCharifyBefore()) { + // Replace all the tokens just added from within VAOPT into a single + // stringified token. This requires token-pasting to eagerly occur + // within these tokens. If either the contents of VAOPT were empty + // or the macro wasn't called with any variadic arguments, the result + // is a token that represents an empty string. + stringifyVAOPTContents(ResultToks, VCtx, + /*ClosingParenLoc*/ Tokens[I].getLocation()); + + } else if (/*No tokens within VAOPT*/ !( + ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) { + // Treat VAOPT as a placemarker token. Eat either the '##' before the + // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that + // hashhash was not a placemarker) or the '##' + // after VAOPT, but not both. + + if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) { + ResultToks.pop_back(); + } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) { + ++I; // Skip the following hashhash. + } + } + VCtx.reset(); + // We processed __VA_OPT__'s closing paren (and the exit out of + // __VA_OPT__), so skip to the next token. + continue; + } + } + + // If we found the stringify operator, get the argument stringified. The + // preprocessor already verified that the following token is a macro + // parameter or __VA_OPT__ when the #define was lexed. + if (CurTok.isOneOf(tok::hash, tok::hashat)) { int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo()); - assert(ArgNo != -1 && "Token following # is not an argument?"); - + assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) && + "Token following # is not an argument or __VA_OPT__!"); + + if (ArgNo == -1) { + // Handle the __VA_OPT__ case. + VCtx.sawHashOrHashAtBefore(NextTokGetsSpace, + CurTok.is(tok::hashat)); + continue; + } + // Else handle the simple argument case. SourceLocation ExpansionLocStart = getExpansionLocForMacroDefLoc(CurTok.getLocation()); SourceLocation ExpansionLocEnd = @@ -232,7 +383,9 @@ void TokenLexer::ExpandFunctionArguments() { !ResultToks.empty() && ResultToks.back().is(tok::hashhash); bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash); bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash); - assert(!NonEmptyPasteBefore || PasteBefore); + + assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) && + "unexpected ## in ResultToks"); // Otherwise, if this is not an argument token, just add the token to the // output buffer. @@ -384,7 +537,13 @@ void TokenLexer::ExpandFunctionArguments() { assert(PasteBefore); if (NonEmptyPasteBefore) { assert(ResultToks.back().is(tok::hashhash)); - ResultToks.pop_back(); + // Do not remove the paste operator if it is the one before __VA_OPT__ + // (and we are still processing tokens within VA_OPT). We handle the case + // of removing the paste operator if __VA_OPT__ reduces to the notional + // placemarker above when we encounter the closing paren of VA_OPT. + if (!VCtx.isInVAOpt() || + ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) + ResultToks.pop_back(); } // If this is the __VA_ARGS__ token, and if the argument wasn't provided, -- cgit v1.2.3 From ef29d1880145e6354858968ee8993d4e1aabdf4b Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Sun, 15 Oct 2017 04:27:37 +0000 Subject: [Lex] Remove unused variables. No functionality change. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@315845 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/TokenLexer.cpp | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib/Lex') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index c0883dd48e..194ceecc07 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -184,18 +184,12 @@ void TokenLexer::stringifyVAOPTContents( // Perform token pasting (concatenation) prior to stringization. for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens; ++CurTokenIdx) { - const unsigned int PrevTokenIdx = CurTokenIdx; - if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) { assert(CurTokenIdx != 0 && "Can not have __VAOPT__ contents begin with a ##"); Token &LHS = VAOPTTokens[CurTokenIdx - 1]; pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens), CurTokenIdx); - // CurTokenIdx is either the same as NumTokens or one past the - // last token concatenated. - // PrevTokenIdx is the index of the hashhash - const unsigned NumTokensPastedTogether = CurTokenIdx - PrevTokenIdx + 1; // Replace the token prior to the first ## in this iteration. ConcatenatedVAOPTResultToks.back() = LHS; if (CurTokenIdx == NumVAOptTokens) -- cgit v1.2.3 From 0b63209b05f8a23134fedfd1e61e8b7423675959 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Sun, 15 Oct 2017 15:01:42 +0000 Subject: Add -f[no-]double-square-bracket-attributes as new driver options to control use of [[]] attributes in all language modes. This is the initial implementation of WG14 N2165, which is a proposal to add [[]] attributes to C2x, but also allows you to enable these attributes in C++98, or disable them in C++11 or later. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@315856 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Lexer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/Lex') diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index b85e0f03dc..5132d0e62c 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -3612,7 +3612,9 @@ LexNextToken: if (LangOpts.Digraphs && Char == '>') { Kind = tok::r_square; // ':>' -> ']' CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); - } else if (LangOpts.CPlusPlus && Char == ':') { + } else if ((LangOpts.CPlusPlus || + LangOpts.DoubleSquareBracketAttributes) && + Char == ':') { Kind = tok::coloncolon; CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); } else { -- cgit v1.2.3