1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "SimpleLexer.h"
#include <cplusplus/ObjectiveCTypeQualifiers.h>
#include <cplusplus/Lexer.h>
#include <cplusplus/Token.h>
#include <QDebug>
using namespace CPlusPlus;
SimpleLexer::SimpleLexer()
: _lastState(0),
_skipComments(false),
_endedJoined(false),
_ppMode(false)
{}
SimpleLexer::~SimpleLexer()
{ }
bool SimpleLexer::skipComments() const
{
return _skipComments;
}
void SimpleLexer::setSkipComments(bool skipComments)
{
_skipComments = skipComments;
}
bool SimpleLexer::endedJoined() const
{
return _endedJoined;
}
Tokens SimpleLexer::operator()(const QString &text, int state)
{
Tokens tokens;
const QByteArray bytes = text.toUtf8();
const char *firstChar = bytes.constData();
const char *lastChar = firstChar + bytes.size();
Lexer lex(firstChar, lastChar);
lex.setExpectedRawStringSuffix(_expectedRawStringSuffix);
lex.setLanguageFeatures(_languageFeatures);
lex.setStartWithNewline(true);
lex.setPreprocessorMode(_ppMode);
if (! _skipComments)
lex.setScanCommentTokens(true);
if (state != -1)
lex.setState(state & 0xff);
bool inPreproc = false;
for (;;) {
Token tk;
lex(&tk);
if (tk.is(T_EOF_SYMBOL)) {
_endedJoined = tk.joined();
break;
}
const QStringView spell = tk.utf16charsBegin() + tk.utf16chars() > text.size()
? QStringView(text).mid(tk.utf16charsBegin())
: QStringView(text).mid(tk.utf16charsBegin(), tk.utf16chars());
lex.setScanAngleStringLiteralTokens(false);
if (tk.newline() && tk.is(T_POUND))
inPreproc = true;
else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("include"))
lex.setScanAngleStringLiteralTokens(true);
else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("include_next"))
lex.setScanAngleStringLiteralTokens(true);
else if (_languageFeatures.objCEnabled
&& inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("import"))
lex.setScanAngleStringLiteralTokens(true);
tokens.append(tk);
}
_lastState = lex.state();
_expectedRawStringSuffix = lex.expectedRawStringSuffix();
return tokens;
}
int SimpleLexer::tokenAt(const Tokens &tokens, int utf16charsOffset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const Token &tk = tokens.at(index);
if (tk.utf16charsBegin() <= utf16charsOffset && tk.utf16charsEnd() >= utf16charsOffset)
return index;
}
return -1;
}
Token SimpleLexer::tokenAt(const QString &text,
int utf16charsOffset,
int state,
const LanguageFeatures &languageFeatures)
{
SimpleLexer tokenize;
tokenize.setLanguageFeatures(languageFeatures);
const QVector<Token> tokens = tokenize(text, state);
const int tokenIdx = tokenAt(tokens, utf16charsOffset);
return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
}
int SimpleLexer::tokenBefore(const Tokens &tokens, int utf16charsOffset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const Token &tk = tokens.at(index);
if (tk.utf16charsBegin() <= utf16charsOffset)
return index;
}
return -1;
}
|
