1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <qmljs/qmljs_global.h>
#include <QStringList>
namespace QmlJS {
class QMLJS_EXPORT Token
{
public:
enum Kind {
EndOfFile,
Keyword,
Identifier,
String,
Comment,
Number,
LeftParenthesis,
RightParenthesis,
LeftBrace,
RightBrace,
LeftBracket,
RightBracket,
Semicolon,
Colon,
Comma,
Dot,
Delimiter,
RegExp
};
inline Token(): offset(0), length(0), kind(EndOfFile) {}
inline Token(int o, int l, Kind k): offset(o), length(l), kind(k) {}
inline int begin() const { return offset; }
inline int end() const { return offset + length; }
inline bool is(int k) const { return k == kind; }
inline bool isNot(int k) const { return k != kind; }
static int compare(const Token &t1, const Token &t2) {
if (int c = t1.offset - t2.offset)
return c;
if (int c = t1.length - t2.length)
return c;
return int(t1.kind) - int(t2.kind);
}
public:
int offset = 0;
int length = 0;
Kind kind = EndOfFile;
};
inline int operator == (const Token &t1, const Token &t2) {
return Token::compare(t1, t2) == 0;
}
inline int operator != (const Token &t1, const Token &t2) {
return Token::compare(t1, t2) != 0;
}
class QMLJS_EXPORT Scanner
{
public:
enum {
FlagsBits = 4,
BraceCounterBits = 7
};
enum {
Normal = 0,
MultiLineComment = 1,
MultiLineStringDQuote = 2,
MultiLineStringSQuote = 3,
MultiLineStringBQuote = 4,
MultiLineMask = 7,
RegexpMayFollow = 8, // flag that may be combined with the above
// templates can be nested, which means that the scanner/lexer cannot
// be a simple state machine anymore, but should have a stack to store
// the state (the number of open braces in the current template
// string).
// The lexer stare is currently stored in an int, so we abuse that and
// store a the number of open braces (maximum 0x7f = 127) for at most 5
// nested templates in the int after the flags for the multiline
// comments and strings.
TemplateExpression = 0x1 << 4,
TemplateExpressionOpenBracesMask0 = 0x7F,
TemplateExpressionOpenBracesMask1 = 0x7F << 4,
TemplateExpressionOpenBracesMask2 = 0x7F << 11,
TemplateExpressionOpenBracesMask3 = 0x7F << 18,
TemplateExpressionOpenBracesMask4 = 0x7F << 25,
TemplateExpressionOpenBracesMask = TemplateExpressionOpenBracesMask1 | TemplateExpressionOpenBracesMask2
| TemplateExpressionOpenBracesMask3 | TemplateExpressionOpenBracesMask4
};
Scanner();
virtual ~Scanner();
bool scanComments() const;
void setScanComments(bool scanComments);
QList<Token> operator()(const QString &text, int startState = Normal);
int state() const;
bool isKeyword(const QString &text) const;
static QStringList keywords();
private:
int _state;
bool _scanComments: 1;
};
} // namespace QmlJS
|
