summaryrefslogtreecommitdiffstats
path: root/src/3rdparty/angle/src/compiler/preprocessor/new/Tokenizer.l
diff options
context:
space:
mode:
Diffstat (limited to 'src/3rdparty/angle/src/compiler/preprocessor/new/Tokenizer.l')
-rw-r--r--src/3rdparty/angle/src/compiler/preprocessor/new/Tokenizer.l340
1 files changed, 340 insertions, 0 deletions
diff --git a/src/3rdparty/angle/src/compiler/preprocessor/new/Tokenizer.l b/src/3rdparty/angle/src/compiler/preprocessor/new/Tokenizer.l
new file mode 100644
index 0000000000..9762988350
--- /dev/null
+++ b/src/3rdparty/angle/src/compiler/preprocessor/new/Tokenizer.l
@@ -0,0 +1,340 @@
+/*
+//
+// Copyright (c) 2002-2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+This file contains the Lex specification for GLSL ES preprocessor.
+Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
+http://msdn.microsoft.com/en-us/library/2scxys89.aspx
+
+IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
+*/
+
+%top{
+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
+}
+
+%{
+#include "Tokenizer.h"
+
+#include "Diagnostics.h"
+#include "Token.h"
+
+#if defined(__GNUC__)
+// Triggered by the auto-generated yy_fatal_error function.
+#pragma GCC diagnostic ignored "-Wmissing-noreturn"
+#endif
+
+typedef std::string YYSTYPE;
+typedef pp::SourceLocation YYLTYPE;
+
+// Use the unused yycolumn variable to track file (string) number.
+#define yyfileno yycolumn
+
+#define YY_USER_INIT \
+ do { \
+ yyfileno = 0; \
+ yylineno = 1; \
+ yyextra->leadingSpace = false; \
+ yyextra->lineStart = true; \
+ } while(0);
+
+#define YY_USER_ACTION \
+ do \
+ { \
+ pp::Input* input = &yyextra->input; \
+ pp::Input::Location* scanLoc = &yyextra->scanLoc; \
+ while ((scanLoc->sIndex < input->count()) && \
+ (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
+ { \
+ scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
+ ++yyfileno; yylineno = 1; \
+ } \
+ yylloc->file = yyfileno; \
+ yylloc->line = yylineno; \
+ scanLoc->cIndex += yyleng; \
+ } while(0);
+
+#define YY_INPUT(buf, result, maxSize) \
+ result = yyextra->input.read(buf, maxSize);
+
+%}
+
+%option noyywrap nounput never-interactive
+%option reentrant bison-bridge bison-locations
+%option prefix="pp"
+%option extra-type="pp::Tokenizer::Context*"
+%x COMMENT
+
+NEWLINE \n|\r|\r\n
+IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]*
+PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?]
+
+DECIMAL_CONSTANT [1-9][0-9]*
+OCTAL_CONSTANT 0[0-7]*
+HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+
+
+DIGIT [0-9]
+EXPONENT_PART [eE][+-]?{DIGIT}+
+FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
+
+%%
+
+ /* Line comment */
+"//"[^\r\n]*
+
+ /* Block comment */
+ /* Line breaks are just counted - not returned. */
+ /* The comment is replaced by a single space. */
+"/*" { BEGIN(COMMENT); }
+<COMMENT>[^*\r\n]+
+<COMMENT>"*"
+<COMMENT>{NEWLINE} { ++yylineno; }
+<COMMENT>"*/" {
+ yyextra->leadingSpace = true;
+ BEGIN(INITIAL);
+}
+
+# {
+ // # is only valid at start of line for preprocessor directives.
+ yylval->assign(1, yytext[0]);
+ return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
+}
+
+{IDENTIFIER} {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::IDENTIFIER;
+}
+
+{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::CONST_INT;
+}
+
+({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::CONST_FLOAT;
+}
+
+ /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
+ /* Rule to catch all invalid integers and floats. */
+({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::PP_NUMBER;
+}
+
+"++" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_INC;
+}
+"--" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_DEC;
+}
+"<<" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LEFT;
+}
+">>" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_RIGHT;
+}
+"<=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LE;
+}
+">=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_GE;
+}
+"==" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_EQ;
+}
+"!=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_NE;
+}
+"&&" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_AND;
+}
+"^^" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_XOR;
+}
+"||" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_OR;
+}
+"+=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_ADD_ASSIGN;
+}
+"-=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_SUB_ASSIGN;
+}
+"*=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_MUL_ASSIGN;
+}
+"/=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_DIV_ASSIGN;
+}
+"%=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_MOD_ASSIGN;
+}
+"<<=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_LEFT_ASSIGN;
+}
+">>=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_RIGHT_ASSIGN;
+}
+"&=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_AND_ASSIGN;
+}
+"^=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_XOR_ASSIGN;
+}
+"|=" {
+ yylval->assign(yytext, yyleng);
+ return pp::Token::OP_OR_ASSIGN;
+}
+
+{PUNCTUATOR} {
+ yylval->assign(1, yytext[0]);
+ return yytext[0];
+}
+
+[ \t\v\f]+ { yyextra->leadingSpace = true; }
+
+{NEWLINE} {
+ ++yylineno;
+ yylval->assign(1, '\n');
+ return '\n';
+}
+
+. {
+ yylval->assign(1, yytext[0]);
+ return pp::Token::PP_OTHER;
+}
+
+<*><<EOF>> {
+ // YY_USER_ACTION is not invoked for handling EOF.
+ // Set the location for EOF token manually.
+ pp::Input* input = &yyextra->input;
+ pp::Input::Location* scanLoc = &yyextra->scanLoc;
+ int sIndexMax = std::max(0, input->count() - 1);
+ if (scanLoc->sIndex != sIndexMax)
+ {
+ // We can only reach here if there are empty strings at the
+ // end of the input.
+ scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
+ yyfileno = sIndexMax; yylineno = 1;
+ }
+ yylloc->file = yyfileno;
+ yylloc->line = yylineno;
+ yylval->clear();
+
+ if (YY_START == COMMENT)
+ {
+ yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+ pp::SourceLocation(yyfileno, yylineno),
+ "");
+ }
+ yyterminate();
+}
+
+%%
+
+namespace pp {
+
+// TODO(alokp): Maximum token length should ideally be specified by
+// the preprocessor client, i.e., the compiler.
+const size_t Tokenizer::kMaxTokenLength = 256;
+
+Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
+{
+ mContext.diagnostics = diagnostics;
+}
+
+Tokenizer::~Tokenizer()
+{
+ destroyScanner();
+}
+
+bool Tokenizer::init(int count, const char* const string[], const int length[])
+{
+ if (count < 0) return false;
+ if ((count > 0) && (string == 0)) return false;
+
+ mContext.input = Input(count, string, length);
+ return initScanner();
+}
+
+void Tokenizer::setFileNumber(int file)
+{
+ // We use column number as file number.
+ // See macro yyfileno.
+ yyset_column(file, mHandle);
+}
+
+void Tokenizer::setLineNumber(int line)
+{
+ yyset_lineno(line, mHandle);
+}
+
+void Tokenizer::lex(Token* token)
+{
+ token->type = yylex(&token->text, &token->location, mHandle);
+ if (token->text.size() > kMaxTokenLength)
+ {
+ mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
+ token->location, token->text);
+ token->text.erase(kMaxTokenLength);
+ }
+
+ token->flags = 0;
+
+ token->setAtStartOfLine(mContext.lineStart);
+ mContext.lineStart = token->type == '\n';
+
+ token->setHasLeadingSpace(mContext.leadingSpace);
+ mContext.leadingSpace = false;
+}
+
+bool Tokenizer::initScanner()
+{
+ if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
+ return false;
+
+ yyrestart(0, mHandle);
+ return true;
+}
+
+void Tokenizer::destroyScanner()
+{
+ if (mHandle == NULL)
+ return;
+
+ yylex_destroy(mHandle);
+ mHandle = NULL;
+}
+
+} // namespace pp
+