summaryrefslogtreecommitdiff
path: root/script/gdscript/gd_tokenizer.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'script/gdscript/gd_tokenizer.cpp')
-rw-r--r--script/gdscript/gd_tokenizer.cpp973
1 files changed, 973 insertions, 0 deletions
diff --git a/script/gdscript/gd_tokenizer.cpp b/script/gdscript/gd_tokenizer.cpp
new file mode 100644
index 0000000000..f7320799a5
--- /dev/null
+++ b/script/gdscript/gd_tokenizer.cpp
@@ -0,0 +1,973 @@
+/*************************************************************************/
+/* gd_tokenizer.cpp */
+/*************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* http://www.godotengine.org */
+/*************************************************************************/
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/*************************************************************************/
+#include "gd_tokenizer.h"
+#include "print_string.h"
+#include "gd_functions.h"
+const char* GDTokenizer::token_names[TK_MAX]={
+"Empty",
+"Identifier",
+"Constant",
+"Self",
+"Built-In Type",
+"Built-In Func",
+"In",
+"'=='",
+"'!='",
+"'<'",
+"'<='",
+"'>'",
+"'>='",
+"'and'",
+"'or'",
+"'not'",
+"'+'",
+"'-'",
+"'*'",
+"'/'",
+"'%'",
+"'<<'",
+"'>>'",
+"'='",
+"'+='",
+"'-='",
+"'*='",
+"'/='",
+"'%='",
+"'<<='",
+"'>>='",
+"'&='",
+"'|='",
+"'^='",
+"'&'",
+"'|'",
+"'^'",
+"'~'",
+//"Plus Plus",
+//"Minus Minus",
+"if",
+"elif",
+"else",
+"for",
+"do",
+"while",
+"switch",
+"case",
+"break",
+"continue",
+"pass",
+"return",
+"func",
+"class",
+"extends",
+"tool",
+"static",
+"export",
+"const",
+"var",
+"preload",
+"assert",
+"'['",
+"']'",
+"'{'",
+"'}'",
+"'('",
+"')'",
+"','",
+"';'",
+"'.'",
+"'?'",
+"':'",
+"'\\n'",
+"Error",
+"EOF"};
+
+const char *GDTokenizer::get_token_name(Token p_token) {
+
+ ERR_FAIL_INDEX_V(p_token,TK_MAX,"<error>");
+ return token_names[p_token];
+}
+
+static bool _is_text_char(CharType c) {
+
+ return (c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='_';
+}
+
+static bool _is_number(CharType c) {
+
+ return (c>='0' && c<='9');
+}
+
+static bool _is_hex(CharType c) {
+
+ return (c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F');
+}
+
+void GDTokenizer::_make_token(Token p_type) {
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+
+ tk.type=p_type;
+ tk.line=line;
+ tk.col=column;
+
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+}
+void GDTokenizer::_make_identifier(const StringName& p_identifier) {
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+
+ tk.type=TK_IDENTIFIER;
+ tk.identifier=p_identifier;
+ tk.line=line;
+ tk.col=column;
+
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+
+}
+
+void GDTokenizer::_make_built_in_func(GDFunctions::Function p_func) {
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+
+ tk.type=TK_BUILT_IN_FUNC;
+ tk.func=p_func;
+ tk.line=line;
+ tk.col=column;
+
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+
+}
+void GDTokenizer::_make_constant(const Variant& p_constant) {
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+
+ tk.type=TK_CONSTANT;
+ tk.constant=p_constant;
+ tk.line=line;
+ tk.col=column;
+
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+
+}
+
+void GDTokenizer::_make_type(const Variant::Type& p_type) {
+
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+
+ tk.type=TK_BUILT_IN_TYPE;
+ tk.vtype=p_type;
+ tk.line=line;
+ tk.col=column;
+
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+
+}
+
+
+void GDTokenizer::_make_error(const String& p_error) {
+
+ error_flag=true;
+ last_error=p_error;
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+ tk.type=TK_ERROR;
+ tk.constant=p_error;
+ tk.line=line;
+ tk.col=column;
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+
+}
+
+
+void GDTokenizer::_make_newline(int p_spaces) {
+
+ TokenData &tk=tk_rb[tk_rb_pos];
+ tk.type=TK_NEWLINE;
+ tk.constant=p_spaces;
+ tk.line=line;
+ tk.col=column;
+ tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
+}
+
+void GDTokenizer::_advance() {
+
+ if (error_flag) {
+ //parser broke
+ _make_error(last_error);
+ return;
+ }
+
+ if (code_pos>=len) {
+ _make_token(TK_EOF);
+ return;
+ }
+#define GETCHAR(m_ofs) ((m_ofs+code_pos)>=len?0:_code[m_ofs+code_pos])
+#define INCPOS(m_amount) { code_pos+=m_amount; column+=m_amount; }
+ while (true) {
+
+
+ bool is_node_path=false;
+
+ switch(GETCHAR(0)) {
+ case 0:
+ _make_token(TK_EOF);
+ break;
+ case '\t':
+ case '\r':
+ case ' ':
+ INCPOS(1);
+ continue;
+ case '\n': {
+ line++;
+ INCPOS(1);
+ column=0;
+ int i=0;
+ while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
+ i++;
+ }
+
+ _make_newline(i);
+ return;
+ }
+#if 1 //py style tokenizer
+ case '#': { // line comment skip
+
+ while(GETCHAR(0)!='\n') {
+ code_pos++;
+ if (GETCHAR(0)==0) { //end of file
+ _make_error("Unterminated Comment");
+ return;
+ }
+ }
+ INCPOS(1);
+ column=0;
+ line++;
+ int i=0;
+ while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
+ i++;
+ }
+ _make_newline(i);
+ return;
+
+ } break;
+#endif
+ case '/': {
+
+ switch(GETCHAR(1)) {
+#if 0 // c style tokenizer
+ case '*': { // block comment
+ int pos = code_pos+2;
+ int new_line=line;
+ int new_col=column+2;
+
+ while(true) {
+ if (_code[pos]=='0') {
+ _make_error("Unterminated Comment");
+ code_pos=pos;
+ return;
+ }
+ if (_code[pos]=='*' && _code[pos+1]=='/') {
+ new_col+=2;
+ pos+=2; //compensate
+ break;
+ } else if (_code[pos]=='\n') {
+ new_line++;
+ new_col=0;
+ } else {
+ new_col++;
+ }
+ pos++;
+ }
+
+ column=new_col;
+ line=new_line;
+ code_pos=pos;
+ continue;
+
+ } break;
+ case '/': { // line comment skip
+
+ while(GETCHAR(0)!='\n') {
+ code_pos++;
+ if (GETCHAR(0)==0) { //end of file
+ _make_error("Unterminated Comment");
+ return;
+ }
+ }
+ INCPOS(1);
+ column=0;
+ line++;
+ continue;
+
+ } break;
+#endif
+ case '=': { // diveq
+
+ _make_token(TK_OP_ASSIGN_DIV);
+ INCPOS(1);
+
+ } break;
+ default:
+ _make_token(TK_OP_DIV);
+
+ }
+ } break;
+ case '=': {
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_EQUAL);
+ INCPOS(1);
+
+ } else
+ _make_token(TK_OP_ASSIGN);
+
+ } break;
+ case '<': {
+ if (GETCHAR(1)=='=') {
+
+ _make_token(TK_OP_LESS_EQUAL);
+ INCPOS(1);
+ } else if (GETCHAR(1)=='<') {
+ if (GETCHAR(2)=='=') {
+ _make_token(TK_OP_ASSIGN_SHIFT_LEFT);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_SHIFT_LEFT);
+ }
+ INCPOS(1);
+ } else
+ _make_token(TK_OP_LESS);
+
+ } break;
+ case '>': {
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_GREATER_EQUAL);
+ INCPOS(1);
+ } else if (GETCHAR(1)=='>') {
+ if (GETCHAR(2)=='=') {
+ _make_token(TK_OP_ASSIGN_SHIFT_RIGHT);
+ INCPOS(1);
+
+ } else {
+ _make_token(TK_OP_SHIFT_RIGHT);
+ }
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_GREATER);
+ }
+
+ } break;
+ case '!': {
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_NOT_EQUAL);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_NOT);
+ }
+
+ } break;
+ //case '"' //string - no strings in shader
+ //case '\'' //string - no strings in shader
+ case '{':
+ _make_token(TK_CURLY_BRACKET_OPEN);
+ break;
+ case '}':
+ _make_token(TK_CURLY_BRACKET_CLOSE);
+ break;
+ case '[':
+ _make_token(TK_BRACKET_OPEN);
+ break;
+ case ']':
+ _make_token(TK_BRACKET_CLOSE);
+ break;
+ case '(':
+ _make_token(TK_PARENTHESIS_OPEN);
+ break;
+ case ')':
+ _make_token(TK_PARENTHESIS_CLOSE);
+ break;
+ case ',':
+ _make_token(TK_COMMA);
+ break;
+ case ';':
+ _make_token(TK_SEMICOLON);
+ break;
+ case '?':
+ _make_token(TK_QUESTION_MARK);
+ break;
+ case ':':
+ _make_token(TK_COLON); //for methods maybe but now useless.
+ break;
+ case '^': {
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_BIT_XOR);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_BIT_XOR);
+ }
+
+ } break;
+ case '~':
+ _make_token(TK_OP_BIT_INVERT);
+ break;
+ case '&': {
+ if (GETCHAR(1)=='&') {
+
+ _make_token(TK_OP_AND);
+ INCPOS(1);
+ } else if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_BIT_AND);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_BIT_AND);
+ }
+ } break;
+ case '|': {
+ if (GETCHAR(1)=='|') {
+
+ _make_token(TK_OP_OR);
+ INCPOS(1);
+ } else if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_BIT_OR);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_BIT_OR);
+ }
+ } break;
+ case '*': {
+
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_MUL);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_MUL);
+ }
+ } break;
+ case '+': {
+
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_ADD);
+ INCPOS(1);
+ //} else if (GETCHAR(1)=='+') {
+ // _make_token(TK_OP_PLUS_PLUS);
+ // INCPOS(1);
+ } else {
+ _make_token(TK_OP_ADD);
+ }
+
+ } break;
+ case '-': {
+
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_SUB);
+ INCPOS(1);
+ //} else if (GETCHAR(1)=='-') {
+ // _make_token(TK_OP_MINUS_MINUS);
+ // INCPOS(1);
+ } else {
+ _make_token(TK_OP_SUB);
+ }
+ } break;
+ case '%': {
+
+ if (GETCHAR(1)=='=') {
+ _make_token(TK_OP_ASSIGN_MOD);
+ INCPOS(1);
+ } else {
+ _make_token(TK_OP_MOD);
+ }
+ } break;
+ case '@':
+ if (CharType(GETCHAR(1))!='"') {
+ _make_error("Unexpected '@'");
+ return;
+ }
+ INCPOS(1);
+ is_node_path=true;
+ case '"': {
+
+ int i=1;
+ String str;
+ while(true) {
+ if (CharType(GETCHAR(i)==0)) {
+
+ _make_error("Unterminated String");
+ return;
+ } else if (CharType(GETCHAR(i)=='"')) {
+ break;
+ } else if (CharType(GETCHAR(i)=='\\')) {
+ //escaped characters...
+ i++;
+ CharType next = GETCHAR(i);
+ if (next==0) {
+ _make_error("Unterminated String");
+ return;
+ }
+ CharType res=0;
+
+ switch(next) {
+
+ case 'a': res=7; break;
+ case 'b': res=8; break;
+ case 't': res=9; break;
+ case 'n': res=10; break;
+ case 'v': res=11; break;
+ case 'f': res=12; break;
+ case 'r': res=13; break;
+ case '\'': res='\''; break;
+ case '\"': res='\"'; break;
+ case '\\': res='\\'; break;
+ case 'x': {
+ //hexnumbarh - oct is deprecated
+
+ int read=0;
+ for(int j=0;j<4;j++) {
+ CharType c = GETCHAR(i+j);
+ if (c==0) {
+ _make_error("Unterminated String");
+ return;
+ }
+ if (!_is_hex(c)) {
+ if (j==0 || !(j&1)) {
+ _make_error("Malformed hex constant in string");
+ return;
+ } else
+ break;
+ }
+ CharType v;
+ if (c>='0' && c<='9') {
+ v=c-'0';
+ } else if (c>='a' && c<='f') {
+ v=c-'a';
+ v+=10;
+ } else if (c>='A' && c<='F') {
+ v=c-'A';
+ v+=10;
+ } else {
+ ERR_PRINT("BUG");
+ v=0;
+ }
+
+ res<<=4;
+ res|=v;
+
+ read++;
+ }
+ i+=read-1;
+
+
+ } break;
+ default: {
+
+ _make_error("Invalid escape sequence");
+ return;
+ } break;
+ }
+
+ str+=res;
+
+ } else {
+ str+=CharType(GETCHAR(i));
+ }
+ i++;
+ }
+ INCPOS(i);
+
+ if (is_node_path) {
+ _make_constant(NodePath(str));
+ } else {
+ _make_constant(str);
+ }
+
+ } break;
+ default: {
+
+ if (_is_number(GETCHAR(0)) || (GETCHAR(0)=='.' && _is_number(GETCHAR(1)))) {
+ // parse number
+ bool period_found=false;
+ bool exponent_found=false;
+ bool hexa_found=false;
+ bool sign_found=false;
+
+ String str;
+ int i=0;
+
+ while(true) {
+ if (GETCHAR(i)=='.') {
+ if (period_found || exponent_found) {
+ _make_error("Invalid numeric constant at '.'");
+ return;
+ }
+ period_found=true;
+ } else if (GETCHAR(i)=='x') {
+ if (hexa_found || str.length()!=1 || !( (i==1 && str[0]=='0') || (i==2 && str[1]=='0' && str[0]=='-') ) ) {
+ _make_error("Invalid numeric constant at 'x'");
+ return;
+ }
+ hexa_found=true;
+ } else if (!hexa_found && GETCHAR(i)=='e') {
+ if (hexa_found || exponent_found) {
+ _make_error("Invalid numeric constant at 'e'");
+ return;
+ }
+ exponent_found=true;
+ } else if (_is_number(GETCHAR(i))) {
+ //all ok
+ } else if (hexa_found && _is_hex(GETCHAR(i))) {
+
+ } else if ((GETCHAR(i)=='-' || GETCHAR(i)=='+') && exponent_found) {
+ if (sign_found) {
+ _make_error("Invalid numeric constant at '-'");
+ return;
+ }
+ sign_found=true;
+ } else
+ break;
+
+ str+=CharType(GETCHAR(i));
+ i++;
+ }
+
+ if (!( _is_number(str[str.length()-1]) || (hexa_found && _is_hex(str[str.length()-1])))) {
+ _make_error("Invalid numeric constant: "+str);
+ return;
+ }
+
+ INCPOS(str.length());
+ if (hexa_found) {
+ int val = str.hex_to_int();
+ _make_constant(val);
+ } else if (period_found) {
+ real_t val = str.to_double();
+ //print_line("*%*%*%*% to convert: "+str+" result: "+rtos(val));
+ _make_constant(val);
+ } else {
+ int val = str.to_int();
+ _make_constant(val);
+
+ }
+
+ return;
+ }
+
+ if (GETCHAR(0)=='.') {
+ //parse period
+ _make_token(TK_PERIOD);
+ break;
+ }
+
+ if (_is_text_char(GETCHAR(0))) {
+ // parse identifier
+ String str;
+ str+=CharType(GETCHAR(0));
+
+ int i=1;
+ while(_is_text_char(GETCHAR(i))) {
+ str+=CharType(GETCHAR(i));
+ i++;
+ }
+
+ bool identifier=false;
+
+ if (str=="null") {
+ _make_constant(Variant());
+
+ } else if (str=="true") {
+ _make_constant(true);
+
+ } else if (str=="false") {
+ _make_constant(false);
+ } else {
+
+ bool found=false;
+
+ struct _bit { Variant::Type type; const char *text;};
+ //built in types
+
+ static const _bit type_list[]={
+ //types
+ {Variant::BOOL,"bool"},
+ {Variant::INT,"int"},
+ {Variant::REAL,"float"},
+ {Variant::STRING,"String"},
+ {Variant::VECTOR2,"vec2"},
+ {Variant::VECTOR2,"Vector2"},
+ {Variant::RECT2,"Rect2"},
+ {Variant::MATRIX32,"Matrix32"},
+ {Variant::MATRIX32,"mat32"},
+ {Variant::VECTOR3,"vec3"},
+ {Variant::VECTOR3,"Vector3"},
+ {Variant::_AABB,"AABB"},
+ {Variant::_AABB,"Rect3"},
+ {Variant::PLANE,"Plane"},
+ {Variant::QUAT,"Quat"},
+ {Variant::MATRIX3,"mat3"},
+ {Variant::MATRIX3,"Matrix3"},
+ {Variant::TRANSFORM,"trn"},
+ {Variant::TRANSFORM,"Transform"},
+ {Variant::COLOR,"Color"},
+ {Variant::IMAGE,"Image"},
+ {Variant::_RID,"RID"},
+ {Variant::OBJECT,"Object"},
+ {Variant::INPUT_EVENT,"InputEvent"},
+ {Variant::DICTIONARY,"dict"},
+ {Variant::DICTIONARY,"Dictionary"},
+ {Variant::ARRAY,"Array"},
+ {Variant::RAW_ARRAY,"RawArray"},
+ {Variant::INT_ARRAY,"IntArray"},
+ {Variant::REAL_ARRAY,"FloatArray"},
+ {Variant::STRING_ARRAY,"StringArray"},
+ {Variant::VECTOR2_ARRAY,"Vector2Array"},
+ {Variant::VECTOR3_ARRAY,"Vector3Array"},
+ {Variant::COLOR_ARRAY,"ColorArray"},
+ {Variant::VARIANT_MAX,NULL},
+ };
+
+ {
+
+
+ int idx=0;
+
+ while(type_list[idx].text) {
+
+ if (str==type_list[idx].text) {
+ _make_type(type_list[idx].type);
+ found=true;
+ break;
+ }
+ idx++;
+ }
+ }
+
+ if (!found) {
+
+ //built in func?
+
+ for(int i=0;i<GDFunctions::FUNC_MAX;i++) {
+
+ if (str==GDFunctions::get_func_name(GDFunctions::Function(i))) {
+
+ _make_built_in_func(GDFunctions::Function(i));
+ found=true;
+ break;
+ }
+ }
+
+ //keywor
+ }
+
+ if (!found) {
+
+
+ struct _kws { Token token; const char *text;};
+
+ static const _kws keyword_list[]={
+ //ops
+ {TK_OP_IN,"in"},
+ {TK_OP_NOT,"not"},
+ {TK_OP_OR,"or"},
+ {TK_OP_AND,"and"},
+ //func
+ {TK_PR_FUNCTION,"func"},
+ {TK_PR_FUNCTION,"function"},
+ {TK_PR_CLASS,"class"},
+ {TK_PR_EXTENDS,"extends"},
+ {TK_PR_TOOL,"tool"},
+ {TK_PR_STATIC,"static"},
+ {TK_PR_EXPORT,"export"},
+ {TK_PR_VAR,"var"},
+ {TK_PR_PRELOAD,"preload"},
+ {TK_PR_ASSERT,"assert"},
+ {TK_PR_CONST,"const"},
+ //controlflow
+ {TK_CF_IF,"if"},
+ {TK_CF_ELIF,"elif"},
+ {TK_CF_ELSE,"else"},
+ {TK_CF_FOR,"for"},
+ {TK_CF_WHILE,"while"},
+ {TK_CF_DO,"do"},
+ {TK_CF_SWITCH,"switch"},
+ {TK_CF_BREAK,"break"},
+ {TK_CF_CONTINUE,"continue"},
+ {TK_CF_RETURN,"return"},
+ {TK_CF_PASS,"pass"},
+ {TK_SELF,"self"},
+ {TK_ERROR,NULL}
+ };
+
+ int idx=0;
+ found=false;
+
+ while(keyword_list[idx].text) {
+
+ if (str==keyword_list[idx].text) {
+ _make_token(keyword_list[idx].token);
+ found=true;
+ break;
+ }
+ idx++;
+ }
+ }
+
+ if (!found)
+ identifier=true;
+ }
+
+
+ if (identifier) {
+ _make_identifier(str);
+ }
+ INCPOS(str.length());
+ return;
+ }
+
+ _make_error("Unknown character");
+ return;
+
+ } break;
+ }
+
+ INCPOS(1);
+ break;
+ }
+
+}
+
+void GDTokenizer::set_code(const String& p_code) {
+
+ code=p_code;
+ len = p_code.length();
+ if (len) {
+ _code=&code[0];
+ } else {
+ _code=NULL;
+ }
+ code_pos=0;
+ line=1; //it is stand-ar-ized that lines begin in 1 in code..
+ column=0;
+ tk_rb_pos=0;
+ error_flag=false;
+ last_error="";
+ for(int i=0;i<MAX_LOOKAHEAD+1;i++)
+ _advance();
+}
+
+GDTokenizer::Token GDTokenizer::get_token(int p_offset) const {
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, TK_ERROR);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, TK_ERROR);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ return tk_rb[ofs].type;
+}
+
+int GDTokenizer::get_token_line(int p_offset) const {
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ return tk_rb[ofs].line;
+}
+
+int GDTokenizer::get_token_column(int p_offset) const {
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ return tk_rb[ofs].col;
+}
+
+const Variant& GDTokenizer::get_token_constant(int p_offset) const {
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, tk_rb[0].constant);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, tk_rb[0].constant);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_CONSTANT,tk_rb[0].constant);
+ return tk_rb[ofs].constant;
+}
+StringName GDTokenizer::get_token_identifier(int p_offset) const {
+
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, StringName());
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, StringName());
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_IDENTIFIER,StringName());
+ return tk_rb[ofs].identifier;
+
+}
+
+GDFunctions::Function GDTokenizer::get_token_built_in_func(int p_offset) const {
+
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, GDFunctions::FUNC_MAX);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, GDFunctions::FUNC_MAX);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_FUNC,GDFunctions::FUNC_MAX);
+ return tk_rb[ofs].func;
+
+}
+
+Variant::Type GDTokenizer::get_token_type(int p_offset) const {
+
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, Variant::NIL);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, Variant::NIL);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_TYPE,Variant::NIL);
+ return tk_rb[ofs].vtype;
+
+}
+
+
+int GDTokenizer::get_token_line_indent(int p_offset) const {
+
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, 0);
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, 0);
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_NEWLINE,0);
+ return tk_rb[ofs].constant;
+
+}
+
+String GDTokenizer::get_token_error(int p_offset) const {
+
+ ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, String());
+ ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, String());
+
+ int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
+ ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_ERROR,String());
+ return tk_rb[ofs].constant;
+}
+
+void GDTokenizer::advance(int p_amount) {
+
+ ERR_FAIL_COND( p_amount <=0 );
+ for(int i=0;i<p_amount;i++)
+ _advance();
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-13 07:58:45 +0000