summaryrefslogtreecommitdiff
path: root/modules/gdscript/gdscript_tokenizer.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'modules/gdscript/gdscript_tokenizer.cpp')
-rw-r--r--modules/gdscript/gdscript_tokenizer.cpp128
1 files changed, 88 insertions, 40 deletions
diff --git a/modules/gdscript/gdscript_tokenizer.cpp b/modules/gdscript/gdscript_tokenizer.cpp
index 77e1b7290e..8b20b0ff48 100644
--- a/modules/gdscript/gdscript_tokenizer.cpp
+++ b/modules/gdscript/gdscript_tokenizer.cpp
@@ -5,8 +5,8 @@
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
-/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */
-/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */
+/* Copyright (c) 2007-2019 Juan Linietsky, Ariel Manzur. */
+/* Copyright (c) 2014-2019 Godot Engine contributors (cf. AUTHORS.md) */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
@@ -80,10 +80,7 @@ const char *GDScriptTokenizer::token_names[TK_MAX] = {
"elif",
"else",
"for",
- "do",
"while",
- "switch (reserved)",
- "case (reserved)",
"break",
"continue",
"pass",
@@ -128,8 +125,8 @@ const char *GDScriptTokenizer::token_names[TK_MAX] = {
"'.'",
"'?'",
"':'",
- "'->'",
"'$'",
+ "'->'",
"'\\n'",
"PI",
"TAU",
@@ -224,9 +221,6 @@ static const _kws _keyword_list[] = {
{ GDScriptTokenizer::TK_CF_ELSE, "else" },
{ GDScriptTokenizer::TK_CF_FOR, "for" },
{ GDScriptTokenizer::TK_CF_WHILE, "while" },
- { GDScriptTokenizer::TK_CF_DO, "do" },
- { GDScriptTokenizer::TK_CF_SWITCH, "switch" },
- { GDScriptTokenizer::TK_CF_CASE, "case" },
{ GDScriptTokenizer::TK_CF_BREAK, "break" },
{ GDScriptTokenizer::TK_CF_CONTINUE, "continue" },
{ GDScriptTokenizer::TK_CF_RETURN, "return" },
@@ -291,9 +285,6 @@ bool GDScriptTokenizer::is_token_literal(int p_offset, bool variable_safe) const
case TK_CF_ELSE:
case TK_CF_FOR:
case TK_CF_WHILE:
- case TK_CF_DO:
- case TK_CF_SWITCH:
- case TK_CF_CASE:
case TK_CF_BREAK:
case TK_CF_CONTINUE:
case TK_CF_RETURN:
@@ -348,7 +339,8 @@ StringName GDScriptTokenizer::get_token_literal(int p_offset) const {
return "null";
case Variant::BOOL:
return value ? "true" : "false";
- default: {}
+ default: {
+ }
}
}
case TK_OP_AND:
@@ -365,8 +357,7 @@ StringName GDScriptTokenizer::get_token_literal(int p_offset) const {
}
}
}
- ERR_EXPLAIN("Failed to get token literal");
- ERR_FAIL_V("");
+ ERR_FAIL_V_MSG("", "Failed to get token literal.");
}
static bool _is_text_char(CharType c) {
@@ -384,6 +375,11 @@ static bool _is_hex(CharType c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
+static bool _is_bin(CharType c) {
+
+ return (c == '0' || c == '1');
+}
+
void GDScriptTokenizerText::_make_token(Token p_type) {
TokenData &tk = tk_rb[tk_rb_pos];
@@ -520,7 +516,22 @@ void GDScriptTokenizerText::_advance() {
INCPOS(1);
column = 1;
int i = 0;
- while (GETCHAR(i) == ' ' || GETCHAR(i) == '\t') {
+ while (true) {
+ if (GETCHAR(i) == ' ') {
+ if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_SPACES;
+ if (file_indent_type != INDENT_SPACES) {
+ _make_error("Spaces used for indentation in tab-indented file!");
+ return;
+ }
+ } else if (GETCHAR(i) == '\t') {
+ if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_TABS;
+ if (file_indent_type != INDENT_TABS) {
+ _make_error("Tabs used for indentation in space-indented file!");
+ return;
+ }
+ } else {
+ break; // not indentation anymore
+ }
i++;
}
@@ -543,13 +554,14 @@ void GDScriptTokenizerText::_advance() {
}
}
#ifdef DEBUG_ENABLED
- if (comment.begins_with("#warning-ignore:")) {
- String code = comment.get_slice(":", 1);
+ String comment_content = comment.trim_prefix("#").trim_prefix(" ");
+ if (comment_content.begins_with("warning-ignore:")) {
+ String code = comment_content.get_slice(":", 1);
warning_skips.push_back(Pair<int, String>(line, code.strip_edges().to_lower()));
- } else if (comment.begins_with("#warning-ignore-all:")) {
- String code = comment.get_slice(":", 1);
+ } else if (comment_content.begins_with("warning-ignore-all:")) {
+ String code = comment_content.get_slice(":", 1);
warning_global_skips.insert(code.strip_edges().to_lower());
- } else if (comment.strip_edges() == "#warnings-disable") {
+ } else if (comment_content.strip_edges() == "warnings-disable") {
ignore_warnings = true;
}
#endif // DEBUG_ENABLED
@@ -557,9 +569,25 @@ void GDScriptTokenizerText::_advance() {
column = 1;
line++;
int i = 0;
- while (GETCHAR(i) == ' ' || GETCHAR(i) == '\t') {
+ while (true) {
+ if (GETCHAR(i) == ' ') {
+ if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_SPACES;
+ if (file_indent_type != INDENT_SPACES) {
+ _make_error("Spaces used for indentation in tab-indented file!");
+ return;
+ }
+ } else if (GETCHAR(i) == '\t') {
+ if (file_indent_type == INDENT_NONE) file_indent_type = INDENT_TABS;
+ if (file_indent_type != INDENT_TABS) {
+ _make_error("Tabs used for indentation in space-indented file!");
+ return;
+ }
+ } else {
+ break; // not indentation anymore
+ }
i++;
}
+
_make_newline(i);
return;
@@ -753,7 +781,7 @@ void GDScriptTokenizerText::_advance() {
}
INCPOS(1);
is_node_path = true;
-
+ FALLTHROUGH;
case '\'':
case '"': {
@@ -884,6 +912,7 @@ void GDScriptTokenizerText::_advance() {
bool period_found = false;
bool exponent_found = false;
bool hexa_found = false;
+ bool bin_found = false;
bool sign_found = false;
String str;
@@ -894,16 +923,28 @@ void GDScriptTokenizerText::_advance() {
if (period_found || exponent_found) {
_make_error("Invalid numeric constant at '.'");
return;
+ } else if (bin_found) {
+ _make_error("Invalid binary constant at '.'");
+ return;
+ } else if (hexa_found) {
+ _make_error("Invalid hexadecimal constant at '.'");
+ return;
}
period_found = true;
} else if (GETCHAR(i) == 'x') {
- if (hexa_found || str.length() != 1 || !((i == 1 && str[0] == '0') || (i == 2 && str[1] == '0' && str[0] == '-'))) {
+ if (hexa_found || bin_found || str.length() != 1 || !((i == 1 && str[0] == '0') || (i == 2 && str[1] == '0' && str[0] == '-'))) {
_make_error("Invalid numeric constant at 'x'");
return;
}
hexa_found = true;
+ } else if (GETCHAR(i) == 'b') {
+ if (hexa_found || bin_found || str.length() != 1 || !((i == 1 && str[0] == '0') || (i == 2 && str[1] == '0' && str[0] == '-'))) {
+ _make_error("Invalid numeric constant at 'b'");
+ return;
+ }
+ bin_found = true;
} else if (!hexa_found && GETCHAR(i) == 'e') {
- if (hexa_found || exponent_found) {
+ if (exponent_found || bin_found) {
_make_error("Invalid numeric constant at 'e'");
return;
}
@@ -912,6 +953,8 @@ void GDScriptTokenizerText::_advance() {
//all ok
} else if (hexa_found && _is_hex(GETCHAR(i))) {
+ } else if (bin_found && _is_bin(GETCHAR(i))) {
+
} else if ((GETCHAR(i) == '-' || GETCHAR(i) == '+') && exponent_found) {
if (sign_found) {
_make_error("Invalid numeric constant at '-'");
@@ -937,6 +980,9 @@ void GDScriptTokenizerText::_advance() {
if (hexa_found) {
int64_t val = str.hex_to_int64();
_make_constant(val);
+ } else if (bin_found) {
+ int64_t val = str.bin_to_int64();
+ _make_constant(val);
} else if (period_found || exponent_found) {
double val = str.to_double();
_make_constant(val);
@@ -998,11 +1044,11 @@ void GDScriptTokenizerText::_advance() {
//built in func?
- for (int i = 0; i < GDScriptFunctions::FUNC_MAX; i++) {
+ for (int j = 0; j < GDScriptFunctions::FUNC_MAX; j++) {
- if (str == GDScriptFunctions::get_func_name(GDScriptFunctions::Function(i))) {
+ if (str == GDScriptFunctions::get_func_name(GDScriptFunctions::Function(j))) {
- _make_built_in_func(GDScriptFunctions::Function(i));
+ _make_built_in_func(GDScriptFunctions::Function(j));
found = true;
break;
}
@@ -1066,6 +1112,7 @@ void GDScriptTokenizerText::set_code(const String &p_code) {
ignore_warnings = false;
#endif // DEBUG_ENABLED
last_error = "";
+ file_indent_type = INDENT_NONE;
for (int i = 0; i < MAX_LOOKAHEAD + 1; i++)
_advance();
}
@@ -1171,10 +1218,8 @@ Error GDScriptTokenizerBuffer::set_code_buffer(const Vector<uint8_t> &p_buffer)
ERR_FAIL_COND_V(p_buffer.size() < 24 || p_buffer[0] != 'G' || p_buffer[1] != 'D' || p_buffer[2] != 'S' || p_buffer[3] != 'C', ERR_INVALID_DATA);
int version = decode_uint32(&buf[4]);
- if (version > BYTECODE_VERSION) {
- ERR_EXPLAIN("Bytecode is too New! Please use a newer engine version.");
- ERR_FAIL_COND_V(version > BYTECODE_VERSION, ERR_INVALID_DATA);
- }
+ ERR_FAIL_COND_V_MSG(version > BYTECODE_VERSION, ERR_INVALID_DATA, "Bytecode is too recent! Please use a newer engine version.");
+
int identifier_count = decode_uint32(&buf[8]);
int constant_count = decode_uint32(&buf[12]);
int line_count = decode_uint32(&buf[16]);
@@ -1208,7 +1253,8 @@ Error GDScriptTokenizerBuffer::set_code_buffer(const Vector<uint8_t> &p_buffer)
Variant v;
int len;
- Error err = decode_variant(v, b, total_len, &len);
+ // An object cannot be constant, never decode objects
+ Error err = decode_variant(v, b, total_len, &len, false);
if (err)
return err;
b += len;
@@ -1286,7 +1332,7 @@ Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code)
} break;
case TK_CONSTANT: {
- Variant c = tt.get_token_constant();
+ const Variant &c = tt.get_token_constant();
if (!constant_map.has(c)) {
int idx = constant_map.size();
constant_map[c] = idx;
@@ -1310,7 +1356,8 @@ Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code)
ERR_FAIL_V(Vector<uint8_t>());
} break;
- default: {}
+ default: {
+ }
};
token_array.push_back(token);
@@ -1376,11 +1423,12 @@ Vector<uint8_t> GDScriptTokenizerBuffer::parse_code_string(const String &p_code)
for (Map<int, Variant>::Element *E = rev_constant_map.front(); E; E = E->next()) {
int len;
- Error err = encode_variant(E->get(), NULL, len);
- ERR_FAIL_COND_V(err != OK, Vector<uint8_t>());
+ // Objects cannot be constant, never encode objects
+ Error err = encode_variant(E->get(), NULL, len, false);
+ ERR_FAIL_COND_V_MSG(err != OK, Vector<uint8_t>(), "Error when trying to encode Variant.");
int pos = buf.size();
buf.resize(pos + len);
- encode_variant(E->get(), &buf.write[pos], len);
+ encode_variant(E->get(), &buf.write[pos], len, false);
}
for (Map<int, uint32_t>::Element *E = rev_line_map.front(); E; E = E->next()) {
@@ -1426,7 +1474,7 @@ StringName GDScriptTokenizerBuffer::get_token_identifier(int p_offset) const {
ERR_FAIL_INDEX_V(offset, tokens.size(), StringName());
uint32_t identifier = tokens[offset] >> TOKEN_BITS;
- ERR_FAIL_INDEX_V(identifier, (uint32_t)identifiers.size(), StringName());
+ ERR_FAIL_UNSIGNED_INDEX_V(identifier, (uint32_t)identifiers.size(), StringName());
return identifiers[identifier];
}
@@ -1482,7 +1530,7 @@ const Variant &GDScriptTokenizerBuffer::get_token_constant(int p_offset) const {
int offset = token + p_offset;
ERR_FAIL_INDEX_V(offset, tokens.size(), nil);
uint32_t constant = tokens[offset] >> TOKEN_BITS;
- ERR_FAIL_INDEX_V(constant, (uint32_t)constants.size(), nil);
+ ERR_FAIL_UNSIGNED_INDEX_V(constant, (uint32_t)constants.size(), nil);
return constants[constant];
}
String GDScriptTokenizerBuffer::get_token_error(int p_offset) const {
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-15 14:58:29 +0000