diff --git a/_extensions/gdscript.py b/_extensions/gdscript.py index f7c1e214c..1dac79fd0 100644 --- a/_extensions/gdscript.py +++ b/_extensions/gdscript.py @@ -9,46 +9,39 @@ :license: MIT. modified by Daniel J. Ramirez based on the original python.py pygment + further expanded and consolidated with the godot-docs lexer by Zackery R. Smith and Ste. """ -import re - -from pygments.lexer import ( - RegexLexer, - include, - bygroups, - default, - words, - combined, -) +from pygments.lexer import RegexLexer, include, bygroups, words, combined from pygments.token import ( - Text, - Comment, - Operator, Keyword, + Literal, Name, + Comment, String, Number, - Punctuation, + Operator, Whitespace, + Punctuation, ) __all__ = ["GDScriptLexer"] -line_re = re.compile(".*?\n") - class GDScriptLexer(RegexLexer): """ - For `GDScript source code `_. + For GDScript source code. """ name = "GDScript" + url = "https://www.godotengine.org" aliases = ["gdscript", "gd"] filenames = ["*.gd"] mimetypes = ["text/x-gdscript", "application/x-gdscript"] - def innerstring_rules(ttype): + # taken from pygments/gdscript.py + @staticmethod + def inner_string_rules(ttype): return [ # the old style '%s' % (...) string formatting ( @@ -56,7 +49,7 @@ class GDScriptLexer(RegexLexer): "[hlL]?[E-GXc-giorsux%]", String.Interpol, ), - # backslashes, quotes and formatting signs must be parsed one at a time + # backslashes, quotes, and formatting signs must be parsed one at a time (r'[^\\\'"%\n]+', ttype), (r'[\'"\\]', ttype), # unhandled string formatting sign @@ -65,78 +58,19 @@ class GDScriptLexer(RegexLexer): ] tokens = { - "root": [ - (r"\n", Whitespace), - ( - r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', - bygroups(Whitespace, String.Affix, String.Doc), - ), - ( - r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", - bygroups(Whitespace, String.Affix, String.Doc), - ), - (r"[^\S\n]+", Whitespace), + "whitespace": [(r"\s+", Whitespace)], + "comment": [ + (r"##.*$", Comment.Doc), + (r"#(?:end)?region.*$", Comment.Region), (r"#.*$", Comment.Single), - (r"[]{}:(),;[]", Punctuation), - (r"(\\)(\n)", Whitespace), - (r"\\", Text), - # modules/gdscript/gdscript.cpp - GDScriptLanguage::get_reserved_words() - # Operators. - (r"(and|as|in|is|not|or)\b", Operator.Word), - ( - r"!=|==|<<|>>|&&|\+=|-=|\*=|/=|%=|&=|\|=|\|\||[-~+/*%=<>&^.!|$]", - Operator, - ), - include("keywords"), - include("control_flow_keywords"), - (r"(func)((?:\s|\\\s)+)", bygroups(Keyword, Whitespace), "funcname"), - (r"(class)((?:\s|\\\s)+)", bygroups(Keyword, Whitespace), "classname"), - include("builtins"), - include("decorators"), - ( - '([rR]|[uUbB][rR]|[rR][uUbB])(""")', - bygroups(String.Affix, String.Double), - "tdqs", - ), - ( - "([rR]|[uUbB][rR]|[rR][uUbB])(''')", - bygroups(String.Affix, String.Single), - "tsqs", - ), - ( - '([rR]|[uUbB][rR]|[rR][uUbB])(")', - bygroups(String.Affix, String.Double), - "dqs", - ), - ( - "([rR]|[uUbB][rR]|[rR][uUbB])(')", - bygroups(String.Affix, String.Single), - "sqs", - ), - ( - '([uUbB]?)(""")', - bygroups(String.Affix, String.Double), - combined("stringescape", "tdqs"), - ), - ( - "([uUbB]?)(''')", - bygroups(String.Affix, String.Single), - combined("stringescape", "tsqs"), - ), - ( - '([uUbB]?)(")', - bygroups(String.Affix, String.Double), - combined("stringescape", "dqs"), - ), - ( - "([uUbB]?)(')", - bygroups(String.Affix, String.Single), - combined("stringescape", "sqs"), - ), - include("name"), - include("numbers"), ], - "keywords": [ + "punctuation": [ + (r"[]{}(),:;[]", Punctuation), + (r":\n", Punctuation), + (r"\\", Punctuation), + ], + # NOTE: from github.com/godotengine/godot-docs + "keyword": [ ( words( ( @@ -148,27 +82,26 @@ class GDScriptLexer(RegexLexer): "enum", "extends", "func", - "namespace", # Reserved for potential future use. + "namespace", # Reserved for potential future use. "signal", "static", - "trait", # Reserved for potential future use. + "trait", # Reserved for potential future use. "var", # Other keywords. "await", "breakpoint", "self", "super", - "yield", # Reserved for potential future use. + "yield", # Reserved for potential future use. # Not really keywords, but used in property syntax. - "set", - "get", + # also colored like functions, not keywords + #"set", + #"get", ), suffix=r"\b", ), Keyword, ), - ], - "control_flow_keywords": [ ( words( ( @@ -185,6 +118,7 @@ class GDScriptLexer(RegexLexer): "return", "when", "while", + "yield", ), suffix=r"\b", ), @@ -193,7 +127,16 @@ class GDScriptLexer(RegexLexer): Keyword.ControlFlow, ), ], - "builtins": [ + "builtin": [ + ( + words( + ("true", "false", "PI", "TAU", "NAN", "INF", "null"), + prefix=r"(?>|&&|\+=|-=|\*=|/=|%=|&=|\|=|\|\||[-~+/*%=<>&^.!|$]", + Operator, ), - (r"(-)?(\d|(?<=\d)_)+[eE][+-]?(\d|(?<=\d)_)+j?", Number.Float), - (r"(-)?0[xX]([a-fA-F0-9]|(?<=[a-fA-F0-9])_)+", Number.Hex), - (r"(-)?0[bB]([01]|(?<=[01])_)+", Number.Bin), - (r"(-)?(\d|(?<=\d)_)+j?", Number.Integer), + (r"(in|is|and|as|or|not)\b", Operator.Word), ], - "name": [(r"@?[a-zA-Z_]\w*", Name)], - "funcname": [(r"[a-zA-Z_]\w*", Name.Function, "#pop"), default("#pop")], - "classname": [(r"[a-zA-Z_]\w*", Name.Class, "#pop")], - "stringescape": [ + "number": [ + (r"([\d_]+\.[\d_]*|[\d_]*\.[\d_]+)([eE][+-]?[\d_]+)?", Number.Float), + (r"[\d_]+[eE][+-]?[\d_]+", Number.Float), + (r"0[xX][a-fA-F\d_]+", Number.Hex), + (r"(-)?0[bB]([01]|(?<=[01])_)+", Number.Bin), + (r"[\d_]+", Number.Integer), + ], + "name": [(r"[a-zA-Z_]\w*", Name)], + "typehint": [ + (r"[a-zA-Z_]\w*", Name.Class, "#pop"), + ], + "string_escape": [ ( r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' - r"U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})", + r"U[a-fA-F0-9]{6}|x[a-fA-F0-9]{2}|[0-7]{1,3})", String.Escape, ) ], - "strings-single": innerstring_rules(String.Single), - "strings-double": innerstring_rules(String.Double), - "dqs": [ + "string_single": inner_string_rules(String.Single), + "string_double": inner_string_rules(String.Double), + "string_other": inner_string_rules(String.Other), + "string_stringname": inner_string_rules(String.StringName), + "string_nodepath": inner_string_rules(String.NodePath), + "double_quotes": [ (r'"', String.Double, "#pop"), (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings - include("strings-double"), + include("string_double"), ], - "sqs": [ + "single_quotes": [ (r"'", String.Single, "#pop"), (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings - include("strings-single"), + include("string_single"), ], - "tdqs": [ + "triple_double_quotes": [ (r'"""', String.Double, "#pop"), - include("strings-double"), - (r"\n", Whitespace), + include("string_double"), + include("whitespace"), ], - "tsqs": [ + "triple_single_quotes": [ (r"'''", String.Single, "#pop"), - include("strings-single"), - (r"\n", Whitespace), + include("string_single"), + include("whitespace"), + ], + "node_reference": [ + (r'[\$%]"', String.Other, include("node_reference_double")), + (r"[\$%]'", String.Other, include("node_reference_single")), + (r"[\$%][A-Za-z_][\w/]*/?", String.Other), + ], + "node_reference_double": [ + (r'"', String.Other, "#pop"), + include("string_other"), + ], + "node_reference_single": [ + (r"'", String.Other, "#pop"), + include("string_other"), + ], + "stringname": [ + (r'[&]"', String.StringName, include("stringname_double")), + (r"[&]'", String.StringName, include("stringname_single")), + ], + "stringname_double": [ + (r'"', String.StringName, "#pop"), + include("string_stringname"), + ], + "stringname_single": [ + (r"'", String.StringName, "#pop"), + include("string_stringname"), + ], + "nodepath": [ + (r'[\^]"', String.NodePath, include("nodepath_double")), + (r"[\^]'", String.NodePath, include("nodepath_single")), + ], + "nodepath_double": [ + (r'"', String.NodePath, "#pop"), + include("string_nodepath"), + ], + "nodepath_single": [ + (r"'", String.NodePath, "#pop"), + include("string_nodepath"), + ], + "function_name": [(r"[a-zA-Z_]\w*", Name.Function.Declaration, "#pop")], + "enum_name": [(r"[a-zA-Z_]\w*", Name, "#pop")], + "function": [ + (r"\b([a-zA-Z_]\w*)\s*(?=\()", Name.Function), + ( + # colored like functions, even without braces + words(("set", "get",), suffix=r"\b", ), + Name.Function, + ), + ], + + ####################################################################### + # LEXER ENTRY POINT + ####################################################################### + "root": [ + include("whitespace"), + include("comment"), + include("punctuation"), + include("builtin"), + # strings + include("stringname"), + include("nodepath"), + include("node_reference"), + ( + '(r)(""")', + bygroups(String.Affix, String.Double), + "triple_double_quotes", + ), + ( + "(r)(''')", + bygroups(String.Affix, String.Single), + "triple_single_quotes", + ), + ( + '(r)(")', + bygroups(String.Affix, String.Double), + "double_quotes", + ), + ( + "(r)(')", + bygroups(String.Affix, String.Single), + "single_quotes", + ), + ( + '(r?)(""")', + bygroups(String.Affix, String.Double), + combined("string_escape", "triple_double_quotes"), + ), + ( + "(r?)(''')", + bygroups(String.Affix, String.Single), + combined("string_escape", "triple_single_quotes"), + ), + ( + '(r?)(")', + bygroups(String.Affix, String.Double), + combined("string_escape", "double_quotes"), + ), + ( + "(r?)(')", + bygroups(String.Affix, String.Single), + combined("string_escape", "single_quotes"), + ), + # consider Name after a . as instance/members variables + (r"(?