From fad15f79b1230b3076be515d6894c8919562809b Mon Sep 17 00:00:00 2001 From: mitchell Date: Sat, 25 Apr 2020 16:26:31 -0400 Subject: Reformatted Lua LPeg lexers and added new convenience functions and pattern. `lexer.range()` replaces `lexer.delimited_range()` and `lexer.nested_pair()`. `lexer.to_eol()` replaces `patt * lexer.nonnewline^0` constructs. `lexer.number` replaces `lexer.float + lexer.integer`. Also added unit tests for lexer functions. --- lexlua/python.lua | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'lexlua/python.lua') diff --git a/lexlua/python.lua b/lexlua/python.lua index 58c6ba308..72e70d70e 100644 --- a/lexlua/python.lua +++ b/lexlua/python.lua @@ -73,19 +73,17 @@ lex:add_style('self', lexer.STYLE_TYPE) lex:add_rule('identifier', token(lexer.IDENTIFIER, lexer.word)) -- Comments. -lex:add_rule('comment', token(lexer.COMMENT, '#' * lexer.nonnewline_esc^0)) +lex:add_rule('comment', token(lexer.COMMENT, lexer.to_eol('#', true))) -- Strings. -local sq_str = P('u')^-1 * lexer.delimited_range("'", true) -local dq_str = P('U')^-1 * lexer.delimited_range('"', true) -local triple_sq_str = "'''" * (lexer.any - "'''")^0 * P("'''")^-1 -local triple_dq_str = '"""' * (lexer.any - '"""')^0 * P('"""')^-1 +local sq_str = P('u')^-1 * lexer.range("'", true) +local dq_str = P('U')^-1 * lexer.range('"', true) +local tq_str = lexer.range("'''") + lexer.range('"""') -- TODO: raw_strs cannot end in single \. -local raw_sq_str = P('u')^-1 * 'r' * lexer.delimited_range("'", false, true) -local raw_dq_str = P('U')^-1 * 'R' * lexer.delimited_range('"', false, true) -lex:add_rule('string', token(lexer.STRING, triple_sq_str + triple_dq_str + - sq_str + dq_str + raw_sq_str + - raw_dq_str)) +local raw_sq_str = P('u')^-1 * 'r' * lexer.range("'", false, false) +local raw_dq_str = P('U')^-1 * 'R' * lexer.range('"', false, false) +lex:add_rule('string', token(lexer.STRING, tq_str + sq_str + dq_str + + raw_sq_str + raw_dq_str)) -- Numbers. local dec = lexer.digit^1 * S('Ll')^-1 @@ -95,7 +93,7 @@ local integer = S('+-')^-1 * (bin + lexer.hex_num + oct + dec) lex:add_rule('number', token(lexer.NUMBER, lexer.float + integer)) -- Decorators. -lex:add_rule('decorator', token('decorator', '@' * lexer.nonnewline^0)) +lex:add_rule('decorator', token('decorator', lexer.to_eol('@'))) lex:add_style('decorator', lexer.STYLE_PREPROCESSOR) -- Operators. -- cgit v1.2.3