Reformatted Lua LPeg lexers and added new convenience functions and pattern.

`lexer.range()` replaces `lexer.delimited_range()` and `lexer.nested_pair()`.
`lexer.to_eol()` replaces `patt * lexer.nonnewline^0` constructs.
`lexer.number` replaces `lexer.float + lexer.integer`.
Also added unit tests for lexer functions.

1 files changed, 11 insertions, 12 deletions

diff --git a/lexlua/tcl.lua b/lexlua/tcl.lua
index 45e3ccf9d..0f0a0d80a 100644
--- a/lexlua/tcl.lua
+++ b/lexlua/tcl.lua
@@ -13,11 +13,12 @@ local lex = lexer.new('tcl')
 lex:add_rule('whitespace', token(lexer.WHITESPACE, lexer.space^1))
 
 -- Comment.
-lex:add_rule('comment', token(lexer.COMMENT, '#' * P(function(input, index)
-  local i = index - 2
-  while i > 0 and input:find('^[ \t]', i) do i = i - 1 end
-  if i < 1 or input:find('^[\r\n;]', i) then return index end
-end) * lexer.nonnewline^0))
+lex:add_rule('comment', token(lexer.COMMENT, lexer.to_eol('#' *
+  P(function(input, index)
+    local i = index - 2
+    while i > 0 and input:find('^[ \t]', i) do i = i - 1 end
+    if i < 1 or input:find('^[\r\n;]', i) then return index end
+  end))))
 
 -- Separator (semicolon).
 lex:add_rule('separator', token(lexer.CLASS, P(';')))
@@ -32,15 +33,13 @@ lex:add_rule('brackets', token(lexer.VARIABLE, S('[]')))
 
 -- Variable substitution.
 lex:add_rule('variable', token(lexer.STRING, '$' *
-                                             (lexer.alnum + '_' + P(':')^2)^0))
+  (lexer.alnum + '_' + P(':')^2)^0))
 
 -- Backslash substitution.
-lex:add_rule('backslash', token(lexer.TYPE,
-                                '\\' * (lexer.digit * lexer.digit^-2 +
-                                        'x' * lexer.xdigit^1 +
-                                        'u' * lexer.xdigit * lexer.xdigit^-3 +
-                                        'U' * lexer.xdigit * lexer.xdigit^-7 +
-                                        1)))
+local oct = lexer.digit * lexer.digit^-2
+local hex = 'x' * lexer.xdigit^1
+local unicode = 'u' * lexer.xdigit * lexer.xdigit^-3
+lex:add_rule('backslash', token(lexer.TYPE, '\\' * (oct + hex + unicode + 1)))
 
 -- Fold points.
 lex:add_fold_point(lexer.KEYWORD, '{', '}')