Reformatted Lua LPeg lexers and added new convenience functions and pattern.

`lexer.range()` replaces `lexer.delimited_range()` and `lexer.nested_pair()`.
`lexer.to_eol()` replaces `patt * lexer.nonnewline^0` constructs.
`lexer.number` replaces `lexer.float + lexer.integer`.
Also added unit tests for lexer functions.

1 files changed, 19 insertions, 20 deletions

diff --git a/lexlua/toml.lua b/lexlua/toml.lua
index 7b32c0c1d..ae6835174 100644
--- a/lexlua/toml.lua
+++ b/lexlua/toml.lua
@@ -9,10 +9,9 @@ local lex = lexer.new('toml', {fold_by_indentation = true})
 
 -- Whitespace
 lex:add_rule('indent', #lexer.starts_line(S(' \t')) *
-                       (token(lexer.WHITESPACE, ' ') +
-                        token('indent_error', '\t'))^1)
+  (token(lexer.WHITESPACE, ' ') + token('indent_error', '\t'))^1)
 lex:add_rule('whitespace', token(lexer.WHITESPACE, S(' \t')^1 +
-                                                   lexer.newline^1))
+  lexer.newline^1))
 lex:add_style('indent_error', 'back:%(color.red)')
 
 -- kewwords.
@@ -22,32 +21,32 @@ lex:add_rule('keyword', token(lexer.KEYWORD, word_match[[true false]]))
 lex:add_rule('identifier', token(lexer.IDENTIFIER, lexer.word))
 
 -- Strings.
-lex:add_rule('string', token(lexer.STRING, lexer.delimited_range("'") +
-                                           lexer.delimited_range('"')))
+local sq_str = lexer.range("'")
+local dq_str = lexer.range('"')
+lex:add_rule('string', token(lexer.STRING, sq_str + dq_str))
 
 -- Comments.
-lex:add_rule('comment', token(lexer.COMMENT, '#' * lexer.nonnewline^0))
+lex:add_rule('comment', token(lexer.COMMENT, lexer.to_eol('#')))
 
 -- Operators.
 lex:add_rule('operator', token(lexer.OPERATOR, S('#=+-,.{}[]()')))
 
 -- Datetime.
-lex:add_rule('datetime',
-             token('timestamp',
-                   lexer.digit * lexer.digit * lexer.digit * lexer.digit * -- yr
-                   '-' * lexer.digit * lexer.digit^-1 * -- month
-                   '-' * lexer.digit * lexer.digit^-1 * -- day
-                   ((S(' \t')^1 + S('tT'))^-1 * -- separator
-                    lexer.digit * lexer.digit^-1 * -- hour
-                    ':' * lexer.digit * lexer.digit * -- minute
-                    ':' * lexer.digit * lexer.digit * -- second
-                    ('.' * lexer.digit^0)^-1 * -- fraction
-                    ('Z' + -- timezone
-                     S(' \t')^0 * S('-+') * lexer.digit * lexer.digit^-1 *
-                     (':' * lexer.digit * lexer.digit)^-1)^-1)^-1))
+local year = lexer.digit * lexer.digit * lexer.digit * lexer.digit
+local month = lexer.digit * lexer.digit^-1
+local day = lexer.digit * lexer.digit^-1
+local date = year * '-' * month * '-' * day
+local hours = lexer.digit * lexer.digit^-1
+local minutes = lexer.digit * lexer.digit
+local seconds = lexer.digit * lexer.digit
+local fraction = '.' * lexer.digit^0
+local time = hours * ':' * minutes * ':' * seconds * fraction^-1
+local T = S(' \t')^1 + S('tT')
+local zone = 'Z' + S(' \t')^0 * S('-+') * hours * (':' * minutes)^-1
+lex:add_rule('datetime', token('timestamp', date * (T * time * zone^-1)))
 lex:add_style('timestamp', lexer.STYLE_NUMBER)
 
 -- Numbers.
-lex:add_rule('number', token(lexer.NUMBER, lexer.float + lexer.integer))
+lex:add_rule('number', token(lexer.NUMBER, lexer.number))
 
 return lex