diff options
Diffstat (limited to 'mods/src/leex.hrl')
| -rw-r--r-- | mods/src/leex.hrl | 217 |
1 files changed, 217 insertions, 0 deletions
diff --git a/mods/src/leex.hrl b/mods/src/leex.hrl new file mode 100644 index 0000000..4845418 --- /dev/null +++ b/mods/src/leex.hrl @@ -0,0 +1,217 @@ +%% THIS IS A PRE-RELEASE OF LEEX - RELEASED ONLY BECAUSE MANY PEOPLE +%% WANTED IT - THE OFFICIAL RELEASE WILL PROVIDE A DIFFERENT INCOMPATIBLE +%% AND BETTER INTERFACE - BE WARNED +%% PLEASE REPORT ALL BUGS TO THE AUTHOR. + +##module + +-export([string/1,string/2,token/2,token/3,tokens/2,tokens/3]). +-export([format_error/1]). + +%% luke +-export([yystate/0, yystate/6, yyaction/4]). + +%% User code. This is placed here to allow extra attributes. +##code + +format_error({illegal,S}) -> ["illegal characters ",io_lib:write_string(S)]; +format_error({user,S}) -> S. + +string(String) -> string(String, 1). + +string(String, Line) -> string(String, Line, String, []). + +%% string(InChars, Line, TokenChars, Tokens) -> +%% {ok,Tokens,Line} | {error,ErrorInfo,Line}. + +string([], L, [], Ts) -> %No partial tokens! + {ok,yyrev(Ts),L}; +string(Ics0, L0, Tcs, Ts) -> + case yystate(yystate(), Ics0, L0, 0, reject, 0) of + {A,Alen,Ics1,L1} -> %Accepting end state + string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L1), Ts); + {A,Alen,Ics1,L1,S1} -> %After an accepting state + string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L1), Ts); + {reject,Alen,Tlen,Ics1,L1,S1} -> + {error,{L1,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},L1}; + {A,Alen,Tlen,Ics1,L1,S1} -> + string_cont(yysuf(Tcs, Alen), L1, yyaction(A, Alen, Tcs, L1), Ts) + end. + +%% string_cont(RestChars, Line, Token, Tokens) +%% Test for and remove the end token wrapper. + +string_cont(Rest, Line, {token,T}, Ts) -> + string(Rest, Line, Rest, [T|Ts]); +string_cont(Rest, Line, {end_token,T}, Ts) -> + string(Rest, Line, Rest, [T|Ts]); +string_cont(Rest, Line, skip_token, Ts) -> + string(Rest, Line, Rest, Ts); +string_cont(Rest, Line, {error,S}, Ts) -> + {error,{Line,?MODULE,{user,S}},Line}. + +%% token(Continuation, Chars, Line) -> +%% {more,Continuation} | {done,ReturnVal,RestChars}. +%% Must be careful when re-entering to append the latest characters to the +%% after characters in an accept. + +token(Cont, Chars) -> token(Cont, Chars, 1). + +token([], Chars, Line) -> + token(Chars, Line, yystate(), Chars, 0, reject, 0); +token({Line,State,Tcs,Tlen,Action,Alen}, Chars, _) -> + token(Chars, Line, State, Tcs ++ Chars, Tlen, Action, Alen). + +%% token(InChars, Line, State, TokenChars, TokenLen, Accept) -> +%% {more,Continuation} | {done,ReturnVal,RestChars}. + +token(Ics0, L0, S0, Tcs, Tlen0, A0, Alen0) -> + case yystate(S0, Ics0, L0, Tlen0, A0, Alen0) of + {A1,Alen1,Ics1,L1} -> %Accepting end state + token_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, L1)); + {A1,Alen1,[],L1,S1} -> %After an accepting state + {more,{L1,S1,Tcs,Alen1,A1,Alen1}}; + {A1,Alen1,Ics1,L1,S1} -> + token_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, L1)); + {A1,Alen1,Tlen1,[],L1,S1} -> %After a non-accepting state + {more,{L1,S1,Tcs,Tlen1,A1,Alen1}}; + {reject,Alen1,Tlen1,eof,L1,S1} -> + {done,{eof,L1},[]}; + {reject,Alen1,Tlen1,Ics1,L1,S1} -> + {done,{error,{L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}},L1},Ics1}; + {A1,Alen1,Tlen1,Ics1,L1,S1} -> + token_cont(yysuf(Tcs, Alen1), L1, yyaction(A1, Alen1, Tcs, L1)) + end. + +%% tokens_cont(RestChars, Line, Token) +%% Test if we have detected the end token, if so return done else continue. + +token_cont(Rest, Line, {token,T}) -> + {done,{ok,T,Line},Rest}; +token_cont(Rest, Line, {end_token,T}) -> + {done,{ok,T,Line},Rest}; +token_cont(Rest, Line, skip_token) -> + token(Rest, Line, yystate(), Rest, 0, reject, 0); +token_cont(Rest, Line, {error,S}) -> + {done,{error,{Line,?MODULE,{user,S}},Line},Rest}. + +%% tokens(Continuation, Chars, Line) -> +%% {more,Continuation} | {done,ReturnVal,RestChars}. +%% Must be careful when re-entering to append the latest characters to the +%% after characters in an accept. + +tokens(Cont, Chars) -> tokens(Cont, Chars, 1). + +tokens([], Chars, Line) -> + tokens(Chars, Line, yystate(), Chars, 0, [], reject, 0); +tokens({tokens,Line,State,Tcs,Tlen,Ts,Action,Alen}, Chars, _) -> + tokens(Chars, Line, State, Tcs ++ Chars, Tlen, Ts, Action, Alen); +tokens({skip_tokens,Line,State,Tcs,Tlen,Error,Action,Alen}, Chars, _) -> + skip_tokens(Chars, Line, State, Tcs ++ Chars, Tlen, Error, Action, Alen). + +%% tokens(InChars, Line, State, TokenChars, TokenLen, Tokens, Accept) -> +%% {more,Continuation} | {done,ReturnVal,RestChars}. + +tokens(Ics0, L0, S0, Tcs, Tlen0, Ts, A0, Alen0) -> + case yystate(S0, Ics0, L0, Tlen0, A0, Alen0) of + {A1,Alen1,Ics1,L1} -> %Accepting end state + tokens_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, L1), Ts); + {A1,Alen1,[],L1,S1} -> %After an accepting state + {more,{tokens,L1,S1,Tcs,Alen1,Ts,A1,Alen1}}; + {A1,Alen1,Ics1,L1,S1} -> + tokens_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, L1), Ts); + {A1,Alen1,Tlen1,[],L1,S1} -> %After a non-accepting state + {more,{tokens,L1,S1,Tcs,Tlen1,Ts,A1,Alen1}}; + {reject,Alen1,Tlen1,eof,L1,S1} -> + {done,if Ts == [] -> {eof,L1}; + true -> {ok,yyrev(Ts),L1} end,[]}; + {reject,Alen1,Tlen1,Ics1,L1,S1} -> + skip_tokens(yysuf(Tcs, Tlen1+1), L1, + {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}); + {A1,Alen1,Tlen1,Ics1,L1,S1} -> + tokens_cont(yysuf(Tcs, Alen1), L1, yyaction(A1, Alen1, Tcs, L1), Ts) + end. + +%% tokens_cont(RestChars, Line, Token, Tokens) +%% Test if we have detected the end token, if so return done else continue. + +tokens_cont(Rest, Line, {token,T}, Ts) -> + tokens(Rest, Line, yystate(), Rest, 0, [T|Ts], reject, 0); +tokens_cont(Rest, Line, {end_token,T}, Ts) -> + {done,{ok,yyrev(Ts, [T]),Line},Rest}; +tokens_cont(Rest, Line, skip_token, Ts) -> + tokens(Rest, Line, yystate(), Rest, 0, Ts, reject, 0); +tokens_cont(Rest, Line, {error,S}, Ts) -> + skip_tokens(Rest, Line, {Line,?MODULE,{user,S}}). + +%% token_skip(InChars, Line, Error) -> {done,ReturnVal,RestChars}. +%% Skip tokens until an end token, junk everything and return the error. + +%%skip_tokens(Ics, Line, Error) -> {done,{error,Error,Line},Ics}. + +skip_tokens(Ics, Line, Error) -> + skip_tokens(Ics, Line, yystate(), Ics, 0, Error, reject, 0). + +%% skip_tokens(InChars, Line, State, TokenChars, TokenLen, Tokens, Accept) -> +%% {more,Continuation} | {done,ReturnVal,RestChars}. + +skip_tokens(Ics0, L0, S0, Tcs, Tlen0, Error, A0, Alen0) -> + case yystate(S0, Ics0, L0, Tlen0, A0, Alen0) of + {A1,Alen1,Ics1,L1} -> %Accepting end state + skip_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, L1), Error); + {A1,Alen1,[],L1,S1} -> %After an accepting state + {more,{skip_tokens,L1,S1,Tcs,Alen1,Error,A1,Alen1}}; + {A1,Alen1,Ics1,L1,S1} -> + skip_cont(Ics1, L1, yyaction(A1, Alen1, Tcs, L1), Error); + {A1,Alen1,Tlen1,[],L1,S1} -> %After a non-accepting state + {more,{skip_tokens,L1,S1,Tcs,Tlen1,Error,A1,Alen1}}; + {reject,Alen1,Tlen1,eof,L1,S1} -> + {done,{error,Error,L1},[]}; + {reject,Alen1,Tlen1,Ics1,L1,S1} -> + skip_tokens(yysuf(Tcs, Tlen1), L1, Error); + {A1,Alen1,Tlen1,Ics1,L1,S1} -> + skip_cont(yysuf(Tcs, Alen1), L1, yyaction(A1, Alen1, Tcs, L1), Error) + end. + +%% skip_cont(RestChars, Line, Token, Error) +%% Test if we have detected the end token, if so return done else continue. + +skip_cont(Rest, Line, {token,T}, Error) -> + skip_tokens(Rest, Line, yystate(), Rest, 0, Error, reject, 0); +skip_cont(Rest, Line, {end_token,T}, Error) -> + {done,{error,Error,Line},Rest}; +skip_cont(Rest, Line, {error,S}, Error) -> + skip_tokens(Rest, Line, yystate(), Rest, 0, Error, reject, 0); +skip_cont(Rest, Line, skip_token, Error) -> + skip_tokens(Rest, Line, yystate(), Rest, 0, Error, reject, 0). + +yyrev(L) -> yyrev(L, []). + +yyrev([H|T], Acc) -> yyrev(T, [H|Acc]); +yyrev([], Acc) -> Acc. + +yypre([H|T], N) when N > 0 -> [H|yypre(T, N-1)]; +yypre(L, N) -> []. + +yysuf([H|T], N) when N > 0 -> yysuf(T, N-1); +yysuf(L, 0) -> L. + +yysplit(L, N) -> yysplit(L, N, []). +yysplit([H|T], N, Acc) when N > 0 -> yysplit(T, N-1, [H|Acc]); +yysplit(L, 0, Acc) -> {lists:reverse(Acc), L}. + +%% yystate() -> InitialState. +%% yystate(State, InChars, Line, Token, ) -> +%% {Action, AcceptLength, RestChars, Line} | Accepting end state +%% {Action, AcceptLength, RestChars, Line, State} | Accepting state +%% {Action, AcceptLength, TokLength, RestChars, Line, State} | +%% {reject, AcceptLength, TokLength, RestChars, Line, State}. +%% Generated state transition functions. + +##dfa + +%% yyaction(Action, TokenLength, TokenChars, Line) -> +%% {token,Token} | {end_token, Token} | skip_token | {error,String}. +%% Generated action function. + +##actions |