added first draft of new modular lexer system

7 files changed, 270 insertions, 0 deletions

diff --git a/lib/lexers/batch.tes b/lib/lexers/batch.tes
new file mode 100644
index 0000000..9453628
--- /dev/null
+++ b/lib/lexers/batch.tes
@@ -0,0 +1,27 @@
+! DOS, Windows, OS/2 Batch Files !
+
+@[lexer.test.batch]{
+  [_ :S.[bat,cmd,nt]
 ]_
+}
+
+@[lexer.set.batch]{
+  ESSETLEXER,SCLEX_BATCH
+  0ESSETKEYWORDS
+    rem set if exist errorlevel for in do
+    break call chcp cd chdir choice cls country ctty date del erase dir echo
+    exit goto loadfix loadhigh mkdir md move path pause prompt rename ren
+    rmdir rd shift time type ver verify vol
+    com con lpt nul
+    color copy defined else not start
+  :M[color.comment],1M[color.set]
+  :M[color.keyword],2M[color.set]
+  :M[color.target],3M[color.set] ! Labels !
+  :M[color.preproc],4M[color.set] ! Hide Cmd @ !
+  :M[color.preproc2],5M[color.set] ! External Cmd !
+  :M[color.variable],6M[color.set]
+  :M[color.operator],7M[color.set]
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.batch]"S Q.fEB :M[lexer.set.batch] Oend '
+}
diff --git a/lib/lexers/cmake.tes b/lib/lexers/cmake.tes
new file mode 100644
index 0000000..8f66750
--- /dev/null
+++ b/lib/lexers/cmake.tes
@@ -0,0 +1,81 @@
+! CMake Lexing !
+
+@[lexer.test.cmake]{
+  [_ :S[CMakeLists.txt,.cmake,.ctest]
 ]_
+}
+
+@[lexer.set.cmake]{
+  ESSETLEXER,SCLEX_CMAKE
+  ! Commands !
+  0ESSETKEYWORDS
+    add_custom_command add_custom_target add_definitions
+    add_dependencies add_executable add_library
+    add_subdirectory add_test aux_source_directory
+    build_command build_name cmake_minimum_required cmake_policy
+    configure_file create_test_sourcelist else
+    elseif enable_language enable_testing
+    endforeach endif endmacro
+    endwhile exec_program execute_process
+    export_library_dependencies file find_file
+    find_library find_package find_path
+    find_program fltk_wrap_ui foreach
+    get_cmake_property get_directory_property get_filename_component
+    get_source_file_property get_target_property get_test_property
+    if include include_directories
+    include_external_msproject include_regular_expression install
+    install_files install_programs install_targets
+    link_directories link_libraries list
+    load_cache load_command macro
+    make_directory mark_as_advanced math
+    message option output_required_files
+    project qt_wrap_cpp qt_wrap_ui
+    remove remove_definitions separate_arguments
+    set set_directory_properties set_source_files_properties
+    set_target_properties set_tests_properties site_name
+    source_group string subdir_depends
+    subdirs target_link_libraries try_compile
+    try_run use_mangled_mesa utility_source
+    variable_requires vtk_make_instantiator vtk_wrap_java
+    vtk_wrap_python vtk_wrap_tcl while
+    write_file
+  ! Special command arguments !
+  1ESSETKEYWORDS
+    ABSOLUTE ABSTRACT ADDITIONAL_MAKE_CLEAN_FILES ALL AND
+    APPEND ARGS ASCII BEFORE CACHE
+    CACHE_VARIABLES CLEAR COMMAND COMMANDS COMMAND_NAME
+    COMMENT COMPARE COMPILE_FLAGS COPYONLY DEFINED
+    DEFINE_SYMBOL DEPENDS DOC EQUAL ESCAPE_QUOTES
+    EXCLUDE EXCLUDE_FROM_ALL EXISTS EXPORT_MACRO EXT
+    EXTRA_INCLUDE FATAL_ERROR FILE FILES FORCE
+    FUNCTION GENERATED GLOB GLOB_RECURSE GREATER
+    GROUP_SIZE HEADER_FILE_ONLY HEADER_LOCATION HINTS IMMEDIATE INCLUDES
+    INCLUDE_DIRECTORIES INCLUDE_INTERNALS INCLUDE_REGULAR_EXPRESSION LESS LINK_DIRECTORIES
+    LINK_FLAGS LOCATION MACOSX_BUNDLE MACROS MAIN_DEPENDENCY
+    MAKE_DIRECTORY MATCH MATCHALL MATCHES MODULE
+    NAME NAME_WE NOT NOTEQUAL NO_SYSTEM_PATH
+    OBJECT_DEPENDS OPTIONAL OR OUTPUT OUTPUT_VARIABLE
+    PATH PATHS POST_BUILD POST_INSTALL_SCRIPT PREFIX
+    PREORDER PRE_BUILD PRE_INSTALL_SCRIPT PRE_LINK PROGRAM
+    PROGRAM_ARGS PROPERTIES QUIET RANGE READ
+    REGEX REGULAR_EXPRESSION REPLACE REQUIRED RETURN_VALUE
+    RUNTIME_DIRECTORY SEND_ERROR SHARED SOURCES STATIC
+    STATUS STREQUAL STRGREATER STRLESS SUFFIX
+    TARGET TOLOWER TOUPPER VAR VARIABLES
+    VERSION WIN32 WRAP_EXCLUDE WRITE APPLE
+    MINGW MSYS CYGWIN BORLAND WATCOM MSVC MSVC_IDE MSVC60
+    MSVC70 MSVC71 MSVC80 CMAKE_COMPILER_2005 OFF
+    ON
+  :M[color.comment],1M[color.set]
+  :M[color.string],2M[color.set]
+  :M[color.string],3M[color.set]
+  :M[color.string],4M[color.set]
+  :M[color.keyword],5M[color.set]
+  :M[color.string],6M[color.set]
+  :M[color.variable],7M[color.set]
+  :M[color.variable],13M[color.set]
+  :M[color.number],14M[color.set]
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.cmake]"S Q.fEB :M[lexer.set.cmake] Oend '
+}
diff --git a/lib/lexers/cpp.tes b/lib/lexers/cpp.tes
new file mode 100644
index 0000000..f979153
--- /dev/null
+++ b/lib/lexers/cpp.tes
@@ -0,0 +1,43 @@
+! C/C++ Lexing !
+
+@[lexer.test.cpp]{
+  [_ :S.[c,cc,cpp,cxx,h,hh,hpp,hxx,ipp,m,mm,sma]
 ]_
+}
+
+@[lexer.set.cpp]{
+  ESSETLEXER,SCLEX_CPP
+  0ESSETKEYWORDS
+    and and_eq asm auto bitand bitor bool break
+    case catch char class compl const const_cast continue
+    default delete do double dynamic_cast else enum explicit export extern false float for
+    friend goto if inline int long mutable namespace new not not_eq
+    operator or or_eq private protected public
+    register reinterpret_cast return short signed sizeof static static_cast struct switch
+    template this throw true try typedef typeid typename union unsigned using
+    virtual void volatile wchar_t while xor xor_eq
+  ! Doxygen keywords !
+  2ESSETKEYWORDS
+    a addindex addtogroup anchor arg attention
+    author b brief bug c class code date def defgroup deprecated dontinclude
+    e em endcode endhtmlonly endif endlatexonly endlink endverbatim enum example exception
+    f$ f[ f] file fn hideinitializer htmlinclude htmlonly
+    if image include ingroup internal invariant interface latexonly li line link
+    mainpage name namespace nosubgrouping note overload
+    p page par param post pre ref relates remarks return retval
+    sa section see showinitializer since skip skipline struct subsection
+    test throw todo typedef union until
+    var verbatim verbinclude version warning weakgroup $ @ \ & < > # { }
+  :M[color.comment],1M[color.set]
+  :M[color.comment],2M[color.set]
+  :M[color.comment],3M[color.set]
+  :M[color.number],4M[color.set]
+  :M[color.keyword],5M[color.set]
+  :M[color.string],6M[color.set]
+  :M[color.string],7M[color.set]
+  :M[color.preproc],9M[color.set]
+  :M[color.operator],10M[color.set]
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.cpp]"S Q.fEB :M[lexer.set.cpp] Oend '
+}
diff --git a/lib/lexers/make.tes b/lib/lexers/make.tes
new file mode 100644
index 0000000..a841f7d
--- /dev/null
+++ b/lib/lexers/make.tes
@@ -0,0 +1,19 @@
+! Makefile Lexing !
+
+@[lexer.test.make]{
+  [_ :S[makefile,.mak]
 ]_
+}
+
+@[lexer.set.make]{
+  ESSETLEXER,SCLEX_MAKEFILE
+  :M[color.comment],1M[color.set]
+  :M[color.preproc],2M[color.set]
+  :M[color.variable],3M[color.set]
+  :M[color.operator],4M[color.set]
+  :M[color.target],5M[color.set]
+  :M[color.error],6M[color.set]
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.make]"S Q.fEB :M[lexer.set.make] Oend '
+}
diff --git a/lib/lexers/patch.tes b/lib/lexers/patch.tes
new file mode 100644
index 0000000..1949712
--- /dev/null
+++ b/lib/lexers/patch.tes
@@ -0,0 +1,20 @@
+! Patch/Diff Files !
+
+@[lexer.test.patch]{
+  [_ :S.[diff,patch]
 ]_
+}
+
+@[lexer.set.patch]{
+  ESSETLEXER,SCLEX_DIFF
+  :M[color.comment],1M[color.set]
+  :M[color.keyword],2M[color.set] ! Command !
+  :M[color.preproc],3M[color.set] ! Source/Dest !
+  :M[color.operator],4M[color.set] ! Position @@ !
+  :M[color.deletion],5M[color.set]
+  :M[color.addition],6M[color.set]
+  :M[color.change],7M[color.set]
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.patch]"S Q.fEB :M[lexer.set.patch] Oend '
+}
diff --git a/lib/lexers/sh.tes b/lib/lexers/sh.tes
new file mode 100644
index 0000000..79bdd8e
--- /dev/null
+++ b/lib/lexers/sh.tes
@@ -0,0 +1,51 @@
+! Unix Shell !
+
+@[lexer.test.sh]{ [_
+  :S[.sh,.bsh,configure,.ksh]

+  ! TODO: test hash-bang line !
+]_ }
+
+@[lexer.set.sh]{
+  ESSETLEXER,SCLEX_BASH
+  0ESSETKEYWORDS
+    alias ar asa awk banner basename bash bc bdiff break
+    bunzip2 bzip2 cal calendar case cat cc cd chmod cksum
+    clear cmp col comm compress continue cp cpio crypt
+    csplit ctags cut date dc dd declare deroff dev df diff diff3
+    dircmp dirname do done du echo ed egrep elif else env
+    esac eval ex exec exit expand export expr false fc
+    fgrep fi file find fmt fold for function functions
+    getconf getopt getopts grep gres hash head help
+    history iconv id if in integer jobs join kill local lc
+    let line ln logname look ls m4 mail mailx make
+    man mkdir more mt mv newgrp nl nm nohup ntps od
+    pack paste patch pathchk pax pcat perl pg pr print
+    printf ps pwd read readonly red return rev rm rmdir
+    sed select set sh shift size sleep sort spell
+    split start stop strings strip stty sum suspend
+    sync tail tar tee test then time times touch tr
+    trap true tsort tty type typeset ulimit umask unalias
+    uname uncompress unexpand uniq unpack unset until
+    uudecode uuencode vi vim vpax wait wc whence which
+    while who wpaste wstart xargs zcat
+
+    chgrp chown chroot dir dircolors
+    factor groups hostid install link md5sum mkfifo
+    mknod nice pinky printenv ptx readlink seq
+    sha1sum shred stat su tac unlink users vdir whoami yes
+  :M[color.error],1M[color.set]
+  :M[color.comment],2M[color.set]
+  :M[color.number],3M[color.set]
+  :M[color.keyword],4M[color.set]
+  :M[color.string],5M[color.set]
+  :M[color.string],6M[color.set]
+  :M[color.operator],7M[color.set]
+  :M[color.target],8M[color.set] ! Identifiers, e.g. FOO=... !
+  :M[color.variable],9M[color.set]
+  :M[color.variable],10M[color.set]
+  :M[color.string2],11M[color.set] ! Backticks !
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.sh]"S Q.fEB :M[lexer.set.sh] Oend '
+}
diff --git a/lib/lexers/xml.tes b/lib/lexers/xml.tes
new file mode 100644
index 0000000..bf82ecb
--- /dev/null
+++ b/lib/lexers/xml.tes
@@ -0,0 +1,29 @@
+! Lexing for XML and its applications !
+
+@[lexer.test.xml]{
+  [_ :S.[xml,xsl,svg,xul,xsd,dtd,xslt,axl,xrc,rdf]
 ]_
+}
+
+@[lexer.set.xml]{
+  ESSETLEXER,SCLEX_XML
+  0ESSETKEYWORDS 
+  ! DTD keywords !
+  5ESSETKEYWORDSELEMENT DOCTYPE ATTLIST ENTITY NOTATION
+  :M[color.keyword],1M[color.set] ! Tags !
+  :M[color.keyword],2M[color.set] ! Tags !
+  :M[color.variable],3M[color.set] ! Attributes !
+  :M[color.variable],4M[color.set] ! Attributes !
+  :M[color.number],5M[color.set]
+  :M[color.string],6M[color.set]
+  :M[color.string],7M[color.set]
+  :M[color.comment],9M[color.set]
+  :M[color.preproc],10M[color.set] ! Entities !
+  :M[color.keyword],11M[color.set] ! Tag Ends /> !
+  :M[color.preproc2],12M[color.set] ! PI !
+  :M[color.preproc2],13M[color.set] ! PI !
+  :M[color.string2],14M[color.set] ! CDATA !
+}
+
+@:[lexer.auto]{
+  :M[lexer.test.xml]"S Q.fEB :M[lexer.set.xml] Oend '
+}