Separate tex lexer.

Metapost lexer.
Export as XML.

3 files changed, 632 insertions, 0 deletions

diff --git a/src/KeyWords.cxx b/src/KeyWords.cxx
index 35bb1f594..7ec5d9922 100644
--- a/src/KeyWords.cxx
+++ b/src/KeyWords.cxx
@@ -155,6 +155,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmLout);
 	LINK_LEXER(lmLua);
 	LINK_LEXER(lmMatlab);
+	LINK_LEXER(lmMETAPOST);
 	LINK_LEXER(lmMMIXAL);
 	LINK_LEXER(lmLot);
 	LINK_LEXER(lmNsis);
@@ -173,6 +174,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmRuby);
 	LINK_LEXER(lmScriptol);
 	LINK_LEXER(lmSQL);
+	LINK_LEXER(lmTeX);
 	LINK_LEXER(lmVB);
 	LINK_LEXER(lmVBScript);
 	LINK_LEXER(lmYAML);
diff --git a/src/LexMetapost.cxx b/src/LexMetapost.cxx
new file mode 100644
index 000000000..728edfc1a
--- /dev/null
+++ b/src/LexMetapost.cxx
@@ -0,0 +1,287 @@
+// Scintilla source code edit control
+
+// File: LexMetapost.cxx - general context conformant metapost coloring scheme
+// Author: Hans Hagen - PRAGMA ADE - Hasselt NL - www.pragma-ade.com
+// Version: August 18, 2003
+
+// Copyright: 1998-2003 by Neil Hodgson <neilh@scintilla.org>
+// The License.txt file describes the conditions under which this software may be distributed.
+
+// This lexer is derived from the one written for the texwork environment (1999++) which in
+// turn is inspired on texedit (1991++) which finds its roots in wdt (1986).
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+#include "Platform.h"
+
+#include "PropSet.h"
+#include "Accessor.h"
+#include "KeyWords.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+
+// Definitions in Scintilla.iface:
+//
+// Lexical states for SCLEX_METAPOST
+//
+// val SCLEX_METAPOST = 46
+//
+// lex METAPOST=SCLEX_METAPOST SCE_METAPOST_
+//
+// val SCE_METAPOST_DEFAULT = 0
+// val SCE_METAPOST_SPECIAL = 1
+// val SCE_METAPOST_GROUP = 2
+// val SCE_METAPOST_SYMBOL = 3
+// val SCE_METAPOST_COMMAND = 4
+// val SCE_METAPOST_TEXT = 5
+
+// Definitions in SciTEGlobal.properties:
+//
+//
+// Metapost Highlighting
+//
+// # Default
+// style.metapost.0=fore:#7F7F00
+// # Special
+// style.metapost.1=fore:#007F7F
+// # Group
+// style.metapost.2=fore:#880000
+// # Symbol
+// style.metapost.3=fore:#7F7F00
+// # Command
+// style.metapost.4=fore:#008800
+// # Text
+// style.metapost.5=fore:#000000
+
+// Auxiliary functions:
+
+static inline bool endOfLine(Accessor &styler, unsigned int i) {
+	return
+      (styler[i] == '\n') || ((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n')) ;
+}
+
+static inline bool isMETAPOSTcomment(char ch, char pc) {
+	return
+      (ch == '%') && (pc != '\\') ;
+}
+
+static inline bool isMETAPOSTone(char ch) {
+	return
+      (ch == '[') || (ch == ']') || (ch == '(') || (ch == ')') ||
+      (ch == ':') || (ch == '=') || (ch == '<') || (ch == '>') ||
+      (ch == '{') || (ch == '}') || (ch == '\'') ; // || (ch == '\"') ;
+}
+
+static inline bool isMETAPOSTtwo(char ch) {
+	return
+      (ch == ';') || (ch == '$') || (ch == '@') || (ch == '#');
+}
+
+static inline bool isMETAPOSTthree(char ch) {
+	return
+      (ch == '.') || (ch == '-') || (ch == '+') || (ch == '/') ||
+      (ch == '*') || (ch == ',') || (ch == '|') || (ch == '`') ||
+      (ch == '!') || (ch == '?') || (ch == '^') || (ch == '&') ||
+      (ch == '%') ;
+}
+
+static inline bool isMETAPOSTidentifier(char ch) {
+	return
+      ((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z')) ||
+      (ch == '_') ;
+}
+
+static inline bool isMETAPOSTnumber(char ch) {
+	return
+      (ch >= '0') && (ch <= '9') ;
+}
+
+static inline bool isMETAPOSTstring(char ch) {
+	return
+      (ch == '\"') ;
+}
+
+
+// Coloring functions:
+
+bool ColourMETAPOSTRange(
+    unsigned int metapostMode,
+    bool texMode,
+    char *key,
+    unsigned int endPos,
+    WordList &keywords,
+    Accessor &styler) {
+
+    switch (metapostMode) {
+        case 0 : // comment mode
+            styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ;
+            break ;
+        case 1 : // special characters mode
+            if (! texMode)
+                { styler.ColourTo(endPos, SCE_METAPOST_SPECIAL) ; }
+            else
+                { styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ; }
+            break ;
+        case 2 : // (kind of) group mode
+            if (! texMode)
+                { styler.ColourTo(endPos, SCE_METAPOST_GROUP) ; }
+            else
+                { styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ; }
+            break ;
+        case 3 : // (more or less) symbol mode
+            if (! texMode)
+                { styler.ColourTo(endPos, SCE_METAPOST_SYMBOL) ; }
+            else
+                { styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ; }
+            break ;
+        case 4 : // command and/or keyword mode
+            if (texMode) {
+                if (0 == strcmp(key,"etex")) {
+                    styler.ColourTo(endPos, SCE_METAPOST_COMMAND) ;
+                    return false ;
+                } else {
+                    styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ;
+                }
+            } else if ((0 == strcmp(key,"btex")) || (0 == strcmp(key,"verbatimtex"))) {
+                styler.ColourTo(endPos, SCE_METAPOST_COMMAND) ;
+                return true ;
+            } else if (keywords.InList(key)) {
+                styler.ColourTo(endPos, SCE_METAPOST_COMMAND) ;
+            } else {
+                styler.ColourTo(endPos, SCE_METAPOST_TEXT) ;
+            }
+            break ;
+        case 5 : // text mode
+            if (! texMode)
+                { styler.ColourTo(endPos, SCE_METAPOST_TEXT) ; }
+            else
+                { styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ; }
+            break ;
+        case 6 : // string mode
+            styler.ColourTo(endPos, SCE_METAPOST_DEFAULT) ;
+            break ;
+    }
+    return texMode ;
+}
+
+static void ColouriseMETAPOSTLine(
+    char *lineBuffer,
+    unsigned int lengthLine,
+    unsigned int startPos,
+    WordList &keywords,
+    Accessor &styler) {
+
+    char ch = ' ' ;
+    char pc;
+    unsigned int offset = 0 ;
+    unsigned int mode = 5 ;
+    unsigned int k = 0 ;
+    char key[1024] ; // length check in calling routine
+    unsigned int start = startPos-1 ;
+
+    bool comment = true ; // does not work: (styler.GetPropertyInt("lexer.metapost.comment.process", 0) == 1) ;
+    bool tex = false ;
+
+    // we may safely assume that pc is either on the same line or a \n \r token
+
+    // we use a cheap append to key method, ugly, but fast and ok
+
+    while (offset < lengthLine) {
+
+        pc = ch ;
+        ch = lineBuffer[offset] ;
+
+        if (!tex && (mode == 6)) {
+            if (isMETAPOSTstring(ch)) {
+                // we've run into the end of the string
+                tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ;
+                mode = 5 ;
+            } else {
+                // we're still in the string, comment is valid
+            }
+        } else if ((comment) && ((mode == 0) || (isMETAPOSTcomment(ch,pc)))) {
+            tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ;
+            mode = 0 ;
+        } else if (isMETAPOSTstring(ch)) {
+            if (mode != 6) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 6 ;
+        } else if (isMETAPOSTone(ch)) {
+            if (mode != 1) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 1 ;
+        } else if (isMETAPOSTtwo(ch)) {
+            if (mode != 2) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 2 ;
+        } else if (isMETAPOSTthree(ch)) {
+            if (mode != 3) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 3 ;
+        } else if (isMETAPOSTidentifier(ch)) {
+            if (mode != 4) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 4 ; key[k] = ch ; ++k ; key[k] = '\0' ;
+        } else if (isMETAPOSTnumber(ch)) {
+            // rather redundant since for the moment we don't handle numbers
+            if (mode != 5) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 5 ;
+        } else {
+            if (mode != 5) { tex = ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ; k = 0 ; }
+            mode = 5 ;
+        }
+
+        ++offset ;
+        ++start ;
+
+    }
+
+    ColourMETAPOSTRange(mode,tex,key,start,keywords,styler) ;
+
+}
+
+// Main handler:
+//
+// The lexer works on a per line basis. I'm not familiar with the internals of scintilla, but
+// since the lexer does not look back or forward beyond the current view, some optimization can
+// be accomplished by providing just the viewport. The following code is more or less copied
+// from the LexOthers.cxx file.
+
+static void ColouriseMETAPOSTDoc(
+    unsigned int startPos,
+    int length,
+    int /*initStyle*/,
+    WordList *keywordlists[],
+    Accessor &styler) {
+
+	char lineBuffer[1024] ;
+    WordList &keywords = *keywordlists[0] ;
+	unsigned int linePos = 0 ;
+
+	styler.StartAt(startPos) ;
+	styler.StartSegment(startPos) ;
+
+    for (unsigned int i = startPos; i < startPos + length; i++) {
+		lineBuffer[linePos++] = styler[i] ;
+		if (endOfLine(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
+			// End of line (or of line buffer) met, colourise it
+			lineBuffer[linePos] = '\0' ;
+			ColouriseMETAPOSTLine(lineBuffer, linePos, i-linePos+1, keywords, styler) ;
+			linePos = 0 ;
+		}
+	}
+
+	if (linePos > 0) {
+        // Last line does not have ending characters
+		ColouriseMETAPOSTLine(lineBuffer, linePos, startPos+length-linePos, keywords, styler) ;
+	}
+
+}
+
+// Hooks info the system:
+
+static const char * const metapostWordListDesc[] = {
+	"Keywords",
+	0
+} ;
+
+LexerModule lmMETAPOST(SCLEX_METAPOST, ColouriseMETAPOSTDoc, "metapost", 0, metapostWordListDesc);
diff --git a/src/LexTeX.cxx b/src/LexTeX.cxx
new file mode 100644
index 000000000..600c64fca
--- /dev/null
+++ b/src/LexTeX.cxx
@@ -0,0 +1,343 @@
+// Scintilla source code edit control
+
+// File: LexTeX.cxx - general context conformant tex coloring scheme
+// Author: Hans Hagen - PRAGMA ADE - Hasselt NL - www.pragma-ade.com
+// Version: August 18, 2003
+
+// Copyright: 1998-2003 by Neil Hodgson <neilh@scintilla.org>
+// The License.txt file describes the conditions under which this software may be distributed.
+
+// This lexer is derived from the one written for the texwork environment (1999++) which in
+// turn is inspired on texedit (1991++) which finds its roots in wdt (1986).
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+#include "Platform.h"
+
+#include "PropSet.h"
+#include "Accessor.h"
+#include "KeyWords.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+#include "StyleContext.h"
+
+// Definitions in Scintilla.iface:
+//
+// Lexical states for SCLEX_TEX
+//
+// val SCLEX_TEX = 45
+//
+// lex TeX=SCLEX_TEX SCE_TEX_
+//
+// val SCE_TEX_DEFAULT = 0
+// val SCE_TEX_SPECIAL = 1
+// val SCE_TEX_GROUP   = 2
+// val SCE_TEX_SYMBOL  = 3
+// val SCE_TEX_COMMAND = 4
+// val SCE_TEX_TEXT    = 5
+
+// in SciTEProps.cxx
+//
+// ForwardPropertyToEditor("lexer.tex.comment.process") ;
+// ForwardPropertyToEditor("lexer.metapost.comment.process") ;
+// ForwardPropertyToEditor("lexer.tex.interface.default") ;
+
+// Definitions in SciTEGlobal.properties:
+//
+//
+// TeX Highlighting
+//
+// # Default
+// style.tex.0=fore:#7F7F00
+// # Special
+// style.tex.1=fore:#007F7F
+// # Group
+// style.tex.2=fore:#880000
+// # Symbol
+// style.tex.3=fore:#7F7F00
+// # Command
+// style.tex.4=fore:#008800
+// # Text
+// style.tex.5=fore:#000000
+
+// Auxiliary functions:
+
+static inline bool endOfLine(Accessor &styler, unsigned int i) {
+	return
+      (styler[i] == '\n') || ((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n')) ;
+}
+
+static inline bool isTeXzero(char ch) {
+	return
+      (ch == '%') ;
+}
+
+static inline bool isTeXone(char ch) {
+	return
+      (ch == '[') || (ch == ']') || (ch == '=') || (ch == '#') ||
+      (ch == '(') || (ch == ')') || (ch == '<') || (ch == '>') ||
+      (ch == '"') ;
+}
+
+static inline bool isTeXtwo(char ch) {
+	return
+      (ch == '{') || (ch == '}') || (ch == '$') ;
+}
+
+static inline bool isTeXthree(char ch) {
+	return
+      (ch == '~') || (ch == '^') || (ch == '_') || (ch == '&') ||
+      (ch == '-') || (ch == '+') || (ch == '\"') || (ch == '`') ||
+      (ch == '/') || (ch == '|') || (ch == '%') ;
+}
+
+static inline bool isTeXfour(char ch) {
+	return
+      (ch == '\\') ;
+}
+
+static inline bool isTeXfive(char ch) {
+	return
+      ((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z')) ||
+      (ch == '@') || (ch == '!') || (ch == '?') ;
+}
+
+static inline bool isTeXsix(char ch) {
+	return
+      (ch == ' ') ;
+}
+
+
+// Coloring functions:
+
+bool newifDone = false ;
+
+static void ColourTeXRange(
+    unsigned int texMode,
+    char *key,
+    unsigned int endPos,
+    WordList &keywords,
+    bool useKeywords,
+    Accessor &styler) {
+
+    bool autoIf = true ;
+
+    switch (texMode) {
+        case 0 :
+            styler.ColourTo(endPos, SCE_TEX_DEFAULT) ;
+			newifDone = false ;
+            break ;
+        case 1 :
+            styler.ColourTo(endPos, SCE_TEX_SPECIAL) ;
+			newifDone = false ;
+            break ;
+        case 2 :
+            styler.ColourTo(endPos, SCE_TEX_GROUP) ;
+			newifDone = false ;
+            break ;
+        case 3 :
+            styler.ColourTo(endPos, SCE_TEX_SYMBOL) ;
+			newifDone = false ;
+            break ;
+        case 4 :
+            if (! keywords || ! useKeywords) {
+                styler.ColourTo(endPos, SCE_TEX_COMMAND) ;
+				newifDone = false ;
+            } else if (key[1] == '\0') {
+                styler.ColourTo(endPos, SCE_TEX_COMMAND) ;
+				newifDone = false ;
+            } else if (keywords.InList(key)) {
+                styler.ColourTo(endPos, SCE_TEX_COMMAND) ;
+				newifDone = autoIf && (strcmp(key,"newif") == 0) ;
+            } else if (autoIf && ! newifDone && (key[0] == 'i') && (key[1] == 'f') && keywords.InList("if")) {
+                styler.ColourTo(endPos, SCE_TEX_COMMAND) ;
+			} else {
+                styler.ColourTo(endPos, SCE_TEX_TEXT) ;
+				newifDone = false ;
+            }
+            break ;
+        case 5 :
+            styler.ColourTo(endPos, SCE_TEX_TEXT) ;
+			newifDone = newifDone || (strspn(key," ") == strlen(key)) ;
+            break ;
+    }
+
+}
+
+static void ColouriseTeXLine(
+    char *lineBuffer,
+    unsigned int lengthLine,
+    unsigned int startPos,
+    WordList &keywords,
+    bool useKeywords,
+    Accessor &styler) {
+
+    char ch;
+    bool cs = false ;
+    unsigned int offset = 0 ;
+    unsigned int mode = 5 ;
+    unsigned int k = 0 ;
+    char key[1024] ; // length check in calling routine
+    unsigned int start = startPos-1 ;
+
+    bool comment = true ; // does not work: (styler.GetPropertyInt("lexer.tex.comment.process", 0) == 1) ;
+
+    // we use a cheap append to key method, ugly, but fast and ok
+
+    while (offset < lengthLine) {
+
+        ch = lineBuffer[offset] ;
+
+        if (cs) {
+			cs = false ;
+			key[k] = ch ; ++k ; key[k] = '\0' ; // ugly but ok
+        } else if ((comment) && ((mode == 0) || (isTeXzero(ch)))) {
+            ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ;
+            mode = 0 ;
+        } else if (isTeXone(ch)) {
+            if (mode != 1) { ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; }
+            mode = 1 ;
+        } else if (isTeXtwo(ch)) {
+            if (mode != 2) { ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; }
+            mode = 2 ;
+        } else if (isTeXthree(ch)) {
+            if (mode != 3) { ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; }
+            mode = 3 ;
+        } else if (isTeXfour(ch)) {
+			if (keywords || (mode != 4)) {
+                ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; cs = true ;
+			}
+			mode = 4 ;
+        } else if (isTeXfive(ch)) {
+            if (mode < 4) {
+				ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; mode = 5 ;
+	            key[k] = ch ; ++k ; key[k] = '\0' ; // ugly but ok
+			} else if ((mode == 4) && (k == 1) && isTeXfour(key[0])) {
+				ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; mode = 5 ;
+			} else {
+	            key[k] = ch ; ++k ; key[k] = '\0' ; // ugly but ok
+			}
+        } else if (isTeXsix(ch)) {
+            if (mode != 5) { ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; }
+            mode = 5 ;
+        } else if (mode != 5) {
+            ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ; k = 0 ; mode = 5 ;
+        }
+
+        ++offset ;
+        ++start ;
+
+    }
+
+    ColourTeXRange(mode,key,start,keywords,useKeywords,styler) ;
+
+}
+
+// Main handler:
+//
+// The lexer works on a per line basis. I'm not familiar with the internals of scintilla, but
+// since the lexer does not look back or forward beyond the current view, some optimization can
+// be accomplished by providing just the viewport. The following code is more or less copied
+// from the LexOthers.cxx file.
+
+static int CheckTeXInterface(
+    unsigned int startPos,
+    int length,
+    Accessor &styler) {
+
+    char lineBuffer[1024] ;
+	unsigned int linePos = 0 ;
+
+    int defaultInterface = styler.GetPropertyInt("lexer.tex.interface.default", 1) ;
+
+    if (styler.SafeGetCharAt(0) == '%') {
+        for (unsigned int i = 0; i < startPos + length; i++) {
+            lineBuffer[linePos++] = styler[i];
+            if (endOfLine(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
+                lineBuffer[linePos] = '\0';
+                if (strstr(lineBuffer, "interface=all")) {
+                    return 0 ;
+                } else if (strstr(lineBuffer, "interface=nl")) {
+                    return 2 ;
+                } else if (strstr(lineBuffer, "interface=en")) {
+                    return 3 ;
+                } else if (strstr(lineBuffer, "interface=de")) {
+                    return 4 ;
+                } else if (strstr(lineBuffer, "interface=cz")) {
+                    return 5 ;
+                } else if (strstr(lineBuffer, "interface=it")) {
+                    return 6 ;
+                } else if (strstr(lineBuffer, "interface=ro")) {
+                    return 7 ;
+				} else if (styler.SafeGetCharAt(1) == 'D' && strstr(lineBuffer, "%D \\module")) {
+					// better would be to limit the search to just one line
+					return 3 ;
+                } else {
+                    return defaultInterface ;
+                }
+            }
+		}
+    }
+
+    return defaultInterface ;
+}
+
+static void ColouriseTeXDoc(
+    unsigned int startPos,
+    int length,
+    int /*initStyle*/,
+    WordList *keywordlists[],
+    Accessor &styler) {
+
+	styler.StartAt(startPos) ;
+	styler.StartSegment(startPos) ;
+
+    int currentInterface = CheckTeXInterface(startPos,length,styler) ;
+
+    bool useKeywords = true ;
+
+    if (currentInterface == 0) {
+        useKeywords = false ;
+        currentInterface = 1 ;
+    }
+
+    WordList &keywords = *keywordlists[currentInterface-1] ;
+
+	char lineBuffer[1024] ;
+	unsigned int linePos = 0 ;
+
+    for (unsigned int i = startPos; i < startPos + length; i++) {
+		lineBuffer[linePos++] = styler[i] ;
+		if (endOfLine(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
+			// End of line (or of line buffer) met, colourise it
+			lineBuffer[linePos] = '\0' ;
+			ColouriseTeXLine(lineBuffer, linePos, i-linePos+1, keywords, useKeywords, styler) ;
+			linePos = 0 ;
+		}
+	}
+
+	if (linePos > 0) {
+        // Last line does not have ending characters
+		ColouriseTeXLine(lineBuffer, linePos, startPos+length-linePos, keywords, useKeywords, styler) ;
+	}
+
+}
+
+// Hooks into the system:
+
+static const char * const texWordListDesc[] = {
+    "Default"
+    "Dutch",
+    "English",
+    "German",
+    "Czech",
+    "Italian",
+    "Romanian",
+	0,
+} ;
+
+LexerModule lmTeX(SCLEX_TEX, ColouriseTeXDoc, "tex", 0, texWordListDesc);