Update of TCL lexer from Andre Arpin adds SCE_TCL_COMMENT_BOX and

SCE_TCL_BLOCK_COMMENT states.

3 files changed, 151 insertions, 98 deletions

diff --git a/include/SciLexer.h b/include/SciLexer.h
index 62dd348eb..8d38c2cac 100644
--- a/include/SciLexer.h
+++ b/include/SciLexer.h
@@ -147,6 +147,8 @@
 #define SCE_TCL_WORD6 17
 #define SCE_TCL_WORD7 18
 #define SCE_TCL_WORD8 19
+#define SCE_TCL_COMMENT_BOX 20
+#define SCE_TCL_BLOCK_COMMENT 21
 #define SCE_H_DEFAULT 0
 #define SCE_H_TAG 1
 #define SCE_H_TAGUNKNOWN 2
diff --git a/include/Scintilla.iface b/include/Scintilla.iface
index 497e00c13..5f2f2ab5a 100644
--- a/include/Scintilla.iface
+++ b/include/Scintilla.iface
@@ -1903,6 +1903,8 @@ val SCE_TCL_WORD5=16
 val SCE_TCL_WORD6=17
 val SCE_TCL_WORD7=18
 val SCE_TCL_WORD8=19
+val SCE_TCL_COMMENT_BOX=20
+val SCE_TCL_BLOCK_COMMENT=21
 # Lexical states for SCLEX_HTML, SCLEX_XML
 lex HTML=SCLEX_HTML SCE_H
 lex XML=SCLEX_XML SCE_H
diff --git a/src/LexTCL.cxx b/src/LexTCL.cxx
index 80bf51aaa..05d225fe3 100644
--- a/src/LexTCL.cxx
+++ b/src/LexTCL.cxx
@@ -23,34 +23,35 @@
 // Extended to accept accented characters
 static inline bool IsAWordChar(int ch) {
 	return ch >= 0x80 ||
-        (isalnum(ch) || ch == '_' || ch ==':'); // : name space separator
+        (isalnum(ch) || ch == '_' || ch ==':' || ch=='.'); // : name space separator
 }
 
 static inline bool IsAWordStart(int ch) {
-	return ch >= 0x80 ||
-	       (isalpha(ch) || ch == '_');
+	return ch >= 0x80 || (ch ==':' || isalpha(ch) || ch == '_');
 }
 
 static inline bool IsANumberChar(int ch) {
 	// Not exactly following number definition (several dots are seen as OK, etc.)
 	// but probably enough in most cases.
 	return (ch < 0x80) &&
-	       (isdigit(ch) || toupper(ch) == 'E' ||
+	       (IsADigit(ch, 0x10) || toupper(ch) == 'E' ||
 	        ch == '.' || ch == '-' || ch == '+');
 }
 
-static void ColouriseTCLDoc(unsigned int startPos, int length, int initStyle, WordList *keywordlists[], Accessor &styler) {
+static void ColouriseTCLDoc(unsigned int startPos, int length, int , WordList *keywordlists[], Accessor &styler) {
+#define  isComment(s) (s==SCE_TCL_COMMENT || s==SCE_TCL_COMMENTLINE || s==SCE_TCL_COMMENT_BOX || s==SCE_TCL_BLOCK_COMMENT)
 	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
 	bool commentLevel = false;
     bool subBrace = false; // substitution begin with a brace ${.....}
-	enum tLineState {LS_DEFAULT, LS_OPEN_COMMENT, LS_OPEN_DOUBLE_QUOTE, LS_MASK_STATE = 0xf, 
+	enum tLineState {LS_DEFAULT, LS_OPEN_COMMENT, LS_OPEN_DOUBLE_QUOTE, LS_COMMENT_BOX, LS_MASK_STATE = 0xf,
         LS_COMMAND_EXPECTED = 16, LS_BRACE_ONLY = 32 } lineState = LS_DEFAULT;
-	bool prevSlash = false;
 	int currentLevel = 0;
     bool expected = 0;
-    int subParen = 0;
+    bool subParen = 0;
 
 	int currentLine = styler.GetLine(startPos);
+    if (currentLine > 0)
+        currentLine--;
 	length += startPos - styler.LineStart(currentLine);
 	// make sure lines overlap
 	startPos = styler.LineStart(currentLine);
@@ -77,77 +78,105 @@ static void ColouriseTCLDoc(unsigned int startPos, int length, int initStyle, Wo
 	bool visibleChars = false;
 
 	int previousLevel = currentLevel;
-    StyleContext sc(startPos, length, initStyle, styler);
+    StyleContext sc(startPos, length, SCE_TCL_DEFAULT, styler);
 	for (; ; sc.Forward()) {
-		bool atEnd = !sc.More();  // make sure we process last word at end of file
 next:
-        if (subBrace) {
+        if (sc.ch=='\r')
+            continue;
+        bool atEnd = !sc.More();  // make sure we coloured the last word
+        if (lineState != LS_DEFAULT) {
+            if (lineState == LS_OPEN_COMMENT)
+                sc.SetState(SCE_TCL_COMMENTLINE);
+            else if (lineState == LS_OPEN_DOUBLE_QUOTE)
+                sc.SetState(SCE_TCL_IN_QUOTE);
+            else if (lineState == LS_COMMENT_BOX)
+                sc.SetState((sc.ch == '#' || sc.chNext=='#') ? SCE_TCL_COMMENT_BOX : SCE_TCL_DEFAULT);
+            lineState = LS_DEFAULT;
+        }
+        if (subBrace) { // ${ overrides every thing even \ except }
             if (sc.ch == '}') {
                 subBrace = false;
-                sc.SetState(SCE_TCL_OPERATOR); // }
                 sc.ForwardSetState(SCE_TCL_DEFAULT);
+                goto next;
             }
             else
                 sc.SetState(SCE_TCL_SUB_BRACE);
             if (!sc.atLineEnd)
                 continue;
         } else if (sc.state == SCE_TCL_DEFAULT || sc.state ==SCE_TCL_OPERATOR) {
-            expected &= isspacechar(static_cast<unsigned char>(sc.ch)) || IsAWordStart(sc.ch);
+            expected &= isspacechar(static_cast<unsigned char>(sc.ch)) || IsAWordStart(sc.ch) || sc.ch =='#';
         } else if (sc.state == SCE_TCL_SUBSTITUTION) {
-            if (sc.ch == '(')
-                subParen++;
-            else if (sc.ch == ')') {
+            switch(sc.ch) {
+            case '(':
+                subParen=true;
+                sc.SetState(SCE_TCL_OPERATOR);
+                sc.ForwardSetState(SCE_TCL_SUBSTITUTION);
+                continue;
+            case ')':
+                sc.SetState(SCE_TCL_OPERATOR);
+                subParen=false;
+                continue;
+            case '$':
+                continue;
+            case ',':
+                sc.SetState(SCE_TCL_OPERATOR);
                 if (subParen)
-                    subParen--;
-                else
-                    sc.SetState(SCE_TCL_DEFAULT); // lets the code below fix it
-            } else if (!IsAWordChar(sc.ch)) {
-                sc.SetState(SCE_TCL_DEFAULT);
-                subParen = 0;
+                    sc.ForwardSetState(SCE_TCL_SUBSTITUTION);
+                continue;
+            default :
+                // maybe spaces should be allowed ???
+                if (!IsAWordChar(sc.ch)) { // probably the code is wrong
+                    sc.SetState(SCE_TCL_DEFAULT);
+                    subParen = 0;
+                }
+                break;
             }
-        }
-        else
-        {
-            if (!IsAWordChar(sc.ch)) {
-                if (sc.state == SCE_TCL_IDENTIFIER ||  sc.state == SCE_TCL_MODIFIER || expected) {
-                    char s[100];
-                    sc.GetCurrent(s, sizeof(s));
-                    bool quote = sc.state == SCE_TCL_IN_QUOTE;
-                    if (commentLevel  || expected) {
-                        if (keywords.InList(s)) {
-                            sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD);
-                        } else if (keywords2.InList(s)) {
-                            sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD2);
-                        } else if (keywords3.InList(s)) {
-                            sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD3);
-                        } else if (keywords4.InList(s)) {
-                            sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD4);
-                        } else if (sc.GetRelative(-static_cast<int>(strlen(s))-1) == '{' &&
-                            keywords5.InList(s) && sc.ch == '}') { // {keyword} exactly no spaces
-                                sc.ChangeState(SCE_TCL_EXPAND);
-                        }
-                        if (keywords6.InList(s)) {
-                            sc.ChangeState(SCE_TCL_WORD5);
-                        } else if (keywords7.InList(s)) {
-                            sc.ChangeState(SCE_TCL_WORD6);
-                        } else if (keywords8.InList(s)) {
-                            sc.ChangeState(SCE_TCL_WORD7);
-                        } else if (keywords9.InList(s)) {
-                            sc.ChangeState(SCE_TCL_WORD8);
-                        } 
+        } else if (isComment(sc.state)) {
+        } else if (!IsAWordChar(sc.ch)) {
+            if ((sc.state == SCE_TCL_IDENTIFIER && expected) ||  sc.state == SCE_TCL_MODIFIER) {
+                char w[100];
+                char *s=w;
+                sc.GetCurrent(w, sizeof(w));
+                if (w[strlen(w)-1]=='\r')
+                    w[strlen(w)-1]=0;
+                while(*s == ':') // ignore leading : like in ::set a 10
+                    ++s;
+                bool quote = sc.state == SCE_TCL_IN_QUOTE;
+                if (commentLevel  || expected) {
+                    if (keywords.InList(s)) {
+                        sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD);
+                    } else if (keywords2.InList(s)) {
+                        sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD2);
+                    } else if (keywords3.InList(s)) {
+                        sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD3);
+                    } else if (keywords4.InList(s)) {
+                        sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD4);
+                    } else if (sc.GetRelative(-static_cast<int>(strlen(s))-1) == '{' &&
+                        keywords5.InList(s) && sc.ch == '}') { // {keyword} exactly no spaces
+                            sc.ChangeState(SCE_TCL_EXPAND);
+                    }
+                    if (keywords6.InList(s)) {
+                        sc.ChangeState(SCE_TCL_WORD5);
+                    } else if (keywords7.InList(s)) {
+                        sc.ChangeState(SCE_TCL_WORD6);
+                    } else if (keywords8.InList(s)) {
+                        sc.ChangeState(SCE_TCL_WORD7);
+                    } else if (keywords9.InList(s)) {
+                        sc.ChangeState(SCE_TCL_WORD8);
                     }
-                    expected = false;
-                    sc.SetState(quote ? SCE_TCL_IN_QUOTE : SCE_TCL_DEFAULT);
-                } else if (sc.state == SCE_TCL_MODIFIER || sc.state == SCE_TCL_SUBSTITUTION) {
-                    sc.SetState(SCE_TCL_DEFAULT);
                 }
+                expected = false;
+                sc.SetState(quote ? SCE_TCL_IN_QUOTE : SCE_TCL_DEFAULT);
+            } else if (sc.state == SCE_TCL_MODIFIER || sc.state == SCE_TCL_IDENTIFIER) {
+                sc.SetState(SCE_TCL_DEFAULT);
             }
         }
 		if (atEnd)
 			break;
-		if (sc.atLineEnd) {
+        if (sc.atLineEnd) {
+            lineState = LS_DEFAULT;
 			currentLine = styler.GetLine(sc.currentPos);
-			if (foldComment && sc.state == SCE_TCL_COMMENTLINE) {
+			if (foldComment && sc.state!=SCE_TCL_COMMENT && isComment(sc.state)) {
 				if (currentLevel == 0) {
 					++currentLevel;
 					commentLevel = true;
@@ -169,35 +198,37 @@ next:
 			// Update the line state, so it can be seen by next line
 			if (sc.state == SCE_TCL_IN_QUOTE)
 				lineState = LS_OPEN_DOUBLE_QUOTE;
-			else if (prevSlash) {
-				if (sc.state == SCE_TCL_COMMENT || sc.state == SCE_TCL_COMMENTLINE)
-					lineState = LS_OPEN_COMMENT;
+            else
+            {
+			    if (sc.chPrev == '\\') {
+				    if (isComment(sc.state))
+					    lineState = LS_OPEN_COMMENT;
+                } else if (sc.state == SCE_TCL_COMMENT_BOX)
+                    lineState = LS_COMMENT_BOX;
 			}
-            styler.SetLineState(currentLine, 
+            styler.SetLineState(currentLine,
                 (subBrace ? LS_BRACE_ONLY : 0) |
                 (expected ? LS_COMMAND_EXPECTED : 0)  | lineState);
-            sc.SetState(SCE_TCL_DEFAULT);
-			prevSlash = false;
+            if (lineState == LS_COMMENT_BOX)
+                sc.ForwardSetState(SCE_TCL_COMMENT_BOX);
+            else if (lineState == LS_OPEN_DOUBLE_QUOTE)
+                sc.ForwardSetState(SCE_TCL_IN_QUOTE);
+            else
+                sc.ForwardSetState(SCE_TCL_DEFAULT);
 			previousLevel = currentLevel;
-			lineState = LS_DEFAULT;
-			continue;
+			goto next;
 		}
 
-		if (prevSlash) {
-			prevSlash = (sc.state == SCE_TCL_COMMENT || sc.state == SCE_TCL_COMMENTLINE) && isspacechar(static_cast<unsigned char>(sc.ch));
+		if (sc.chPrev == '\\') {
+            if (sc.ch == '#' && IsANumberChar(sc.chNext))
+                sc.ForwardSetState(SCE_TCL_NUMBER);
 			continue;
 		}
-
+        if (isComment(sc.state))
+            continue;
 		if (sc.atLineStart) {
 			visibleChars = false;
-			if (lineState == LS_OPEN_COMMENT) {
-				sc.SetState(SCE_TCL_COMMENT);
-				lineState = LS_DEFAULT;
-				continue;
-			}
-			if (lineState == LS_OPEN_DOUBLE_QUOTE)
-				sc.SetState(SCE_TCL_IN_QUOTE);
-			else
+			if (sc.state!=SCE_TCL_IN_QUOTE && !isComment(sc.state))
             {
 				sc.SetState(SCE_TCL_DEFAULT);
                 expected = IsAWordStart(sc.ch)|| isspacechar(static_cast<unsigned char>(sc.ch));
@@ -219,39 +250,53 @@ next:
                 expected = sc.ch == '[';
                 sc.ForwardSetState(SCE_TCL_IN_QUOTE);
 				goto next;
-			} 
-			prevSlash = sc.ch == '\\';
+			}
+        case SCE_TCL_COMMENT:
+        case SCE_TCL_COMMENTLINE:
+        case SCE_TCL_BLOCK_COMMENT:
 			continue;
 		case SCE_TCL_OPERATOR:
 			sc.SetState(SCE_TCL_DEFAULT);
 			break;
+        case SCE_TCL_COMMENT_BOX:
+             if (sc.atLineStart && sc.chNext!='#')
+             {
+                 sc.SetState(SCE_TCL_DEFAULT);
+                 break;
+             }
+             continue;
 		}
 
 		if (sc.ch == '#') {
 			if (visibleChars) {
-                if (sc.state != SCE_TCL_IN_QUOTE && expected) {
+                if (sc.state != SCE_TCL_IN_QUOTE && expected)
 					sc.SetState(SCE_TCL_COMMENT);
-                    expected = false;
+			} else {
+                sc.SetState(SCE_TCL_COMMENTLINE);
+                if (sc.atLineStart)
+                {
+                    if (sc.chNext == '~')
+                        sc.SetState(SCE_TCL_BLOCK_COMMENT);
+                    else if (sc.chNext == '#' || sc.chNext == '-')
+                        sc.SetState(SCE_TCL_COMMENT_BOX);
                 }
-			} else
-				sc.SetState(SCE_TCL_COMMENTLINE);
-		}
+            }
+        }
 
 		if (!isspacechar(static_cast<unsigned char>(sc.ch))) {
 			visibleChars = true;
 		}
 
-		if (sc.ch == '\\') {
-			prevSlash = true;
+		if (sc.ch == '\\')
 			continue;
-		}
 
 		// Determine if a new state should be entered.
 		if (sc.state == SCE_TCL_DEFAULT) {
-			if (IsADigit(sc.ch) || (sc.ch == '.' && IsADigit(sc.chNext))) {
-				sc.SetState(SCE_TCL_NUMBER);
-			} else if (IsAWordStart(sc.ch) & expected) {
+            if (IsAWordStart(sc.ch)) {
 				sc.SetState(SCE_TCL_IDENTIFIER);
+			} else if (IsADigit(sc.ch) && !IsAWordChar(sc.chPrev))
+			{
+				sc.SetState(SCE_TCL_NUMBER);
 			} else {
 				switch (sc.ch) {
 				case '\"':
@@ -267,10 +312,8 @@ next:
 					--currentLevel;
 					break;
 				case '[':
+                    expected = true;
 				case ']':
-					sc.SetState(SCE_TCL_OPERATOR);
-					expected = true;
-					break;
 				case '(':
 				case ')':
 					sc.SetState(SCE_TCL_OPERATOR);
@@ -282,17 +325,23 @@ next:
                     subParen = 0;
                     if (sc.chNext != '{') {
                         sc.SetState(SCE_TCL_SUBSTITUTION);
-                    } 
+                    }
                     else {
-                        sc.ForwardSetState(SCE_TCL_OPERATOR);  // {
-                        sc.ForwardSetState(SCE_TCL_SUB_BRACE);     
+                        sc.SetState(SCE_TCL_SUB_BRACE);  // {
                         subBrace = true;
                     }
                     break;
+                case '#':
+                    if ((isspacechar(static_cast<unsigned char>(sc.chPrev))||isoperator(static_cast<unsigned char>(sc.chPrev))) && IsADigit(sc.chNext,0x10))
+                        sc.SetState(SCE_TCL_NUMBER);
+                    break;
                 case '-':
-                    if (!IsADigit(sc.chNext))
-                        sc.SetState(SCE_TCL_MODIFIER);
+                    sc.SetState(IsADigit(sc.chNext)? SCE_TCL_NUMBER: SCE_TCL_MODIFIER);
                     break;
+                default:
+                    if (isoperator(static_cast<char>(sc.ch))) {
+                        sc.SetState(SCE_TCL_OPERATOR);
+                    }
 				}
 			}
 		}