Patch from Steve Menard improves batch file lexing, recognizing more

builtin syntax.

2 files changed, 376 insertions, 67 deletions

diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html
index e95f080e4..8a04361e6 100644
--- a/doc/ScintillaHistory.html
+++ b/doc/ScintillaHistory.html
@@ -206,6 +206,7 @@
 	<li>Ben Harper</li>
 	<li>Adam Strzelecki</li>
 	<li>Kamen Stanev</li>
+	<li>Steve Menard</li>
     </ul>
     <p>
        Images used in GTK+ version
diff --git a/src/LexOthers.cxx b/src/LexOthers.cxx
index 4ad58495a..662b36308 100644
--- a/src/LexOthers.cxx
+++ b/src/LexOthers.cxx
@@ -33,6 +33,18 @@ static inline bool AtEOL(Accessor &styler, unsigned int i) {
 	       ((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n'));
 }
 
+// Tests for BATCH Operators
+static bool IsBOperator(char ch) {
+	return (ch == '=') || (ch == '+') || (ch == '>') || (ch == '<') ||
+		(ch == '|') || (ch == '?') || (ch == '*');
+}
+
+// Tests for BATCH Separators
+static bool IsBSeparator(char ch) {
+	return (ch == ':') || (ch == '\\') || (ch == '.') || (ch == ';') ||
+		(ch == '\"') || (ch == '\'') || (ch == '/') || (ch == ')');
+}
+
 static void ColouriseBatchLine(
     char *lineBuffer,
     unsigned int lengthLine,
@@ -41,95 +53,391 @@ static void ColouriseBatchLine(
     WordList &keywords,
     Accessor &styler) {
 
-	unsigned int i = 0;
-	unsigned int state = SCE_BAT_DEFAULT;
+	unsigned int offset = 0;	// Line Buffer Offset
+	unsigned int enVarEnd;		// Environment Variable End point
+	unsigned int cmdLoc;		// External Command / Program Location
+	char wordBuffer[81];		// Word Buffer - large to catch long paths
+	unsigned int wbl;		// Word Buffer Length
+	unsigned int wbo;		// Word Buffer Offset - also Special Keyword Buffer Length
+	bool forFound = false;		// No Local Variable without FOR statement
+	// CHOICE, ECHO, GOTO, PROMPT and SET have Default Text that may contain Regular Keywords
+	//   Toggling Regular Keyword Checking off improves readability
+	// Other Regular Keywords and External Commands / Programs might also benefit from toggling
+	//   Need a more robust algorithm to properly toggle Regular Keyword Checking
+	bool continueProcessing = true;	// Used to toggle Regular Keyword Checking
+	// Special Keywords are those that allow certain characters without whitespace after the command
+	// Examples are: cd. cd\ md. rd. dir| dir> echo: echo. path=
+	// Special Keyword Buffer used to determine if the first n characters is a Keyword
+	char sKeywordBuffer[10];	// Special Keyword Buffer
+	bool sKeywordFound;		// Exit Special Keyword for-loop if found
 
-	while ((i < lengthLine) && isspacechar(lineBuffer[i])) {	// Skip initial spaces
-		i++;
-	}
-	if (lineBuffer[i] == '@') {	// Hide command (ECHO OFF)
-		styler.ColourTo(startLine + i, SCE_BAT_HIDE);
-		i++;
-		while ((i < lengthLine) && isspacechar(lineBuffer[i])) {	// Skip next spaces
-			i++;
-		}
+	// Skip initial spaces
+	while ((offset < lengthLine) && (isspacechar(lineBuffer[offset]))) {
+		offset++;
 	}
-	if (lineBuffer[i] == ':') {
-		// Label
-		if (lineBuffer[i + 1] == ':') {
-			// :: is a fake label, similar to REM, see http://content.techweb.com/winmag/columns/explorer/2000/21.htm
+	// Colorize Default Text
+	styler.ColourTo(startLine + offset - 1, SCE_BAT_DEFAULT);
+	// Set External Command / Program Location
+	cmdLoc = offset;
+
+	// Check for Fake Label (Comment) or Real Label - return if found
+	if (lineBuffer[offset] == ':') {
+		if (lineBuffer[offset + 1] == ':') {
+			// Colorize Fake Label (Comment) - :: is similar to REM, see http://content.techweb.com/winmag/columns/explorer/2000/21.htm
 			styler.ColourTo(endPos, SCE_BAT_COMMENT);
-		} else {	// Real label
+		} else {
+			// Colorize Real Label
 			styler.ColourTo(endPos, SCE_BAT_LABEL);
 		}
-	} else {
-		// Check if initial word is a keyword
-		char wordBuffer[21];
-		unsigned int wbl = 0, offset = i;
-		// Copy word in buffer
-		for (; offset < lengthLine && wbl < 20 &&
+		return;
+	// Check for Drive Change (Drive Change is internal command) - return if found
+	} else if ((isalpha(lineBuffer[offset])) &&
+		(lineBuffer[offset + 1] == ':') &&
+		((isspacechar(lineBuffer[offset + 2])) ||
+		(((lineBuffer[offset + 2] == '\\')) &&
+		(isspacechar(lineBuffer[offset + 3]))))) {
+		// Colorize Regular Keyword
+		styler.ColourTo(endPos, SCE_BAT_WORD);
+		return;
+	}
+
+	// Check for Hide Command (@ECHO OFF/ON)
+	if (lineBuffer[offset] == '@') {
+		styler.ColourTo(startLine + offset, SCE_BAT_HIDE);
+		offset++;
+	// Check for Argument (%n) or Environment Variable (%x...%)
+	} else if (lineBuffer[offset] == '%') {
+		enVarEnd = offset + 1;
+		// Search end of word for second % (can be a long path)
+		while ((enVarEnd < lengthLine) &&
+			(!isspacechar(lineBuffer[enVarEnd])) &&
+			(lineBuffer[enVarEnd] != '%') &&
+			(!IsBOperator(lineBuffer[enVarEnd])) &&
+			(!IsBSeparator(lineBuffer[enVarEnd]))) {
+			enVarEnd++;
+		}
+		// Check for Argument (%n)
+		if ((Is0To9(lineBuffer[offset + 1])) &&
+			(lineBuffer[enVarEnd] != '%')) {
+			// Colorize Argument
+			styler.ColourTo(startLine + offset + 1, SCE_BAT_IDENTIFIER);
+			offset += 2;
+			// Check for External Command / Program
+			if (!isspacechar(lineBuffer[offset])) {
+				cmdLoc = offset;
+			}
+		// Check for Environment Variable (%x...%)
+		} else if ((lineBuffer[offset + 1] != '%') &&
+			(lineBuffer[enVarEnd] == '%')) {
+			offset = enVarEnd;
+			// Colorize Environment Variable
+			styler.ColourTo(startLine + offset, SCE_BAT_IDENTIFIER);
+			offset++;
+			// Check for External Command / Program
+			if (!isspacechar(lineBuffer[offset])) {
+				cmdLoc = offset;
+			}
+		}
+	}
+	// Skip next spaces
+	while ((offset < lengthLine) && (isspacechar(lineBuffer[offset]))) {
+		offset++;
+	}
+
+	// Read remainder of line word-at-a-time or remainder-of-word-at-a-time
+	while (offset < lengthLine) {
+		if (offset > startLine) {
+			// Colorize Default Text
+			styler.ColourTo(startLine + offset - 1, SCE_BAT_DEFAULT);
+		}
+		// Copy word from Line Buffer into Word Buffer
+		wbl = 0;
+		for (; offset < lengthLine && wbl < 80 &&
 		        !isspacechar(lineBuffer[offset]); wbl++, offset++) {
 			wordBuffer[wbl] = static_cast<char>(tolower(lineBuffer[offset]));
 		}
 		wordBuffer[wbl] = '\0';
-		// Check if it is a comment
+		wbo = 0;
+
+		// Check for Comment - return if found
 		if (CompareCaseInsensitive(wordBuffer, "rem") == 0) {
 			styler.ColourTo(endPos, SCE_BAT_COMMENT);
 			return;
 		}
-		// Check if it is in the list
-		if (keywords.InList(wordBuffer)) {
-			styler.ColourTo(startLine + offset - 1, SCE_BAT_WORD);	// Regular keyword
-		} else {
-			// Search end of word (can be a long path)
-			while (offset < lengthLine &&
-			        !isspacechar(lineBuffer[offset])) {
-				offset++;
+		// Check for Separator
+		if (IsBSeparator(wordBuffer[0])) {
+			// Check for External Command / Program
+			if ((cmdLoc == offset - wbl) &&
+				((wordBuffer[0] == ':') ||
+				(wordBuffer[0] == '\\') ||
+				(wordBuffer[0] == '.'))) {
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 1);
+				// Colorize External Command / Program
+				styler.ColourTo(startLine + offset - 1, SCE_BAT_COMMAND);
+				// Reset External Command / Program Location
+				cmdLoc = offset;
+			} else {
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 1);
+				// Colorize Default Text
+				styler.ColourTo(startLine + offset - 1, SCE_BAT_DEFAULT);
+			}
+		// Check for Regular Keyword in list
+		} else if ((keywords.InList(wordBuffer)) &&
+			(continueProcessing)) {
+			// Local Variables do not exist if no FOR statement
+			if (CompareCaseInsensitive(wordBuffer, "for") == 0) {
+				forFound = true;
+			}
+			// ECHO, GOTO, PROMPT and SET require no further Regular Keyword Checking
+			if ((CompareCaseInsensitive(wordBuffer, "echo") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "goto") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "prompt") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "set") == 0)) {
+				continueProcessing = false;
+			}
+			// Identify External Command / Program Location for ERRORLEVEL, and EXIST
+			if ((CompareCaseInsensitive(wordBuffer, "errorlevel") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "exist") == 0)) {
+				// Reset External Command / Program Location
+				cmdLoc = offset;
+				// Skip next spaces
+				while ((cmdLoc < lengthLine) &&
+					(isspacechar(lineBuffer[cmdLoc]))) {
+					cmdLoc++;
+				}
+				// Skip comparison
+				while ((cmdLoc < lengthLine) &&
+					(!isspacechar(lineBuffer[cmdLoc]))) {
+					cmdLoc++;
+				}
+				// Skip next spaces
+				while ((cmdLoc < lengthLine) &&
+					(isspacechar(lineBuffer[cmdLoc]))) {
+					cmdLoc++;
+				}
+			// Identify External Command / Program Location for CALL, DO, LOADHIGH and LH
+			} else if ((CompareCaseInsensitive(wordBuffer, "call") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "do") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "loadhigh") == 0) ||
+				(CompareCaseInsensitive(wordBuffer, "lh") == 0)) {
+				// Reset External Command / Program Location
+				cmdLoc = offset;
+				// Skip next spaces
+				while ((cmdLoc < lengthLine) &&
+					(isspacechar(lineBuffer[cmdLoc]))) {
+					cmdLoc++;
+				}
+			}
+			// Colorize Regular keyword
+			styler.ColourTo(startLine + offset - 1, SCE_BAT_WORD);
+			// No need to Reset Offset
+		// Check for Special Keyword in list, External Command / Program, or Default Text
+		} else if ((wordBuffer[0] != '%') &&
+			(!IsBOperator(wordBuffer[0])) &&
+			(continueProcessing)) {
+			// Check for Special Keyword
+			//     Affected Commands are in Length range 2-6
+			//     Good that ERRORLEVEL, EXIST, CALL, DO, LOADHIGH, and LH are unaffected
+			sKeywordFound = false;
+			for (unsigned int keywordLength = 2; keywordLength < wbl && keywordLength < 7 && !sKeywordFound; keywordLength++) {
+				wbo = 0;
+				// Copy Keyword Length from Word Buffer into Special Keyword Buffer
+				for (; wbo < keywordLength; wbo++) {
+					sKeywordBuffer[wbo] = static_cast<char>(wordBuffer[wbo]);
+				}
+				sKeywordBuffer[wbo] = '\0';
+				// Check for Special Keyword in list
+				if ((keywords.InList(sKeywordBuffer)) &&
+					((IsBOperator(wordBuffer[wbo])) ||
+					(IsBSeparator(wordBuffer[wbo])))) {
+					sKeywordFound = true;
+					// ECHO requires no further Regular Keyword Checking
+					if (CompareCaseInsensitive(sKeywordBuffer, "echo") == 0) {
+						continueProcessing = false;
+					}
+					// Colorize Special Keyword as Regular Keyword
+					styler.ColourTo(startLine + offset - 1 - (wbl - wbo), SCE_BAT_WORD);
+					// Reset Offset to re-process remainder of word
+					offset -= (wbl - wbo);
 			}
-			styler.ColourTo(startLine + offset - 1, SCE_BAT_COMMAND);	// External command / program
 		}
-		// Remainder of the line: colourise the variables.
-
-		while (offset < lengthLine) {
-			if (state == SCE_BAT_DEFAULT && lineBuffer[offset] == '%') {
-				styler.ColourTo(startLine + offset - 1, state);
-				if (Is0To9(lineBuffer[offset + 1])) {
-					styler.ColourTo(startLine + offset + 1, SCE_BAT_IDENTIFIER);
-					offset += 2;
-				} else if (lineBuffer[offset + 1] == '%' &&
-				           !isspacechar(lineBuffer[offset + 2])) {
-					// Should be safe, as there is CRLF at the end of the line...
-					styler.ColourTo(startLine + offset + 2, SCE_BAT_IDENTIFIER);
-					offset += 3;
+			// Check for External Command / Program or Default Text
+			if (!sKeywordFound) {
+				wbo = 0;
+				// Check for External Command / Program
+				if (cmdLoc == offset - wbl) {
+					// Read up to %, Operator or Separator
+					while ((wbo < wbl) &&
+						(wordBuffer[wbo] != '%') &&
+						(!IsBOperator(wordBuffer[wbo])) &&
+						(!IsBSeparator(wordBuffer[wbo]))) {
+						wbo++;
+					}
+					// Reset External Command / Program Location
+					cmdLoc = offset - (wbl - wbo);
+					// Reset Offset to re-process remainder of word
+					offset -= (wbl - wbo);
+					// CHOICE requires no further Regular Keyword Checking
+					if (CompareCaseInsensitive(wordBuffer, "choice") == 0) {
+						continueProcessing = false;
+					}
+					// Check for START (and its switches) - What follows is External Command \ Program
+					if (CompareCaseInsensitive(wordBuffer, "start") == 0) {
+						// Reset External Command / Program Location
+						cmdLoc = offset;
+						// Skip next spaces
+						while ((cmdLoc < lengthLine) &&
+							(isspacechar(lineBuffer[cmdLoc]))) {
+							cmdLoc++;
+						}
+						// Reset External Command / Program Location if command switch detected
+						if (lineBuffer[cmdLoc] == '/') {
+							// Skip command switch
+							while ((cmdLoc < lengthLine) &&
+								(!isspacechar(lineBuffer[cmdLoc]))) {
+								cmdLoc++;
+							}
+							// Skip next spaces
+							while ((cmdLoc < lengthLine) &&
+								(isspacechar(lineBuffer[cmdLoc]))) {
+								cmdLoc++;
+							}
+						}
+					}
+					// Colorize External command / program
+					styler.ColourTo(startLine + offset - 1, SCE_BAT_COMMAND);
+					// No need to Reset Offset
+				// Check for Default Text
 				} else {
-					state = SCE_BAT_IDENTIFIER;
+					// Read up to %, Operator or Separator
+					while ((wbo < wbl) &&
+						(wordBuffer[wbo] != '%') &&
+						(!IsBOperator(wordBuffer[wbo])) &&
+						(!IsBSeparator(wordBuffer[wbo]))) {
+						wbo++;
+					}
+					// Colorize Default Text
+					styler.ColourTo(startLine + offset - 1 - (wbl - wbo), SCE_BAT_DEFAULT);
+					// Reset Offset to re-process remainder of word
+					offset -= (wbl - wbo);
 				}
-			} else if (state == SCE_BAT_IDENTIFIER && lineBuffer[offset] == '%') {
-				styler.ColourTo(startLine + offset, state);
-				state = SCE_BAT_DEFAULT;
-			} else if (state == SCE_BAT_DEFAULT &&
-			           (lineBuffer[offset] == '*' ||
-			            lineBuffer[offset] == '?' ||
-			            lineBuffer[offset] == '=' ||
-			            lineBuffer[offset] == '<' ||
-			            lineBuffer[offset] == '>' ||
-			            lineBuffer[offset] == '|')) {
-				styler.ColourTo(startLine + offset - 1, state);
-				styler.ColourTo(startLine + offset, SCE_BAT_OPERATOR);
 			}
+		// Check for Argument  (%n), Environment Variable (%x...%) or Local Variable (%%a)
+		} else if (wordBuffer[0] == '%') {
+			// Colorize Default Text
+			styler.ColourTo(startLine + offset - 1 - wbl, SCE_BAT_DEFAULT);
+			wbo++;
+			// Search to end of word for second % (can be a long path)
+			while ((wbo < wbl) &&
+				(wordBuffer[wbo] != '%') &&
+				(!IsBOperator(wordBuffer[wbo])) &&
+				(!IsBSeparator(wordBuffer[wbo]))) {
+				wbo++;
+			}
+			// Check for Argument (%n)
+			if ((Is0To9(wordBuffer[1])) &&
+				(wordBuffer[wbo] != '%')) {
+				// Check for External Command / Program
+				if (cmdLoc == offset - wbl) {
+					cmdLoc = offset - (wbl - 2);
+				}
+				// Colorize Argument
+				styler.ColourTo(startLine + offset - 1 - (wbl - 2), SCE_BAT_IDENTIFIER);
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 2);
+			// Check for Environment Variable (%x...%)
+			} else if ((wordBuffer[1] != '%') &&
+				(wordBuffer[wbo] == '%')) {
+				wbo++;
+				// Check for External Command / Program
+				if (cmdLoc == offset - wbl) {
+					cmdLoc = offset - (wbl - wbo);
+			}
+				// Colorize Environment Variable
+				styler.ColourTo(startLine + offset - 1 - (wbl - wbo), SCE_BAT_IDENTIFIER);
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - wbo);
+			// Check for Local Variable (%%a)
+			} else if ((forFound) &&
+				(wordBuffer[1] == '%') &&
+				(wordBuffer[2] != '%') &&
+				(!IsBOperator(wordBuffer[2])) &&
+				(!IsBSeparator(wordBuffer[2]))) {
+				// Check for External Command / Program
+				if (cmdLoc == offset - wbl) {
+					cmdLoc = offset - (wbl - 3);
+				}
+				// Colorize Local Variable
+				styler.ColourTo(startLine + offset - 1 - (wbl - 3), SCE_BAT_IDENTIFIER);
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 3);
+			}
+		// Check for Operator
+		} else if (IsBOperator(wordBuffer[0])) {
+			// Colorize Default Text
+			styler.ColourTo(startLine + offset - 1 - wbl, SCE_BAT_DEFAULT);
+			// Check for Comparison Operator
+			if ((wordBuffer[0] == '=') && (wordBuffer[1] == '=')) {
+				// Identify External Command / Program Location for IF
+				cmdLoc = offset;
+				// Skip next spaces
+				while ((cmdLoc < lengthLine) &&
+					(isspacechar(lineBuffer[cmdLoc]))) {
+					cmdLoc++;
+				}
+				// Colorize Comparison Operator
+				styler.ColourTo(startLine + offset - 1 - (wbl - 2), SCE_BAT_OPERATOR);
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 2);
+			// Check for Pipe Operator
+			} else if (wordBuffer[0] == '|') {
+				// Reset External Command / Program Location
+				cmdLoc = offset - wbl + 1;
+				// Skip next spaces
+				while ((cmdLoc < lengthLine) &&
+					(isspacechar(lineBuffer[cmdLoc]))) {
+					cmdLoc++;
+				}
+				// Colorize Pipe Operator
+				styler.ColourTo(startLine + offset - 1 - (wbl - 1), SCE_BAT_OPERATOR);
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 1);
+			// Check for Other Operator
+			} else {
+				// Check for > Operator
+				if (wordBuffer[0] == '>') {
+					// Turn Keyword and External Command / Program checking back on
+					continueProcessing = true;
+				}
+				// Colorize Other Operator
+				styler.ColourTo(startLine + offset - 1 - (wbl - 1), SCE_BAT_OPERATOR);
+				// Reset Offset to re-process remainder of word
+				offset -= (wbl - 1);
+			}
+		// Check for Default Text
+		} else {
+			// Read up to %, Operator or Separator
+			while ((wbo < wbl) &&
+				(wordBuffer[wbo] != '%') &&
+				(!IsBOperator(wordBuffer[wbo])) &&
+				(!IsBSeparator(wordBuffer[wbo]))) {
+				wbo++;
+			}
+			// Colorize Default Text
+			styler.ColourTo(startLine + offset - 1 - (wbl - wbo), SCE_BAT_DEFAULT);
+			// Reset Offset to re-process remainder of word
+			offset -= (wbl - wbo);
+		}
+		// Skip next spaces - nothing happens if Offset was Reset
+		while ((offset < lengthLine) && (isspacechar(lineBuffer[offset]))) {
 			offset++;
 		}
-		//		if (endPos > startLine + offset - 1) {
-		styler.ColourTo(endPos, SCE_BAT_DEFAULT);		// Remainder of line, currently not lexed
-		//		}
 	}
-
+	// Colorize Default Text for remainder of line - currently not lexed
+	styler.ColourTo(endPos, SCE_BAT_DEFAULT);
 }
-// ToDo: (not necessarily at beginning of line) GOTO, [IF] NOT, ERRORLEVEL
-// IF [NO] (test) (command) -- test is EXIST (filename) | (string1)==(string2) | ERRORLEVEL (number)
-// FOR %%(variable) IN (set) DO (command) -- variable is [a-zA-Z] -- eg for %%X in (*.txt) do type %%X
-// ToDo: %n (parameters), %EnvironmentVariable% colourising
-// ToDo: Colourise = > >> < | "
 
 static void ColouriseBatchDoc(
     unsigned int startPos,