Updated YAML lexer.

1 files changed, 215 insertions, 93 deletions

diff --git a/src/LexYAML.cxx b/src/LexYAML.cxx
index 156ca279d..e3053f814 100644
--- a/src/LexYAML.cxx
+++ b/src/LexYAML.cxx
@@ -9,11 +9,13 @@
 #include <string.h>
 #include <ctype.h>
 #include <stdio.h>
+#include <stdarg.h>
 
 #include "Platform.h"
 
 #include "PropSet.h"
 #include "Accessor.h"
+#include "StyleContext.h"
 #include "KeyWords.h"
 #include "Scintilla.h"
 #include "SciLexer.h"
@@ -28,68 +30,126 @@ static inline bool AtEOL(Accessor &styler, unsigned int i) {
 		((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n'));
 }
 
-static void ColouriseYAMLLine(
-    char *lineBuffer,
-    unsigned int lengthLine,
-    unsigned int startLine,
-    unsigned int endPos,
-    WordList &keywords,
-    Accessor &styler) {
+static unsigned int SpaceCount(char* lineBuffer) {
+	if (lineBuffer == NULL)
+		return 0;
+	
+	char* headBuffer = lineBuffer;
+	
+	while (*headBuffer == ' ')
+		headBuffer++;
+	
+	return headBuffer - lineBuffer;
+}
 
+#define YAML_STATE_BITSIZE 16
+#define YAML_STATE_MASK			(0xFFFF0000)
+#define YAML_STATE_DOCUMENT		(1 << YAML_STATE_BITSIZE)
+#define YAML_STATE_VALUE			(2 << YAML_STATE_BITSIZE)
+#define YAML_STATE_COMMENT		(3 << YAML_STATE_BITSIZE)
+#define YAML_STATE_TEXT_PARENT	(4 << YAML_STATE_BITSIZE)
+#define YAML_STATE_TEXT			(5 << YAML_STATE_BITSIZE)
+
+static void ColouriseYAMLLine(
+	char *lineBuffer,
+	unsigned int currentLine,
+	unsigned int lengthLine,
+	unsigned int startLine,
+	unsigned int endPos,
+	WordList &keywords,
+	Accessor &styler) {
+	    
 	unsigned int i = 0;
 	bool bInQuotes = false;
-	unsigned int startValue, endValue, valueLen;
-	char scalar[256];
-	if (lineBuffer[0] == '#') {	// Comment
-		styler.ColourTo(endPos, SCE_YAML_COMMENT);
-		return;
+	unsigned int indentAmount = SpaceCount(lineBuffer);
+		
+	if (currentLine > 0) {
+		int parentLineState = styler.GetLineState(currentLine - 1);
+	
+		if ((parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT || (parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT_PARENT) {
+			unsigned int parentIndentAmount = parentLineState&(~YAML_STATE_MASK);
+			if (indentAmount > parentIndentAmount) {
+				styler.SetLineState(currentLine, YAML_STATE_TEXT | parentIndentAmount);
+				styler.ColourTo(endPos, SCE_YAML_TEXT);
+				return;
+			}
+		}
 	}
+	styler.SetLineState(currentLine, 0);
 	if (strncmp(lineBuffer, "---", 3) == 0) {	// Document marker
+		styler.SetLineState(currentLine, YAML_STATE_DOCUMENT);
 		styler.ColourTo(endPos, SCE_YAML_DOCUMENT);
 		return;
 	}
 	// Skip initial spaces
-	while ((i < lengthLine) && isspacechar(lineBuffer[i])) {
+	while ((i < lengthLine) && lineBuffer[i] == ' ') { // YAML always uses space, never TABS or anything else
 		i++;
 	}
+	if (lineBuffer[i] == '\t') { // if we skipped all spaces, and we are NOT inside a text block, this is wrong
+		styler.ColourTo(endPos, SCE_YAML_ERROR);
+		return;
+	}
+	if (lineBuffer[i] == '#') {	// Comment
+		styler.SetLineState(currentLine, YAML_STATE_COMMENT);
+		styler.ColourTo(endPos, SCE_YAML_COMMENT);
+		return;
+	}
 	while (i < lengthLine) {
 		if (lineBuffer[i] == '\'' || lineBuffer[i] == '\"') {
 			bInQuotes = !bInQuotes;
 		} else if (lineBuffer[i] == ':' && !bInQuotes) {
 			styler.ColourTo(startLine + i, SCE_YAML_IDENTIFIER);
 			// Non-folding scalar
-			startValue = i + 1;
-			while ((startValue < lengthLine) && isspacechar(lineBuffer[startValue])) {
-				startValue++;
-			}
-			endValue = lengthLine - 1;
-			while ((endValue >= startValue) && isspacechar(lineBuffer[endValue])) {
+			i++;
+			while ((i < lengthLine) && isspacechar(lineBuffer[i]))
+				i++;
+			unsigned int endValue = lengthLine - 1;
+			while ((endValue >= i) && isspacechar(lineBuffer[endValue]))
 				endValue--;
+			lineBuffer[endValue + 1] = '\0';
+			if (lineBuffer[i] == '|' || lineBuffer[i] == '>') {
+				i++;
+				if (lineBuffer[i] == '+' || lineBuffer[i] == '-')
+					i++;
+				while ((i < lengthLine) && isspacechar(lineBuffer[i]))
+					i++;
+				if (lineBuffer[i] == '\0') {
+					styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
+					styler.ColourTo(endPos, SCE_YAML_DEFAULT);
+					return;
+				} else if (lineBuffer[i] == '#') {
+					styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
+					styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
+					styler.ColourTo(endPos, SCE_YAML_COMMENT);
+					return;
+				} else {
+					styler.ColourTo(endPos, SCE_YAML_ERROR);
+					return;
+				}
 			}
-			valueLen = endValue - startValue + 1;
-			if (endValue < startValue || valueLen > sizeof(scalar)) {
-				break;
-			}
-			strncpy(scalar,  &lineBuffer[startValue], valueLen);
-			scalar[valueLen] = '\0';
-			if (scalar[0] == '&' || scalar[0] == '*') {
-				styler.ColourTo(startLine + endValue, SCE_YAML_REFERENCE);
+			styler.SetLineState(currentLine, YAML_STATE_VALUE);
+			if (lineBuffer[i] == '&' || lineBuffer[i] == '*') {
+				styler.ColourTo(endPos, SCE_YAML_REFERENCE);
+				return;
 			}
-			else if (keywords.InList(scalar)) { // Convertible value (true/false, etc.)
-				styler.ColourTo(startLine + endValue, SCE_YAML_KEYWORD);
+			if (keywords.InList(&lineBuffer[i])) { // Convertible value (true/false, etc.)
+				styler.ColourTo(endPos, SCE_YAML_KEYWORD);
+				return;
 			} else {
-				startValue = 0;
-				while (startValue < valueLen) {
-					if (!isdigit(scalar[startValue]) && scalar[startValue] != '-' && scalar[startValue] != '.' && scalar[startValue] != ',') {
-						break;
+				unsigned int i2 = i;
+				while ((i < lengthLine) && lineBuffer[i]) {
+					if (!isdigit(lineBuffer[i]) && lineBuffer[i] != '-' && lineBuffer[i] != '.' && lineBuffer[i] != ',') {
+						styler.ColourTo(endPos, SCE_YAML_DEFAULT);
+						return;
 					}
-					startValue++;
+					i++;
 				}
-				if (startValue >= valueLen) {
-					styler.ColourTo(startLine + endValue, SCE_YAML_NUMBER);
+				if (i > i2) {
+					styler.ColourTo(endPos, SCE_YAML_NUMBER);
+					return;
 				}
 			}
-			break; // the rest of the line is coloured the default
+			break; // shouldn't get here, but just in case, the rest of the line is coloured the default
 		}
 		i++;
 	}
@@ -102,82 +162,144 @@ static void ColouriseYAMLDoc(unsigned int startPos, int length, int, WordList *k
 	styler.StartSegment(startPos);
 	unsigned int linePos = 0;
 	unsigned int startLine = startPos;
-	for (unsigned int i = startPos; i < startPos + length; i++) {
+	unsigned int endPos = startPos + length;
+	unsigned int maxPos = styler.Length();
+	unsigned int lineCurrent = styler.GetLine(startPos);
+	
+	for (unsigned int i = startPos; i < maxPos && i < endPos; i++) {
 		lineBuffer[linePos++] = styler[i];
 		if (AtEOL(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
 			// End of line (or of line buffer) met, colourise it
 			lineBuffer[linePos] = '\0';
-			ColouriseYAMLLine(lineBuffer, linePos, startLine, i, *keywordLists[0], styler);
+			ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, i, *keywordLists[0], styler);
 			linePos = 0;
 			startLine = i + 1;
+			lineCurrent++;
 		}
 	}
 	if (linePos > 0) {	// Last line does not have ending characters
-		ColouriseYAMLLine(lineBuffer, linePos, startLine, startPos + length - 1, *keywordLists[0], styler);
+		ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, startPos + length - 1, *keywordLists[0], styler);
 	}
 }
 
-static int IdentifierLevelYAMLLine(
-    char *lineBuffer,
-    unsigned int lengthLine) {
+static bool IsCommentLine(int line, Accessor &styler) {
+	int pos = styler.LineStart(line);
+	if (styler[pos] == '#')
+		return true;
+	return false;
+}
 
-	unsigned int i = 0;
-	bool bInQuotes = false;
-	int level;
-	if (lineBuffer[0] == '#') {	// Comment
-		return 0xFFFFFFFF;
-	}
-	if (strncmp(lineBuffer, "---", 3) == 0) {	// Document marker
-		return 0;
+static void FoldYAMLDoc(unsigned int startPos, int length, int /*initStyle - unused*/,
+                      WordList *[], Accessor &styler) {
+	const int maxPos = startPos + length;
+	const int maxLines = styler.GetLine(maxPos - 1);             // Requested last line
+	const int docLines = styler.GetLine(styler.Length() - 1);  // Available last line
+	const bool foldComment = styler.GetPropertyInt("fold.comment.yaml") != 0;
+
+	// Backtrack to previous non-blank line so we can determine indent level
+	// for any white space lines
+	// and so we can fix any preceding fold level (which is why we go back
+	// at least one line in all cases)
+	int spaceFlags = 0;
+	int lineCurrent = styler.GetLine(startPos);
+	int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
+	while (lineCurrent > 0) {
+		lineCurrent--;
+		indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
+		if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) &&
+		        (!IsCommentLine(lineCurrent, styler)))
+			break;
 	}
-	// Count initial spaces and '-' character
-	while ((i < lengthLine) && (isspacechar(lineBuffer[i]) || lineBuffer[i] == '-')) {
-		i++;
+	int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
+
+	// Set up initial loop state
+	int prevComment = 0;
+	if (lineCurrent >= 1)
+		prevComment = foldComment && IsCommentLine(lineCurrent - 1, styler);
+
+	// Process all characters to end of requested range
+	// or comment that hangs over the end of the range.  Cap processing in all cases
+	// to end of document (in case of unclosed comment at end).
+	while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) || prevComment)) {
+
+		// Gather info
+		int lev = indentCurrent;
+		int lineNext = lineCurrent + 1;
+		int indentNext = indentCurrent;
+		if (lineNext <= docLines) {
+			// Information about next line is only available if not at end of document
+			indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 	}
-	level = i;
-	while (i < lengthLine) {
-		if (lineBuffer[i] == '\'' || lineBuffer[i] == '\"') {
-			bInQuotes = !bInQuotes;
-		} else if (lineBuffer[i] == ':' && !bInQuotes) {
-			return level;
+		const int comment = foldComment && IsCommentLine(lineCurrent, styler);
+		const int comment_start = (comment && !prevComment && (lineNext <= docLines) &&
+		                           IsCommentLine(lineNext, styler) && (lev > SC_FOLDLEVELBASE));
+		const int comment_continue = (comment && prevComment);
+		if (!comment)
+			indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
+		if (indentNext & SC_FOLDLEVELWHITEFLAG)
+			indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel;
+
+		if (comment_start) {
+			// Place fold point at start of a block of comments
+			lev |= SC_FOLDLEVELHEADERFLAG;
+		} else if (comment_continue) {
+			// Add level to rest of lines in the block
+			lev = lev + 1;
 		}
-		i++;
-	}
-	return -level; // Scalar
-}
 
-static void FoldYAMLDoc(unsigned int startPos, int length, int, WordList *[], Accessor &styler) {
-	char lineBuffer[1024];
-	styler.StartAt(startPos);
-	styler.StartSegment(startPos);
-	unsigned int linePos = 0;
-	int currentLevel;
-	int lastLevel = 0xFFFFFFFF;
-	unsigned int lineCurrent = 0;
-	for (unsigned int i = startPos; i < startPos + length; i++) {
-		lineBuffer[linePos++] = styler[i];
-		if (AtEOL(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
-			// End of line (or of line buffer) met, colourise it
-			lineBuffer[linePos] = '\0';
-			currentLevel = IdentifierLevelYAMLLine(lineBuffer, linePos);
-	 		if (currentLevel != static_cast<int>(0xFFFFFFFF)) {
-				if (abs(currentLevel) > abs(lastLevel) && lastLevel >= 0) { // indented higher than last, and last was an identifier line
-					styler.SetLevel(lineCurrent - 1, SC_FOLDLEVELHEADERFLAG);
-				}
-				lastLevel = currentLevel;
-			}
-			linePos = 0;
-			lineCurrent++;
+		// Skip past any blank lines for next indent level info; we skip also
+		// comments (all comments, not just those starting in column 0)
+		// which effectively folds them into surrounding code rather
+		// than screwing up folding.
+
+		while ((lineNext < docLines) &&
+		        ((indentNext & SC_FOLDLEVELWHITEFLAG) ||
+		         (lineNext <= docLines && IsCommentLine(lineNext, styler)))) {
+
+			lineNext++;
+			indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 		}
-	}
-	if (linePos > 0) {	// Last line does not have ending characters
-		currentLevel = IdentifierLevelYAMLLine(lineBuffer, linePos);
-		if (currentLevel != static_cast<int>(0xFFFFFFFF)) {
-			if (abs(currentLevel) > abs(lastLevel) && lastLevel >= 0) {
-				styler.SetLevel(lineCurrent - 1, SC_FOLDLEVELHEADERFLAG);
-			}
+
+		const int levelAfterComments = indentNext & SC_FOLDLEVELNUMBERMASK;
+		const int levelBeforeComments = Platform::Maximum(indentCurrentLevel,levelAfterComments);
+
+		// Now set all the indent levels on the lines we skipped
+		// Do this from end to start.  Once we encounter one line
+		// which is indented more than the line after the end of
+		// the comment-block, use the level of the block before
+
+		int skipLine = lineNext;
+		int skipLevel = levelAfterComments;
+
+		while (--skipLine > lineCurrent) {
+			int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, NULL);
+
+			if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments)
+				skipLevel = levelBeforeComments;
+
+			int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
+
+			styler.SetLevel(skipLine, skipLevel | whiteFlag);
+		}
+
+		// Set fold header on non-comment line
+		if (!comment && !(indentCurrent & SC_FOLDLEVELWHITEFLAG) ) {
+			if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
+				lev |= SC_FOLDLEVELHEADERFLAG;
 		}
+
+		// Keep track of block comment state of previous line
+		prevComment = comment_start || comment_continue;
+
+		// Set fold level for this line and move to next line
+		styler.SetLevel(lineCurrent, lev);
+		indentCurrent = indentNext;
+		lineCurrent = lineNext;
 	}
+
+	// NOTE: Cannot set level of last line here because indentCurrent doesn't have
+	// header flag set; the loop above is crafted to take care of this case!
+	//styler.SetLevel(lineCurrent, indentCurrent);
 }
 
 LexerModule lmYAML(SCLEX_YAML, ColouriseYAMLDoc, "yaml", FoldYAMLDoc, yamlWordListDesc);