From 269bff4307f3130d1873130baf13b963ef34da0b Mon Sep 17 00:00:00 2001
From: nkmathew <unknown>
Date: Sun, 20 Mar 2016 23:19:59 +1100
Subject: Feature [feature-requests:#1140]. JSON lexer added.

---
 doc/ScintillaHistory.html |   6 +-
 include/SciLexer.h        |  15 ++
 include/Scintilla.iface   |  17 ++
 lexers/LexJSON.cxx        | 497 ++++++++++++++++++++++++++++++++++++++++++++++
 src/Catalogue.cxx         |   1 +
 win32/scintilla.mak       |   3 +
 6 files changed, 538 insertions(+), 1 deletion(-)
 create mode 100644 lexers/LexJSON.cxx

diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html
index d1331ab10..50029dd8f 100644
--- a/doc/ScintillaHistory.html
+++ b/doc/ScintillaHistory.html
@@ -500,7 +500,11 @@
     </h3>
     <ul>
 	<li>
-	Released 18 May 2016.
+	Released 13 March 2016.
+	</li>
+	<li>
+	JSON lexer added.
+	<a href="http://sourceforge.net/p/scintilla/feature-requests/1140/">Feature #1140.</a>
 	</li>
 	<li>
 	The Python lexer treats '@' as an operator except when it is the first visible character on a line.
diff --git a/include/SciLexer.h b/include/SciLexer.h
index 2103b47b9..6b98b09fc 100644
--- a/include/SciLexer.h
+++ b/include/SciLexer.h
@@ -132,6 +132,7 @@
 #define SCLEX_SREC 117
 #define SCLEX_IHEX 118
 #define SCLEX_TEHEX 119
+#define SCLEX_JSON 120
 #define SCLEX_AUTOMATIC 1000
 #define SCE_P_DEFAULT 0
 #define SCE_P_COMMENTLINE 1
@@ -1787,6 +1788,20 @@
 #define SCE_HEX_CHECKSUM 16
 #define SCE_HEX_CHECKSUM_WRONG 17
 #define SCE_HEX_GARBAGE 18
+#define SCE_JSON_DEFAULT 0
+#define SCE_JSON_NUMBER 1
+#define SCE_JSON_STRING 2
+#define SCE_JSON_STRINGEOL 3
+#define SCE_JSON_PROPERTYNAME 4
+#define SCE_JSON_ESCAPESEQUENCE 5
+#define SCE_JSON_LINECOMMENT 6
+#define SCE_JSON_BLOCKCOMMENT 7
+#define SCE_JSON_OPERATOR 8
+#define SCE_JSON_URI 9
+#define SCE_JSON_COMPACTIRI 10
+#define SCE_JSON_KEYWORD 11
+#define SCE_JSON_LDKEYWORD 12
+#define SCE_JSON_ERROR 13
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
 #endif
diff --git a/include/Scintilla.iface b/include/Scintilla.iface
index 91b3066b5..71defe09d 100644
--- a/include/Scintilla.iface
+++ b/include/Scintilla.iface
@@ -2799,6 +2799,7 @@ val SCLEX_BIBTEX=116
 val SCLEX_SREC=117
 val SCLEX_IHEX=118
 val SCLEX_TEHEX=119
+val SCLEX_JSON=120
 
 # When a lexer specifies its language as SCLEX_AUTOMATIC it receives a
 # value assigned in sequence from SCLEX_AUTOMATIC+1.
@@ -4677,6 +4678,22 @@ val SCE_HEX_GARBAGE=18
 lex IHex=SCLEX_IHEX SCE_HEX_
 # Lexical state for SCLEX_TEHEX (shared with Srec)
 lex TEHex=SCLEX_TEHEX SCE_HEX_
+# Lexical states for SCLEX_JSON
+lex JSON=SCLEX_JSON SCE_JSON_
+val SCE_JSON_DEFAULT=0
+val SCE_JSON_NUMBER=1
+val SCE_JSON_STRING=2
+val SCE_JSON_STRINGEOL=3
+val SCE_JSON_PROPERTYNAME=4
+val SCE_JSON_ESCAPESEQUENCE=5
+val SCE_JSON_LINECOMMENT=6
+val SCE_JSON_BLOCKCOMMENT=7
+val SCE_JSON_OPERATOR=8
+val SCE_JSON_URI=9
+val SCE_JSON_COMPACTIRI=10
+val SCE_JSON_KEYWORD=11
+val SCE_JSON_LDKEYWORD=12
+val SCE_JSON_ERROR=13
 
 # Events
 
diff --git a/lexers/LexJSON.cxx b/lexers/LexJSON.cxx
new file mode 100644
index 000000000..9c044e52c
--- /dev/null
+++ b/lexers/LexJSON.cxx
@@ -0,0 +1,497 @@
+// Scintilla source code edit control
+/**
+ * @file LexJSON.cxx
+ * @date February 19, 2016
+ * @brief Lexer for JSON and JSON-LD formats
+ * @author nkmathew
+ *
+ * The License.txt file describes the conditions under which this software may
+ * be distributed.
+ *
+ */
+
+#include <cstdlib>
+#include <cassert>
+#include <cctype>
+#include <cstdio>
+#include <string>
+#include <vector>
+#include <map>
+
+#include "ILexer.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+#include "WordList.h"
+#include "LexAccessor.h"
+#include "StyleContext.h"
+#include "CharacterSet.h"
+#include "LexerModule.h"
+#include "OptionSet.h"
+
+#ifdef SCI_NAMESPACE
+using namespace Scintilla;
+#endif
+
+static const char *const JSONWordListDesc[] = {
+	"JSON Keywords",
+	"JSON-LD Keywords",
+	0
+};
+
+/**
+ * Used to detect compact IRI/URLs in JSON-LD without first looking ahead for the
+ * colon separating the prefix and suffix
+ *
+ * https://www.w3.org/TR/json-ld/#dfn-compact-iri
+ */
+struct CompactIRI {
+	int colonCount;
+	bool foundInvalidChar;
+	CharacterSet setCompactIRI;
+	CompactIRI() {
+		colonCount = 0;
+		foundInvalidChar = false;
+		setCompactIRI = CharacterSet(CharacterSet::setAlpha, "$_-");
+	}
+	void resetState() {
+		colonCount = 0;
+		foundInvalidChar = false;
+	}
+	void checkChar(int ch) {
+		if (ch == ':') {
+			colonCount++;
+		} else {
+			foundInvalidChar |= !setCompactIRI.Contains(ch);
+		}
+	}
+	bool shouldHighlight() const {
+		return !foundInvalidChar && colonCount == 1;
+	}
+};
+
+/**
+ * Keeps track of escaped characters in strings as per:
+ *
+ * https://tools.ietf.org/html/rfc7159#section-7
+ */
+struct EscapeSequence {
+	int digitsLeft;
+	CharacterSet setHexDigits;
+	CharacterSet setEscapeChars;
+	EscapeSequence() {
+		digitsLeft = 0;
+		setHexDigits = CharacterSet(CharacterSet::setDigits, "ABCDEFabcdef");
+		setEscapeChars = CharacterSet(CharacterSet::setNone, "\\\"tnbfru/");
+	}
+	// Returns true if the following character is a valid escaped character
+	bool newSequence(int nextChar) {
+		digitsLeft = 0;
+		if (nextChar == 'u') {
+			digitsLeft = 5;
+		} else if (!setEscapeChars.Contains(nextChar)) {
+			return false;
+		}
+		return true;
+	}
+	bool atEscapeEnd() const {
+		return digitsLeft <= 0;
+	}
+	bool isInvalidChar(int currChar) const {
+		return !setHexDigits.Contains(currChar);
+	}
+};
+
+struct OptionsJSON {
+	bool foldCompact;
+	bool fold;
+	bool allowComments;
+	bool escapeSequence;
+	OptionsJSON() {
+		foldCompact = false;
+		fold = false;
+		allowComments = false;
+		escapeSequence = false;
+	}
+};
+
+struct OptionSetJSON : public OptionSet<OptionsJSON> {
+	OptionSetJSON() {
+		DefineProperty("lexer.json.escape.sequence", &OptionsJSON::escapeSequence,
+					   "Set to 1 to enable highlighting of escape sequences in strings");
+
+		DefineProperty("lexer.json.allow.comments", &OptionsJSON::allowComments,
+					   "Set to 1 to enable highlighting of line/block comments in JSON");
+
+		DefineProperty("fold.compact", &OptionsJSON::foldCompact);
+		DefineProperty("fold", &OptionsJSON::fold);
+		DefineWordListSets(JSONWordListDesc);
+	}
+};
+
+class LexerJSON : public ILexer {
+	OptionsJSON options;
+	OptionSetJSON optSetJSON;
+	EscapeSequence escapeSeq;
+	WordList keywordsJSON;
+	WordList keywordsJSONLD;
+	CharacterSet setOperators;
+	CharacterSet setURL;
+	CharacterSet setKeywordJSONLD;
+	CharacterSet setKeywordJSON;
+	CompactIRI compactIRI;
+
+	static bool IsNextNonWhitespace(LexAccessor &styler, Sci_Position start, char ch) {
+		Sci_Position i = 0;
+		while (i < 50) {
+			i++;
+			char curr = styler.SafeGetCharAt(start+i, '\0');
+			char next = styler.SafeGetCharAt(start+i+1, '\0');
+			bool atEOL = (curr == '\r' && next != '\n') || (curr == '\n');
+			if (curr == ch) {
+				return true;
+			} else if (!isspacechar(curr) || atEOL) {
+				return false;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Looks for the colon following the end quote
+	 *
+	 * Assumes property names of lengths no longer than a 100 characters.
+	 * The colon is also expected to be less than 50 spaces after the end
+	 * quote for the string to be considered a property name
+	 */
+	static bool AtPropertyName(LexAccessor &styler, Sci_Position start) {
+		Sci_Position i = 0;
+		bool escaped = false;
+		while (i < 100) {
+			i++;
+			char curr = styler.SafeGetCharAt(start+i, '\0');
+			if (escaped) {
+				escaped = false;
+				continue;
+			}
+			escaped = curr == '\\';
+			if (curr == '"') {
+				return IsNextNonWhitespace(styler, start+i, ':');
+			} else if (!curr) {
+				return false;
+			}
+		}
+		return false;
+	}
+
+	static bool IsNextWordInList(WordList &keywordList, CharacterSet wordSet,
+								 StyleContext &context, LexAccessor &styler) {
+		char word[51];
+		Sci_Position currPos = (Sci_Position) context.currentPos;
+		int i = 0;
+		while (i < 50) {
+			char ch = styler.SafeGetCharAt(currPos + i);
+			if (!wordSet.Contains(ch)) {
+				break;
+			}
+			word[i] = ch;
+			i++;
+		}
+		word[i] = '\0';
+		return keywordList.InList(word);
+	}
+
+	public:
+	LexerJSON() :
+		setOperators(CharacterSet::setNone, "[{}]:,"),
+		setURL(CharacterSet::setAlphaNum, "-._~:/?#[]@!$&'()*+,),="),
+		setKeywordJSONLD(CharacterSet::setAlpha, ":@"),
+		setKeywordJSON(CharacterSet::setAlpha, "$_") {
+	}
+	virtual ~LexerJSON() {}
+	virtual int SCI_METHOD Version() const {
+		return lvOriginal;
+	}
+	virtual void SCI_METHOD Release() {
+		delete this;
+	}
+	virtual const char *SCI_METHOD PropertyNames() {
+		return optSetJSON.PropertyNames();
+	}
+	virtual int SCI_METHOD PropertyType(const char *name) {
+		return optSetJSON.PropertyType(name);
+	}
+	virtual const char *SCI_METHOD DescribeProperty(const char *name) {
+		return optSetJSON.DescribeProperty(name);
+	}
+	virtual Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) {
+		if (optSetJSON.PropertySet(&options, key, val)) {
+			return 0;
+		}
+		return -1;
+	}
+	virtual Sci_Position SCI_METHOD WordListSet(int n, const char *wl) {
+		WordList *wordListN = 0;
+		switch (n) {
+			case 0:
+				wordListN = &keywordsJSON;
+				break;
+			case 1:
+				wordListN = &keywordsJSONLD;
+				break;
+		}
+		Sci_Position firstModification = -1;
+		if (wordListN) {
+			WordList wlNew;
+			wlNew.Set(wl);
+			if (*wordListN != wlNew) {
+				wordListN->Set(wl);
+				firstModification = 0;
+			}
+		}
+		return firstModification;
+	}
+	virtual void *SCI_METHOD PrivateCall(int, void *) {
+		return 0;
+	}
+	static ILexer *LexerFactoryJSON() {
+		return new LexerJSON;
+	}
+	virtual const char *SCI_METHOD DescribeWordListSets() {
+		return optSetJSON.DescribeWordListSets();
+	}
+	virtual void SCI_METHOD Lex(Sci_PositionU startPos,
+								Sci_Position length,
+								int initStyle,
+								IDocument *pAccess);
+	virtual void SCI_METHOD Fold(Sci_PositionU startPos,
+								 Sci_Position length,
+								 int initStyle,
+								 IDocument *pAccess);
+};
+
+void SCI_METHOD LexerJSON::Lex(Sci_PositionU startPos,
+							   Sci_Position length,
+							   int initStyle,
+							   IDocument *pAccess) {
+	LexAccessor styler(pAccess);
+	StyleContext context(startPos, length, initStyle, styler);
+	int stringStyleBefore = SCE_JSON_STRING;
+	while (context.More()) {
+		switch (context.state) {
+			case SCE_JSON_BLOCKCOMMENT:
+				if (context.Match("*/")) {
+					context.Forward();
+					context.ForwardSetState(SCE_JSON_DEFAULT);
+				}
+				break;
+			case SCE_JSON_LINECOMMENT:
+				if (context.atLineEnd) {
+					context.SetState(SCE_JSON_DEFAULT);
+				}
+				break;
+			case SCE_JSON_STRINGEOL:
+				if (context.atLineStart) {
+					context.SetState(SCE_JSON_DEFAULT);
+				}
+				break;
+			case SCE_JSON_ESCAPESEQUENCE:
+				escapeSeq.digitsLeft--;
+				if (!escapeSeq.atEscapeEnd()) {
+					if (escapeSeq.isInvalidChar(context.ch)) {
+						context.SetState(SCE_JSON_ERROR);
+					}
+					break;
+				}
+				if (context.ch == '"') {
+					context.SetState(stringStyleBefore);
+					context.ForwardSetState(SCE_C_DEFAULT);
+				} else if (context.ch == '\\') {
+					if (!escapeSeq.newSequence(context.chNext)) {
+						context.SetState(SCE_JSON_ERROR);
+					}
+					context.Forward();
+				} else {
+					context.SetState(stringStyleBefore);
+					if (context.atLineEnd) {
+						context.ChangeState(SCE_JSON_STRINGEOL);
+					}
+				}
+				break;
+			case SCE_JSON_PROPERTYNAME:
+			case SCE_JSON_STRING:
+				if (context.ch == '"') {
+					if (compactIRI.shouldHighlight()) {
+						context.ChangeState(SCE_JSON_COMPACTIRI);
+						context.ForwardSetState(SCE_JSON_DEFAULT);
+						compactIRI.resetState();
+					} else {
+						context.ForwardSetState(SCE_JSON_DEFAULT);
+					}
+				} else if (context.atLineEnd) {
+					context.ChangeState(SCE_JSON_STRINGEOL);
+				} else if (context.ch == '\\') {
+					stringStyleBefore = context.state;
+					if (options.escapeSequence) {
+						context.SetState(SCE_JSON_ESCAPESEQUENCE);
+						if (!escapeSeq.newSequence(context.chNext)) {
+							context.SetState(SCE_JSON_ERROR);
+						}
+					}
+					context.Forward();
+				} else if (context.Match("https://") ||
+						   context.Match("http://") ||
+						   context.Match("ssh://") ||
+						   context.Match("git://") ||
+						   context.Match("svn://") ||
+						   context.Match("ftp://") ||
+						   context.Match("mailto:")) {
+					// Handle most common URI schemes only
+					stringStyleBefore = context.state;
+					context.SetState(SCE_JSON_URI);
+				} else if (context.ch == '@') {
+					// https://www.w3.org/TR/json-ld/#dfn-keyword
+					if (IsNextWordInList(keywordsJSONLD, setKeywordJSONLD, context, styler)) {
+						stringStyleBefore = context.state;
+						context.SetState(SCE_JSON_LDKEYWORD);
+					}
+				} else {
+					compactIRI.checkChar(context.ch);
+				}
+				break;
+			case SCE_JSON_LDKEYWORD:
+			case SCE_JSON_URI:
+				if ((!setKeywordJSONLD.Contains(context.ch) &&
+					 (context.state == SCE_JSON_LDKEYWORD)) ||
+					(!setURL.Contains(context.ch))) {
+					context.SetState(stringStyleBefore);
+				}
+				if (context.ch == '"') {
+					context.ForwardSetState(SCE_JSON_DEFAULT);
+				} else if (context.atLineEnd) {
+					context.ChangeState(SCE_JSON_STRINGEOL);
+				}
+				break;
+			case SCE_JSON_OPERATOR:
+			case SCE_JSON_NUMBER:
+				context.SetState(SCE_JSON_DEFAULT);
+				break;
+			case SCE_JSON_ERROR:
+				if (context.atLineEnd) {
+					context.SetState(SCE_JSON_DEFAULT);
+				}
+				break;
+			case SCE_JSON_KEYWORD:
+				if (!setKeywordJSON.Contains(context.ch)) {
+					context.SetState(SCE_JSON_DEFAULT);
+				}
+				break;
+		}
+		if (context.state == SCE_JSON_DEFAULT) {
+			if (context.ch == '"') {
+				compactIRI.resetState();
+				context.SetState(SCE_JSON_STRING);
+				Sci_Position currPos = static_cast<Sci_Position>(context.currentPos);
+				if (AtPropertyName(styler, currPos)) {
+					context.SetState(SCE_JSON_PROPERTYNAME);
+				}
+			} else if (setOperators.Contains(context.ch)) {
+				context.SetState(SCE_JSON_OPERATOR);
+			} else if (options.allowComments && context.Match("/*")) {
+				context.SetState(SCE_JSON_BLOCKCOMMENT);
+				context.Forward();
+			} else if (options.allowComments && context.Match("//")) {
+				context.SetState(SCE_JSON_LINECOMMENT);
+			} else if (setKeywordJSON.Contains(context.ch)) {
+				if (IsNextWordInList(keywordsJSON, setKeywordJSON, context, styler)) {
+					context.SetState(SCE_JSON_KEYWORD);
+				}
+			}
+			bool numberStart =
+				IsADigit(context.ch) && (context.chPrev == '+'||
+										 context.chPrev == '-' ||
+										 context.atLineStart ||
+										 IsASpace(context.chPrev) ||
+										 setOperators.Contains(context.chPrev));
+			bool exponentPart =
+				tolower(context.ch) == 'e' &&
+				IsADigit(context.chPrev) &&
+				(IsADigit(context.chNext) ||
+				 context.chNext == '+' ||
+				 context.chNext == '-');
+			bool signPart =
+				(context.ch == '-' || context.ch == '+') &&
+				((tolower(context.chPrev) == 'e' && IsADigit(context.chNext)) ||
+				 ((IsASpace(context.chPrev) || setOperators.Contains(context.chPrev))
+				  && IsADigit(context.chNext)));
+			bool adjacentDigit =
+				IsADigit(context.ch) && IsADigit(context.chPrev);
+			bool afterExponent = IsADigit(context.ch) && tolower(context.chPrev) == 'e';
+			bool dotPart = context.ch == '.' &&
+				IsADigit(context.chPrev) &&
+				IsADigit(context.chNext);
+			bool afterDot = IsADigit(context.ch) && context.chPrev == '.';
+			if (numberStart ||
+				exponentPart ||
+				signPart ||
+				adjacentDigit ||
+				dotPart ||
+				afterExponent ||
+				afterDot) {
+				context.SetState(SCE_JSON_NUMBER);
+			} else if (context.state == SCE_JSON_DEFAULT && !IsASpace(context.ch)) {
+				context.SetState(SCE_JSON_ERROR);
+			}
+		}
+		context.Forward();
+	}
+	context.Complete();
+}
+
+void SCI_METHOD LexerJSON::Fold(Sci_PositionU startPos,
+								Sci_Position length,
+								int,
+								IDocument *pAccess) {
+	if (!options.fold) {
+		return;
+	}
+	LexAccessor styler(pAccess);
+	Sci_PositionU currLine = styler.GetLine(startPos);
+	Sci_PositionU endPos = startPos + length;
+	int currLevel = styler.LevelAt(currLine) & SC_FOLDLEVELNUMBERMASK;
+	int nextLevel = currLevel;
+	int visibleChars = 0;
+	for (Sci_PositionU i = startPos; i < endPos; i++) {
+		char curr = styler.SafeGetCharAt(i);
+		char next = styler.SafeGetCharAt(i+1);
+		bool atEOL = (curr == '\r' && next != '\n') || (curr == '\n');
+		if (styler.StyleAt(i) == SCE_JSON_OPERATOR) {
+			if (curr == '{' || curr == '[') {
+				nextLevel++;
+			} else if (curr == '}' || curr == ']') {
+				nextLevel--;
+			}
+		}
+		if (atEOL || i == (endPos-1)) {
+			int level = currLevel;
+			if (!visibleChars && options.foldCompact) {
+				level |= SC_FOLDLEVELWHITEFLAG;
+			} else if (nextLevel > currLevel) {
+				level |= SC_FOLDLEVELHEADERFLAG;
+			}
+			if (level != styler.LevelAt(currLine)) {
+				styler.SetLevel(currLine, level);
+			}
+			currLine++;
+			currLevel = nextLevel;
+			visibleChars = 0;
+		}
+		if (!isspacechar(curr)) {
+			visibleChars++;
+		}
+	}
+}
+
+LexerModule lmJSON(SCLEX_JSON,
+				   LexerJSON::LexerFactoryJSON,
+				   "json",
+				   JSONWordListDesc);
diff --git a/src/Catalogue.cxx b/src/Catalogue.cxx
index ed47aa8b7..e6aa2587e 100644
--- a/src/Catalogue.cxx
+++ b/src/Catalogue.cxx
@@ -126,6 +126,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmHTML);
 	LINK_LEXER(lmIHex);
 	LINK_LEXER(lmInno);
+	LINK_LEXER(lmJSON);
 	LINK_LEXER(lmKix);
 	LINK_LEXER(lmKVIrc);
 	LINK_LEXER(lmLatex);
diff --git a/win32/scintilla.mak b/win32/scintilla.mak
index e0d16f387..ad72c51c7 100644
--- a/win32/scintilla.mak
+++ b/win32/scintilla.mak
@@ -149,6 +149,7 @@ LEXOBJS=\
 	$(DIR_O)\LexHex.obj \
 	$(DIR_O)\LexHTML.obj \
 	$(DIR_O)\LexInno.obj \
+	$(DIR_O)\LexJSON.obj \
 	$(DIR_O)\LexKix.obj \
 	$(DIR_O)\LexKVIrc.obj \
 	$(DIR_O)\LexLaTeX.obj \
@@ -581,6 +582,8 @@ $(DIR_O)\LexHTML.obj: ..\lexers\LexHTML.cxx $(LEX_HEADERS)
 
 $(DIR_O)\LexInno.obj: ..\lexers\LexInno.cxx $(LEX_HEADERS)
 
+$(DIR_O)\LexJSON.obj: ..\lexers\LexJSON.cxx $(LEX_HEADERS)
+
 $(DIR_O)\LexKix.obj: ..\lexers\LexKix.cxx $(LEX_HEADERS)
 
 $(DIR_O)\LexKVIrc.obj: ..\lexers\LexKVIrc.cxx $(LEX_HEADERS)
-- 
cgit v1.2.3