Feature: [feature-requests:#959]. Added lexer and folder for Structured Text language.

from Pavel Bulochkin

6 files changed, 460 insertions, 0 deletions

diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html
index 9b8bfa9ac..6a3174c95 100644
--- a/doc/ScintillaHistory.html
+++ b/doc/ScintillaHistory.html
@@ -423,6 +423,7 @@
       </tr><tr>
 	<td>kudah</td>
 	<td>Igor Shaula</td>
+	<td>Pavel Bulochkin</td>
     </tr>
     </table>
     <p>
@@ -435,6 +436,18 @@
       </li>
     </ul>
     <h3>
+       <a href="http://prdownloads.sourceforge.net/scintilla/scite333.zip?download">Release 3.3.3</a>
+    </h3>
+    <ul>
+	<li>
+	Released 22 December 2013.
+	</li>
+	<li>
+	Lexer and folder added for Structured Text language.
+	<a href="http://sourceforge.net/p/scintilla/feature-requests/959/">Feature #959.</a>
+	</li>
+    </ul>
+    <h3>
        <a href="http://prdownloads.sourceforge.net/scintilla/scite332.zip?download">Release 3.3.2</a>
     </h3>
     <ul>
diff --git a/include/SciLexer.h b/include/SciLexer.h
index 1ddf628bd..18cdb984c 100644
--- a/include/SciLexer.h
+++ b/include/SciLexer.h
@@ -121,6 +121,7 @@
 #define SCLEX_OSCRIPT 106
 #define SCLEX_VISUALPROLOG 107
 #define SCLEX_LITERATEHASKELL 108
+#define SCLEX_STTXT 109
 #define SCLEX_AUTOMATIC 1000
 #define SCE_P_DEFAULT 0
 #define SCE_P_COMMENTLINE 1
@@ -1627,6 +1628,25 @@
 #define SCE_VISUALPROLOG_STRING_VERBATIM 20
 #define SCE_VISUALPROLOG_STRING_VERBATIM_SPECIAL 21
 #define SCE_VISUALPROLOG_STRING_VERBATIM_EOL 22
+#define SCE_STTXT_DEFAULT 0
+#define SCE_STTXT_COMMENT 1
+#define SCE_STTXT_COMMENTLINE 2
+#define SCE_STTXT_KEYWORD 3
+#define SCE_STTXT_TYPE 4
+#define SCE_STTXT_FUNCTION 5
+#define SCE_STTXT_FB 6
+#define SCE_STTXT_NUMBER 7
+#define SCE_STTXT_HEXNUMBER 8
+#define SCE_STTXT_PRAGMA 9
+#define SCE_STTXT_OPERATOR 10
+#define SCE_STTXT_CHARACTER 11
+#define SCE_STTXT_STRING1 12
+#define SCE_STTXT_STRING2 13
+#define SCE_STTXT_STRINGEOL 14
+#define SCE_STTXT_IDENTIFIER 15
+#define SCE_STTXT_DATETIME 16
+#define SCE_STTXT_VARS 17
+#define SCE_STTXT_PRAGMAS 18
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
 #endif
diff --git a/include/Scintilla.iface b/include/Scintilla.iface
index d15050e1d..120a5433d 100644
--- a/include/Scintilla.iface
+++ b/include/Scintilla.iface
@@ -2579,6 +2579,7 @@ val SCLEX_ECL=105
 val SCLEX_OSCRIPT=106
 val SCLEX_VISUALPROLOG=107
 val SCLEX_LITERATEHASKELL=108
+val SCLEX_STTXT=109
 
 # When a lexer specifies its language as SCLEX_AUTOMATIC it receives a
 # value assigned in sequence from SCLEX_AUTOMATIC+1.
@@ -4283,6 +4284,27 @@ val SCE_VISUALPROLOG_STRING_EOL_OPEN=19
 val SCE_VISUALPROLOG_STRING_VERBATIM=20
 val SCE_VISUALPROLOG_STRING_VERBATIM_SPECIAL=21
 val SCE_VISUALPROLOG_STRING_VERBATIM_EOL=22
+# Lexical states for SCLEX_STTXT
+lex StructuredText=SCLEX_STTXT SCE_STTXT_
+val SCE_STTXT_DEFAULT=0
+val SCE_STTXT_COMMENT=1
+val SCE_STTXT_COMMENTLINE=2
+val SCE_STTXT_KEYWORD=3
+val SCE_STTXT_TYPE=4
+val SCE_STTXT_FUNCTION=5
+val SCE_STTXT_FB=6
+val SCE_STTXT_NUMBER=7
+val SCE_STTXT_HEXNUMBER=8
+val SCE_STTXT_PRAGMA=9
+val SCE_STTXT_OPERATOR=10
+val SCE_STTXT_CHARACTER=11
+val SCE_STTXT_STRING1=12
+val SCE_STTXT_STRING2=13
+val SCE_STTXT_STRINGEOL=14
+val SCE_STTXT_IDENTIFIER=15
+val SCE_STTXT_DATETIME=16
+val SCE_STTXT_VARS=17
+val SCE_STTXT_PRAGMAS=18
 
 # Events
 
diff --git a/lexers/LexSTTXT.cxx b/lexers/LexSTTXT.cxx
new file mode 100644
index 000000000..c58d054aa
--- /dev/null
+++ b/lexers/LexSTTXT.cxx
@@ -0,0 +1,401 @@
+// Scintilla source code edit control
+/** @file LexSTTXT.cxx
+ ** Lexer for Structured Text language.
+ ** Written by Pavel Bulochkin
+ **/
+// The License.txt file describes the conditions under which this software may be distributed.
+
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <ctype.h>
+
+#include "ILexer.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+
+#include "WordList.h"
+#include "LexAccessor.h"
+#include "Accessor.h"
+#include "StyleContext.h"
+#include "CharacterSet.h"
+#include "LexerModule.h"
+
+#ifdef SCI_NAMESPACE
+using namespace Scintilla;
+#endif
+
+static void ClassifySTTXTWord(WordList *keywordlists[], StyleContext &sc)
+{
+	char s[256] = { 0 };
+	sc.GetCurrentLowered(s, sizeof(s));
+
+ 	if ((*keywordlists[0]).InList(s)) {
+ 		sc.ChangeState(SCE_STTXT_KEYWORD);
+ 	}
+
+	else if ((*keywordlists[1]).InList(s)) {
+		sc.ChangeState(SCE_STTXT_TYPE);
+	}
+
+	else if ((*keywordlists[2]).InList(s)) {
+		sc.ChangeState(SCE_STTXT_FUNCTION);
+	}
+
+	else if ((*keywordlists[3]).InList(s)) {
+		sc.ChangeState(SCE_STTXT_FB);
+	}
+
+	else if ((*keywordlists[4]).InList(s)) {
+		sc.ChangeState(SCE_STTXT_VARS);
+	}
+
+	else if ((*keywordlists[5]).InList(s)) {
+		sc.ChangeState(SCE_STTXT_PRAGMAS);
+	}
+
+	sc.SetState(SCE_STTXT_DEFAULT);
+}
+
+static void ColouriseSTTXTDoc (unsigned int startPos, int length, int initStyle,
+							  WordList *keywordlists[], Accessor &styler)
+{
+	StyleContext sc(startPos, length, initStyle, styler);
+
+	CharacterSet setWord(CharacterSet::setAlphaNum, "_", 0x80, true);
+	CharacterSet setWordStart(CharacterSet::setAlpha, "_", 0x80, true);
+	CharacterSet setNumber(CharacterSet::setDigits, "_.eE");
+	CharacterSet setHexNumber(CharacterSet::setDigits, "_abcdefABCDEF");
+	CharacterSet setOperator(CharacterSet::setNone,",.+-*/:;<=>[]()%&");
+	CharacterSet setDataTime(CharacterSet::setDigits,"_.-:dmshDMSH");
+
+ 	for ( ; sc.More() ; sc.Forward())
+ 	{
+		if(sc.atLineStart && sc.state != SCE_STTXT_COMMENT)
+			sc.SetState(SCE_STTXT_DEFAULT);
+
+		switch(sc.state)
+		{
+			case SCE_STTXT_NUMBER: {
+				if(!setNumber.Contains(sc.ch))
+					sc.SetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_HEXNUMBER: {
+				if(setHexNumber.Contains(sc.ch))
+					continue;
+				else if(setDataTime.Contains(sc.ch))
+					sc.SetState(SCE_STTXT_DATETIME);
+				else
+					sc.SetState(SCE_STTXT_DEFAULT);
+
+				break;
+			}
+			case SCE_STTXT_DATETIME: {
+				if(!setDataTime.Contains(sc.ch))
+					sc.SetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_OPERATOR: {
+				sc.SetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_PRAGMA: {
+				if (sc.ch == '}')
+					sc.ForwardSetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_COMMENTLINE: {
+				if (sc.atLineStart)
+					sc.SetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_COMMENT: {
+				if(sc.Match('*',')'))
+				{
+					sc.Forward();
+					sc.ForwardSetState(SCE_STTXT_DEFAULT);
+				}
+				break;
+			}
+			case SCE_STTXT_STRING1: {
+				if(sc.atLineEnd)
+					sc.SetState(SCE_STTXT_STRINGEOL);
+				else if(sc.ch == '\'' && sc.chPrev != '$')
+					sc.ForwardSetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_STRING2: {
+				if (sc.atLineEnd)
+					sc.SetState(SCE_STTXT_STRINGEOL);
+				else if(sc.ch == '\"' && sc.chPrev != '$')
+					sc.ForwardSetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_STRINGEOL: {
+				if(sc.atLineStart)
+					sc.SetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_CHARACTER: {
+				if(setHexNumber.Contains(sc.ch))
+					sc.SetState(SCE_STTXT_HEXNUMBER);
+				else if(setDataTime.Contains(sc.ch))
+					sc.SetState(SCE_STTXT_DATETIME);
+				else sc.SetState(SCE_STTXT_DEFAULT);
+				break;
+			}
+			case SCE_STTXT_IDENTIFIER: {
+				if(!setWord.Contains(sc.ch))
+					ClassifySTTXTWord(keywordlists, sc);
+				break;
+			}
+		}
+
+		if(sc.state == SCE_STTXT_DEFAULT)
+		{
+			if(IsADigit(sc.ch))
+				sc.SetState(SCE_STTXT_NUMBER);
+			else if (setWordStart.Contains(sc.ch))
+				sc.SetState(SCE_STTXT_IDENTIFIER);
+			else if (sc.Match('/', '/'))
+				sc.SetState(SCE_STTXT_COMMENTLINE);
+			else if(sc.Match('(', '*'))
+				sc.SetState(SCE_STTXT_COMMENT);
+			else if (sc.ch == '{')
+				sc.SetState(SCE_STTXT_PRAGMA);
+			else if (sc.ch == '\'')
+				sc.SetState(SCE_STTXT_STRING1);
+			else if (sc.ch == '\"')
+				sc.SetState(SCE_STTXT_STRING2);
+			else if(sc.ch == '#')
+				sc.SetState(SCE_STTXT_CHARACTER);
+			else if (setOperator.Contains(sc.ch))
+				sc.SetState(SCE_STTXT_OPERATOR);
+		}
+ 	}
+
+	if (sc.state == SCE_STTXT_IDENTIFIER && setWord.Contains(sc.chPrev))
+		ClassifySTTXTWord(keywordlists, sc);
+
+	sc.Complete();
+}
+
+static const char * const STTXTWordListDesc[] = {
+	"Keywords",
+	"Types",
+	"Functions",
+	"FB",
+	"Local_Var",
+	"Local_Pragma",
+	0
+};
+
+static bool IsCommentLine(int line, Accessor &styler, bool type)
+{
+	int pos = styler.LineStart(line);
+	int eolPos = styler.LineStart(line + 1) - 1;
+
+	for (int i = pos; i < eolPos; i++)
+	{
+		char ch = styler[i];
+		char chNext = styler.SafeGetCharAt(i + 1);
+		int style = styler.StyleAt(i);
+
+		if(type) {
+			 if (ch == '/' && chNext == '/' && style == SCE_STTXT_COMMENTLINE)
+				return true;
+		}
+		else if (ch == '(' && chNext == '*' && style == SCE_STTXT_COMMENT)
+			break;
+
+		if (!IsASpaceOrTab(ch))
+			return false;
+	}
+
+	for (int i = eolPos-2; i>pos; i--)
+	{
+		char ch = styler[i];
+		char chPrev = styler.SafeGetCharAt(i-1);
+		int style = styler.StyleAt(i);
+
+		if(ch == ')' && chPrev == '*' && style == SCE_STTXT_COMMENT)
+			return true;
+		if(!IsASpaceOrTab(ch))
+			return false;
+	}
+
+	return false;
+}
+
+static bool IsPragmaLine(int line, Accessor &styler)
+{
+	int pos = styler.LineStart(line);
+	int eolPos = styler.LineStart(line+1) - 1;
+
+	for (int i = pos ; i < eolPos ; i++)
+	{
+		char ch = styler[i];
+		int style = styler.StyleAt(i);
+
+		if(ch == '{' && style == SCE_STTXT_PRAGMA)
+			return true;
+		else if (!IsASpaceOrTab(ch))
+			return false;
+	}
+	return false;
+}
+
+static void GetRangeUpper(unsigned int start,unsigned int end,Accessor &styler,char *s,unsigned int len)
+{
+	unsigned int i = 0;
+	while ((i < end - start + 1) && (i < len-1)) {
+		s[i] = static_cast<char>(toupper(styler[start + i]));
+		i++;
+	}
+	s[i] = '\0';
+}
+
+static void ClassifySTTXTWordFoldPoint(int &levelCurrent,unsigned int lastStart,
+									 unsigned int currentPos, Accessor &styler)
+{
+	char s[256];
+	GetRangeUpper(lastStart, currentPos, styler, s, sizeof(s));
+
+	// See Table C.2 - Keywords
+	if (!strcmp(s, "ACTION") ||
+		!strcmp(s, "CASE") ||
+		!strcmp(s, "CONFIGURATION") ||
+		!strcmp(s, "FOR") ||
+		!strcmp(s, "FUNCTION") ||
+		!strcmp(s, "FUNCTION_BLOCK") ||
+		!strcmp(s, "IF") ||
+		!strcmp(s, "INITIAL_STEP") ||
+		!strcmp(s, "REPEAT") ||
+		!strcmp(s, "RESOURCE") ||
+		!strcmp(s, "STEP") ||
+		!strcmp(s, "STRUCT") ||
+		!strcmp(s, "TRANSITION") ||
+		!strcmp(s, "TYPE") ||
+		!strcmp(s, "VAR") ||
+		!strcmp(s, "VAR_INPUT") ||
+		!strcmp(s, "VAR_OUTPUT") ||
+		!strcmp(s, "VAR_IN_OUT") ||
+		!strcmp(s, "VAR_TEMP") ||
+		!strcmp(s, "VAR_EXTERNAL") ||
+		!strcmp(s, "VAR_ACCESS") ||
+		!strcmp(s, "VAR_CONFIG") ||
+		!strcmp(s, "VAR_GLOBAL") ||
+		!strcmp(s, "WHILE"))
+	{
+		levelCurrent++;
+	}
+	else if (!strcmp(s, "END_ACTION") ||
+		!strcmp(s, "END_CASE") ||
+		!strcmp(s, "END_CONFIGURATION") ||
+		!strcmp(s, "END_FOR") ||
+		!strcmp(s, "END_FUNCTION") ||
+		!strcmp(s, "END_FUNCTION_BLOCK") ||
+		!strcmp(s, "END_IF") ||
+		!strcmp(s, "END_REPEAT") ||
+		!strcmp(s, "END_RESOURCE") ||
+		!strcmp(s, "END_STEP") ||
+		!strcmp(s, "END_STRUCT") ||
+		!strcmp(s, "END_TRANSITION") ||
+		!strcmp(s, "END_TYPE") ||
+		!strcmp(s, "END_VAR") ||
+		!strcmp(s, "END_WHILE"))
+	{
+		levelCurrent--;
+		if (levelCurrent < SC_FOLDLEVELBASE) {
+			levelCurrent = SC_FOLDLEVELBASE;
+		}
+	}
+}
+
+static void FoldSTTXTDoc(unsigned int startPos, int length, int initStyle, WordList *[],Accessor &styler)
+{
+	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
+	bool foldPreprocessor = styler.GetPropertyInt("fold.preprocessor") != 0;
+	bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
+	unsigned int endPos = startPos + length;
+	int visibleChars = 0;
+	int lineCurrent = styler.GetLine(startPos);
+	int levelPrev = styler.LevelAt(lineCurrent) & SC_FOLDLEVELNUMBERMASK;
+	int levelCurrent = levelPrev;
+	char chNext = styler[startPos];
+	int styleNext = styler.StyleAt(startPos);
+	int style = initStyle;
+	int lastStart = 0;
+
+	CharacterSet setWord(CharacterSet::setAlphaNum, "_", 0x80, true);
+
+	for (unsigned int i = startPos; i < endPos; i++)
+	{
+		char ch = chNext;
+		chNext = styler.SafeGetCharAt(i + 1);
+		int stylePrev = style;
+		style = styleNext;
+		styleNext = styler.StyleAt(i + 1);
+		bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
+
+		if (foldComment && style == SCE_STTXT_COMMENT) {
+			if(stylePrev != SCE_STTXT_COMMENT)
+				levelCurrent++;
+			else if(styleNext != SCE_STTXT_COMMENT && !atEOL)
+				levelCurrent--;
+		}
+		if ( foldComment && atEOL && ( IsCommentLine(lineCurrent, styler,false)
+			|| IsCommentLine(lineCurrent,styler,true))) {
+ 			if(!IsCommentLine(lineCurrent-1, styler,true) && IsCommentLine(lineCurrent+1, styler,true))
+				levelCurrent++;
+			if (IsCommentLine(lineCurrent-1, styler,true) && !IsCommentLine(lineCurrent+1, styler,true))
+				levelCurrent--;
+			if (!IsCommentLine(lineCurrent-1, styler,false) && IsCommentLine(lineCurrent+1, styler,false))
+				levelCurrent++;
+			if (IsCommentLine(lineCurrent-1, styler,false) && !IsCommentLine(lineCurrent+1, styler,false))
+				levelCurrent--;
+		}
+		if(foldPreprocessor && atEOL && IsPragmaLine(lineCurrent, styler)) {
+			if(!IsPragmaLine(lineCurrent-1, styler) && IsPragmaLine(lineCurrent+1, styler ))
+				levelCurrent++;
+			else if(IsPragmaLine(lineCurrent-1, styler) && !IsPragmaLine(lineCurrent+1, styler))
+				levelCurrent--;
+		}
+		if (stylePrev != SCE_STTXT_KEYWORD && style == SCE_STTXT_KEYWORD) {
+				lastStart = i;
+		}
+		if(stylePrev == SCE_STTXT_KEYWORD) {
+			if(setWord.Contains(ch) && !setWord.Contains(chNext))
+				ClassifySTTXTWordFoldPoint(levelCurrent,lastStart, i, styler);
+		}
+		if (!IsASpace(ch)) {
+			visibleChars++;
+		}
+		if (atEOL) {
+			int lev = levelPrev;
+			if (visibleChars == 0 && foldCompact)
+				lev |= SC_FOLDLEVELWHITEFLAG;
+			if ((levelCurrent > levelPrev) && (visibleChars > 0))
+				lev |= SC_FOLDLEVELHEADERFLAG;
+			if (lev != styler.LevelAt(lineCurrent))
+				styler.SetLevel(lineCurrent, lev);
+
+			lineCurrent++;
+			levelPrev = levelCurrent;
+			visibleChars = 0;
+		}
+
+		// If we didn't reach the EOL in previous loop, store line level and whitespace information.
+		// The rest will be filled in later...
+		int lev = levelPrev;
+		if (visibleChars == 0 && foldCompact)
+			lev |= SC_FOLDLEVELWHITEFLAG;
+		styler.SetLevel(lineCurrent, lev);
+	}
+}
+
+LexerModule lmSTTXT(SCLEX_STTXT, ColouriseSTTXTDoc, "fcST", FoldSTTXTDoc, STTXTWordListDesc);
diff --git a/src/Catalogue.cxx b/src/Catalogue.cxx
index 0ca6e67a5..84d003e08 100644
--- a/src/Catalogue.cxx
+++ b/src/Catalogue.cxx
@@ -172,6 +172,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmSpecman);
 	LINK_LEXER(lmSpice);
 	LINK_LEXER(lmSQL);
+	LINK_LEXER(lmSTTXT);
 	LINK_LEXER(lmTACL);
 	LINK_LEXER(lmTADS3);
 	LINK_LEXER(lmTAL);
diff --git a/win32/scintilla.mak b/win32/scintilla.mak
index a943609b1..708aac07d 100644
--- a/win32/scintilla.mak
+++ b/win32/scintilla.mak
@@ -164,6 +164,7 @@ LEXOBJS=\
 	$(DIR_O)\LexSpecman.obj \
 	$(DIR_O)\LexSpice.obj \
 	$(DIR_O)\LexSQL.obj \
+	$(DIR_O)\LexSTTXT.obj \
 	$(DIR_O)\LexTACL.obj \
 	$(DIR_O)\LexTADS3.obj \
 	$(DIR_O)\LexTAL.obj \
@@ -446,6 +447,8 @@ $(DIR_O)\LexSpice.obj: ..\lexers\LexSpice.cxx $(LEX_HEADERS)
 
 $(DIR_O)\LexSQL.obj: ..\lexers\LexSQL.cxx $(LEX_HEADERS)
 
+$(DIR_O)\LexSTTXT.obj: ..\lexers\LexSTTXT.cxx $(LEX_HEADERS)
+
 $(DIR_O)\LexTACL.obj: ..\lexers\LexTACL.cxx $(LEX_HEADERS)
 
 $(DIR_O)\LexTADS3.obj: ..\lexers\LexTADS3.cxx $(LEX_HEADERS)