Backport: Feature [feature-requests:#1265]. Common Intermediate Language (CIL) lexer.

Backport of changeset 7283:aaeca7f7cf9d.

7 files changed, 442 insertions, 0 deletions

diff --git a/cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj b/cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj
index fc816550e..b5c04018a 100644
--- a/cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj
+++ b/cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj
@@ -230,6 +230,7 @@
 		FDC7442CAD70B9A67EF1639D /* LexSAS.cxx in Sources */ = {isa = PBXBuildFile; fileRef = A95147A1AB7CADB00DAFE724 /* LexSAS.cxx */; };
 		5F804AA6B60FE695863A39FE /* LexStata.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 7623427695416AB1270EE023 /* LexStata.cxx */; };
 		0ED84236A703D57578EBFD2F /* LexNim.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 47814937A6B72D2B0F065B61 /* LexNim.cxx */; }; 
+		00724A59981D34F11A3D162F /* LexCIL.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 577F46B88F633198B56D088D /* LexCIL.cxx */; };
 		4D0C4365AB6DF998CD48B1FC /* LexLPeg.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 5EB3467789767C0ACE40A46A /* LexLPeg.cxx */; }; */; };
 /* End PBXBuildFile section */
 
@@ -464,6 +465,7 @@
 		A95147A1AB7CADB00DAFE724 /* LexSAS.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexSAS.cxx; path = ../../lexers/LexSAS.cxx; sourceTree = SOURCE_ROOT; };
 		7623427695416AB1270EE023 /* LexStata.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexStata.cxx; path = ../../lexers/LexStata.cxx; sourceTree = SOURCE_ROOT; };
 		47814937A6B72D2B0F065B61 /* LexNim.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexNim.cxx; path = ../../lexers/LexNim.cxx; sourceTree = SOURCE_ROOT; };
+		577F46B88F633198B56D088D /* LexCIL.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexCIL.cxx; path = ../../lexers/LexCIL.cxx; sourceTree = SOURCE_ROOT; };
 		5EB3467789767C0ACE40A46A /* LexLPeg.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexLPeg.cxx; path = ../../lexers/LexLPeg.cxx; sourceTree = SOURCE_ROOT; };
 /* End PBXFileReference section */
 
@@ -579,6 +581,7 @@
 				28FDA42019B6967B00BE27D7 /* LexBibTeX.cxx */,
 				114B6EC911FA7526004FB6AB /* LexBullant.cxx */,
 				114B6ECA11FA7526004FB6AB /* LexCaml.cxx */,
+				577F46B88F633198B56D088D /* LexCIL.cxx */,
 				114B6ECB11FA7526004FB6AB /* LexCLW.cxx */,
 				114B6ECC11FA7526004FB6AB /* LexCmake.cxx */,
 				114B6ECD11FA7526004FB6AB /* LexCOBOL.cxx */,
@@ -1128,6 +1131,7 @@
 				FDC7442CAD70B9A67EF1639D /* LexSAS.cxx in Sources */,
 				5F804AA6B60FE695863A39FE /* LexStata.cxx in Sources */,
 				0ED84236A703D57578EBFD2F /* LexNim.cxx in Sources */,
+				00724A59981D34F11A3D162F /* LexCIL.cxx in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html
index 239280a85..22398797d 100644
--- a/doc/ScintillaHistory.html
+++ b/doc/ScintillaHistory.html
@@ -558,6 +558,10 @@
 	<a href="https://sourceforge.net/p/scintilla/feature-requests/1217/">Feature #1217</a>.
 	</li>
  	<li>
+	Lexer added for .NET's Common Intermediate Language CIL.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1265/">Feature #1265</a>.
+	</li>
+ 	<li>
 	The C++ lexer, with styling.within.preprocessor on, now interprets "(" in preprocessor "#if("
 	as an operator instead of part of the directive. This improves folding as well which could become
 	unbalanced.
diff --git a/include/SciLexer.h b/include/SciLexer.h
index 4613e48de..ae20985ce 100644
--- a/include/SciLexer.h
+++ b/include/SciLexer.h
@@ -139,6 +139,7 @@
 #define SCLEX_STATA 124
 #define SCLEX_SAS 125
 #define SCLEX_NIM 126
+#define SCLEX_CIL 127
 #define SCLEX_LPEG 999
 #define SCLEX_AUTOMATIC 1000
 #define SCE_P_DEFAULT 0
@@ -1874,6 +1875,17 @@
 #define SCE_NIM_NUMERROR 14
 #define SCE_NIM_OPERATOR 15
 #define SCE_NIM_IDENTIFIER 16
+#define SCE_CIL_DEFAULT 0
+#define SCE_CIL_COMMENT 1
+#define SCE_CIL_COMMENTLINE 2
+#define SCE_CIL_WORD 3
+#define SCE_CIL_WORD2 4
+#define SCE_CIL_WORD3 5
+#define SCE_CIL_STRING 6
+#define SCE_CIL_LABEL 7
+#define SCE_CIL_OPERATOR 8
+#define SCE_CIL_IDENTIFIER 9
+#define SCE_CIL_STRINGEOL 10
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
 #endif
diff --git a/include/Scintilla.iface b/include/Scintilla.iface
index 8efe13647..a26f7057d 100644
--- a/include/Scintilla.iface
+++ b/include/Scintilla.iface
@@ -2959,6 +2959,7 @@ val SCLEX_MAXIMA=123
 val SCLEX_STATA=124
 val SCLEX_SAS=125
 val SCLEX_NIM=126
+val SCLEX_CIL=127
 val SCLEX_LPEG=999
 
 # When a lexer specifies its language as SCLEX_AUTOMATIC it receives a
@@ -4932,6 +4933,19 @@ val SCE_NIM_STRINGEOL=13
 val SCE_NIM_NUMERROR=14
 val SCE_NIM_OPERATOR=15
 val SCE_NIM_IDENTIFIER=16
+# Lexical states for SCLEX_CIL
+lex CIL=SCLEX_CIL SCE_CIL_
+val SCE_CIL_DEFAULT=0
+val SCE_CIL_COMMENT=1
+val SCE_CIL_COMMENTLINE=2
+val SCE_CIL_WORD=3
+val SCE_CIL_WORD2=4
+val SCE_CIL_WORD3=5
+val SCE_CIL_STRING=6
+val SCE_CIL_LABEL=7
+val SCE_CIL_OPERATOR=8
+val SCE_CIL_IDENTIFIER=9
+val SCE_CIL_STRINGEOL=10
 
 # Events
 
diff --git a/lexers/LexCIL.cxx b/lexers/LexCIL.cxx
new file mode 100644
index 000000000..994d0a023
--- /dev/null
+++ b/lexers/LexCIL.cxx
@@ -0,0 +1,404 @@
+// Scintilla source code edit control
+/** @file LexCIL.cxx
+ ** Lexer for Common Intermediate Language
+ ** Written by Jad Altahan (github.com/xv)
+ ** CIL manual: https://www.ecma-international.org/publications/standards/Ecma-335.htm
+ **/
+// Copyright 1998-2001 by Neil Hodgson <neilh@scintilla.org>
+// The License.txt file describes the conditions under which this software may be distributed.
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <ctype.h>
+
+#include <string>
+#include <map>
+#include <algorithm>
+
+#include "ILexer.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+
+#include "StringCopy.h"
+#include "WordList.h"
+#include "LexAccessor.h"
+#include "Accessor.h"
+#include "StyleContext.h"
+#include "CharacterSet.h"
+#include "CharacterCategory.h"
+#include "LexerModule.h"
+#include "OptionSet.h"
+#include "DefaultLexer.h"
+
+using namespace Scintilla;
+
+namespace {
+    // Use an unnamed namespace to protect the functions and classes from name conflicts
+
+bool IsAWordChar(const int ch) {
+    return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '.');
+}
+
+bool IsOperator(const int ch) {
+    if ((ch < 0x80) && (isalnum(ch)))
+        return false;
+
+    if (strchr("!%&*+-/<=>@^|~()[]{}", ch)) {
+        return true;
+    }
+
+    return false;
+}
+
+constexpr bool IsStreamCommentStyle(const int style) noexcept {
+    return style == SCE_CIL_COMMENT;
+}
+
+struct OptionsCIL {
+    bool fold;
+    bool foldComment;
+    bool foldCommentMultiline;
+    bool foldCompact;
+
+    OptionsCIL() {
+        fold = true;
+        foldComment = false;
+        foldCommentMultiline = true;
+        foldCompact = true;
+    }
+};
+
+static const char *const cilWordListDesc[] = {
+    "Primary CIL keywords",
+    "Metadata",
+    "Opcode instructions",
+    0
+};
+
+struct OptionSetCIL : public OptionSet<OptionsCIL> {
+    OptionSetCIL() {
+        DefineProperty("fold", &OptionsCIL::fold);
+        DefineProperty("fold.comment", &OptionsCIL::foldComment);
+
+        DefineProperty("fold.cil.comment.multiline", &OptionsCIL::foldCommentMultiline,
+            "Set this property to 0 to disable folding multi-line comments when fold.comment=1.");
+
+        DefineProperty("fold.compact", &OptionsCIL::foldCompact);
+
+        DefineWordListSets(cilWordListDesc);
+    }
+};
+
+LexicalClass lexicalClasses[] = {
+    // Lexer CIL SCLEX_CIL SCE_CIL_:
+    0,  "SCE_CIL_DEFAULT",     "default",              "White space",
+    1,  "SCE_CIL_COMMENT",     "comment",              "Multi-line comment",
+    2,  "SCE_CIL_COMMENTLINE", "comment line",         "Line comment",
+    3,  "SCE_CIL_WORD",        "keyword",              "Keyword 1",
+    4,  "SCE_CIL_WORD2",       "keyword",              "Keyword 2",
+    5,  "SCE_CIL_WORD3",       "keyword",              "Keyword 3",
+    6,  "SCE_CIL_STRING",      "literal string",       "Double quoted string",
+    7,  "SCE_CIL_LABEL",       "label",                "Code label",
+    8,  "SCE_CIL_OPERATOR",    "operator",             "Operators",
+    9,  "SCE_CIL_STRINGEOL",   "error literal string", "String is not closed",
+    10, "SCE_CIL_IDENTIFIER",  "identifier",           "Identifiers",
+};
+
+}
+
+class LexerCIL : public DefaultLexer {
+    WordList keywords, keywords2, keywords3;
+    OptionsCIL options;
+    OptionSetCIL osCIL;
+
+public:
+    LexerCIL() : DefaultLexer(lexicalClasses, ELEMENTS(lexicalClasses)) { }
+
+    virtual ~LexerCIL() { }
+
+    void SCI_METHOD Release() override {
+        delete this;
+    }
+
+    int SCI_METHOD Version() const override {
+        return lvMetaData;
+    }
+
+    const char * SCI_METHOD PropertyNames() override {
+        return osCIL.PropertyNames();
+    }
+
+    int SCI_METHOD PropertyType(const char *name) override {
+        return osCIL.PropertyType(name);
+    }
+
+    const char * SCI_METHOD DescribeProperty(const char *name) override {
+        return osCIL.DescribeProperty(name);
+    }
+
+    Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override;
+
+    const char * SCI_METHOD DescribeWordListSets() override {
+        return osCIL.DescribeWordListSets();
+    }
+
+    Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override;
+
+    void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
+    void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
+
+    void * SCI_METHOD PrivateCall(int, void *) override {
+        return 0;
+    }
+
+    int SCI_METHOD LineEndTypesSupported() override {
+        return SC_LINE_END_TYPE_UNICODE;
+    }
+
+    int SCI_METHOD PrimaryStyleFromStyle(int style) override {
+        return style;
+    }
+
+    static ILexer *LexerFactoryCIL() {
+        return new LexerCIL();
+    }
+};
+
+Sci_Position SCI_METHOD LexerCIL::PropertySet(const char *key, const char *val) {
+    if (osCIL.PropertySet(&options, key, val)) {
+        return 0;
+    }
+
+    return -1;
+}
+
+Sci_Position SCI_METHOD LexerCIL::WordListSet(int n, const char *wl) {
+    WordList *wordListN = 0;
+
+    switch (n) {
+        case 0:
+            wordListN = &keywords;
+            break;
+        case 1:
+            wordListN = &keywords2;
+            break;
+        case 2:
+            wordListN = &keywords3;
+            break;
+    }
+
+    Sci_Position firstModification = -1;
+
+    if (wordListN) {
+        WordList wlNew;
+        wlNew.Set(wl);
+
+        if (*wordListN != wlNew) {
+            wordListN->Set(wl);
+            firstModification = 0;
+        }
+    }
+
+    return firstModification;
+}
+
+void SCI_METHOD LexerCIL::Lex(Sci_PositionU startPos, Sci_Position length,
+                              int initStyle, IDocument *pAccess) {
+    if (initStyle == SCE_CIL_STRINGEOL) {
+        initStyle = SCE_CIL_DEFAULT;
+    }
+
+    Accessor styler(pAccess, NULL);
+    StyleContext sc(startPos, length, initStyle, styler);
+
+    bool identAtLineStart = false, // Checks if an identifier is at line start (ignoring spaces)
+         canStyleLabels = false;   // Checks if conditions are met to style SCE_CIL_LABEL
+
+    for (; sc.More(); sc.Forward()) {
+        if (sc.atLineStart) {
+            if (sc.state == SCE_CIL_STRING) {
+                sc.SetState(SCE_CIL_STRING);
+            }
+
+            identAtLineStart = true;
+        }
+
+        // Handle string line continuation
+        if (sc.ch == '\\' && (sc.chNext == '\n' || sc.chNext == '\r') &&
+           (sc.state == SCE_CIL_STRING)) {
+            sc.Forward();
+
+            if (sc.ch == '\r' && sc.chNext == '\n') {
+                sc.Forward();
+            }
+
+            continue;
+        }
+
+        switch (sc.state) {
+            case SCE_CIL_OPERATOR:
+                sc.SetState(SCE_CIL_DEFAULT);
+                break;
+            case SCE_CIL_IDENTIFIER:
+                if (!IsAWordChar(sc.ch)) {
+                    if (canStyleLabels && (sc.ch == ':' && sc.chNext != ':')) {
+                        sc.ChangeState(SCE_CIL_LABEL);
+                        sc.ForwardSetState(SCE_CIL_DEFAULT);
+                    } else {
+                        char kwSize[100];
+                        sc.GetCurrent(kwSize, sizeof(kwSize));
+                        int style = SCE_CIL_IDENTIFIER;
+
+                        if (keywords.InList(kwSize)) {
+                            style = SCE_CIL_WORD;
+                        } else if (keywords2.InList(kwSize)) {
+                            style = SCE_CIL_WORD2;
+                        } else if (keywords3.InList(kwSize)) {
+                            style = SCE_CIL_WORD3;
+                        }
+
+                        sc.ChangeState(style);
+                        sc.SetState(SCE_CIL_DEFAULT);
+                    }
+                }
+                break;
+            case SCE_CIL_COMMENT:
+                if (sc.Match('*', '/')) {
+                    sc.Forward();
+                    sc.ForwardSetState(SCE_CIL_DEFAULT);
+                }
+                break;
+            case SCE_CIL_COMMENTLINE:
+                if (sc.atLineStart) {
+                    sc.SetState(SCE_CIL_DEFAULT);
+                }
+                break;
+            case SCE_CIL_STRING:
+                if (sc.ch == '\\') {
+                    if (sc.chNext == '"' || sc.chNext == '\\') {
+                        sc.Forward();
+                    }
+                } else if (sc.ch == '"') {
+                    sc.ForwardSetState(SCE_CIL_DEFAULT);
+                } else if (sc.atLineEnd) {
+                    sc.ChangeState(SCE_CIL_STRINGEOL);
+                    sc.ForwardSetState(SCE_CIL_DEFAULT);
+                }
+                break;
+        }
+
+        if (sc.state == SCE_CIL_DEFAULT) {
+            // String
+            if (sc.ch == '"') {
+                sc.SetState(SCE_CIL_STRING);
+            }
+            // Keyword
+            else if (IsAWordChar(sc.ch)) {
+                // Allow setting SCE_CIL_LABEL style only if the label is the
+                // first token in the line and does not start with a dot or a digit
+                canStyleLabels = identAtLineStart && !(sc.ch == '.' || IsADigit(sc.ch));
+                sc.SetState(SCE_CIL_IDENTIFIER);
+            }
+            // Multi-line comment
+            else if (sc.Match('/', '*')) {
+                sc.SetState(SCE_CIL_COMMENT);
+                sc.Forward();
+            }
+            // Line comment
+            else if (sc.Match('/', '/')) {
+                sc.SetState(SCE_CIL_COMMENTLINE);
+            }
+            // Operators
+            else if (IsOperator(sc.ch)) {
+                sc.SetState(SCE_CIL_OPERATOR);
+            }
+        }
+
+        if (!IsASpace(sc.ch)) {
+            identAtLineStart = false;
+        }
+    }
+
+    sc.Complete();
+}
+
+void SCI_METHOD LexerCIL::Fold(Sci_PositionU startPos, Sci_Position length, 
+                               int initStyle, IDocument *pAccess) {
+    if (!options.fold) {
+        return;
+    }
+
+    LexAccessor styler(pAccess);
+
+    const Sci_PositionU endPos = startPos + length;
+    Sci_Position lineCurrent = styler.GetLine(startPos);
+
+    int levelCurrent = SC_FOLDLEVELBASE;
+    if (lineCurrent > 0)
+        levelCurrent = styler.LevelAt(lineCurrent - 1) >> 16;
+    
+    int style = initStyle;
+    int styleNext = styler.StyleAt(startPos);
+    int levelNext = levelCurrent;
+    int visibleChars = 0;
+
+    char chNext = styler[startPos];
+
+    for (Sci_PositionU i = startPos; i < endPos; i++) {
+        const char ch = chNext;
+        int stylePrev = style;
+
+        chNext = styler.SafeGetCharAt(i + 1);
+        style = styleNext;
+        styleNext = styler.StyleAt(i + 1);
+
+        const bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
+
+        if (options.foldComment && 
+            options.foldCommentMultiline && IsStreamCommentStyle(style)) {
+            if (!IsStreamCommentStyle(stylePrev)) {
+                levelNext++;
+            } else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
+                levelNext--;
+            }
+        }
+
+        if (style == SCE_CIL_OPERATOR) {
+            if (ch == '{') {
+                levelNext++;
+            } else if (ch == '}') {
+                levelNext--;
+            }
+        }
+
+        if (!IsASpace(ch)) {
+            visibleChars++;
+        }
+
+        if (atEOL || (i == endPos - 1)) {
+            int lev = levelCurrent | levelNext << 16;
+            if (visibleChars == 0 && options.foldCompact)
+                lev |= SC_FOLDLEVELWHITEFLAG;
+            if (levelCurrent < levelNext)
+                lev |= SC_FOLDLEVELHEADERFLAG;
+            if (lev != styler.LevelAt(lineCurrent)) {
+                styler.SetLevel(lineCurrent, lev);
+            }
+            
+            lineCurrent++;
+            levelCurrent = levelNext;
+            
+            if (options.foldCompact &&
+                i == static_cast<Sci_PositionU>(styler.Length() - 1)) {
+                styler.SetLevel(lineCurrent, lev | SC_FOLDLEVELWHITEFLAG);
+            }
+
+            visibleChars = 0;
+        }
+    }
+}
+
+LexerModule lmCIL(SCLEX_CIL, LexerCIL::LexerFactoryCIL, "cil", cilWordListDesc);
\ No newline at end of file
diff --git a/src/Catalogue.cxx b/src/Catalogue.cxx
index b7b722a51..6b70a9225 100644
--- a/src/Catalogue.cxx
+++ b/src/Catalogue.cxx
@@ -90,6 +90,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmBlitzBasic);
 	LINK_LEXER(lmBullant);
 	LINK_LEXER(lmCaml);
+	LINK_LEXER(lmCIL);
 	LINK_LEXER(lmClw);
 	LINK_LEXER(lmClwNoCase);
 	LINK_LEXER(lmCmake);
diff --git a/win32/scintilla.mak b/win32/scintilla.mak
index de4256b56..0517c4055 100644
--- a/win32/scintilla.mak
+++ b/win32/scintilla.mak
@@ -120,6 +120,7 @@ LEXOBJS=\
 	$(DIR_O)\LexBibTeX.obj \
 	$(DIR_O)\LexBullant.obj \
 	$(DIR_O)\LexCaml.obj \
+	$(DIR_O)\LexCIL.obj \
 	$(DIR_O)\LexCLW.obj \
 	$(DIR_O)\LexCmake.obj \
 	$(DIR_O)\LexCOBOL.obj \
@@ -564,6 +565,8 @@ $(DIR_O)\LexBullant.obj: ..\lexers\LexBullant.cxx $(LEX_HEADERS)
 
 $(DIR_O)\LexCaml.obj: ..\lexers\LexCaml.cxx $(LEX_HEADERS)
 
+$(DIR_O)\LexCIL.obj: ..\lexers\LexCIL.cxx $(LEX_HEADERS)
+
 $(DIR_O)\LexCLW.obj: ..\lexers\LexCLW.cxx $(LEX_HEADERS)
 
 $(DIR_O)\LexCmake.obj: ..\lexers\LexCmake.cxx $(LEX_HEADERS)