Implement APIs for Unicode line ends and substyles and expose to

message interface.
Expose IDocumentWithLineEnd interface on Document.

6 files changed, 220 insertions, 9 deletions

diff --git a/doc/ScintillaDoc.html b/doc/ScintillaDoc.html
index 1e5352506..436a931f2 100644
--- a/doc/ScintillaDoc.html
+++ b/doc/ScintillaDoc.html
@@ -2176,17 +2176,29 @@ struct Sci_TextToFind {
 
     <h2 id="LineEndings">Line endings</h2>
 
-    <p>Scintilla can interpret any of the three major line end conventions, Macintosh (\r), Unix
-    (\n) and CP/M / DOS / Windows (\r\n). When the user presses the Enter key, one of these line
+    <p>Scintilla can handle the major line end conventions and, depending on settings and
+    the current lexer also support additional Unicode line ends.</p>
+    
+    <p>Scintilla can interpret any of the Macintosh (\r), Unix (\n) and Windows (\r\n)
+    line ends.
+    When the user presses the Enter key, one of these line
     end strings is inserted into the buffer. The default is \r\n in Windows and \n in Unix, but
     this can be changed with the <code>SCI_SETEOLMODE</code> message. You can also convert the
     entire document to one of these line endings with <code>SCI_CONVERTEOLS</code>. Finally, you
     can choose to display the line endings with <code>SCI_SETVIEWEOL</code>.</p>
+
+    <p>For the UTF-8 encoding, three additional Unicode line ends,
+    Next Line (<code>NEL=U+0085</code>), Line Separator (<code>LS=U+2028</code>), and Paragraph Separator (<code>PS=U+2029</code>)
+    may optionally be interpreted when Unicode line ends is turned on and the current lexer also supports
+    Unicode line ends.</p>
+
     <code><a class="message" href="#SCI_SETEOLMODE">SCI_SETEOLMODE(int eolMode)</a><br />
      <a class="message" href="#SCI_GETEOLMODE">SCI_GETEOLMODE</a><br />
      <a class="message" href="#SCI_CONVERTEOLS">SCI_CONVERTEOLS(int eolMode)</a><br />
      <a class="message" href="#SCI_SETVIEWEOL">SCI_SETVIEWEOL(bool visible)</a><br />
      <a class="message" href="#SCI_GETVIEWEOL">SCI_GETVIEWEOL</a><br />
+
+     <a class="message" href="#SCI_GETLINEENDTYPESSUPPORTED">SCI_GETLINEENDTYPESSUPPORTED</a><br />
     </code>
 
     <p><b id="SCI_SETEOLMODE">SCI_SETEOLMODE(int eolMode)</b><br />
@@ -2209,6 +2221,12 @@ struct Sci_TextToFind {
     <code>(CR)</code>, <code>(LF)</code>, or <code>(CR)(LF)</code>. <code>SCI_GETVIEWEOL</code>
     returns the current state.</p>
 
+    <p><b id="SCI_GETLINEENDTYPESSUPPORTED">SCI_GETLINEENDTYPESSUPPORTED</b><br />
+     <code>SCI_GETLINEENDTYPESSUPPORTED</code> reports the different types of line ends supported
+     by the current lexer. This is a bit set although there is currently only a single choice
+     with either <code>SC_LINE_END_TYPE_DEFAULT</code> (0) or <code>SC_LINE_END_TYPE_UNICODE</code> (1).
+     These values are also used by the other messages concerned with Unicode line ends.</p>
+
     <h2 id="Styling">Styling</h2>
 
     <p>The styling messages allow you to assign styles to text. The standard Scintilla settings
@@ -5638,8 +5656,16 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){
      <a class="message" href="#SCI_DESCRIBEKEYWORDSETS">SCI_DESCRIBEKEYWORDSETS(&lt;unused&gt;, char *descriptions)</a><br />
      <a class="message" href="#SCI_SETKEYWORDS">SCI_SETKEYWORDS(int keyWordSet, const char
     *keyWordList)</a><br />
-     <a class="message" href="#SCI_GETSTYLEBITSNEEDED">SCI_GETSTYLEBITSNEEDED</a>
-    <br />
+     <a class="message" href="#SCI_GETSTYLEBITSNEEDED">SCI_GETSTYLEBITSNEEDED</a><br />
+
+     <a class="message" href="#SCI_GETSUBSTYLEBASES">SCI_GETSUBSTYLEBASES(&lt;unused&gt;, char *styles)</a><br />
+     <a class="message" href="#SCI_DISTANCETOSECONDARYSTYLES">SCI_DISTANCETOSECONDARYSTYLES</a><br />
+     <a class="message" href="#SCI_ALLOCATESUBSTYLES">SCI_ALLOCATESUBSTYLES(int styleBase, int numberStyles)</a><br />
+     <a class="message" href="#SCI_FREESUBSTYLES">SCI_FREESUBSTYLES</a><br />
+     <a class="message" href="#SCI_GETSUBSTYLESSTART">SCI_GETSUBSTYLESSTART(int styleBase)</a><br />
+     <a class="message" href="#SCI_GETSUBSTYLESLENGTH">SCI_GETSUBSTYLESLENGTH(int styleBase)</a><br />
+     <a class="message" href="#SCI_SETIDENTIFIERS">SCI_SETIDENTIFIERS(int style, const char *identifiers)</a><br />
+
     </code>
 
     <p><b id="SCI_SETLEXER">SCI_SETLEXER(int lexer)</b><br />
@@ -5785,6 +5811,34 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){
      to <a class="message" href="#SCI_SETSTYLEBITS">SCI_SETSTYLEBITS</a>.
      </p>
 
+    <h3 id="Substyles">Substyles</h3>
+    <p>Lexers may support several different sublanguages and each sublanguage may want to style some number of
+    sets of identifiers (or similar lexemes such as documentation keywords) uniquely. Preallocating a large number for each
+    purpose would exhaust the number of allowed styles quickly.
+    This is alleviated by substyles which allow the application to determine how many sets of identifiers to allocate for
+    each purpose.
+    Lexers have to explicitly support this feature by implementing the methods in <code>ILexerWithSubStyles</code>.</p>
+
+    <p><b id="SCI_GETSUBSTYLEBASES">SCI_GETSUBSTYLEBASES(&lt;unused&gt;, char *styles)</b><br />
+     Fill <code>styles</code> with a byte for each style that can be split into substyles.</p>
+
+    <p><b id="SCI_DISTANCETOSECONDARYSTYLES">SCI_DISTANCETOSECONDARYSTYLES</b><br />
+     Returns the distance between a primary style and its corresponding secondary style.</p>
+
+    <p><b id="SCI_ALLOCATESUBSTYLES">SCI_ALLOCATESUBSTYLES(int styleBase, int numberStyles)</b><br />
+     Allocate some number of substyles for a particular base style returning the first substyle number allocated.
+     Substyles are allocated contiguously.</p>
+
+    <p><b id="SCI_FREESUBSTYLES">SCI_FREESUBSTYLES</b><br />
+     Free all allocated substyles.</p>
+
+    <p><b id="SCI_GETSUBSTYLESSTART">SCI_GETSUBSTYLESSTART(int styleBase)</b><br />
+    <b id="SCI_GETSUBSTYLESLENGTH">SCI_GETSUBSTYLESLENGTH(int styleBase)</b><br />
+     Return the start and length of the substyles allocated for a base style.</p>
+
+    <p><b id="SCI_SETIDENTIFIERS">SCI_SETIDENTIFIERS(int style, const char *identifiers)</b><br />
+     Similar to <code>SCI_SETKEYWORDS</code> but for substyles.</p>
+
     <h2 id="LexerObjects">Lexer Objects</h2>
 
     <p>Lexers are programmed as objects that implement the ILexer interface and that interact
diff --git a/include/Scintilla.h b/include/Scintilla.h
index 029102641..8c35cf696 100644
--- a/include/Scintilla.h
+++ b/include/Scintilla.h
@@ -855,6 +855,8 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_VCHOMEDISPLAYEXTEND 2653
 #define SCI_GETCARETLINEVISIBLEALWAYS 2654
 #define SCI_SETCARETLINEVISIBLEALWAYS 2655
+#define SC_LINE_END_TYPE_DEFAULT 0
+#define SC_LINE_END_TYPE_UNICODE 1
 #define SCI_STARTRECORD 3001
 #define SCI_STOPRECORD 3002
 #define SCI_SETLEXER 4001
@@ -878,6 +880,14 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_PROPERTYTYPE 4015
 #define SCI_DESCRIBEPROPERTY 4016
 #define SCI_DESCRIBEKEYWORDSETS 4017
+#define SCI_GETLINEENDTYPESSUPPORTED 4018
+#define SCI_ALLOCATESUBSTYLES 4020
+#define SCI_GETSUBSTYLESSTART 4021
+#define SCI_GETSUBSTYLESLENGTH 4022
+#define SCI_FREESUBSTYLES 4023
+#define SCI_SETIDENTIFIERS 4024
+#define SCI_DISTANCETOSECONDARYSTYLES 4025
+#define SCI_GETSUBSTYLEBASES 4026
 #define SC_MOD_INSERTTEXT 0x1
 #define SC_MOD_DELETETEXT 0x2
 #define SC_MOD_CHANGESTYLE 0x4
diff --git a/include/Scintilla.iface b/include/Scintilla.iface
index ac0b78781..eb77a584a 100644
--- a/include/Scintilla.iface
+++ b/include/Scintilla.iface
@@ -2273,6 +2273,14 @@ get bool GetCaretLineVisibleAlways=2654(,)
 # Sets the caret line to always visible.
 set void SetCaretLineVisibleAlways=2655(bool alwaysVisible,)
 
+# Line end types which may be used in addition to LF, CR, and CRLF
+# SC_LINE_END_TYPE_UNICODE includes U+2028 Line Separator,
+# U+2029 Paragraph Separator, and U+0085 Next Line
+enu LineEndType=SC_LINE_END_TYPE_
+val SC_LINE_END_TYPE_DEFAULT=0
+val SC_LINE_END_TYPE_UNICODE=1
+
+
 # Start notifying the container of all key presses and commands.
 fun void StartRecord=3001(,)
 
@@ -2341,6 +2349,32 @@ fun int DescribeProperty=4016(string name, stringresult description)
 # Retrieve a '\n' separated list of descriptions of the keyword sets understood by the current lexer.
 fun int DescribeKeyWordSets=4017(, stringresult descriptions)
 
+# Bit set of LineEndType enumertion for which line ends beyond the standard
+# LF, CR, and CRLF are supported by the lexer.
+get int GetLineEndTypesSupported=4018(,)
+
+# Allocate a set of sub styles for a particular base style, returning start of range
+fun int AllocateSubStyles=4020(int styleBase, int numberStyles)
+
+# The starting style number for the sub styles associated with a base style
+get int GetSubStylesStart=4021(int styleBase,)
+
+# The number of sub styles associated with a base style
+get int GetSubStylesLength=4022(int styleBase,)
+
+# Free allocated sub styles
+fun void FreeSubStyles=4023(,)
+
+# Set the identifiers that are shown in a particular style
+set void SetIdentifiers=4024(int style, string identifiers)
+
+# Where styles are duplicated by a feature such as active/inactive code
+# return the distance between the two types.
+get int DistanceToSecondaryStyles=4025(,)
+
+# Get the set of base styles that can be extended with sub styles
+get int GetSubStyleBases=4026(, stringresult styles)
+
 # Notifications
 # Type of modification and the action which caused the modification.
 # These are defined as a bit mask to make it easy to specify which notifications are wanted.
diff --git a/src/Document.cxx b/src/Document.cxx
index 1938149e7..500259d8d 100644
--- a/src/Document.cxx
+++ b/src/Document.cxx
@@ -67,6 +67,17 @@ void LexInterface::Colourise(int start, int end) {
 	}
 }
 
+int LexInterface::LineEndTypesSupported() {
+	if (instance) {
+		int interfaceVersion = instance->Version();
+		if (interfaceVersion >= lvSubStyles) {
+			ILexerWithSubStyles *ssinstance = static_cast<ILexerWithSubStyles *>(instance);
+			return ssinstance->LineEndTypesSupported();
+		}
+	}
+	return 0;
+}
+
 Document::Document() {
 	refCount = 0;
 	pcf = NULL;
@@ -135,6 +146,13 @@ void Document::Init() {
 	}
 }
 
+int Document::LineEndTypesSupported() const {
+	if ((SC_CP_UTF8 == dbcsCodePage) && pli)
+		return pli->LineEndTypesSupported();
+	else
+		return 0;
+}
+
 bool Document::SetDBCSCodePage(int dbcsCodePage_) {
 	if (dbcsCodePage != dbcsCodePage_) {
 		dbcsCodePage = dbcsCodePage_;
@@ -245,7 +263,7 @@ int SCI_METHOD Document::LineStart(int line) const {
 	return cb.LineStart(line);
 }
 
-int Document::LineEnd(int line) const {
+int SCI_METHOD Document::LineEnd(int line) const {
 	if (line == LinesTotal() - 1) {
 		return LineStart(line + 1);
 	} else {
diff --git a/src/Document.h b/src/Document.h
index 36d670937..2830d626e 100644
--- a/src/Document.h
+++ b/src/Document.h
@@ -186,6 +186,7 @@ public:
 	virtual ~LexInterface() {
 	}
 	void Colourise(int start, int end);
+	int LineEndTypesSupported();
 	bool UseContainerLexing() const {
 		return instance == 0;
 	}
@@ -193,7 +194,7 @@ public:
 
 /**
  */
-class Document : PerLine, public IDocument, public ILoader {
+class Document : PerLine, public IDocumentWithLineEnd, public ILoader {
 
 public:
 	/** Used to pair watcher pointer with user data. */
@@ -256,12 +257,13 @@ public:
 	int SCI_METHOD Release();
 
 	virtual void Init();
+	int LineEndTypesSupported() const;
 	bool SetDBCSCodePage(int dbcsCodePage_);
 	virtual void InsertLine(int line);
 	virtual void RemoveLine(int line);
 
 	int SCI_METHOD Version() const {
-		return dvOriginal;
+		return dvLineEnd;
 	}
 
 	void SCI_METHOD SetErrorStatus(int status);
@@ -338,7 +340,7 @@ public:
 	void DeleteAllMarks(int markerNum);
 	int LineFromHandle(int markerHandle);
 	int SCI_METHOD LineStart(int line) const;
-	int LineEnd(int line) const;
+	int SCI_METHOD LineEnd(int line) const;
 	int LineEndPosition(int position) const;
 	bool IsLineEndPosition(int position) const;
 	int VCHomePosition(int position) const;
diff --git a/src/ScintillaBase.cxx b/src/ScintillaBase.cxx
index 5fc908e91..4a0146f7b 100644
--- a/src/ScintillaBase.cxx
+++ b/src/ScintillaBase.cxx
@@ -473,6 +473,7 @@ class LexState : public LexInterface {
 	const LexerModule *lexCurrent;
 	void SetLexerModule(const LexerModule *lex);
 	PropSetSimple props;
+	int interfaceVersion;
 public:
 	int lexLanguage;
 
@@ -492,6 +493,15 @@ public:
 	const char *PropGet(const char *key) const;
 	int PropGetInt(const char *key, int defaultValue=0) const;
 	int PropGetExpanded(const char *key, char *result) const;
+
+	int LineEndTypesSupported();
+	int AllocateSubStyles(int styleBase, int numberStyles);
+	int SubStylesStart(int styleBase);
+	int SubStylesLength(int styleBase);
+	void FreeSubStyles();
+	void SetIdentifiers(int style, const char *identifiers);
+	int DistanceToSecondaryStyles();
+	const char *GetSubStyleBases();
 };
 
 #ifdef SCI_NAMESPACE
@@ -501,6 +511,7 @@ public:
 LexState::LexState(Document *pdoc_) : LexInterface(pdoc_) {
 	lexCurrent = 0;
 	performingStyle = false;
+	interfaceVersion = lvOriginal;
 	lexLanguage = SCLEX_CONTAINER;
 }
 
@@ -524,9 +535,12 @@ void LexState::SetLexerModule(const LexerModule *lex) {
 			instance->Release();
 			instance = 0;
 		}
+		interfaceVersion = lvOriginal;
 		lexCurrent = lex;
-		if (lexCurrent)
+		if (lexCurrent) {
 			instance = lexCurrent->Create();
+			interfaceVersion = instance->Version();
+		}
 		pdoc->LexerChanged();
 	}
 }
@@ -631,6 +645,60 @@ int LexState::PropGetExpanded(const char *key, char *result) const {
 	return props.GetExpanded(key, result);
 }
 
+int LexState::LineEndTypesSupported() {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		return static_cast<ILexerWithSubStyles *>(instance)->LineEndTypesSupported();
+	}
+	return 0;
+}
+
+int LexState::AllocateSubStyles(int styleBase, int numberStyles) {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		return static_cast<ILexerWithSubStyles *>(instance)->AllocateSubStyles(styleBase, numberStyles);
+	}
+	return -1;
+}
+
+int LexState::SubStylesStart(int styleBase) {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		return static_cast<ILexerWithSubStyles *>(instance)->SubStylesStart(styleBase);
+	}
+	return -1;
+}
+
+int LexState::SubStylesLength(int styleBase) {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		return static_cast<ILexerWithSubStyles *>(instance)->SubStylesLength(styleBase);
+	}
+	return 0;
+}
+
+void LexState::FreeSubStyles() {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		static_cast<ILexerWithSubStyles *>(instance)->FreeSubStyles();
+	}
+}
+
+void LexState::SetIdentifiers(int style, const char *identifiers) {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		static_cast<ILexerWithSubStyles *>(instance)->SetIdentifiers(style, identifiers);
+	}
+}
+
+int LexState::DistanceToSecondaryStyles() {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		return static_cast<ILexerWithSubStyles *>(instance)->DistanceToSecondaryStyles();
+	}
+	return 0;
+}
+
+const char *LexState::GetSubStyleBases() {
+	if (instance && (interfaceVersion >= lvSubStyles)) {
+		return static_cast<ILexerWithSubStyles *>(instance)->GetSubStyleBases();
+	}
+	return "";
+}
+
 #endif
 
 void ScintillaBase::NotifyStyleToNeeded(int endStyleNeeded) {
@@ -891,6 +959,31 @@ sptr_t ScintillaBase::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lPara
 	case SCI_DESCRIBEKEYWORDSETS:
 		return StringResult(lParam, DocumentLexState()->DescribeWordListSets());
 
+	case SCI_GETLINEENDTYPESSUPPORTED:
+		return DocumentLexState()->LineEndTypesSupported();
+
+	case SCI_ALLOCATESUBSTYLES:
+		return DocumentLexState()->AllocateSubStyles(wParam, lParam);
+
+	case SCI_GETSUBSTYLESSTART:
+		return DocumentLexState()->SubStylesStart(wParam);
+
+	case SCI_GETSUBSTYLESLENGTH:
+		return DocumentLexState()->SubStylesLength(wParam);
+
+	case SCI_FREESUBSTYLES:
+		DocumentLexState()->FreeSubStyles();
+		break;
+
+	case SCI_SETIDENTIFIERS:
+		DocumentLexState()->SetIdentifiers(wParam, reinterpret_cast<const char *>(lParam));
+		break;
+
+	case SCI_DISTANCETOSECONDARYSTYLES:
+		return DocumentLexState()->DistanceToSecondaryStyles();
+
+	case SCI_GETSUBSTYLEBASES:
+		return StringResult(lParam, DocumentLexState()->GetSubStyleBases());
 #endif
 
 	default: