Initial, experimental support for upstream ILexer5/lexilla lexers.

7 files changed, 120 insertions, 19 deletions

diff --git a/doc/ScintillaDoc.html b/doc/ScintillaDoc.html
index 5a83b8d92..b3fe1e553 100644
--- a/doc/ScintillaDoc.html
+++ b/doc/ScintillaDoc.html
@@ -6857,7 +6857,7 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){
      <a class="message" href="#SCI_SETLEXERLANGUAGE">SCI_SETLEXERLANGUAGE(&lt;unused&gt;, const char
     *language)</a><br />
      <a class="message" href="#SCI_GETLEXERLANGUAGE">SCI_GETLEXERLANGUAGE(&lt;unused&gt;, char *language) &rarr; int</a><br />
-     <a class="message" href="#SCI_LOADLEXERLIBRARY">SCI_LOADLEXERLIBRARY(&lt;unused&gt;, const char
+     <a class="message" href="#SCI_LOADLEXERLIBRARY">SCI_LOADLEXERLIBRARY(bool iLexer5, const char
     *path)</a><br />
      <a class="message" href="#SCI_COLOURISE">SCI_COLOURISE(position start, position end)</a><br />
      <a class="message" href="#SCI_CHANGELEXERSTATE">SCI_CHANGELEXERSTATE(position start, position end) &rarr; int</a><br />
@@ -6913,9 +6913,13 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){
 
     <p><code>SCI_GETLEXERLANGUAGE</code> retrieves the name of the lexer.</p>
 
-    <p><b id="SCI_LOADLEXERLIBRARY">SCI_LOADLEXERLIBRARY(&lt;unused&gt;, const char *path)</b><br />
+    <p><b id="SCI_LOADLEXERLIBRARY">SCI_LOADLEXERLIBRARY(bool iLexer5, const char *path)</b><br />
      Load a lexer implemented in a shared library. This is a .so file on GTK/Linux, a .dylib file on macOS, or a .DLL file on Windows.
-     </p>
+    <code>iLexer5</code> indicates whether or not the library contains Scintilla lexers that implement
+    the ILexer5 interface defined in Scintilla 4.3+ (e.g. Lexilla). The default is to assume lexers
+    implement the ILexer interface defined in Scintilla 3.x. This flag must be set explicitly and
+    cannot be inferred.
+    </p>
 
     <p><b id="SCI_COLOURISE">SCI_COLOURISE(position start, position end)</b><br />
      This requests the current lexer or the container (if the lexer is set to
diff --git a/include/ILexer.h b/include/ILexer.h
index f7f188972..ee83939ea 100644
--- a/include/ILexer.h
+++ b/include/ILexer.h
@@ -12,7 +12,7 @@
 
 namespace Scintilla {
 
-enum { dvOriginal=0, dvLineEnd=1 };
+enum { dvOriginal=0, dvLineEnd=1, dvRelease4=2 };
 
 class IDocument {
 public:
@@ -92,6 +92,82 @@ public:
 	virtual const char * SCI_METHOD PropertyGet(const char *key) = 0;
 };
 
+// For upstream lexer/lexilla support.
+
+enum { lvRelease4=2, lvRelease5=3 };
+
+class ILexer4 {
+public:
+	virtual int SCI_METHOD Version() const = 0;
+	virtual void SCI_METHOD Release() = 0;
+	virtual const char * SCI_METHOD PropertyNames() = 0;
+	virtual int SCI_METHOD PropertyType(const char *name) = 0;
+	virtual const char * SCI_METHOD DescribeProperty(const char *name) = 0;
+	virtual Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) = 0;
+	virtual const char * SCI_METHOD DescribeWordListSets() = 0;
+	virtual Sci_Position SCI_METHOD WordListSet(int n, const char *wl) = 0;
+	virtual void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) = 0;
+	virtual void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) = 0;
+	virtual void * SCI_METHOD PrivateCall(int operation, void *pointer) = 0;
+	virtual int SCI_METHOD LineEndTypesSupported() = 0;
+	virtual int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) = 0;
+	virtual int SCI_METHOD SubStylesStart(int styleBase) = 0;
+	virtual int SCI_METHOD SubStylesLength(int styleBase) = 0;
+	virtual int SCI_METHOD StyleFromSubStyle(int subStyle) = 0;
+	virtual int SCI_METHOD PrimaryStyleFromStyle(int style) = 0;
+	virtual void SCI_METHOD FreeSubStyles() = 0;
+	virtual void SCI_METHOD SetIdentifiers(int style, const char *identifiers) = 0;
+	virtual int SCI_METHOD DistanceToSecondaryStyles() = 0;
+	virtual const char * SCI_METHOD GetSubStyleBases() = 0;
+	virtual int SCI_METHOD NamedStyles() = 0;
+	virtual const char * SCI_METHOD NameOfStyle(int style) = 0;
+	virtual const char * SCI_METHOD TagsOfStyle(int style) = 0;
+	virtual const char * SCI_METHOD DescriptionOfStyle(int style) = 0;
+};
+
+class ILexer5 : public ILexer4 {
+public:
+	virtual const char * SCI_METHOD GetName() = 0;
+	virtual int SCI_METHOD  GetIdentifier() = 0;
+	virtual const char * SCI_METHOD PropertyGet(const char *key) = 0;
+};
+
+class LexillaLexer : public ILexerWithIdentity {
+ILexer5 *lex;
+public:
+	LexillaLexer(ILexer5 *lex_) : lex(lex_) {}
+	virtual ~LexillaLexer() {}
+	void SCI_METHOD Release() override { lex->Release(); };
+	int SCI_METHOD Version() const override { return lex->Version(); }
+	const char * SCI_METHOD PropertyNames() override { return lex->PropertyNames(); }
+	int SCI_METHOD PropertyType(const char *name) override { return lex->PropertyType(name); }
+	const char * SCI_METHOD DescribeProperty(const char *name) override { return lex->DescribeProperty(name); }
+	Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override { return lex->PropertySet(key, val); }
+	const char * SCI_METHOD DescribeWordListSets() override { return lex->DescribeWordListSets(); }
+	Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override { return lex->WordListSet(n, wl); }
+	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) override { lex->Lex(startPos, lengthDoc, initStyle, pAccess); }
+	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) override { lex->Fold(startPos, lengthDoc, initStyle, pAccess); }
+	void * SCI_METHOD PrivateCall(int operation, void *pointer) override { return lex->PrivateCall(operation, pointer); }
+	int SCI_METHOD LineEndTypesSupported() override { return lex->LineEndTypesSupported(); }
+	int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override { return lex->AllocateSubStyles(styleBase, numberStyles); }
+	int SCI_METHOD SubStylesStart(int styleBase) override { return lex->SubStylesStart(styleBase); }
+	int SCI_METHOD SubStylesLength(int styleBase) override { return lex->SubStylesLength(styleBase); }
+	int SCI_METHOD StyleFromSubStyle(int subStyle) override { return lex->StyleFromSubStyle(subStyle); }
+	int SCI_METHOD PrimaryStyleFromStyle(int style) override { return lex->PrimaryStyleFromStyle(style); }
+	void SCI_METHOD FreeSubStyles() override { lex->FreeSubStyles(); }
+	void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override { lex->SetIdentifiers(style, identifiers); }
+	int SCI_METHOD DistanceToSecondaryStyles() override { return lex->DistanceToSecondaryStyles(); }
+	const char * SCI_METHOD GetSubStyleBases() override { return lex->GetSubStyleBases(); }
+	int SCI_METHOD NamedStyles() override { return lex->NamedStyles(); }
+	const char * SCI_METHOD NameOfStyle(int style) override { return lex->NameOfStyle(style); }
+	const char * SCI_METHOD TagsOfStyle(int style) override { return lex->TagsOfStyle(style); }
+	const char * SCI_METHOD DescriptionOfStyle(int style) override { return lex->DescriptionOfStyle(style); }
+	// ILexerWithIdentity methods
+	const char * SCI_METHOD GetName() override { return lex->GetName(); }
+	int SCI_METHOD GetIdentifier() override { return lex->GetIdentifier(); }
+	const char * SCI_METHOD PropertyGet(const char *key) override { return lex->PropertyGet(key); }
+};
+
 }
 
 #endif
diff --git a/lexlib/LexerModule.cxx b/lexlib/LexerModule.cxx
index 30e8cf041..592b53871 100644
--- a/lexlib/LexerModule.cxx
+++ b/lexlib/LexerModule.cxx
@@ -35,6 +35,7 @@ LexerModule::LexerModule(int language_,
 	fnLexer(fnLexer_),
 	fnFolder(fnFolder_),
 	fnFactory(nullptr),
+	fnFactory5(nullptr),
 	wordListDescriptions(wordListDescriptions_),
 	lexClasses(lexClasses_),
 	nClasses(nClasses_),
@@ -58,7 +59,7 @@ LexerModule::LexerModule(int language_,
 LexerModule::~LexerModule() {
 }
 
-int LexerModule::GetLanguage() const { 
+int LexerModule::GetLanguage() const {
 	return language;
 }
 
@@ -96,6 +97,8 @@ size_t LexerModule::NamedStyles() const {
 ILexer *LexerModule::Create() const {
 	if (fnFactory)
 		return fnFactory();
+	else if (fnFactory5)
+		return new LexillaLexer(fnFactory5());
 	else
 		return new LexerSimple(this);
 }
diff --git a/lexlib/LexerModule.h b/lexlib/LexerModule.h
index 096df5042..f22729af6 100644
--- a/lexlib/LexerModule.h
+++ b/lexlib/LexerModule.h
@@ -17,6 +17,7 @@ struct LexicalClass;
 typedef void (*LexerFunction)(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle,
                   WordList *keywordlists[], Accessor &styler);
 typedef ILexer *(*LexerFactoryFunction)();
+typedef ILexer5 *(*LexerFactoryFunction5)();
 
 /**
  * A LexerModule is responsible for lexing and folding a particular language.
@@ -30,6 +31,7 @@ protected:
 	LexerFunction fnLexer;
 	LexerFunction fnFolder;
 	LexerFactoryFunction fnFactory;
+	LexerFactoryFunction5 fnFactory5;
 	const char * const * wordListDescriptions;
 	const LexicalClass *lexClasses;
 	size_t nClasses;
diff --git a/src/ExternalLexer.cxx b/src/ExternalLexer.cxx
index acb18339d..eae5806ee 100644
--- a/src/ExternalLexer.cxx
+++ b/src/ExternalLexer.cxx
@@ -39,6 +39,7 @@ int nextLanguage = SCLEX_AUTOMATIC + 1;
 typedef int (EXT_LEXER_DECL *GetLexerCountFn)();
 typedef void (EXT_LEXER_DECL *GetLexerNameFn)(unsigned int Index, char *name, int buflength);
 typedef LexerFactoryFunction(EXT_LEXER_DECL *GetLexerFactoryFunction)(unsigned int Index);
+typedef LexerFactoryFunction5(EXT_LEXER_DECL *GetLexerFactoryFunction5)(unsigned int Index);
 
 /// Generic function to convert from a void* to a function pointer.
 /// This avoids undefined and conditionally defined behaviour.
@@ -54,15 +55,17 @@ T FunctionPointer(void *function) noexcept {
 class ExternalLexerModule : public LexerModule {
 protected:
 	GetLexerFactoryFunction fneFactory;
+	GetLexerFactoryFunction5 fneFactory5;
 	std::string name;
 public:
 	ExternalLexerModule(int language_, LexerFunction fnLexer_,
 		const char *languageName_=nullptr, LexerFunction fnFolder_=nullptr) :
 		LexerModule(language_, fnLexer_, nullptr, fnFolder_),
-		fneFactory(nullptr), name(languageName_){
+		fneFactory(nullptr), fneFactory5(nullptr), name(languageName_){
 		languageName = name.c_str();
 	}
 	void SetExternal(GetLexerFactoryFunction fFactory, int index) noexcept;
+	void SetExternal(GetLexerFactoryFunction5 fFactory, int index) noexcept;
 };
 
 /// LexerLibrary exists for every External Lexer DLL, contains ExternalLexerModules.
@@ -70,7 +73,7 @@ class LexerLibrary {
 	std::unique_ptr<DynamicLibrary> lib;
 	std::vector<std::unique_ptr<ExternalLexerModule>> modules;
 public:
-	explicit LexerLibrary(const char *moduleName_);
+	explicit LexerLibrary(const char *moduleName_, bool iLexer5=false);
 	~LexerLibrary();
 
 	std::string moduleName;
@@ -84,7 +87,7 @@ public:
 	static LexerManager *GetInstance();
 	static void DeleteInstance() noexcept;
 
-	void Load(const char *path);
+	void Load(const char *path, bool iLexer5=false);
 	void Clear() noexcept;
 
 private:
@@ -111,13 +114,18 @@ void ExternalLexerModule::SetExternal(GetLexerFactoryFunction fFactory, int inde
 	fnFactory = fFactory(index);
 }
 
+void ExternalLexerModule::SetExternal(GetLexerFactoryFunction5 fFactory5, int index) noexcept {
+	fneFactory5 = fFactory5;
+	fnFactory5 = fFactory5(index);
+}
+
 //------------------------------------------
 //
 // LexerLibrary
 //
 //------------------------------------------
 
-LexerLibrary::LexerLibrary(const char *moduleName_) {
+LexerLibrary::LexerLibrary(const char *moduleName_, bool iLexer5) {
 	// Load the DLL
 	lib.reset(DynamicLibrary::Load(moduleName_));
 	if (lib->IsValid()) {
@@ -127,9 +135,14 @@ LexerLibrary::LexerLibrary(const char *moduleName_) {
 		if (GetLexerCount) {
 			// Find functions in the DLL
 			GetLexerNameFn GetLexerName = FunctionPointer<GetLexerNameFn>(lib->FindFunction("GetLexerName"));
-			GetLexerFactoryFunction fnFactory = FunctionPointer<GetLexerFactoryFunction>(lib->FindFunction("GetLexerFactory"));
-
-			if (!GetLexerName || !fnFactory) {
+			GetLexerFactoryFunction fnFactory = nullptr;
+			GetLexerFactoryFunction5 fnFactory5 = nullptr;
+			if (!iLexer5)
+				fnFactory = FunctionPointer<GetLexerFactoryFunction>(lib->FindFunction("GetLexerFactory"));
+			else
+				fnFactory5 = FunctionPointer<GetLexerFactoryFunction5>(lib->FindFunction("GetLexerFactory"));
+
+			if (!GetLexerName || (!fnFactory && !fnFactory5)) {
 				return;
 			}
 
@@ -151,7 +164,10 @@ LexerLibrary::LexerLibrary(const char *moduleName_) {
 
 				// The external lexer needs to know how to call into its DLL to
 				// do its lexing and folding, we tell it here.
-				lex->SetExternal(fnFactory, i);
+				if (!iLexer5)
+					lex->SetExternal(fnFactory, i);
+				else
+					lex->SetExternal(fnFactory5, i);
 			}
 		}
 	}
@@ -186,12 +202,12 @@ LexerManager::~LexerManager() {
 	Clear();
 }
 
-void LexerManager::Load(const char *path) {
+void LexerManager::Load(const char *path, bool iLexer5) {
 	for (const std::unique_ptr<LexerLibrary> &ll : libraries) {
 		if (ll->moduleName == path)
 			return;
 	}
-	libraries.push_back(Sci::make_unique<LexerLibrary>(path));
+	libraries.push_back(Sci::make_unique<LexerLibrary>(path, iLexer5));
 }
 
 void LexerManager::Clear() noexcept {
@@ -214,8 +230,8 @@ LMMinder minder;
 
 namespace Scintilla {
 
-void ExternalLexerLoad(const char *path) {
-	LexerManager::GetInstance()->Load(path);
+void ExternalLexerLoad(const char *path, bool iLexer5) {
+	LexerManager::GetInstance()->Load(path, iLexer5);
 }
 
 }
diff --git a/src/ExternalLexer.h b/src/ExternalLexer.h
index d0a615e31..736b91531 100644
--- a/src/ExternalLexer.h
+++ b/src/ExternalLexer.h
@@ -10,7 +10,7 @@
 
 namespace Scintilla {
 
-void ExternalLexerLoad(const char *path);
+void ExternalLexerLoad(const char *path, bool iLexer5=false);
 
 }
 
diff --git a/src/ScintillaBase.cxx b/src/ScintillaBase.cxx
index 082cb82d5..09736d9f7 100644
--- a/src/ScintillaBase.cxx
+++ b/src/ScintillaBase.cxx
@@ -1103,7 +1103,7 @@ sptr_t ScintillaBase::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lPara
 		return StringResult(lParam, DocumentLexState()->GetName());
 
 	case SCI_LOADLEXERLIBRARY:
-		ExternalLexerLoad(ConstCharPtrFromSPtr(lParam));
+		ExternalLexerLoad(ConstCharPtrFromSPtr(lParam), wParam != 0);
 		break;
 
 	case SCI_PRIVATELEXERCALL: