14 files changed, 925 insertions, 1 deletions
diff --git a/cppcheck.suppress b/cppcheck.suppress
index 444adba26..eaba434fe 100644
--- a/cppcheck.suppress
+++ b/cppcheck.suppress
@@ -71,7 +71,6 @@ constParameter:scintilla/lexers/LexFortran.cxx
 variableScope:scintilla/lexers/LexGui4Cli.cxx
 constParameter:scintilla/lexers/LexHaskell.cxx
 constParameter:scintilla/lexers/LexHex.cxx
-missingOverride:scintilla/lexers/LexHollywood.cxx
 constParameter:scintilla/lexers/LexHTML.cxx
 variableScope:scintilla/lexers/LexInno.cxx
 constParameter:scintilla/lexers/LexJSON.cxx
diff --git a/doc/ScintillaDoc.html b/doc/ScintillaDoc.html
index b4eae90ea..4ad921cf5 100644
--- a/doc/ScintillaDoc.html
+++ b/doc/ScintillaDoc.html
@@ -291,6 +291,9 @@
     <p>A lexer created by Lexilla may be used in Scintilla by calling
     <a class="seealso" href="#SCI_SETILEXER">SCI_SETILEXER</a>.</p>
 
+    <p>Lexilla and its contained lexers can be tested with the TestLexers program in lexilla/test.
+    Read lexilla/test/README for information on building and using TestLexers.</p>
+
     <h2 id="MessageCategories">Contents</h2>
 
     <table class="categories" summary="Message categories">
diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html
index af9c8a49c..4edde2556 100644
--- a/doc/ScintillaHistory.html
+++ b/doc/ScintillaHistory.html
@@ -568,6 +568,7 @@
 	</li>
 	<li>
 	Lexers made available as Lexilla library.
+	TestLexers program with tests for Lexilla and lexers added in lexilla/test.
 	</li>
 	<li>
 	SCI_SETILEXER implemented to use lexers from Lexilla or other sources.
diff --git a/lexilla/test/LexillaAccess.cxx b/lexilla/test/LexillaAccess.cxx
new file mode 100644
index 000000000..ea62c9605
--- /dev/null
+++ b/lexilla/test/LexillaAccess.cxx
@@ -0,0 +1,95 @@
+// Scintilla source code edit control
+/** @file LexillaAccess.cxx
+ ** Interface to Lexilla shared library.
+ **/
+ // Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+ // The License.txt file describes the conditions under which this software may be distributed.
+
+#include <cstring>
+
+#include <iterator>
+
+#include <iostream>
+#include <filesystem>
+
+#if !_WIN32
+#include <dlfcn.h>
+#else
+#include <windows.h>
+#endif
+
+#include "ILexer.h"
+
+#include "Lexilla.h"
+
+#include "LexillaAccess.h"
+
+#if _WIN32
+#define EXT_LEXER_DECL __stdcall
+typedef FARPROC Function;
+typedef HMODULE Module;
+#else
+#define EXT_LEXER_DECL
+typedef void *Function;
+typedef void *Module;
+#endif
+
+typedef Scintilla::ILexer5 *(EXT_LEXER_DECL *CreateLexerFn)(const char *name);
+
+Module lexillaDL {};
+
+/// Generic function to convert from a Function(void* or FARPROC) to a function pointer.
+/// This avoids undefined and conditionally defined behaviour.
+template<typename T>
+T FunctionPointer(Function function) noexcept {
+	static_assert(sizeof(T) == sizeof(function));
+	T fp;
+	memcpy(&fp, &function, sizeof(T));
+	return fp;
+}
+
+CreateLexerFn fnCL;
+
+Function FindSymbol(const char *symbol) noexcept {
+#if _WIN32
+	return ::GetProcAddress(lexillaDL, symbol);
+#else
+	return dlsym(lexillaDL, symbol);
+#endif
+}
+
+Scintilla::ILexer5 *CreateLexer(std::string languageName) {
+#ifdef LEXILLA_STATIC
+	return CreateLexer(languageName.c_str());
+#else
+	return fnCL(languageName.c_str());
+#endif
+}
+
+bool LoadLexilla([[maybe_unused]] std::filesystem::path path) {
+#ifdef LEXILLA_STATIC
+	return true;
+#else
+	std::filesystem::path sharedLibrary = path;
+	sharedLibrary.append("bin");
+#if _WIN32
+	sharedLibrary.append("lexilla.dll");
+	lexillaDL = ::LoadLibraryW(sharedLibrary.c_str());
+#else
+#if defined(__APPLE__)
+	sharedLibrary.append("lexilla.dylib");
+#else
+	sharedLibrary.append("lexilla.so");
+#endif
+	lexillaDL = dlopen(sharedLibrary.c_str(), RTLD_LAZY);
+#endif
+
+	if (lexillaDL) {
+		fnCL = FunctionPointer<CreateLexerFn>(FindSymbol("CreateLexer"));
+	} else {
+		std::cout << "Cannot load " << sharedLibrary.string() << "\n";
+	}
+
+	return lexillaDL && fnCL;
+#endif
+}
diff --git a/lexilla/test/LexillaAccess.h b/lexilla/test/LexillaAccess.h
new file mode 100644
index 000000000..3e4e087cb
--- /dev/null
+++ b/lexilla/test/LexillaAccess.h
@@ -0,0 +1,9 @@
+// Scintilla source code edit control
+/** @file LexillaAccess.h
+ ** Interface to Lexilla shared library.
+ **/
+// Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+// The License.txt file describes the conditions under which this software may be distributed.
+
+Scintilla::ILexer5 *CreateLexer(std::string languageName);
+bool LoadLexilla(std::filesystem::path path);
diff --git a/lexilla/test/README b/lexilla/test/README
new file mode 100644
index 000000000..119f08cf2
--- /dev/null
+++ b/lexilla/test/README
@@ -0,0 +1,65 @@
+README for testing lexers with lexilla/test.
+
+The TestLexers application is run to test the lexing of a set of example files
+and thus ensure that the lexers are working correctly.
+
+Lexers are accessed through the Lexilla shared library which must be built first
+in the lexilla/src directory.
+
+TestLexers works on Windows, Linux, or macOS and requires a C++20 compiler.
+MSVC 2019.4, GCC 9.0, Clang 9.0, and Apple Clang 11.0 are known to work.
+
+MSVC is only available on Windows.
+
+GCC and Clang work on Windows and Linux.
+
+On macOS, only Apple Clang is available.
+
+To use GCC run lexilla/test/makefile:
+	make test
+
+To use Clang run lexilla/test/makefile:
+	make CLANG=1 test
+On macOS, CLANG is set automatically so this can just be
+	make test
+
+To use MSVC:
+	nmake -f testlexers.mak test
+There is also a project file TestLexers.vcxproj that can be loaded into the Visual
+C++ IDE.
+
+
+
+Adding or Changing Tests
+
+The lexilla/test/examples directory contains a set of tests located in a tree of
+subdirectories.
+
+Each directory contains example files along with control files called
+SciTE.properties and expected result files with a .styled suffix.
+If an unexpected result occurs then files with the suffix .new may be created.
+
+Each file in the examples tree that does not have an extension of .properties, .styled, or
+.new is an example file that will be lexed according to settings found in SciTE.properties.
+The results of the lex will be compared to the corresponding .styled file and if different
+the result will be saved to a .new file for checking.
+So, if x.cxx is the example, its lexed form will be checked against x.cxx.styled and a
+x.cxx.new file may be created. The .new and .styled files contain the text of the original
+file along with style number changes in {} like:
+	{5}function{0} {11}first{10}(){0}
+After checking that the .new file is correct, it can be promoted to .styled and committed
+to the repository.
+
+The SciTE.properties file is similar to properties files used for SciTE but are simpler.
+The lexer to be run is defined with a lexer.{filepattern} statement like:
+	lexer.*.d=d
+
+Keywords may be defined with keywords settings like:
+	keywords.*.cxx=int char
+	keywords2.*.cxx=open
+
+Other settings are treated as lexer properties and forwarded to the lexer:
+	lexer.cpp.track.preprocessor=1
+
+If there is a need to test additional configurations of keywords or properties then
+create another subdirectory with the different settings in a new SciTE.properties.
diff --git a/lexilla/test/TestDocument.cxx b/lexilla/test/TestDocument.cxx
new file mode 100644
index 000000000..780265305
--- /dev/null
+++ b/lexilla/test/TestDocument.cxx
@@ -0,0 +1,222 @@
+// Scintilla source code edit control
+/** @file TestDocument.cxx
+ ** Lexer testing.
+ **/
+ // Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+ // The License.txt file describes the conditions under which this software may be distributed.
+
+#include <cassert>
+
+#include <string>
+#include <string_view>
+#include <vector>
+#include <algorithm>
+
+#include <iostream>
+
+#include "ILexer.h"
+
+#include "TestDocument.h"
+
+namespace {
+
+	const unsigned char UTF8BytesOfLead[256] = {
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 00 - 0F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 10 - 1F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 20 - 2F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 30 - 3F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 40 - 4F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 50 - 5F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 60 - 6F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 70 - 7F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 80 - 8F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 90 - 9F
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A0 - AF
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B0 - BF
+	1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0 - CF
+	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D0 - DF
+	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E0 - EF
+	4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // F0 - FF
+	};
+
+	int UnicodeFromUTF8(const unsigned char *us) noexcept {
+		switch (UTF8BytesOfLead[us[0]]) {
+		case 1:
+			return us[0];
+		case 2:
+			return ((us[0] & 0x1F) << 6) + (us[1] & 0x3F);
+		case 3:
+			return ((us[0] & 0xF) << 12) + ((us[1] & 0x3F) << 6) + (us[2] & 0x3F);
+		default:
+			return ((us[0] & 0x7) << 18) + ((us[1] & 0x3F) << 12) + ((us[2] & 0x3F) << 6) + (us[3] & 0x3F);
+		}
+	}
+
+}
+
+void TestDocument::Set(std::string_view sv) {
+	text = sv;
+	textStyles.resize(text.size());
+	lineStarts.clear();
+	endStyled = 0;
+	lineStarts.push_back(0);
+	for (size_t pos = 0; pos < text.length(); pos++) {
+		if (text[pos] == '\n') {
+			lineStarts.push_back(pos + 1);
+		}
+	}
+	lineStarts.push_back(text.length());
+	lineStates.resize(lineStarts.size());
+}
+
+int SCI_METHOD TestDocument::Version() const {
+	return Scintilla::dvRelease4;
+}
+
+void SCI_METHOD TestDocument::SetErrorStatus(int) {
+}
+
+Sci_Position SCI_METHOD TestDocument::Length() const {
+	return text.length();
+}
+
+void SCI_METHOD TestDocument::GetCharRange(char *buffer, Sci_Position position, Sci_Position lengthRetrieve) const {
+	text.copy(buffer, lengthRetrieve, position);
+}
+
+char SCI_METHOD TestDocument::StyleAt(Sci_Position position) const {
+	return textStyles.at(position);
+}
+
+Sci_Position SCI_METHOD TestDocument::LineFromPosition(Sci_Position position) const {
+	if (position >= static_cast<Sci_Position>(text.length())) {
+		return lineStarts.size() - 1 - 1;
+	}
+
+	std::vector<Sci_Position>::const_iterator it = std::lower_bound(lineStarts.begin(), lineStarts.end(), position);
+	Sci_Position line = it - lineStarts.begin();
+	if (*it > position)
+		line--;
+	return line;
+}
+
+Sci_Position SCI_METHOD TestDocument::LineStart(Sci_Position line) const {
+	if (line >= static_cast<Sci_Position>(lineStarts.size())) {
+		return text.length();
+	}
+	return lineStarts.at(line);
+}
+
+int SCI_METHOD TestDocument::GetLevel(Sci_Position) const {
+	// Only for folding so not implemented yet
+	return 0;
+}
+
+int SCI_METHOD TestDocument::SetLevel(Sci_Position, int) {
+	// Only for folding so not implemented yet
+	return 0;
+}
+
+int SCI_METHOD TestDocument::GetLineState(Sci_Position line) const {
+	return lineStates.at(line);
+}
+
+int SCI_METHOD TestDocument::SetLineState(Sci_Position line, int state) {
+	return lineStates.at(line) = state;
+}
+
+void SCI_METHOD TestDocument::StartStyling(Sci_Position position) {
+	endStyled = position;
+}
+
+bool SCI_METHOD TestDocument::SetStyleFor(Sci_Position length, char style) {
+	for (Sci_Position i = 0; i < length; i++) {
+		textStyles[endStyled] = style;
+		endStyled++;
+	}
+	return true;
+}
+
+bool SCI_METHOD TestDocument::SetStyles(Sci_Position length, const char *styles) {
+	for (Sci_Position i = 0; i < length; i++) {
+		textStyles[endStyled] = styles[i];
+		endStyled++;
+	}
+	return true;
+}
+
+void SCI_METHOD TestDocument::DecorationSetCurrentIndicator(int) {
+	// Not implemented as no way to read decorations
+}
+
+void SCI_METHOD TestDocument::DecorationFillRange(Sci_Position, int, Sci_Position) {
+	// Not implemented as no way to read decorations
+}
+
+void SCI_METHOD TestDocument::ChangeLexerState(Sci_Position, Sci_Position) {
+	// Not implemented as no watcher to trigger
+}
+
+int SCI_METHOD TestDocument::CodePage() const {
+	// Always UTF-8 for now
+	return 65001;
+}
+
+bool SCI_METHOD TestDocument::IsDBCSLeadByte(char) const {
+	// Always UTF-8 for now
+	return false;
+}
+
+const char *SCI_METHOD TestDocument::BufferPointer() {
+	return text.c_str();
+}
+
+int SCI_METHOD TestDocument::GetLineIndentation(Sci_Position) {
+	// Never actually called - lexers use Accessor::IndentAmount
+	return 0;
+}
+
+Sci_Position SCI_METHOD TestDocument::LineEnd(Sci_Position line) const {
+	Sci_Position position = LineStart(line + 1);
+	position--; // Back over CR or LF
+	// When line terminator is CR+LF, may need to go back one more
+	if ((position > LineStart(line)) && (text.at(position - 1) == '\r')) {
+		position--;
+	}
+	return position;
+}
+
+Sci_Position SCI_METHOD TestDocument::GetRelativePosition(Sci_Position positionStart, Sci_Position characterOffset) const {
+	// TODO: negative characterOffset
+	assert(characterOffset >= 0);
+	// TODO: invalid UTF-8
+	Sci_Position pos = positionStart;
+	while (characterOffset > 0) {
+		Sci_Position width = 0;
+		GetCharacterAndWidth(pos, &width);
+		pos += width;
+		characterOffset--;
+	}
+	return pos;
+}
+
+int SCI_METHOD TestDocument::GetCharacterAndWidth(Sci_Position position, Sci_Position *pWidth) const {
+	// TODO: invalid UTF-8
+	if (position >= static_cast<Sci_Position>(text.length())) {
+		// Return NULs after document end
+		if (pWidth) {
+			*pWidth = 1;
+		}
+		return '\0';
+	}
+	const unsigned char leadByte = text.at(position);
+	const int widthCharBytes = UTF8BytesOfLead[leadByte];
+	unsigned char charBytes[] = { leadByte,0,0,0 };
+	for (int b = 1; b < widthCharBytes; b++)
+		charBytes[b] = text[position + b];
+
+	if (pWidth) {
+		*pWidth = widthCharBytes;
+	}
+	return UnicodeFromUTF8(charBytes);
+}
diff --git a/lexilla/test/TestDocument.h b/lexilla/test/TestDocument.h
new file mode 100644
index 000000000..fd7181319
--- /dev/null
+++ b/lexilla/test/TestDocument.h
@@ -0,0 +1,43 @@
+// Scintilla source code edit control
+/** @file TestDocument.h
+ ** Lexer testing.
+ **/
+// Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+// The License.txt file describes the conditions under which this software may be distributed.
+
+class TestDocument : public Scintilla::IDocument {
+	std::string text;
+	std::string textStyles;
+	std::vector<Sci_Position> lineStarts;
+	std::vector<int> lineStates;
+	Sci_Position endStyled=0;
+public:
+	void Set(std::string_view sv);
+	virtual ~TestDocument() = default;
+
+	int SCI_METHOD Version() const override;
+	void SCI_METHOD SetErrorStatus(int status) override;
+	Sci_Position SCI_METHOD Length() const override;
+	void SCI_METHOD GetCharRange(char *buffer, Sci_Position position, Sci_Position lengthRetrieve) const override;
+	char SCI_METHOD StyleAt(Sci_Position position) const override;
+	Sci_Position SCI_METHOD LineFromPosition(Sci_Position position) const override;
+	Sci_Position SCI_METHOD LineStart(Sci_Position line) const override;
+	int SCI_METHOD GetLevel(Sci_Position line) const override;
+	int SCI_METHOD SetLevel(Sci_Position line, int level) override;
+	int SCI_METHOD GetLineState(Sci_Position line) const override;
+	int SCI_METHOD SetLineState(Sci_Position line, int state) override;
+	void SCI_METHOD StartStyling(Sci_Position position) override;
+	bool SCI_METHOD SetStyleFor(Sci_Position length, char style) override;
+	bool SCI_METHOD SetStyles(Sci_Position length, const char *styles) override;
+	void SCI_METHOD DecorationSetCurrentIndicator(int indicator) override;
+	void SCI_METHOD DecorationFillRange(Sci_Position position, int value, Sci_Position fillLength) override;
+	void SCI_METHOD ChangeLexerState(Sci_Position start, Sci_Position end) override;
+	int SCI_METHOD CodePage() const override;
+	bool SCI_METHOD IsDBCSLeadByte(char ch) const override;
+	const char *SCI_METHOD BufferPointer() override;
+	int SCI_METHOD GetLineIndentation(Sci_Position line) override;
+	Sci_Position SCI_METHOD LineEnd(Sci_Position line) const override;
+	Sci_Position SCI_METHOD GetRelativePosition(Sci_Position positionStart, Sci_Position characterOffset) const override;
+	int SCI_METHOD GetCharacterAndWidth(Sci_Position position, Sci_Position *pWidth) const override;
+};
+
diff --git a/lexilla/test/TestLexers.cxx b/lexilla/test/TestLexers.cxx
new file mode 100644
index 000000000..542ca212c
--- /dev/null
+++ b/lexilla/test/TestLexers.cxx
@@ -0,0 +1,195 @@
+// TestLexers.cxx : Test lexers through Lexilla
+//
+
+#include <cassert>
+
+#include <string>
+#include <string_view>
+#include <vector>
+#include <map>
+
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include <filesystem>
+
+#include "ILexer.h"
+
+#include "TestDocument.h"
+#include "LexillaAccess.h"
+
+namespace {
+
+std::string ReadFile(std::filesystem::path path) {
+	std::ifstream ifs(path, std::ios::binary);
+	std::string content((std::istreambuf_iterator<char>(ifs)),
+		(std::istreambuf_iterator<char>()));
+	return content;
+}
+
+std::string MarkedDocument(const Scintilla::IDocument *pdoc) {
+	std::ostringstream os(std::ios::binary);
+	char prevStyle = -1;
+	for (Sci_Position pos = 0; pos < pdoc->Length(); pos++) {
+		const char styleNow = pdoc->StyleAt(pos);
+		if (styleNow != prevStyle) {
+			os << "{" << static_cast<unsigned int>(styleNow) << "}";
+			prevStyle = styleNow;
+		}
+		char ch = '\0';
+		pdoc->GetCharRange(&ch, pos, 1);
+		os << ch;
+	}
+	return os.str();
+}
+
+std::map<std::string, std::string> PropertiesFromFile(std::filesystem::path path) {
+	std::map<std::string, std::string> m;
+	std::ifstream ifs(path);
+	std::string line;
+	std::string logicalLine;
+	while (std::getline(ifs, line)) {
+		logicalLine += line;
+		if (logicalLine.ends_with("\\")) {
+			logicalLine.pop_back();
+		} else {
+			const size_t positionEquals = logicalLine.find("=");
+			if (positionEquals != std::string::npos) {
+				const std::string key = logicalLine.substr(0, positionEquals);
+				const std::string value = logicalLine.substr(positionEquals+1);
+				m[key] = value;
+			}
+			logicalLine.clear();
+		}
+	}
+	return m;
+}
+
+const std::string BOM = "\xEF\xBB\xBF";
+
+bool TestFile(std::filesystem::path path,
+	std::map<std::string, std::string> properties) {
+	// Find and create correct lexer
+	std::string language;
+	Scintilla::ILexer5 *plex = nullptr;
+	for (auto const &[key, val] : properties) {
+		if (key.starts_with("lexer.*")) {
+			language = val;
+			plex = CreateLexer(language);
+			break;
+		}
+	}
+	if (!plex) {
+		return false;
+	}
+
+	// Set parameters of lexer
+	const std::string keywords = "keywords";
+	for (auto const &[key, val] : properties) {
+		if (key.starts_with("#")) {
+			// Ignore comments
+		} else if (key.starts_with("lexer.*")) {
+			// Ignore
+		} else if (key.starts_with("keywords")) {
+			// Get character after keywords
+			std::string afterKeywords = key.substr(keywords.length(), 1);
+			char characterAfterKeywords = afterKeywords.empty() ? '1' : afterKeywords[0];
+			if (characterAfterKeywords < '1' || characterAfterKeywords > '9')
+				characterAfterKeywords = '1';
+			const int wordSet = characterAfterKeywords - '1';
+			plex->WordListSet(wordSet, val.c_str());
+		} else {
+			plex->PropertySet(key.c_str(), val.c_str());
+		}
+	}
+	std::string text = ReadFile(path);
+	if (text.starts_with(BOM)) {
+		text.erase(0, BOM.length());
+	}
+
+	TestDocument doc;
+	doc.Set(text);
+	Scintilla::IDocument *pdoc = &doc;
+	plex->Lex(0, pdoc->Length(), 0, pdoc);
+	const std::string styledTextNew = MarkedDocument(pdoc);
+	std::filesystem::path pathStyled = path;
+	pathStyled += ".styled";
+	const std::string styledText = ReadFile(pathStyled);
+	if (styledTextNew != styledText) {
+		std::cout << "\n" << path.string() << ":1: is different\n\n";
+		std::filesystem::path pathNew = path;
+		pathNew += ".new";
+		std::ofstream ofs(pathNew, std::ios::binary);
+		ofs << styledTextNew;
+	}
+	return true;
+}
+
+bool TestDirectory(std::filesystem::path directory, std::filesystem::path basePath) {
+	const std::map<std::string, std::string> properties = PropertiesFromFile(directory / "SciTE.properties");
+	bool success = true;
+	for (auto &p : std::filesystem::directory_iterator(directory)) {
+		if (!p.is_directory()) {
+			const std::string extension = p.path().extension().string();
+			if (extension != ".properties" && extension != ".styled" && extension != ".new") {
+				const std::filesystem::path relativePath = p.path().lexically_relative(basePath);
+				std::cout << "Lexing " << relativePath.string() << '\n';
+				if (!TestFile(p, properties)) {
+					success = false;
+				}
+			}
+		}
+	}
+	return success;
+}
+
+bool AccessLexilla(std::filesystem::path basePath) {
+	if (!std::filesystem::exists(basePath)) {
+		std::cout << "No examples at " << basePath.string() << "\n";
+		return false;
+	}
+
+	bool success = true;
+	for (auto &p : std::filesystem::recursive_directory_iterator(basePath)) {
+		if (p.is_directory()) {
+			//std::cout << p.path().string() << '\n';
+			if (!TestDirectory(p, basePath)) {
+				success = false;
+			}
+		}
+	}
+	return success;
+}
+
+std::filesystem::path FindScintillaDirectory(std::filesystem::path startDirectory) {
+	std::filesystem::path directory = startDirectory;
+	while (!directory.empty()) {
+		const std::filesystem::path localScintilla = directory / "scintilla";
+		const std::filesystem::directory_entry entry(localScintilla);
+		if (entry.is_directory()) {
+			std::cout << "Found Scintilla at " << entry.path().string() << "\n";
+			return localScintilla;
+		}
+		const std::filesystem::path parent = directory.parent_path();
+		if (parent == directory) {
+			std::cout << "Reached root at " << directory.string() << "\n";
+			return std::filesystem::path();
+		}
+		directory = parent;
+	}
+	return std::filesystem::path();
+}
+
+}
+
+
+
+int main() {
+	// TODO: Allow specifying the base directory through a command line argument
+	const std::filesystem::path baseDirectory = FindScintillaDirectory(std::filesystem::current_path());
+	if (!baseDirectory.empty()) {
+		if (LoadLexilla(baseDirectory)) {
+			AccessLexilla(baseDirectory / "lexilla" / "test" / "examples");
+		}
+	}
+}
diff --git a/lexilla/test/TestLexers.vcxproj b/lexilla/test/TestLexers.vcxproj
new file mode 100644
index 000000000..e2a910a63
--- /dev/null
+++ b/lexilla/test/TestLexers.vcxproj
@@ -0,0 +1,176 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <VCProjectVersion>16.0</VCProjectVersion>
+    <ProjectGuid>{2E0BBD6B-4BC8-4A6C-9DDA-199C27899335}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>lexillatest</RootNamespace>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <CodeAnalysisRuleSet>..\..\..\..\..\..\..\Users\Neil\NeilRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <CodeAnalysisRuleSet>..\..\..\..\..\..\..\Users\Neil\NeilRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <CodeAnalysisRuleSet>..\..\..\..\..\..\..\Users\Neil\NeilRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <CodeAnalysisRuleSet>..\..\..\..\..\..\..\Users\Neil\NeilRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <AdditionalIncludeDirectories>..\..\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <AdditionalIncludeDirectories>..\..\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <AdditionalIncludeDirectories>..\..\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <AdditionalIncludeDirectories>..\..\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="LexillaAccess.cxx" />
+    <ClCompile Include="TestLexers.cxx" />
+    <ClCompile Include="TestDocument.cxx" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\..\bin\Lexilla.dll" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
+\ No newline at end of file
diff --git a/lexilla/test/makefile b/lexilla/test/makefile
new file mode 100644
index 000000000..f89f75640
--- /dev/null
+++ b/lexilla/test/makefile
@@ -0,0 +1,68 @@
+# Build all the lexer tests using GNU make and either g++ or Clang
+# @file makefile
+# Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+# The License.txt file describes the conditions under which this software may be distributed.
+# Should be run using mingw32-make on Windows, not nmake
+# On Windows g++ is used, on OS X clang, and on Linux g++ is used by default
+# but clang can be used by defining CLANG when invoking make
+# clang works only with libc++, not libstdc++
+
+.PHONY: all test clean
+
+.SUFFIXES: .cxx
+
+WARNINGS = -Wpedantic -Wall -Wextra
+
+ifndef windir
+LIBS += -ldl
+ifeq ($(shell uname),Darwin)
+# On macOS always use Clang
+CLANG = 1
+endif
+endif
+
+EXE = $(if $(windir),TestLexers.exe,TestLexers)
+
+BASE_FLAGS += --std=c++2a
+
+ifdef CLANG
+    CXX = clang++
+    BASE_FLAGS += -fsanitize=address
+endif
+
+ifdef LEXILLA_STATIC
+    DEFINES += -D LEXILLA_STATIC
+    LIBS += ../../bin/liblexilla.a
+endif
+
+ifdef windir
+    DEL = $(if $(wildcard $(dir $(SHELL))rm.exe), $(dir $(SHELL))rm.exe -f, del /q)
+else
+    DEL = rm -f
+endif
+
+DEFINES += -D$(if $(DEBUG),DEBUG,NDEBUG)
+BASE_FLAGS += $(if $(DEBUG),-g,-Os)
+
+INCLUDES = -I ../../include -I ../src
+BASE_FLAGS += $(WARNINGS)
+
+all: $(EXE)
+
+test: $(EXE)
+	./$(EXE)
+
+clean:
+	$(DEL) *.o *.obj $(EXE)
+
+%.o: %.cxx
+	$(CXX) $(DEFINES) $(INCLUDES) $(BASE_FLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@
+
+OBJS = TestLexers.o TestDocument.o LexillaAccess.o
+
+$(EXE): $(OBJS)
+	$(CXX) $(BASE_FLAGS) $(CPPFLAGS) $(CXXFLAGS) $^ $(LIBS) $(LDLIBS) -o $@
+
+TestLexers.o: TestLexers.cxx TestDocument.h LexillaAccess.h
+TestDocument.o: TestDocument.cxx TestDocument.h
+LexillaAccess.o: LexillaAccess.cxx LexillaAccess.h
diff --git a/lexilla/test/testlexers.mak b/lexilla/test/testlexers.mak
new file mode 100644
index 000000000..dd89318c3
--- /dev/null
+++ b/lexilla/test/testlexers.mak
@@ -0,0 +1,40 @@
+# Build the lexers test with Microsoft Visual C++ using nmake
+# Tested with Visual C++ 2019
+
+DEL = del /q
+EXE = TestLexers.exe
+
+INCLUDEDIRS = -I ../../include -I ../src
+
+!IFDEF LEXILLA_STATIC
+STATIC_FLAG = -D LEXILLA_STATIC
+LIBS = ../../bin/liblexilla.lib
+!ENDIF
+
+!IFDEF DEBUG
+DEBUG_OPTIONS = -Od -MTd -DDEBUG $(STATIC_FLAG)
+!ELSE
+DEBUG_OPTIONS=-O1 -MT -DNDEBUG $(STATIC_FLAG) -GL
+!ENDIF
+
+CXXFLAGS = /EHsc /std:c++latest $(DEBUG_OPTIONS) $(INCLUDEDIRS)
+
+OBJS = TestLexers.obj TestDocument.obj LexillaAccess.obj
+
+all: $(EXE)
+
+test: $(EXE)
+	$(EXE)
+
+clean:
+	$(DEL) *.o *.obj *.exe
+
+$(EXE): $(OBJS) $(LIBS)
+	$(CXX) $(CXXFLAGS) $(LIBS) /Fe$@ $**
+
+.cxx.obj::
+	$(CXX) $(CXXFLAGS) -c $<
+
+TestLexers.obj: $*.cxx TestDocument.h LexillaAccess.h
+TestDocument.obj: $*.cxx $*.h
+LexillaAccess.obj: $*.cxx $*.h
diff --git a/scripts/HeaderCheck.py b/scripts/HeaderCheck.py
index b25dd105c..5378717a6 100644
--- a/scripts/HeaderCheck.py
+++ b/scripts/HeaderCheck.py
@@ -42,6 +42,8 @@ def CheckFiles(root):
     filePaths += glob.glob(root + "/cocoa/*.mm")
     filePaths += glob.glob(root + "/cocoa/*.h")
     filePaths += glob.glob(root + "/test/unit/*.cxx")
+    filePaths += glob.glob(root + "/lexilla/src/*.cxx")
+    filePaths += glob.glob(root + "/lexilla/test/*.cxx")
     # The Qt platform code interleaves system and Scintilla headers
     #~ filePaths += glob.glob(root + "/qt/ScintillaEditBase/*.cpp")
     #~ filePaths += glob.glob(root + "/qt/ScintillaEdit/*.cpp")
diff --git a/scripts/HeaderOrder.txt b/scripts/HeaderOrder.txt
index 23c16081a..03eb3d539 100644
--- a/scripts/HeaderOrder.txt
+++ b/scripts/HeaderOrder.txt
@@ -49,6 +49,8 @@
 #include <regex>
 #include <iostream>
 #include <sstream>
+#include <fstream>
+#include <filesystem>
 
 // POSIX
 #include <dlfcn.h>
@@ -156,6 +158,10 @@
 
 #include "ExternalLexer.h"
 
+#include "Lexilla.h"
+#include "TestDocument.h"
+#include "LexillaAccess.h"
+
 // Platform-specific headers
 
 // win32