Move non-platform-specific scripts into the scripts directory.

Use FileGenerator module for file generation instead of code in each script.

7 files changed, 361 insertions, 570 deletions

diff --git a/include/HFacer.py b/include/HFacer.py
deleted file mode 100755
index aa6b0f0a2..000000000
--- a/include/HFacer.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-# HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface
-# definition file.
-# The header files are copied to a temporary file apart from the section between a /* ++Autogenerated*/
-# comment and a /* --Autogenerated*/ comment which is generated by the printHFile and printLexHFile
-# functions. After the temporary file is created, it is copied back to the original file name.
-
-import sys
-import os
-import Face
-
-def Contains(s,sub):
-	return s.find(sub) != -1
-
-def printLexHFile(f,out):
-	for name in f.order:
-		v = f.features[name]
-		if v["FeatureType"] in ["val"]:
-			if Contains(name, "SCE_") or Contains(name, "SCLEX_"):
-				out.write("#define " + name + " " + v["Value"] + "\n")
-
-def printHFile(f,out):
-	previousCategory = ""
-	for name in f.order:
-		v = f.features[name]
-		if v["Category"] != "Deprecated":
-			if v["Category"] == "Provisional" and previousCategory != "Provisional":
-				out.write("#ifndef SCI_DISABLE_PROVISIONAL\n")
-			previousCategory = v["Category"]
-			if v["FeatureType"] in ["fun", "get", "set"]:
-				featureDefineName = "SCI_" + name.upper()
-				out.write("#define " + featureDefineName + " " + v["Value"] + "\n")
-			elif v["FeatureType"] in ["evt"]:
-				featureDefineName = "SCN_" + name.upper()
-				out.write("#define " + featureDefineName + " " + v["Value"] + "\n")
-			elif v["FeatureType"] in ["val"]:
-				if not (Contains(name, "SCE_") or Contains(name, "SCLEX_")):
-					out.write("#define " + name + " " + v["Value"] + "\n")
-	out.write("#endif\n")
-
-def CopyWithInsertion(input, output, genfn, definition):
-	copying = 1
-	for line in input.readlines():
-		if copying:
-			output.write(line)
-		if Contains(line, "/* ++Autogenerated"):
-			copying = 0
-			genfn(definition, output)
-		if Contains(line, "/* --Autogenerated"):
-			copying = 1
-			output.write(line)
-
-def contents(filename):
-	f = open(filename)
-	t = f.read()
-	f.close()
-	return t
-
-def Regenerate(filename, genfn, definition):
-	inText = contents(filename)
-	tempname = "HFacer.tmp"
-	out = open(tempname,"w")
-	hfile = open(filename)
-	CopyWithInsertion(hfile, out, genfn, definition)
-	out.close()
-	hfile.close()
-	outText = contents(tempname)
-	if inText == outText:
-		os.unlink(tempname)
-	else:
-		os.unlink(filename)
-		os.rename(tempname, filename)
-
-f = Face.Face()
-try:
-	f.ReadFromFile("Scintilla.iface")
-	Regenerate("Scintilla.h", printHFile, f)
-	Regenerate("SciLexer.h", printLexHFile, f)
-	print("Maximum ID is %s" % max([x for x in f.values if int(x) < 3000]))
-except:
-	raise
diff --git a/qt/ScintillaEdit/WidgetGen.py b/qt/ScintillaEdit/WidgetGen.py
index 322c7dc7d..b53fe988c 100644
--- a/qt/ScintillaEdit/WidgetGen.py
+++ b/qt/ScintillaEdit/WidgetGen.py
@@ -7,12 +7,10 @@ import os
 import getopt
 
 scintillaDirectory = "../.."
-scintillaIncludeDirectory = os.path.join(scintillaDirectory, "include")
-sys.path.append(scintillaIncludeDirectory)
+scintillaScriptsDirectory = os.path.join(scintillaDirectory, "scripts")
+sys.path.append(scintillaScriptsDirectory)
 import Face
-
-def Contains(s,sub):
-	return s.find(sub) != -1
+from FileGenerator import GenerateFile
 
 def underscoreName(s):
 	# Name conversion fixes to match gtkscintilla2
@@ -82,17 +80,20 @@ def arguments(v, stringResult, options):
 		ret = ret + p2Type + " " + normalisedName(v["Param2Name"], options)
 	return ret
 
-def printPyFile(f,out, options):
+def printPyFile(f, options):
+	out = []
 	for name in f.order:
 		v = f.features[name]
 		if v["Category"] != "Deprecated":
 			feat = v["FeatureType"]
 			if feat in ["val"]:
-				out.write(name + "=" + v["Value"] + "\n")
+				out.append(name + "=" + v["Value"])
 			if feat in ["evt"]:
-				out.write("SCN_" + name.upper() + "=" + v["Value"] + "\n")
+				out.append("SCN_" + name.upper() + "=" + v["Value"])
+	return out
 
-def printHFile(f,out, options):
+def printHFile(f, options):
+	out = []
 	for name in f.order:
 		v = f.features[name]
 		if v["Category"] != "Deprecated":
@@ -104,9 +105,10 @@ def printHFile(f,out, options):
 					stringResult = v["Param2Type"] == "stringresult"
 					if stringResult:
 						returnType = "QByteArray"
-					out.write("\t" + returnType + " " + normalisedName(name, options, feat) + "(")
-					out.write(arguments(v, stringResult, options))
-					out.write(")" + constDeclarator + ";\n")
+					out.append("\t" + returnType + " " + normalisedName(name, options, feat) + "(" +
+						arguments(v, stringResult, options)+
+						")" + constDeclarator + ";")
+	return out
 
 def methodNames(f, options):
 	for name in f.order:
@@ -117,7 +119,8 @@ def methodNames(f, options):
 				if checkTypes(name, v):
 					yield normalisedName(name, options)
 
-def printCPPFile(f,out, options):
+def printCPPFile(f, options):
+	out = []
 	for name in f.order:
 		v = f.features[name]
 		if v["Category"] != "Deprecated":
@@ -133,75 +136,39 @@ def printCPPFile(f,out, options):
 					returnStatement = ""
 					if returnType != "void":
 						returnStatement = "return "
-					out.write(returnType + " ScintillaEdit::" + normalisedName(name, options, feat) + "(")
-					out.write(arguments(v, stringResult, options))
-					out.write(")" + constDeclarator + " {\n")
+					out.append(returnType + " ScintillaEdit::" + normalisedName(name, options, feat) + "(" +
+						arguments(v, stringResult, options) +
+						")" + constDeclarator + " {")
+					returns = ""
 					if stringResult:
-						out.write("    " + returnStatement + "TextReturner(" + featureDefineName + ", ")
+						returns += "    " + returnStatement + "TextReturner(" + featureDefineName + ", "
 						if "*" in cppAlias(v["Param1Type"]):
-							out.write("(uptr_t)")
+							returns += "(uptr_t)"
 						if v["Param1Name"]:
-							out.write(normalisedName(v["Param1Name"], options))
+							returns += normalisedName(v["Param1Name"], options)
 						else:
-							out.write("0")
-						out.write(");\n")
+							returns += "0"
+						returns += ");"
 					else:
-						out.write("    " + returnStatement + "send(" + featureDefineName + ", ")
+						returns += "    " + returnStatement + "send(" + featureDefineName + ", "
 						if "*" in cppAlias(v["Param1Type"]):
-							out.write("(uptr_t)")
+							returns += "(uptr_t)"
 						if v["Param1Name"]:
-							out.write(normalisedName(v["Param1Name"], options))
+							returns += normalisedName(v["Param1Name"], options)
 						else:
-							out.write("0")
-						out.write(", ")
+							returns += "0"
+						returns += ", "
 						if "*" in cppAlias(v["Param2Type"]):
-							out.write("(sptr_t)")
+							returns += "(sptr_t)"
 						if v["Param2Name"]:
-							out.write(normalisedName(v["Param2Name"], options))
+							returns += normalisedName(v["Param2Name"], options)
 						else:
-							out.write("0")
-						out.write(");\n")
-					out.write("}\n")
-					out.write("\n")
-
-def CopyWithInsertion(input, output, genfn, definition, options):
-	copying = 1
-	for line in input.readlines():
-		if copying:
-			output.write(line)
-		if "/* ++Autogenerated" in line or "# ++Autogenerated" in line or "<!-- ++Autogenerated" in line:
-			copying = 0
-			genfn(definition, output, options)
-		# ~~ form needed as XML comments can not contain --
-		if "/* --Autogenerated" in line or "# --Autogenerated" in line or "<!-- ~~Autogenerated" in line:
-			copying = 1
-			output.write(line)
-
-def contents(filename):
-	with open(filename, "U") as f:
-		t = f.read()
-		return t
-
-def Generate(templateFile, destinationFile, genfn, definition, options):
-	inText = contents(templateFile)
-	try:
-		currentText = contents(destinationFile)
-	except IOError:
-		currentText = ""
-	tempname = "WidgetGen.tmp"
-	with open(tempname, "w") as out:
-		with open(templateFile, "U") as hfile:
-			CopyWithInsertion(hfile, out, genfn, definition, options)
-	outText = contents(tempname)
-	if currentText == outText:
-		os.unlink(tempname)
-	else:
-		try:
-			os.unlink(destinationFile)
-		except OSError:
-			# Will see failure if file does not yet exist
-			pass
-		os.rename(tempname, destinationFile)
+							returns += "0"
+						returns += ");"
+					out.append(returns)
+					out.append("}")
+					out.append("")
+	return out
 
 def gtkNames():
 	# The full path on my machine: should be altered for anyone else
@@ -253,11 +220,13 @@ def main(argv):
 	options = {"qtStyle": qtStyleInterface}
 	f = readInterface(cleanGenerated)
 	try:
-		Generate("ScintillaEdit.cpp.template", "ScintillaEdit.cpp", printCPPFile, f, options)
-		Generate("ScintillaEdit.h.template", "ScintillaEdit.h", printHFile, f, options)
-		Generate("../ScintillaEditPy/ScintillaConstants.py.template",
+		GenerateFile("ScintillaEdit.cpp.template", "ScintillaEdit.cpp", 
+			"/* ", True, printCPPFile(f, options))
+		GenerateFile("ScintillaEdit.h.template", "ScintillaEdit.h",
+			"/* ", True, printHFile(f, options))
+		GenerateFile("../ScintillaEditPy/ScintillaConstants.py.template",
 			"../ScintillaEditPy/ScintillaConstants.py",
-			printPyFile, f, options)
+			"# ", True, printPyFile(f, options))
 		if checkGTK:
 			names = set(methodNames(f))
 			#~ print("\n".join(names))
diff --git a/qt/ScintillaEditPy/sepbuild.py b/qt/ScintillaEditPy/sepbuild.py
index a823a96b7..77d2cb221 100644
--- a/qt/ScintillaEditPy/sepbuild.py
+++ b/qt/ScintillaEditPy/sepbuild.py
@@ -11,6 +11,11 @@ import sys
 sys.path.append(os.path.join("..", "ScintillaEdit"))
 import WidgetGen
 
+scintillaDirectory = "../.."
+scintillaScriptsDirectory = os.path.join(scintillaDirectory, "scripts")
+sys.path.append(scintillaScriptsDirectory)
+from FileGenerator import GenerateFile
+
 # Decide up front which platform, treat anything other than Windows or OS X as Linux
 PLAT_WINDOWS = platform.system() == "Windows"
 PLAT_DARWIN = platform.system() == "Darwin"
@@ -56,8 +61,7 @@ def usage():
 	print("-u --underscore-names  use method_names consistent with GTK+ standards")
 
 modifyFunctionElement = """		<modify-function signature="%s">%s
-		</modify-function>
-"""
+		</modify-function>"""
 
 injectCode = """
 			<inject-code class="target" position="beginning">%s
@@ -83,7 +87,8 @@ def methodSignature(name, v, options):
 	constDeclarator = " const" if v["FeatureType"] == "get" else ""
 	return methodName + "(" + argTypes + ")" + constDeclarator
 
-def printTypeSystemFile(f,out, options):
+def printTypeSystemFile(f, options):
+	out = []
 	for name in f.order:
 		v = f.features[name]
 		if v["Category"] != "Deprecated":
@@ -99,9 +104,10 @@ def printTypeSystemFile(f,out, options):
 						checks = checks + (injectCheckN % 1)
 				if checks:
 					inject = injectCode % checks
-					out.write(modifyFunctionElement % (methodSignature(name, v, options), inject))
+					out.append(modifyFunctionElement % (methodSignature(name, v, options), inject))
 				#if v["Param1Type"] == "string":
-				#	out.write("<string-xml>" + name + "</string-xml>\n")
+				#	out.append("<string-xml>" + name + "</string-xml>\n")
+	return out
 
 def doubleBackSlashes(s):
 	# Quote backslashes so qmake does not produce warnings
@@ -193,7 +199,8 @@ class SepBuilder:
 		f = WidgetGen.readInterface(False)
 		os.chdir(os.path.join("..", "ScintillaEditPy"))
 		options = {"qtStyle": self.qtStyleInterface}
-		WidgetGen.Generate("typesystem_ScintillaEdit.xml.template", "typesystem_ScintillaEdit.xml", printTypeSystemFile, f, options)
+		GenerateFile("typesystem_ScintillaEdit.xml.template", "typesystem_ScintillaEdit.xml", 
+			"<!-- ", True, printTypeSystemFile(f, options))
 
 	def runGenerator(self):
 		generatorrunner = "shiboken"
diff --git a/include/Face.py b/scripts/Face.py
index 855d6321a..855d6321a 100644
--- a/include/Face.py
+++ b/scripts/Face.py
diff --git a/scripts/HFacer.py b/scripts/HFacer.py
new file mode 100755
index 000000000..27537fec3
--- /dev/null
+++ b/scripts/HFacer.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface
+# definition file.
+
+import sys
+import os
+import Face
+
+from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd
+
+def printLexHFile(f):
+	out = []
+	for name in f.order:
+		v = f.features[name]
+		if v["FeatureType"] in ["val"]:
+			if "SCE_" in name or "SCLEX_" in name:
+				out.append("#define " + name + " " + v["Value"])
+	return out
+
+def printHFile(f):
+	out = []
+	previousCategory = ""
+	for name in f.order:
+		v = f.features[name]
+		if v["Category"] != "Deprecated":
+			if v["Category"] == "Provisional" and previousCategory != "Provisional":
+				out.append("#ifndef SCI_DISABLE_PROVISIONAL")
+			previousCategory = v["Category"]
+			if v["FeatureType"] in ["fun", "get", "set"]:
+				featureDefineName = "SCI_" + name.upper()
+				out.append("#define " + featureDefineName + " " + v["Value"])
+			elif v["FeatureType"] in ["evt"]:
+				featureDefineName = "SCN_" + name.upper()
+				out.append("#define " + featureDefineName + " " + v["Value"])
+			elif v["FeatureType"] in ["val"]:
+				if not ("SCE_" in name or "SCLEX_" in name):
+					out.append("#define " + name + " " + v["Value"])
+	out.append("#endif")
+	return out
+
+f = Face.Face()
+try:
+	f.ReadFromFile("../include/Scintilla.iface")
+	Regenerate("../include/Scintilla.h", "/* ", printHFile(f))
+	Regenerate("../include/SciLexer.h", "/* ", printLexHFile(f))
+	print("Maximum ID is %s" % max([x for x in f.values if int(x) < 3000]))
+except:
+	raise
diff --git a/scripts/LexGen.py b/scripts/LexGen.py
new file mode 100755
index 000000000..4068ee485
--- /dev/null
+++ b/scripts/LexGen.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python
+# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
+# Released to the public domain.
+
+# Regenerate the Scintilla and SciTE source files that list
+# all the lexers and all the properties files.
+# Should be run whenever a new lexer is added or removed.
+# Requires Python 2.4 or later
+# Most files are regenerated in place with templates stored in comments.
+# The VS .NET project file is generated into a different file as the
+# VS .NET environment will not retain comments when modifying the file.
+# The format of generation comments is documented in FileGenerator.py.
+
+import string
+import sys
+import os
+import glob
+import codecs
+import datetime
+
+from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd
+
+def FindModules(lexFile):
+    modules = []
+    with open(lexFile) as f:
+        for l in f.readlines():
+            if l.startswith("LexerModule"):
+                l = l.replace("(", " ")
+                modules.append(l.split()[1])
+    return modules
+
+# Properties that start with lexer. or fold. are automatically found but there are some
+# older properties that don't follow this pattern so must be explicitly listed.
+knownIrregularProperties = [
+    "fold",
+    "styling.within.preprocessor",
+    "tab.timmy.whinge.level",
+    "asp.default.language",
+    "html.tags.case.sensitive",
+    "ps.level",
+    "ps.tokenize",
+    "sql.backslash.escapes",
+    "nsis.uservars",
+    "nsis.ignorecase"
+]
+
+def FindProperties(lexFile):
+    properties = {}
+    with open(lexFile) as f:
+        for l in f.readlines():
+            if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l:
+                l = l.strip()
+                if not l.startswith("//"):	# Drop comments
+                    propertyName = l.split("\"")[1]
+                    if propertyName.lower() == propertyName:
+                        # Only allow lower case property names
+                        if propertyName in knownIrregularProperties or \
+                            propertyName.startswith("fold.") or \
+                            propertyName.startswith("lexer."):
+                            properties[propertyName] = 1
+    return properties
+
+def FindPropertyDocumentation(lexFile):
+    documents = {}
+    with open(lexFile) as f:
+        name = ""
+        for l in f.readlines():
+            l = l.strip()
+            if "// property " in l:
+                propertyName = l.split()[2]
+                if propertyName.lower() == propertyName:
+                    # Only allow lower case property names
+                    name = propertyName
+                    documents[name] = ""
+            elif "DefineProperty" in l and "\"" in l:
+                propertyName = l.split("\"")[1]
+                if propertyName.lower() == propertyName:
+                    # Only allow lower case property names
+                    name = propertyName
+                    documents[name] = ""
+            elif name:
+                if l.startswith("//"):
+                    if documents[name]:
+                        documents[name] += " "
+                    documents[name] += l[2:].strip()
+                elif l.startswith("\""):
+                    l = l[1:].strip()
+                    if l.endswith(";"):
+                        l = l[:-1].strip()
+                    if l.endswith(")"):
+                        l = l[:-1].strip()
+                    if l.endswith("\""):
+                        l = l[:-1]
+                    # Fix escaped double quotes
+                    l = l.replace("\\\"", "\"")
+                    documents[name] += l
+                else:
+                    name = ""
+    for name in list(documents.keys()):
+        if documents[name] == "":
+            del documents[name]
+    return documents
+
+def ciCompare(a,b):
+    return cmp(a.lower(), b.lower())
+
+def ciKey(a):
+    return a.lower()
+
+def sortListInsensitive(l):
+    try:    # Try key function
+        l.sort(key=ciKey)
+    except TypeError:    # Earlier version of Python, so use comparison function
+        l.sort(ciCompare)
+
+host = "prdownloads.sourceforge.net/"
+def UpdateDownloadLinks(path, version):
+    lines = []
+    with open(path, "r") as f:
+        for l in f.readlines():
+            l = l.rstrip()
+            if host in l:
+                start, prd, rest = l.partition(host)
+                pth, dot, ending = rest.partition(".")
+                pthNew = pth[:-3] + version.rstrip()
+                lineWithNewVersion = start + prd +pthNew + dot + ending
+                lines.append(lineWithNewVersion)
+            else:
+                lines.append(l)
+    contents = lineEnd.join(lines) + lineEnd
+    UpdateFile(path, contents)
+
+def UpdateVersionNumbers(root):
+    with open(root + "scintilla/version.txt") as f:
+        version = f.read()
+    versionDotted = version[0] + '.' + version[1] + '.' + version[2]
+    versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0'
+    with open(root + "scintilla/doc/index.html") as f:
+        dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3]
+        # 20130602
+        # index.html, SciTE.html
+        dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d")
+        yearModified = dateModified[0:4]
+        monthModified = dtModified.strftime("%B")
+        dayModified = "%d" % dtModified.day
+        mdyModified = monthModified + " " + dayModified + " " + yearModified
+        # May 22 2013
+        # index.html, SciTE.html
+        dmyModified = dayModified + " " + monthModified + " " + yearModified
+        # 22 May 2013
+        # ScintillaHistory.html -- only first should change
+        myModified = monthModified + " " + yearModified
+        # scite/src/SciTE.h
+        #define COPYRIGHT_DATES "December 1998-May 2013"
+        #define COPYRIGHT_YEARS "1998-2013"
+        dateLine = f.readlines()
+
+    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA",
+        "#define VERSION_SCINTILLA \"" + versionDotted + "\"")
+    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS",
+        "#define VERSION_WORDS " + versionCommad)
+    UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro",
+        "VERSION =",
+        "VERSION = " + versionDotted)
+    UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro",
+        "VERSION =",
+        "VERSION = " + versionDotted)
+    UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", "       Release",
+        "       Release " + versionDotted)
+    UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version)
+    UpdateLineInFile(root + "scintilla/doc/index.html",
+        '          <font color="#FFCC99" size="3"> Release version',
+        '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
+    UpdateLineInFile(root + "scintilla/doc/index.html",
+        '           Site last modified',
+        '           Site last modified ' + mdyModified + '</font>')
+    UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html",
+        '	Released ',
+        '	Released ' + dmyModified + '.')
+
+    if os.path.exists(root + "scite"):
+        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE",
+            "#define VERSION_SCITE \"" + versionDotted + "\"")
+        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS",
+            "#define VERSION_WORDS " + versionCommad)
+        UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES",
+            '#define COPYRIGHT_DATES "December 1998-' + myModified + '"')
+        UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS",
+            '#define COPYRIGHT_YEARS "1998-' + yearModified + '"')
+        UpdateLineInFile(root + "scite/doc/SciTEDownload.html", "       Release",
+            "       Release " + versionDotted)
+        UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version)
+        UpdateLineInFile(root + "scite/doc/SciTE.html",
+            '          <font color="#FFCC99" size="3"> Release version',
+            '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
+        UpdateLineInFile(root + "scite/doc/SciTE.html",
+            '           Site last modified',
+            '           Site last modified ' + mdyModified + '</font>')
+        UpdateLineInFile(root + "scite/doc/SciTE.html",
+            '    <meta name="Date.Modified"',
+            '    <meta name="Date.Modified" content="' + dateModified + '" />')
+
+def RegenerateAll():
+    root="../../"
+
+    # Find all the lexer source code files
+    lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx")
+    sortListInsensitive(lexFilePaths)
+    lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
+    print(lexFiles)
+    lexerModules = []
+    lexerProperties = {}
+    propertyDocuments = {}
+    for lexFile in lexFilePaths:
+        lexerModules.extend(FindModules(lexFile))
+        for k in FindProperties(lexFile).keys():
+            lexerProperties[k] = 1
+        documents = FindPropertyDocumentation(lexFile)
+        for k in documents.keys():
+            if k not in propertyDocuments:
+                propertyDocuments[k] = documents[k]
+    sortListInsensitive(lexerModules)
+    lexerProperties = list(lexerProperties.keys())
+    sortListInsensitive(lexerProperties)
+
+    # Generate HTML to document each property
+    # This is done because tags can not be safely put inside comments in HTML
+    documentProperties = list(propertyDocuments.keys())
+    sortListInsensitive(documentProperties)
+    propertiesHTML = []
+    for k in documentProperties:
+        propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" %
+            (k, k, propertyDocuments[k]))
+
+    # Find all the SciTE properties files
+    otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
+    if os.path.exists(root + "scite"):
+        propFilePaths = glob.glob(root + "scite/src/*.properties")
+        sortListInsensitive(propFilePaths)
+        propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps]
+        sortListInsensitive(propFiles)
+        print(propFiles)
+
+    Regenerate(root + "scintilla/src/Catalogue.cxx", "//", lexerModules)
+    Regenerate(root + "scintilla/win32/scintilla.mak", "#", lexFiles)
+    if os.path.exists(root + "scite"):
+        Regenerate(root + "scite/win32/makefile", "#", propFiles)
+        Regenerate(root + "scite/win32/scite.mak", "#", propFiles)
+        Regenerate(root + "scite/src/SciTEProps.cxx", "//", lexerProperties)
+        Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", propertiesHTML)
+        Generate(root + "scite/boundscheck/vcproj.gen",
+         root + "scite/boundscheck/SciTE.vcproj", "#", lexFiles)
+
+    UpdateVersionNumbers(root)
+
+RegenerateAll()
diff --git a/src/LexGen.py b/src/LexGen.py
deleted file mode 100755
index 579468abe..000000000
--- a/src/LexGen.py
+++ /dev/null
@@ -1,408 +0,0 @@
-#!/usr/bin/env python
-# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
-# Released to the public domain.
-
-# Regenerate the Scintilla and SciTE source files that list
-# all the lexers and all the properties files.
-# Should be run whenever a new lexer is added or removed.
-# Requires Python 2.4 or later
-# Most files are regenerated in place with templates stored in comments.
-# The VS .NET project file is generated into a different file as the
-# VS .NET environment will not retain comments when modifying the file.
-# The files are copied to a string apart from sections between a
-# ++Autogenerated comment and a --Autogenerated comment which is
-# generated by the CopyWithInsertion function. After the whole
-# string is instantiated, it is compared with the target file and
-# if different the file is rewritten.
-# Does not regenerate the Visual C++ 6 project files but does the VS .NET
-# project file.
-
-import string
-import sys
-import os
-import glob
-import codecs
-import datetime
-
-# EOL constants
-CR = "\r"
-LF = "\n"
-CRLF = "\r\n"
-if sys.platform == "win32":
-    NATIVE = CRLF
-else:
-    # Yes, LF is the native EOL even on Mac OS X. CR is just for
-    # Mac OS <=9 (a.k.a. "Mac Classic")
-    NATIVE = LF
-
-# Automatically generated sections contain start and end comments,
-# a definition line and the results.
-# The results are replaced by regenerating based on the definition line.
-# The definition line is a comment prefix followed by "**".
-# If there is a digit after the ** then this indicates which list to use
-# and the digit and next character are not part of the definition
-# Backslash is used as an escape within the definition line.
-# The part between \( and \) is repeated for each item in the list.
-# \* is replaced by each list item. \t, and \n are tab and newline.
-def CopyWithInsertion(input, commentPrefix, retainDefs, eolType, *lists):
-    copying = 1
-    listid = 0
-    output = []
-    for line in input.splitlines(0):
-        isStartGenerated = line.startswith(commentPrefix + "++Autogenerated")
-        if copying and not isStartGenerated:
-            output.append(line)
-        if isStartGenerated:
-            if retainDefs:
-                output.append(line)
-            copying = 0
-            definition = ""
-        elif not copying and line.startswith(commentPrefix + "**"):
-            if retainDefs:
-                output.append(line)
-            definition = line[len(commentPrefix + "**"):]
-            if (commentPrefix == "<!--") and (" -->" in definition):
-                definition = definition.replace(" -->", "")
-            listid = 0
-            if definition[0] in string.digits:
-                listid = int(definition[:1])
-                definition = definition[2:]
-            # Hide double slashes as a control character
-            definition = definition.replace("\\\\", "\001")
-            # Do some normal C style transforms
-            definition = definition.replace("\\n", "\n")
-            definition = definition.replace("\\t", "\t")
-            # Get the doubled backslashes back as single backslashes
-            definition = definition.replace("\001", "\\")
-            startRepeat = definition.find("\\(")
-            endRepeat = definition.find("\\)")
-            intro = definition[:startRepeat]
-            out = ""
-            if intro.endswith("\n"):
-                pos = 0
-            else:
-                pos = len(intro)
-            out += intro
-            middle = definition[startRepeat+2:endRepeat]
-            for i in lists[listid]:
-                item = middle.replace("\\*", i)
-                if pos and (pos + len(item) >= 80):
-                    out += "\\\n"
-                    pos = 0
-                out += item
-                pos += len(item)
-                if item.endswith("\n"):
-                    pos = 0
-            outro = definition[endRepeat+2:]
-            out += outro
-            out = out.replace("\n", eolType) # correct EOLs in generated content
-            output.append(out)
-        elif line.startswith(commentPrefix + "--Autogenerated"):
-            copying = 1
-            if retainDefs:
-                output.append(line)
-    output = [line.rstrip(" \t") for line in output] # trim trailing whitespace
-    return eolType.join(output) + eolType
-
-def UpdateFile(filename, updated):
-    """ If the file is different to updated then copy updated
-    into the file else leave alone so CVS and make don't treat
-    it as modified. """
-    try:
-        infile = open(filename, "rb")
-    except IOError:	# File is not there yet
-        out = open(filename, "wb")
-        out.write(updated.encode('utf-8'))
-        out.close()
-        print("New %s" % filename)
-        return
-    original = infile.read()
-    infile.close()
-    original = original.decode('utf-8')
-    if updated != original:
-        os.unlink(filename)
-        out = open(filename, "wb")
-        out.write(updated.encode('utf-8'))
-        out.close()
-        print("Changed %s " % filename)
-    #~ else:
-        #~ print "Unchanged", filename
-
-def Generate(inpath, outpath, commentPrefix, eolType, *lists):
-    """Generate 'outpath' from 'inpath'.
-
-        "eolType" indicates the type of EOLs to use in the generated
-            file. It should be one of following constants: LF, CRLF,
-            CR, or NATIVE.
-    """
-    #print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\
-    #      % (inpath, outpath, commentPrefix, eolType)
-    try:
-        infile = open(inpath, "rb")
-    except IOError:
-        print("Can not open %s" % inpath)
-        return
-    original = infile.read()
-    infile.close()
-    original = original.decode('utf-8')
-    updated = CopyWithInsertion(original, commentPrefix,
-        inpath == outpath, eolType, *lists)
-    UpdateFile(outpath, updated)
-
-def Regenerate(filename, commentPrefix, eolType, *lists):
-    """Regenerate the given file.
-
-        "eolType" indicates the type of EOLs to use in the generated
-            file. It should be one of following constants: LF, CRLF,
-            CR, or NATIVE.
-    """
-    Generate(filename, filename, commentPrefix, eolType, *lists)
-
-def FindModules(lexFile):
-    modules = []
-    f = open(lexFile)
-    for l in f.readlines():
-        if l.startswith("LexerModule"):
-            l = l.replace("(", " ")
-            modules.append(l.split()[1])
-    return modules
-
-# Properties that start with lexer. or fold. are automatically found but there are some
-# older properties that don't follow this pattern so must be explicitly listed.
-knownIrregularProperties = [
-    "fold",
-    "styling.within.preprocessor",
-    "tab.timmy.whinge.level",
-    "asp.default.language",
-    "html.tags.case.sensitive",
-    "ps.level",
-    "ps.tokenize",
-    "sql.backslash.escapes",
-    "nsis.uservars",
-    "nsis.ignorecase"
-]
-
-def FindProperties(lexFile):
-    properties = {}
-    f = open(lexFile)
-    for l in f.readlines():
-        if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l:
-            l = l.strip()
-            if not l.startswith("//"):	# Drop comments
-                propertyName = l.split("\"")[1]
-                if propertyName.lower() == propertyName:
-                    # Only allow lower case property names
-                    if propertyName in knownIrregularProperties or \
-                        propertyName.startswith("fold.") or \
-                        propertyName.startswith("lexer."):
-                        properties[propertyName] = 1
-    return properties
-
-def FindPropertyDocumentation(lexFile):
-    documents = {}
-    f = open(lexFile)
-    name = ""
-    for l in f.readlines():
-        l = l.strip()
-        if "// property " in l:
-            propertyName = l.split()[2]
-            if propertyName.lower() == propertyName:
-                # Only allow lower case property names
-                name = propertyName
-                documents[name] = ""
-        elif "DefineProperty" in l and "\"" in l:
-            propertyName = l.split("\"")[1]
-            if propertyName.lower() == propertyName:
-                # Only allow lower case property names
-                name = propertyName
-                documents[name] = ""
-        elif name:
-            if l.startswith("//"):
-                if documents[name]:
-                    documents[name] += " "
-                documents[name] += l[2:].strip()
-            elif l.startswith("\""):
-                l = l[1:].strip()
-                if l.endswith(";"):
-                    l = l[:-1].strip()
-                if l.endswith(")"):
-                    l = l[:-1].strip()
-                if l.endswith("\""):
-                    l = l[:-1]
-                # Fix escaped double quotes
-                l = l.replace("\\\"", "\"")
-                documents[name] += l
-            else:
-                name = ""
-    for name in list(documents.keys()):
-        if documents[name] == "":
-            del documents[name]
-    return documents
-
-def ciCompare(a,b):
-    return cmp(a.lower(), b.lower())
-
-def ciKey(a):
-    return a.lower()
-
-def sortListInsensitive(l):
-    try:    # Try key function
-        l.sort(key=ciKey)
-    except TypeError:    # Earlier version of Python, so use comparison function
-        l.sort(ciCompare)
-
-def UpdateLineInFile(path, linePrefix, lineReplace):
-    lines = []
-    updated = False
-    with codecs.open(path, "r", "utf-8") as f:
-        for l in f.readlines():
-            l = l.rstrip()
-            if not updated and l.startswith(linePrefix):
-                lines.append(lineReplace)
-                updated = True
-            else:
-                lines.append(l)
-    contents = NATIVE.join(lines) + NATIVE
-    UpdateFile(path, contents)
-
-host = "prdownloads.sourceforge.net/"
-def UpdateDownloadLinks(path, version):
-    lines = []
-    with open(path, "r") as f:
-        for l in f.readlines():
-            l = l.rstrip()
-            if host in l:
-                start, prd, rest = l.partition(host)
-                pth, dot, ending = rest.partition(".")
-                pthNew = pth[:-3] + version.rstrip()
-                lineWithNewVersion = start + prd +pthNew + dot + ending
-                lines.append(lineWithNewVersion)
-            else:
-                lines.append(l)
-    contents = NATIVE.join(lines) + NATIVE
-    UpdateFile(path, contents)
-
-def UpdateVersionNumbers(root):
-    with open(root + "scintilla/version.txt") as f:
-        version = f.read()
-    versionDotted = version[0] + '.' + version[1] + '.' + version[2]
-    versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0'
-    with open(root + "scintilla/doc/index.html") as f:
-        dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3]
-        # 20130602
-        # index.html, SciTE.html
-        dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d")
-        yearModified = dateModified[0:4]
-        monthModified = dtModified.strftime("%B")
-        dayModified = "%d" % dtModified.day
-        mdyModified = monthModified + " " + dayModified + " " + yearModified
-        # May 22 2013
-        # index.html, SciTE.html
-        dmyModified = dayModified + " " + monthModified + " " + yearModified
-        # 22 May 2013
-        # ScintillaHistory.html -- only first should change
-        myModified = monthModified + " " + yearModified
-        # scite/src/SciTE.h
-        #define COPYRIGHT_DATES "December 1998-May 2013"
-        #define COPYRIGHT_YEARS "1998-2013"
-        dateLine = f.readlines()
-
-    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA",
-        "#define VERSION_SCINTILLA \"" + versionDotted + "\"")
-    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS",
-        "#define VERSION_WORDS " + versionCommad)
-    UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro",
-        "VERSION =",
-        "VERSION = " + versionDotted)
-    UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro",
-        "VERSION =",
-        "VERSION = " + versionDotted)
-    UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", "       Release",
-        "       Release " + versionDotted)
-    UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version)
-    UpdateLineInFile(root + "scintilla/doc/index.html",
-        '          <font color="#FFCC99" size="3"> Release version',
-        '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
-    UpdateLineInFile(root + "scintilla/doc/index.html",
-        '           Site last modified',
-        '           Site last modified ' + mdyModified + '</font>')
-    UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html",
-        '	Released ',
-        '	Released ' + dmyModified + '.')
-
-    if os.path.exists(root + "scite"):
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE",
-            "#define VERSION_SCITE \"" + versionDotted + "\"")
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS",
-            "#define VERSION_WORDS " + versionCommad)
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES",
-            '#define COPYRIGHT_DATES "December 1998-' + myModified + '"')
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS",
-            '#define COPYRIGHT_YEARS "1998-' + yearModified + '"')
-        UpdateLineInFile(root + "scite/doc/SciTEDownload.html", "       Release",
-            "       Release " + versionDotted)
-        UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version)
-        UpdateLineInFile(root + "scite/doc/SciTE.html",
-            '          <font color="#FFCC99" size="3"> Release version',
-            '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
-        UpdateLineInFile(root + "scite/doc/SciTE.html",
-            '           Site last modified',
-            '           Site last modified ' + mdyModified + '</font>')
-        UpdateLineInFile(root + "scite/doc/SciTE.html",
-            '    <meta name="Date.Modified"',
-            '    <meta name="Date.Modified" content="' + dateModified + '" />')
-
-def RegenerateAll():
-    root="../../"
-
-    # Find all the lexer source code files
-    lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx")
-    sortListInsensitive(lexFilePaths)
-    lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
-    print(lexFiles)
-    lexerModules = []
-    lexerProperties = {}
-    propertyDocuments = {}
-    for lexFile in lexFilePaths:
-        lexerModules.extend(FindModules(lexFile))
-        for k in FindProperties(lexFile).keys():
-            lexerProperties[k] = 1
-        documents = FindPropertyDocumentation(lexFile)
-        for k in documents.keys():
-            if k not in propertyDocuments:
-                propertyDocuments[k] = documents[k]
-    sortListInsensitive(lexerModules)
-    lexerProperties = list(lexerProperties.keys())
-    sortListInsensitive(lexerProperties)
-
-    # Generate HTML to document each property
-    # This is done because tags can not be safely put inside comments in HTML
-    documentProperties = list(propertyDocuments.keys())
-    sortListInsensitive(documentProperties)
-    propertiesHTML = []
-    for k in documentProperties:
-        propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" %
-            (k, k, propertyDocuments[k]))
-
-    # Find all the SciTE properties files
-    otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
-    if os.path.exists(root + "scite"):
-        propFilePaths = glob.glob(root + "scite/src/*.properties")
-        sortListInsensitive(propFilePaths)
-        propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps]
-        sortListInsensitive(propFiles)
-        print(propFiles)
-
-    Regenerate(root + "scintilla/src/Catalogue.cxx", "//", NATIVE, lexerModules)
-    Regenerate(root + "scintilla/win32/scintilla.mak", "#", NATIVE, lexFiles)
-    if os.path.exists(root + "scite"):
-        Regenerate(root + "scite/win32/makefile", "#", NATIVE, propFiles)
-        Regenerate(root + "scite/win32/scite.mak", "#", NATIVE, propFiles)
-        Regenerate(root + "scite/src/SciTEProps.cxx", "//", NATIVE, lexerProperties)
-        Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", NATIVE, propertiesHTML)
-        Generate(root + "scite/boundscheck/vcproj.gen",
-         root + "scite/boundscheck/SciTE.vcproj", "#", NATIVE, lexFiles)
-
-    UpdateVersionNumbers(root)
-
-RegenerateAll()