diff options
| -rwxr-xr-x | include/HFacer.py | 81 | ||||
| -rw-r--r-- | qt/ScintillaEdit/WidgetGen.py | 119 | ||||
| -rw-r--r-- | qt/ScintillaEditPy/sepbuild.py | 19 | ||||
| -rw-r--r-- | scripts/Face.py (renamed from include/Face.py) | 0 | ||||
| -rwxr-xr-x | scripts/HFacer.py | 48 | ||||
| -rwxr-xr-x | scripts/LexGen.py | 256 | ||||
| -rwxr-xr-x | src/LexGen.py | 408 |
7 files changed, 361 insertions, 570 deletions
diff --git a/include/HFacer.py b/include/HFacer.py deleted file mode 100755 index aa6b0f0a2..000000000 --- a/include/HFacer.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface -# definition file. -# The header files are copied to a temporary file apart from the section between a /* ++Autogenerated*/ -# comment and a /* --Autogenerated*/ comment which is generated by the printHFile and printLexHFile -# functions. After the temporary file is created, it is copied back to the original file name. - -import sys -import os -import Face - -def Contains(s,sub): - return s.find(sub) != -1 - -def printLexHFile(f,out): - for name in f.order: - v = f.features[name] - if v["FeatureType"] in ["val"]: - if Contains(name, "SCE_") or Contains(name, "SCLEX_"): - out.write("#define " + name + " " + v["Value"] + "\n") - -def printHFile(f,out): - previousCategory = "" - for name in f.order: - v = f.features[name] - if v["Category"] != "Deprecated": - if v["Category"] == "Provisional" and previousCategory != "Provisional": - out.write("#ifndef SCI_DISABLE_PROVISIONAL\n") - previousCategory = v["Category"] - if v["FeatureType"] in ["fun", "get", "set"]: - featureDefineName = "SCI_" + name.upper() - out.write("#define " + featureDefineName + " " + v["Value"] + "\n") - elif v["FeatureType"] in ["evt"]: - featureDefineName = "SCN_" + name.upper() - out.write("#define " + featureDefineName + " " + v["Value"] + "\n") - elif v["FeatureType"] in ["val"]: - if not (Contains(name, "SCE_") or Contains(name, "SCLEX_")): - out.write("#define " + name + " " + v["Value"] + "\n") - out.write("#endif\n") - -def CopyWithInsertion(input, output, genfn, definition): - copying = 1 - for line in input.readlines(): - if copying: - output.write(line) - if Contains(line, "/* ++Autogenerated"): - copying = 0 - genfn(definition, output) - if Contains(line, "/* --Autogenerated"): - copying = 1 - output.write(line) - -def contents(filename): - f = open(filename) - t = f.read() - f.close() - return t - -def Regenerate(filename, genfn, definition): - inText = contents(filename) - tempname = "HFacer.tmp" - out = open(tempname,"w") - hfile = open(filename) - CopyWithInsertion(hfile, out, genfn, definition) - out.close() - hfile.close() - outText = contents(tempname) - if inText == outText: - os.unlink(tempname) - else: - os.unlink(filename) - os.rename(tempname, filename) - -f = Face.Face() -try: - f.ReadFromFile("Scintilla.iface") - Regenerate("Scintilla.h", printHFile, f) - Regenerate("SciLexer.h", printLexHFile, f) - print("Maximum ID is %s" % max([x for x in f.values if int(x) < 3000])) -except: - raise diff --git a/qt/ScintillaEdit/WidgetGen.py b/qt/ScintillaEdit/WidgetGen.py index 322c7dc7d..b53fe988c 100644 --- a/qt/ScintillaEdit/WidgetGen.py +++ b/qt/ScintillaEdit/WidgetGen.py @@ -7,12 +7,10 @@ import os import getopt scintillaDirectory = "../.." -scintillaIncludeDirectory = os.path.join(scintillaDirectory, "include") -sys.path.append(scintillaIncludeDirectory) +scintillaScriptsDirectory = os.path.join(scintillaDirectory, "scripts") +sys.path.append(scintillaScriptsDirectory) import Face - -def Contains(s,sub): - return s.find(sub) != -1 +from FileGenerator import GenerateFile def underscoreName(s): # Name conversion fixes to match gtkscintilla2 @@ -82,17 +80,20 @@ def arguments(v, stringResult, options): ret = ret + p2Type + " " + normalisedName(v["Param2Name"], options) return ret -def printPyFile(f,out, options): +def printPyFile(f, options): + out = [] for name in f.order: v = f.features[name] if v["Category"] != "Deprecated": feat = v["FeatureType"] if feat in ["val"]: - out.write(name + "=" + v["Value"] + "\n") + out.append(name + "=" + v["Value"]) if feat in ["evt"]: - out.write("SCN_" + name.upper() + "=" + v["Value"] + "\n") + out.append("SCN_" + name.upper() + "=" + v["Value"]) + return out -def printHFile(f,out, options): +def printHFile(f, options): + out = [] for name in f.order: v = f.features[name] if v["Category"] != "Deprecated": @@ -104,9 +105,10 @@ def printHFile(f,out, options): stringResult = v["Param2Type"] == "stringresult" if stringResult: returnType = "QByteArray" - out.write("\t" + returnType + " " + normalisedName(name, options, feat) + "(") - out.write(arguments(v, stringResult, options)) - out.write(")" + constDeclarator + ";\n") + out.append("\t" + returnType + " " + normalisedName(name, options, feat) + "(" + + arguments(v, stringResult, options)+ + ")" + constDeclarator + ";") + return out def methodNames(f, options): for name in f.order: @@ -117,7 +119,8 @@ def methodNames(f, options): if checkTypes(name, v): yield normalisedName(name, options) -def printCPPFile(f,out, options): +def printCPPFile(f, options): + out = [] for name in f.order: v = f.features[name] if v["Category"] != "Deprecated": @@ -133,75 +136,39 @@ def printCPPFile(f,out, options): returnStatement = "" if returnType != "void": returnStatement = "return " - out.write(returnType + " ScintillaEdit::" + normalisedName(name, options, feat) + "(") - out.write(arguments(v, stringResult, options)) - out.write(")" + constDeclarator + " {\n") + out.append(returnType + " ScintillaEdit::" + normalisedName(name, options, feat) + "(" + + arguments(v, stringResult, options) + + ")" + constDeclarator + " {") + returns = "" if stringResult: - out.write(" " + returnStatement + "TextReturner(" + featureDefineName + ", ") + returns += " " + returnStatement + "TextReturner(" + featureDefineName + ", " if "*" in cppAlias(v["Param1Type"]): - out.write("(uptr_t)") + returns += "(uptr_t)" if v["Param1Name"]: - out.write(normalisedName(v["Param1Name"], options)) + returns += normalisedName(v["Param1Name"], options) else: - out.write("0") - out.write(");\n") + returns += "0" + returns += ");" else: - out.write(" " + returnStatement + "send(" + featureDefineName + ", ") + returns += " " + returnStatement + "send(" + featureDefineName + ", " if "*" in cppAlias(v["Param1Type"]): - out.write("(uptr_t)") + returns += "(uptr_t)" if v["Param1Name"]: - out.write(normalisedName(v["Param1Name"], options)) + returns += normalisedName(v["Param1Name"], options) else: - out.write("0") - out.write(", ") + returns += "0" + returns += ", " if "*" in cppAlias(v["Param2Type"]): - out.write("(sptr_t)") + returns += "(sptr_t)" if v["Param2Name"]: - out.write(normalisedName(v["Param2Name"], options)) + returns += normalisedName(v["Param2Name"], options) else: - out.write("0") - out.write(");\n") - out.write("}\n") - out.write("\n") - -def CopyWithInsertion(input, output, genfn, definition, options): - copying = 1 - for line in input.readlines(): - if copying: - output.write(line) - if "/* ++Autogenerated" in line or "# ++Autogenerated" in line or "<!-- ++Autogenerated" in line: - copying = 0 - genfn(definition, output, options) - # ~~ form needed as XML comments can not contain -- - if "/* --Autogenerated" in line or "# --Autogenerated" in line or "<!-- ~~Autogenerated" in line: - copying = 1 - output.write(line) - -def contents(filename): - with open(filename, "U") as f: - t = f.read() - return t - -def Generate(templateFile, destinationFile, genfn, definition, options): - inText = contents(templateFile) - try: - currentText = contents(destinationFile) - except IOError: - currentText = "" - tempname = "WidgetGen.tmp" - with open(tempname, "w") as out: - with open(templateFile, "U") as hfile: - CopyWithInsertion(hfile, out, genfn, definition, options) - outText = contents(tempname) - if currentText == outText: - os.unlink(tempname) - else: - try: - os.unlink(destinationFile) - except OSError: - # Will see failure if file does not yet exist - pass - os.rename(tempname, destinationFile) + returns += "0" + returns += ");" + out.append(returns) + out.append("}") + out.append("") + return out def gtkNames(): # The full path on my machine: should be altered for anyone else @@ -253,11 +220,13 @@ def main(argv): options = {"qtStyle": qtStyleInterface} f = readInterface(cleanGenerated) try: - Generate("ScintillaEdit.cpp.template", "ScintillaEdit.cpp", printCPPFile, f, options) - Generate("ScintillaEdit.h.template", "ScintillaEdit.h", printHFile, f, options) - Generate("../ScintillaEditPy/ScintillaConstants.py.template", + GenerateFile("ScintillaEdit.cpp.template", "ScintillaEdit.cpp", + "/* ", True, printCPPFile(f, options)) + GenerateFile("ScintillaEdit.h.template", "ScintillaEdit.h", + "/* ", True, printHFile(f, options)) + GenerateFile("../ScintillaEditPy/ScintillaConstants.py.template", "../ScintillaEditPy/ScintillaConstants.py", - printPyFile, f, options) + "# ", True, printPyFile(f, options)) if checkGTK: names = set(methodNames(f)) #~ print("\n".join(names)) diff --git a/qt/ScintillaEditPy/sepbuild.py b/qt/ScintillaEditPy/sepbuild.py index a823a96b7..77d2cb221 100644 --- a/qt/ScintillaEditPy/sepbuild.py +++ b/qt/ScintillaEditPy/sepbuild.py @@ -11,6 +11,11 @@ import sys sys.path.append(os.path.join("..", "ScintillaEdit")) import WidgetGen +scintillaDirectory = "../.." +scintillaScriptsDirectory = os.path.join(scintillaDirectory, "scripts") +sys.path.append(scintillaScriptsDirectory) +from FileGenerator import GenerateFile + # Decide up front which platform, treat anything other than Windows or OS X as Linux PLAT_WINDOWS = platform.system() == "Windows" PLAT_DARWIN = platform.system() == "Darwin" @@ -56,8 +61,7 @@ def usage(): print("-u --underscore-names use method_names consistent with GTK+ standards") modifyFunctionElement = """ <modify-function signature="%s">%s - </modify-function> -""" + </modify-function>""" injectCode = """ <inject-code class="target" position="beginning">%s @@ -83,7 +87,8 @@ def methodSignature(name, v, options): constDeclarator = " const" if v["FeatureType"] == "get" else "" return methodName + "(" + argTypes + ")" + constDeclarator -def printTypeSystemFile(f,out, options): +def printTypeSystemFile(f, options): + out = [] for name in f.order: v = f.features[name] if v["Category"] != "Deprecated": @@ -99,9 +104,10 @@ def printTypeSystemFile(f,out, options): checks = checks + (injectCheckN % 1) if checks: inject = injectCode % checks - out.write(modifyFunctionElement % (methodSignature(name, v, options), inject)) + out.append(modifyFunctionElement % (methodSignature(name, v, options), inject)) #if v["Param1Type"] == "string": - # out.write("<string-xml>" + name + "</string-xml>\n") + # out.append("<string-xml>" + name + "</string-xml>\n") + return out def doubleBackSlashes(s): # Quote backslashes so qmake does not produce warnings @@ -193,7 +199,8 @@ class SepBuilder: f = WidgetGen.readInterface(False) os.chdir(os.path.join("..", "ScintillaEditPy")) options = {"qtStyle": self.qtStyleInterface} - WidgetGen.Generate("typesystem_ScintillaEdit.xml.template", "typesystem_ScintillaEdit.xml", printTypeSystemFile, f, options) + GenerateFile("typesystem_ScintillaEdit.xml.template", "typesystem_ScintillaEdit.xml", + "<!-- ", True, printTypeSystemFile(f, options)) def runGenerator(self): generatorrunner = "shiboken" diff --git a/include/Face.py b/scripts/Face.py index 855d6321a..855d6321a 100644 --- a/include/Face.py +++ b/scripts/Face.py diff --git a/scripts/HFacer.py b/scripts/HFacer.py new file mode 100755 index 000000000..27537fec3 --- /dev/null +++ b/scripts/HFacer.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface +# definition file. + +import sys +import os +import Face + +from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd + +def printLexHFile(f): + out = [] + for name in f.order: + v = f.features[name] + if v["FeatureType"] in ["val"]: + if "SCE_" in name or "SCLEX_" in name: + out.append("#define " + name + " " + v["Value"]) + return out + +def printHFile(f): + out = [] + previousCategory = "" + for name in f.order: + v = f.features[name] + if v["Category"] != "Deprecated": + if v["Category"] == "Provisional" and previousCategory != "Provisional": + out.append("#ifndef SCI_DISABLE_PROVISIONAL") + previousCategory = v["Category"] + if v["FeatureType"] in ["fun", "get", "set"]: + featureDefineName = "SCI_" + name.upper() + out.append("#define " + featureDefineName + " " + v["Value"]) + elif v["FeatureType"] in ["evt"]: + featureDefineName = "SCN_" + name.upper() + out.append("#define " + featureDefineName + " " + v["Value"]) + elif v["FeatureType"] in ["val"]: + if not ("SCE_" in name or "SCLEX_" in name): + out.append("#define " + name + " " + v["Value"]) + out.append("#endif") + return out + +f = Face.Face() +try: + f.ReadFromFile("../include/Scintilla.iface") + Regenerate("../include/Scintilla.h", "/* ", printHFile(f)) + Regenerate("../include/SciLexer.h", "/* ", printLexHFile(f)) + print("Maximum ID is %s" % max([x for x in f.values if int(x) < 3000])) +except: + raise diff --git a/scripts/LexGen.py b/scripts/LexGen.py new file mode 100755 index 000000000..4068ee485 --- /dev/null +++ b/scripts/LexGen.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python +# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org +# Released to the public domain. + +# Regenerate the Scintilla and SciTE source files that list +# all the lexers and all the properties files. +# Should be run whenever a new lexer is added or removed. +# Requires Python 2.4 or later +# Most files are regenerated in place with templates stored in comments. +# The VS .NET project file is generated into a different file as the +# VS .NET environment will not retain comments when modifying the file. +# The format of generation comments is documented in FileGenerator.py. + +import string +import sys +import os +import glob +import codecs +import datetime + +from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd + +def FindModules(lexFile): + modules = [] + with open(lexFile) as f: + for l in f.readlines(): + if l.startswith("LexerModule"): + l = l.replace("(", " ") + modules.append(l.split()[1]) + return modules + +# Properties that start with lexer. or fold. are automatically found but there are some +# older properties that don't follow this pattern so must be explicitly listed. +knownIrregularProperties = [ + "fold", + "styling.within.preprocessor", + "tab.timmy.whinge.level", + "asp.default.language", + "html.tags.case.sensitive", + "ps.level", + "ps.tokenize", + "sql.backslash.escapes", + "nsis.uservars", + "nsis.ignorecase" +] + +def FindProperties(lexFile): + properties = {} + with open(lexFile) as f: + for l in f.readlines(): + if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l: + l = l.strip() + if not l.startswith("//"): # Drop comments + propertyName = l.split("\"")[1] + if propertyName.lower() == propertyName: + # Only allow lower case property names + if propertyName in knownIrregularProperties or \ + propertyName.startswith("fold.") or \ + propertyName.startswith("lexer."): + properties[propertyName] = 1 + return properties + +def FindPropertyDocumentation(lexFile): + documents = {} + with open(lexFile) as f: + name = "" + for l in f.readlines(): + l = l.strip() + if "// property " in l: + propertyName = l.split()[2] + if propertyName.lower() == propertyName: + # Only allow lower case property names + name = propertyName + documents[name] = "" + elif "DefineProperty" in l and "\"" in l: + propertyName = l.split("\"")[1] + if propertyName.lower() == propertyName: + # Only allow lower case property names + name = propertyName + documents[name] = "" + elif name: + if l.startswith("//"): + if documents[name]: + documents[name] += " " + documents[name] += l[2:].strip() + elif l.startswith("\""): + l = l[1:].strip() + if l.endswith(";"): + l = l[:-1].strip() + if l.endswith(")"): + l = l[:-1].strip() + if l.endswith("\""): + l = l[:-1] + # Fix escaped double quotes + l = l.replace("\\\"", "\"") + documents[name] += l + else: + name = "" + for name in list(documents.keys()): + if documents[name] == "": + del documents[name] + return documents + +def ciCompare(a,b): + return cmp(a.lower(), b.lower()) + +def ciKey(a): + return a.lower() + +def sortListInsensitive(l): + try: # Try key function + l.sort(key=ciKey) + except TypeError: # Earlier version of Python, so use comparison function + l.sort(ciCompare) + +host = "prdownloads.sourceforge.net/" +def UpdateDownloadLinks(path, version): + lines = [] + with open(path, "r") as f: + for l in f.readlines(): + l = l.rstrip() + if host in l: + start, prd, rest = l.partition(host) + pth, dot, ending = rest.partition(".") + pthNew = pth[:-3] + version.rstrip() + lineWithNewVersion = start + prd +pthNew + dot + ending + lines.append(lineWithNewVersion) + else: + lines.append(l) + contents = lineEnd.join(lines) + lineEnd + UpdateFile(path, contents) + +def UpdateVersionNumbers(root): + with open(root + "scintilla/version.txt") as f: + version = f.read() + versionDotted = version[0] + '.' + version[1] + '.' + version[2] + versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0' + with open(root + "scintilla/doc/index.html") as f: + dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3] + # 20130602 + # index.html, SciTE.html + dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d") + yearModified = dateModified[0:4] + monthModified = dtModified.strftime("%B") + dayModified = "%d" % dtModified.day + mdyModified = monthModified + " " + dayModified + " " + yearModified + # May 22 2013 + # index.html, SciTE.html + dmyModified = dayModified + " " + monthModified + " " + yearModified + # 22 May 2013 + # ScintillaHistory.html -- only first should change + myModified = monthModified + " " + yearModified + # scite/src/SciTE.h + #define COPYRIGHT_DATES "December 1998-May 2013" + #define COPYRIGHT_YEARS "1998-2013" + dateLine = f.readlines() + + UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA", + "#define VERSION_SCINTILLA \"" + versionDotted + "\"") + UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS", + "#define VERSION_WORDS " + versionCommad) + UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro", + "VERSION =", + "VERSION = " + versionDotted) + UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro", + "VERSION =", + "VERSION = " + versionDotted) + UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", " Release", + " Release " + versionDotted) + UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version) + UpdateLineInFile(root + "scintilla/doc/index.html", + ' <font color="#FFCC99" size="3"> Release version', + ' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />') + UpdateLineInFile(root + "scintilla/doc/index.html", + ' Site last modified', + ' Site last modified ' + mdyModified + '</font>') + UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html", + ' Released ', + ' Released ' + dmyModified + '.') + + if os.path.exists(root + "scite"): + UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE", + "#define VERSION_SCITE \"" + versionDotted + "\"") + UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS", + "#define VERSION_WORDS " + versionCommad) + UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES", + '#define COPYRIGHT_DATES "December 1998-' + myModified + '"') + UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS", + '#define COPYRIGHT_YEARS "1998-' + yearModified + '"') + UpdateLineInFile(root + "scite/doc/SciTEDownload.html", " Release", + " Release " + versionDotted) + UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version) + UpdateLineInFile(root + "scite/doc/SciTE.html", + ' <font color="#FFCC99" size="3"> Release version', + ' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />') + UpdateLineInFile(root + "scite/doc/SciTE.html", + ' Site last modified', + ' Site last modified ' + mdyModified + '</font>') + UpdateLineInFile(root + "scite/doc/SciTE.html", + ' <meta name="Date.Modified"', + ' <meta name="Date.Modified" content="' + dateModified + '" />') + +def RegenerateAll(): + root="../../" + + # Find all the lexer source code files + lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx") + sortListInsensitive(lexFilePaths) + lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths] + print(lexFiles) + lexerModules = [] + lexerProperties = {} + propertyDocuments = {} + for lexFile in lexFilePaths: + lexerModules.extend(FindModules(lexFile)) + for k in FindProperties(lexFile).keys(): + lexerProperties[k] = 1 + documents = FindPropertyDocumentation(lexFile) + for k in documents.keys(): + if k not in propertyDocuments: + propertyDocuments[k] = documents[k] + sortListInsensitive(lexerModules) + lexerProperties = list(lexerProperties.keys()) + sortListInsensitive(lexerProperties) + + # Generate HTML to document each property + # This is done because tags can not be safely put inside comments in HTML + documentProperties = list(propertyDocuments.keys()) + sortListInsensitive(documentProperties) + propertiesHTML = [] + for k in documentProperties: + propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" % + (k, k, propertyDocuments[k])) + + # Find all the SciTE properties files + otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"] + if os.path.exists(root + "scite"): + propFilePaths = glob.glob(root + "scite/src/*.properties") + sortListInsensitive(propFilePaths) + propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps] + sortListInsensitive(propFiles) + print(propFiles) + + Regenerate(root + "scintilla/src/Catalogue.cxx", "//", lexerModules) + Regenerate(root + "scintilla/win32/scintilla.mak", "#", lexFiles) + if os.path.exists(root + "scite"): + Regenerate(root + "scite/win32/makefile", "#", propFiles) + Regenerate(root + "scite/win32/scite.mak", "#", propFiles) + Regenerate(root + "scite/src/SciTEProps.cxx", "//", lexerProperties) + Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", propertiesHTML) + Generate(root + "scite/boundscheck/vcproj.gen", + root + "scite/boundscheck/SciTE.vcproj", "#", lexFiles) + + UpdateVersionNumbers(root) + +RegenerateAll() diff --git a/src/LexGen.py b/src/LexGen.py deleted file mode 100755 index 579468abe..000000000 --- a/src/LexGen.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python -# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org -# Released to the public domain. - -# Regenerate the Scintilla and SciTE source files that list -# all the lexers and all the properties files. -# Should be run whenever a new lexer is added or removed. -# Requires Python 2.4 or later -# Most files are regenerated in place with templates stored in comments. -# The VS .NET project file is generated into a different file as the -# VS .NET environment will not retain comments when modifying the file. -# The files are copied to a string apart from sections between a -# ++Autogenerated comment and a --Autogenerated comment which is -# generated by the CopyWithInsertion function. After the whole -# string is instantiated, it is compared with the target file and -# if different the file is rewritten. -# Does not regenerate the Visual C++ 6 project files but does the VS .NET -# project file. - -import string -import sys -import os -import glob -import codecs -import datetime - -# EOL constants -CR = "\r" -LF = "\n" -CRLF = "\r\n" -if sys.platform == "win32": - NATIVE = CRLF -else: - # Yes, LF is the native EOL even on Mac OS X. CR is just for - # Mac OS <=9 (a.k.a. "Mac Classic") - NATIVE = LF - -# Automatically generated sections contain start and end comments, -# a definition line and the results. -# The results are replaced by regenerating based on the definition line. -# The definition line is a comment prefix followed by "**". -# If there is a digit after the ** then this indicates which list to use -# and the digit and next character are not part of the definition -# Backslash is used as an escape within the definition line. -# The part between \( and \) is repeated for each item in the list. -# \* is replaced by each list item. \t, and \n are tab and newline. -def CopyWithInsertion(input, commentPrefix, retainDefs, eolType, *lists): - copying = 1 - listid = 0 - output = [] - for line in input.splitlines(0): - isStartGenerated = line.startswith(commentPrefix + "++Autogenerated") - if copying and not isStartGenerated: - output.append(line) - if isStartGenerated: - if retainDefs: - output.append(line) - copying = 0 - definition = "" - elif not copying and line.startswith(commentPrefix + "**"): - if retainDefs: - output.append(line) - definition = line[len(commentPrefix + "**"):] - if (commentPrefix == "<!--") and (" -->" in definition): - definition = definition.replace(" -->", "") - listid = 0 - if definition[0] in string.digits: - listid = int(definition[:1]) - definition = definition[2:] - # Hide double slashes as a control character - definition = definition.replace("\\\\", "\001") - # Do some normal C style transforms - definition = definition.replace("\\n", "\n") - definition = definition.replace("\\t", "\t") - # Get the doubled backslashes back as single backslashes - definition = definition.replace("\001", "\\") - startRepeat = definition.find("\\(") - endRepeat = definition.find("\\)") - intro = definition[:startRepeat] - out = "" - if intro.endswith("\n"): - pos = 0 - else: - pos = len(intro) - out += intro - middle = definition[startRepeat+2:endRepeat] - for i in lists[listid]: - item = middle.replace("\\*", i) - if pos and (pos + len(item) >= 80): - out += "\\\n" - pos = 0 - out += item - pos += len(item) - if item.endswith("\n"): - pos = 0 - outro = definition[endRepeat+2:] - out += outro - out = out.replace("\n", eolType) # correct EOLs in generated content - output.append(out) - elif line.startswith(commentPrefix + "--Autogenerated"): - copying = 1 - if retainDefs: - output.append(line) - output = [line.rstrip(" \t") for line in output] # trim trailing whitespace - return eolType.join(output) + eolType - -def UpdateFile(filename, updated): - """ If the file is different to updated then copy updated - into the file else leave alone so CVS and make don't treat - it as modified. """ - try: - infile = open(filename, "rb") - except IOError: # File is not there yet - out = open(filename, "wb") - out.write(updated.encode('utf-8')) - out.close() - print("New %s" % filename) - return - original = infile.read() - infile.close() - original = original.decode('utf-8') - if updated != original: - os.unlink(filename) - out = open(filename, "wb") - out.write(updated.encode('utf-8')) - out.close() - print("Changed %s " % filename) - #~ else: - #~ print "Unchanged", filename - -def Generate(inpath, outpath, commentPrefix, eolType, *lists): - """Generate 'outpath' from 'inpath'. - - "eolType" indicates the type of EOLs to use in the generated - file. It should be one of following constants: LF, CRLF, - CR, or NATIVE. - """ - #print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\ - # % (inpath, outpath, commentPrefix, eolType) - try: - infile = open(inpath, "rb") - except IOError: - print("Can not open %s" % inpath) - return - original = infile.read() - infile.close() - original = original.decode('utf-8') - updated = CopyWithInsertion(original, commentPrefix, - inpath == outpath, eolType, *lists) - UpdateFile(outpath, updated) - -def Regenerate(filename, commentPrefix, eolType, *lists): - """Regenerate the given file. - - "eolType" indicates the type of EOLs to use in the generated - file. It should be one of following constants: LF, CRLF, - CR, or NATIVE. - """ - Generate(filename, filename, commentPrefix, eolType, *lists) - -def FindModules(lexFile): - modules = [] - f = open(lexFile) - for l in f.readlines(): - if l.startswith("LexerModule"): - l = l.replace("(", " ") - modules.append(l.split()[1]) - return modules - -# Properties that start with lexer. or fold. are automatically found but there are some -# older properties that don't follow this pattern so must be explicitly listed. -knownIrregularProperties = [ - "fold", - "styling.within.preprocessor", - "tab.timmy.whinge.level", - "asp.default.language", - "html.tags.case.sensitive", - "ps.level", - "ps.tokenize", - "sql.backslash.escapes", - "nsis.uservars", - "nsis.ignorecase" -] - -def FindProperties(lexFile): - properties = {} - f = open(lexFile) - for l in f.readlines(): - if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l: - l = l.strip() - if not l.startswith("//"): # Drop comments - propertyName = l.split("\"")[1] - if propertyName.lower() == propertyName: - # Only allow lower case property names - if propertyName in knownIrregularProperties or \ - propertyName.startswith("fold.") or \ - propertyName.startswith("lexer."): - properties[propertyName] = 1 - return properties - -def FindPropertyDocumentation(lexFile): - documents = {} - f = open(lexFile) - name = "" - for l in f.readlines(): - l = l.strip() - if "// property " in l: - propertyName = l.split()[2] - if propertyName.lower() == propertyName: - # Only allow lower case property names - name = propertyName - documents[name] = "" - elif "DefineProperty" in l and "\"" in l: - propertyName = l.split("\"")[1] - if propertyName.lower() == propertyName: - # Only allow lower case property names - name = propertyName - documents[name] = "" - elif name: - if l.startswith("//"): - if documents[name]: - documents[name] += " " - documents[name] += l[2:].strip() - elif l.startswith("\""): - l = l[1:].strip() - if l.endswith(";"): - l = l[:-1].strip() - if l.endswith(")"): - l = l[:-1].strip() - if l.endswith("\""): - l = l[:-1] - # Fix escaped double quotes - l = l.replace("\\\"", "\"") - documents[name] += l - else: - name = "" - for name in list(documents.keys()): - if documents[name] == "": - del documents[name] - return documents - -def ciCompare(a,b): - return cmp(a.lower(), b.lower()) - -def ciKey(a): - return a.lower() - -def sortListInsensitive(l): - try: # Try key function - l.sort(key=ciKey) - except TypeError: # Earlier version of Python, so use comparison function - l.sort(ciCompare) - -def UpdateLineInFile(path, linePrefix, lineReplace): - lines = [] - updated = False - with codecs.open(path, "r", "utf-8") as f: - for l in f.readlines(): - l = l.rstrip() - if not updated and l.startswith(linePrefix): - lines.append(lineReplace) - updated = True - else: - lines.append(l) - contents = NATIVE.join(lines) + NATIVE - UpdateFile(path, contents) - -host = "prdownloads.sourceforge.net/" -def UpdateDownloadLinks(path, version): - lines = [] - with open(path, "r") as f: - for l in f.readlines(): - l = l.rstrip() - if host in l: - start, prd, rest = l.partition(host) - pth, dot, ending = rest.partition(".") - pthNew = pth[:-3] + version.rstrip() - lineWithNewVersion = start + prd +pthNew + dot + ending - lines.append(lineWithNewVersion) - else: - lines.append(l) - contents = NATIVE.join(lines) + NATIVE - UpdateFile(path, contents) - -def UpdateVersionNumbers(root): - with open(root + "scintilla/version.txt") as f: - version = f.read() - versionDotted = version[0] + '.' + version[1] + '.' + version[2] - versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0' - with open(root + "scintilla/doc/index.html") as f: - dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3] - # 20130602 - # index.html, SciTE.html - dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d") - yearModified = dateModified[0:4] - monthModified = dtModified.strftime("%B") - dayModified = "%d" % dtModified.day - mdyModified = monthModified + " " + dayModified + " " + yearModified - # May 22 2013 - # index.html, SciTE.html - dmyModified = dayModified + " " + monthModified + " " + yearModified - # 22 May 2013 - # ScintillaHistory.html -- only first should change - myModified = monthModified + " " + yearModified - # scite/src/SciTE.h - #define COPYRIGHT_DATES "December 1998-May 2013" - #define COPYRIGHT_YEARS "1998-2013" - dateLine = f.readlines() - - UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA", - "#define VERSION_SCINTILLA \"" + versionDotted + "\"") - UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS", - "#define VERSION_WORDS " + versionCommad) - UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro", - "VERSION =", - "VERSION = " + versionDotted) - UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro", - "VERSION =", - "VERSION = " + versionDotted) - UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", " Release", - " Release " + versionDotted) - UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version) - UpdateLineInFile(root + "scintilla/doc/index.html", - ' <font color="#FFCC99" size="3"> Release version', - ' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />') - UpdateLineInFile(root + "scintilla/doc/index.html", - ' Site last modified', - ' Site last modified ' + mdyModified + '</font>') - UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html", - ' Released ', - ' Released ' + dmyModified + '.') - - if os.path.exists(root + "scite"): - UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE", - "#define VERSION_SCITE \"" + versionDotted + "\"") - UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS", - "#define VERSION_WORDS " + versionCommad) - UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES", - '#define COPYRIGHT_DATES "December 1998-' + myModified + '"') - UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS", - '#define COPYRIGHT_YEARS "1998-' + yearModified + '"') - UpdateLineInFile(root + "scite/doc/SciTEDownload.html", " Release", - " Release " + versionDotted) - UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version) - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' <font color="#FFCC99" size="3"> Release version', - ' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />') - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' Site last modified', - ' Site last modified ' + mdyModified + '</font>') - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' <meta name="Date.Modified"', - ' <meta name="Date.Modified" content="' + dateModified + '" />') - -def RegenerateAll(): - root="../../" - - # Find all the lexer source code files - lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx") - sortListInsensitive(lexFilePaths) - lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths] - print(lexFiles) - lexerModules = [] - lexerProperties = {} - propertyDocuments = {} - for lexFile in lexFilePaths: - lexerModules.extend(FindModules(lexFile)) - for k in FindProperties(lexFile).keys(): - lexerProperties[k] = 1 - documents = FindPropertyDocumentation(lexFile) - for k in documents.keys(): - if k not in propertyDocuments: - propertyDocuments[k] = documents[k] - sortListInsensitive(lexerModules) - lexerProperties = list(lexerProperties.keys()) - sortListInsensitive(lexerProperties) - - # Generate HTML to document each property - # This is done because tags can not be safely put inside comments in HTML - documentProperties = list(propertyDocuments.keys()) - sortListInsensitive(documentProperties) - propertiesHTML = [] - for k in documentProperties: - propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" % - (k, k, propertyDocuments[k])) - - # Find all the SciTE properties files - otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"] - if os.path.exists(root + "scite"): - propFilePaths = glob.glob(root + "scite/src/*.properties") - sortListInsensitive(propFilePaths) - propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps] - sortListInsensitive(propFiles) - print(propFiles) - - Regenerate(root + "scintilla/src/Catalogue.cxx", "//", NATIVE, lexerModules) - Regenerate(root + "scintilla/win32/scintilla.mak", "#", NATIVE, lexFiles) - if os.path.exists(root + "scite"): - Regenerate(root + "scite/win32/makefile", "#", NATIVE, propFiles) - Regenerate(root + "scite/win32/scite.mak", "#", NATIVE, propFiles) - Regenerate(root + "scite/src/SciTEProps.cxx", "//", NATIVE, lexerProperties) - Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", NATIVE, propertiesHTML) - Generate(root + "scite/boundscheck/vcproj.gen", - root + "scite/boundscheck/SciTE.vcproj", "#", NATIVE, lexFiles) - - UpdateVersionNumbers(root) - -RegenerateAll() |