diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/Face.py | 5 | ||||
| -rw-r--r-- | scripts/FileGenerator.py | 12 | ||||
| -rwxr-xr-x | scripts/HFacer.py | 2 | ||||
| -rwxr-xr-x | scripts/LexGen.py | 273 | ||||
| -rw-r--r-- | scripts/ScintillaData.py | 193 |
5 files changed, 243 insertions, 242 deletions
diff --git a/scripts/Face.py b/scripts/Face.py index 855d6321a..17d161fd7 100644 --- a/scripts/Face.py +++ b/scripts/Face.py @@ -1,4 +1,7 @@ -# Module for reading and parsing Scintilla.iface file +# Face.py - module for reading and parsing Scintilla.iface file +# Implemented 2000 by Neil Hodgson neilh@scintilla.org +# Released to the public domain. +# Requires Python 2.5 or later def sanitiseLine(line): if line[-1:] == '\n': line = line[:-1] diff --git a/scripts/FileGenerator.py b/scripts/FileGenerator.py index 601030cbb..01a79bf99 100644 --- a/scripts/FileGenerator.py +++ b/scripts/FileGenerator.py @@ -4,14 +4,16 @@ # Generate or regenerate source files based on comments in those files. # May be modified in-place or a template may be generated into a complete file. -# Requires Python 2.4 or later +# Requires Python 2.5 or later # The files are copied to a string apart from sections between a # ++Autogenerated comment and a --Autogenerated comment which is # generated by the CopyWithInsertion function. After the whole string is # instantiated, it is compared with the target file and if different the file # is rewritten. -import codecs, os, string, sys +from __future__ import with_statement + +import codecs, os, re, string, sys lineEnd = "\r\n" if sys.platform == "win32" else "\n" @@ -148,3 +150,9 @@ def UpdateLineInFile(path, linePrefix, lineReplace): lines.append(l) contents = lineEnd.join(lines) + lineEnd UpdateFile(path, contents) + +def ReplaceREInFile(path, match, replace): + with codecs.open(path, "r", "utf-8") as f: + contents = f.read() + contents = re.sub(match, replace, contents) + UpdateFile(path, contents) diff --git a/scripts/HFacer.py b/scripts/HFacer.py index 27537fec3..05da04ee6 100755 --- a/scripts/HFacer.py +++ b/scripts/HFacer.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface # definition file. +# Implemented 2000 by Neil Hodgson neilh@scintilla.org +# Requires Python 2.5 or later import sys import os diff --git a/scripts/LexGen.py b/scripts/LexGen.py index 4068ee485..ef29b8def 100755 --- a/scripts/LexGen.py +++ b/scripts/LexGen.py @@ -2,255 +2,50 @@ # LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org # Released to the public domain. -# Regenerate the Scintilla and SciTE source files that list -# all the lexers and all the properties files. +# Regenerate the Scintilla source files that list all the lexers. # Should be run whenever a new lexer is added or removed. -# Requires Python 2.4 or later -# Most files are regenerated in place with templates stored in comments. -# The VS .NET project file is generated into a different file as the -# VS .NET environment will not retain comments when modifying the file. +# Requires Python 2.5 or later +# Files are regenerated in place with templates stored in comments. # The format of generation comments is documented in FileGenerator.py. -import string -import sys -import os -import glob -import codecs -import datetime +from FileGenerator import Regenerate, UpdateLineInFile, ReplaceREInFile +import ScintillaData -from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd - -def FindModules(lexFile): - modules = [] - with open(lexFile) as f: - for l in f.readlines(): - if l.startswith("LexerModule"): - l = l.replace("(", " ") - modules.append(l.split()[1]) - return modules - -# Properties that start with lexer. or fold. are automatically found but there are some -# older properties that don't follow this pattern so must be explicitly listed. -knownIrregularProperties = [ - "fold", - "styling.within.preprocessor", - "tab.timmy.whinge.level", - "asp.default.language", - "html.tags.case.sensitive", - "ps.level", - "ps.tokenize", - "sql.backslash.escapes", - "nsis.uservars", - "nsis.ignorecase" -] - -def FindProperties(lexFile): - properties = {} - with open(lexFile) as f: - for l in f.readlines(): - if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l: - l = l.strip() - if not l.startswith("//"): # Drop comments - propertyName = l.split("\"")[1] - if propertyName.lower() == propertyName: - # Only allow lower case property names - if propertyName in knownIrregularProperties or \ - propertyName.startswith("fold.") or \ - propertyName.startswith("lexer."): - properties[propertyName] = 1 - return properties - -def FindPropertyDocumentation(lexFile): - documents = {} - with open(lexFile) as f: - name = "" - for l in f.readlines(): - l = l.strip() - if "// property " in l: - propertyName = l.split()[2] - if propertyName.lower() == propertyName: - # Only allow lower case property names - name = propertyName - documents[name] = "" - elif "DefineProperty" in l and "\"" in l: - propertyName = l.split("\"")[1] - if propertyName.lower() == propertyName: - # Only allow lower case property names - name = propertyName - documents[name] = "" - elif name: - if l.startswith("//"): - if documents[name]: - documents[name] += " " - documents[name] += l[2:].strip() - elif l.startswith("\""): - l = l[1:].strip() - if l.endswith(";"): - l = l[:-1].strip() - if l.endswith(")"): - l = l[:-1].strip() - if l.endswith("\""): - l = l[:-1] - # Fix escaped double quotes - l = l.replace("\\\"", "\"") - documents[name] += l - else: - name = "" - for name in list(documents.keys()): - if documents[name] == "": - del documents[name] - return documents - -def ciCompare(a,b): - return cmp(a.lower(), b.lower()) - -def ciKey(a): - return a.lower() - -def sortListInsensitive(l): - try: # Try key function - l.sort(key=ciKey) - except TypeError: # Earlier version of Python, so use comparison function - l.sort(ciCompare) - -host = "prdownloads.sourceforge.net/" -def UpdateDownloadLinks(path, version): - lines = [] - with open(path, "r") as f: - for l in f.readlines(): - l = l.rstrip() - if host in l: - start, prd, rest = l.partition(host) - pth, dot, ending = rest.partition(".") - pthNew = pth[:-3] + version.rstrip() - lineWithNewVersion = start + prd +pthNew + dot + ending - lines.append(lineWithNewVersion) - else: - lines.append(l) - contents = lineEnd.join(lines) + lineEnd - UpdateFile(path, contents) - -def UpdateVersionNumbers(root): - with open(root + "scintilla/version.txt") as f: - version = f.read() - versionDotted = version[0] + '.' + version[1] + '.' + version[2] - versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0' - with open(root + "scintilla/doc/index.html") as f: - dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3] - # 20130602 - # index.html, SciTE.html - dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d") - yearModified = dateModified[0:4] - monthModified = dtModified.strftime("%B") - dayModified = "%d" % dtModified.day - mdyModified = monthModified + " " + dayModified + " " + yearModified - # May 22 2013 - # index.html, SciTE.html - dmyModified = dayModified + " " + monthModified + " " + yearModified - # 22 May 2013 - # ScintillaHistory.html -- only first should change - myModified = monthModified + " " + yearModified - # scite/src/SciTE.h - #define COPYRIGHT_DATES "December 1998-May 2013" - #define COPYRIGHT_YEARS "1998-2013" - dateLine = f.readlines() - - UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA", - "#define VERSION_SCINTILLA \"" + versionDotted + "\"") - UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS", - "#define VERSION_WORDS " + versionCommad) - UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro", +def UpdateVersionNumbers(sci, root): + UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_SCINTILLA", + "#define VERSION_SCINTILLA \"" + sci.versionDotted + "\"") + UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_WORDS", + "#define VERSION_WORDS " + sci.versionCommad) + UpdateLineInFile(root + "qt/ScintillaEditBase/ScintillaEditBase.pro", "VERSION =", - "VERSION = " + versionDotted) - UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro", + "VERSION = " + sci.versionDotted) + UpdateLineInFile(root + "qt/ScintillaEdit/ScintillaEdit.pro", "VERSION =", - "VERSION = " + versionDotted) - UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", " Release", - " Release " + versionDotted) - UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version) - UpdateLineInFile(root + "scintilla/doc/index.html", + "VERSION = " + sci.versionDotted) + UpdateLineInFile(root + "doc/ScintillaDownload.html", " Release", + " Release " + sci.versionDotted) + ReplaceREInFile(root + "doc/ScintillaDownload.html", + r"/scintilla/([a-zA-Z]+)\d\d\d", + r"/scintilla/\g<1>" + sci.version) + UpdateLineInFile(root + "doc/index.html", ' <font color="#FFCC99" size="3"> Release version', - ' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />') - UpdateLineInFile(root + "scintilla/doc/index.html", + ' <font color="#FFCC99" size="3"> Release version ' +\ + sci.versionDotted + '<br />') + UpdateLineInFile(root + "doc/index.html", ' Site last modified', - ' Site last modified ' + mdyModified + '</font>') - UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html", + ' Site last modified ' + sci.mdyModified + '</font>') + UpdateLineInFile(root + "doc/ScintillaHistory.html", ' Released ', - ' Released ' + dmyModified + '.') - - if os.path.exists(root + "scite"): - UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE", - "#define VERSION_SCITE \"" + versionDotted + "\"") - UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS", - "#define VERSION_WORDS " + versionCommad) - UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES", - '#define COPYRIGHT_DATES "December 1998-' + myModified + '"') - UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS", - '#define COPYRIGHT_YEARS "1998-' + yearModified + '"') - UpdateLineInFile(root + "scite/doc/SciTEDownload.html", " Release", - " Release " + versionDotted) - UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version) - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' <font color="#FFCC99" size="3"> Release version', - ' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />') - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' Site last modified', - ' Site last modified ' + mdyModified + '</font>') - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' <meta name="Date.Modified"', - ' <meta name="Date.Modified" content="' + dateModified + '" />') - -def RegenerateAll(): - root="../../" - - # Find all the lexer source code files - lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx") - sortListInsensitive(lexFilePaths) - lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths] - print(lexFiles) - lexerModules = [] - lexerProperties = {} - propertyDocuments = {} - for lexFile in lexFilePaths: - lexerModules.extend(FindModules(lexFile)) - for k in FindProperties(lexFile).keys(): - lexerProperties[k] = 1 - documents = FindPropertyDocumentation(lexFile) - for k in documents.keys(): - if k not in propertyDocuments: - propertyDocuments[k] = documents[k] - sortListInsensitive(lexerModules) - lexerProperties = list(lexerProperties.keys()) - sortListInsensitive(lexerProperties) - - # Generate HTML to document each property - # This is done because tags can not be safely put inside comments in HTML - documentProperties = list(propertyDocuments.keys()) - sortListInsensitive(documentProperties) - propertiesHTML = [] - for k in documentProperties: - propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" % - (k, k, propertyDocuments[k])) + ' Released ' + sci.dmyModified + '.') - # Find all the SciTE properties files - otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"] - if os.path.exists(root + "scite"): - propFilePaths = glob.glob(root + "scite/src/*.properties") - sortListInsensitive(propFilePaths) - propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps] - sortListInsensitive(propFiles) - print(propFiles) +def RegenerateAll(root): + + sci = ScintillaData.ScintillaData(root) - Regenerate(root + "scintilla/src/Catalogue.cxx", "//", lexerModules) - Regenerate(root + "scintilla/win32/scintilla.mak", "#", lexFiles) - if os.path.exists(root + "scite"): - Regenerate(root + "scite/win32/makefile", "#", propFiles) - Regenerate(root + "scite/win32/scite.mak", "#", propFiles) - Regenerate(root + "scite/src/SciTEProps.cxx", "//", lexerProperties) - Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", propertiesHTML) - Generate(root + "scite/boundscheck/vcproj.gen", - root + "scite/boundscheck/SciTE.vcproj", "#", lexFiles) + Regenerate(root + "src/Catalogue.cxx", "//", sci.lexerModules) + Regenerate(root + "win32/scintilla.mak", "#", sci.lexFiles) - UpdateVersionNumbers(root) + UpdateVersionNumbers(sci, root) -RegenerateAll() +if __name__=="__main__": + RegenerateAll("../") diff --git a/scripts/ScintillaData.py b/scripts/ScintillaData.py new file mode 100644 index 000000000..1c999a119 --- /dev/null +++ b/scripts/ScintillaData.py @@ -0,0 +1,193 @@ +# ScintillaData.py - implemented 2013 by Neil Hodgson neilh@scintilla.org +# Released to the public domain. + +# Common code used by Scintilla and SciTE for source file regeneration. +# The ScintillaData object exposes information about Scintilla as properties: +# Version properties +# version +# versionDotted +# versionCommad +# +# Date last modified +# dateModified +# yearModified +# mdyModified +# dmyModified +# myModified +# +# Information about lexers and properties defined in lexers +# lexFiles +# sorted list of lexer files +# lexerModules +# sorted list of module names +# lexerProperties +# sorted list of lexer properties +# propertyDocuments +# dictionary of property documentation { name: document string } + +# This file can be run to see the data it provides. +# Requires Python 2.5 or later + +from __future__ import with_statement + +import datetime, glob, os, textwrap + +import FileGenerator + +def FindModules(lexFile): + modules = [] + with open(lexFile) as f: + for l in f.readlines(): + if l.startswith("LexerModule"): + l = l.replace("(", " ") + modules.append(l.split()[1]) + return modules + +# Properties that start with lexer. or fold. are automatically found but there are some +# older properties that don't follow this pattern so must be explicitly listed. +knownIrregularProperties = [ + "fold", + "styling.within.preprocessor", + "tab.timmy.whinge.level", + "asp.default.language", + "html.tags.case.sensitive", + "ps.level", + "ps.tokenize", + "sql.backslash.escapes", + "nsis.uservars", + "nsis.ignorecase" +] + +def FindProperties(lexFile): + properties = {} + with open(lexFile) as f: + for l in f.readlines(): + if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l: + l = l.strip() + if not l.startswith("//"): # Drop comments + propertyName = l.split("\"")[1] + if propertyName.lower() == propertyName: + # Only allow lower case property names + if propertyName in knownIrregularProperties or \ + propertyName.startswith("fold.") or \ + propertyName.startswith("lexer."): + properties[propertyName] = 1 + return properties + +def FindPropertyDocumentation(lexFile): + documents = {} + with open(lexFile) as f: + name = "" + for l in f.readlines(): + l = l.strip() + if "// property " in l: + propertyName = l.split()[2] + if propertyName.lower() == propertyName: + # Only allow lower case property names + name = propertyName + documents[name] = "" + elif "DefineProperty" in l and "\"" in l: + propertyName = l.split("\"")[1] + if propertyName.lower() == propertyName: + # Only allow lower case property names + name = propertyName + documents[name] = "" + elif name: + if l.startswith("//"): + if documents[name]: + documents[name] += " " + documents[name] += l[2:].strip() + elif l.startswith("\""): + l = l[1:].strip() + if l.endswith(";"): + l = l[:-1].strip() + if l.endswith(")"): + l = l[:-1].strip() + if l.endswith("\""): + l = l[:-1] + # Fix escaped double quotes + l = l.replace("\\\"", "\"") + documents[name] += l + else: + name = "" + for name in list(documents.keys()): + if documents[name] == "": + del documents[name] + return documents + +def ciCompare(a,b): + return cmp(a.lower(), b.lower()) + +def ciKey(a): + return a.lower() + +def SortListInsensitive(l): + try: # Try key function + l.sort(key=ciKey) + except TypeError: # Earlier version of Python, so use comparison function + l.sort(ciCompare) + +class ScintillaData: + def __init__(self, scintillaRoot): + # Discover verion information + with open(scintillaRoot + "version.txt") as f: + self.version = f.read().strip() + self.versionDotted = self.version[0] + '.' + self.version[1] + '.' + \ + self.version[2] + self.versionCommad = self.version[0] + ', ' + self.version[1] + ', ' + \ + self.version[2] + ', 0' + + with open(scintillaRoot + "doc/index.html") as f: + self.dateModified = [l for l in f.readlines() if "Date.Modified" in l]\ + [0].split('\"')[3] + # 20130602 + # index.html, SciTE.html + dtModified = datetime.datetime.strptime(self.dateModified, "%Y%m%d") + self.yearModified = self.dateModified[0:4] + monthModified = dtModified.strftime("%B") + dayModified = "%d" % dtModified.day + self.mdyModified = monthModified + " " + dayModified + " " + self.yearModified + # May 22 2013 + # index.html, SciTE.html + self.dmyModified = dayModified + " " + monthModified + " " + self.yearModified + # 22 May 2013 + # ScintillaHistory.html -- only first should change + self.myModified = monthModified + " " + self.yearModified + + # Find all the lexer source code files + lexFilePaths = glob.glob(scintillaRoot + "lexers/Lex*.cxx") + SortListInsensitive(lexFilePaths) + self.lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths] + self.lexerModules = [] + lexerProperties = set() + self.propertyDocuments = {} + for lexFile in lexFilePaths: + self.lexerModules.extend(FindModules(lexFile)) + for k in FindProperties(lexFile).keys(): + lexerProperties.add(k) + documents = FindPropertyDocumentation(lexFile) + for k in documents.keys(): + if k not in self.propertyDocuments: + self.propertyDocuments[k] = documents[k] + SortListInsensitive(self.lexerModules) + self.lexerProperties = list(lexerProperties) + SortListInsensitive(self.lexerProperties) + +def printWrapped(text): + print(textwrap.fill(text, subsequent_indent=" ")) + +if __name__=="__main__": + sci = ScintillaData("../") + print("Version %s %s %s" % (sci.version, sci.versionDotted, sci.versionCommad)) + print("Date last modified %s %s %s %s %s" % ( + sci.dateModified, sci.yearModified, sci.mdyModified, sci.dmyModified, sci.myModified)) + printWrapped(str(len(sci.lexFiles)) + " lexer files: " + ", ".join(sci.lexFiles)) + printWrapped(str(len(sci.lexerModules)) + " lexer modules: " + ", ".join(sci.lexerModules)) + printWrapped("Lexer properties: " + ", ".join(sci.lexerProperties)) + print("Lexer property documentation:") + documentProperties = list(sci.propertyDocuments.keys()) + SortListInsensitive(documentProperties) + for k in documentProperties: + print(" " + k) + print(textwrap.fill(sci.propertyDocuments[k], initial_indent=" ", + subsequent_indent=" ")) |