From db454bdf6d717c9775079b292ee1a70b65cb449c Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 3 Jul 2013 09:34:20 +1000 Subject: Restructured LexGen.py by moving discovery code into ScintillaData.py and SciTE regeneration into ../scite/scripts/RegenerateSource.py. FileGenerator.py adds ReplaceREInFile function. --- scripts/Face.py | 5 +- scripts/FileGenerator.py | 12 ++- scripts/HFacer.py | 2 + scripts/LexGen.py | 273 ++++++----------------------------------------- scripts/ScintillaData.py | 193 +++++++++++++++++++++++++++++++++ 5 files changed, 243 insertions(+), 242 deletions(-) create mode 100644 scripts/ScintillaData.py (limited to 'scripts') diff --git a/scripts/Face.py b/scripts/Face.py index 855d6321a..17d161fd7 100644 --- a/scripts/Face.py +++ b/scripts/Face.py @@ -1,4 +1,7 @@ -# Module for reading and parsing Scintilla.iface file +# Face.py - module for reading and parsing Scintilla.iface file +# Implemented 2000 by Neil Hodgson neilh@scintilla.org +# Released to the public domain. +# Requires Python 2.5 or later def sanitiseLine(line): if line[-1:] == '\n': line = line[:-1] diff --git a/scripts/FileGenerator.py b/scripts/FileGenerator.py index 601030cbb..01a79bf99 100644 --- a/scripts/FileGenerator.py +++ b/scripts/FileGenerator.py @@ -4,14 +4,16 @@ # Generate or regenerate source files based on comments in those files. # May be modified in-place or a template may be generated into a complete file. -# Requires Python 2.4 or later +# Requires Python 2.5 or later # The files are copied to a string apart from sections between a # ++Autogenerated comment and a --Autogenerated comment which is # generated by the CopyWithInsertion function. After the whole string is # instantiated, it is compared with the target file and if different the file # is rewritten. -import codecs, os, string, sys +from __future__ import with_statement + +import codecs, os, re, string, sys lineEnd = "\r\n" if sys.platform == "win32" else "\n" @@ -148,3 +150,9 @@ def UpdateLineInFile(path, linePrefix, lineReplace): lines.append(l) contents = lineEnd.join(lines) + lineEnd UpdateFile(path, contents) + +def ReplaceREInFile(path, match, replace): + with codecs.open(path, "r", "utf-8") as f: + contents = f.read() + contents = re.sub(match, replace, contents) + UpdateFile(path, contents) diff --git a/scripts/HFacer.py b/scripts/HFacer.py index 27537fec3..05da04ee6 100755 --- a/scripts/HFacer.py +++ b/scripts/HFacer.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface # definition file. +# Implemented 2000 by Neil Hodgson neilh@scintilla.org +# Requires Python 2.5 or later import sys import os diff --git a/scripts/LexGen.py b/scripts/LexGen.py index 4068ee485..ef29b8def 100755 --- a/scripts/LexGen.py +++ b/scripts/LexGen.py @@ -2,255 +2,50 @@ # LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org # Released to the public domain. -# Regenerate the Scintilla and SciTE source files that list -# all the lexers and all the properties files. +# Regenerate the Scintilla source files that list all the lexers. # Should be run whenever a new lexer is added or removed. -# Requires Python 2.4 or later -# Most files are regenerated in place with templates stored in comments. -# The VS .NET project file is generated into a different file as the -# VS .NET environment will not retain comments when modifying the file. +# Requires Python 2.5 or later +# Files are regenerated in place with templates stored in comments. # The format of generation comments is documented in FileGenerator.py. -import string -import sys -import os -import glob -import codecs -import datetime +from FileGenerator import Regenerate, UpdateLineInFile, ReplaceREInFile +import ScintillaData -from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd - -def FindModules(lexFile): - modules = [] - with open(lexFile) as f: - for l in f.readlines(): - if l.startswith("LexerModule"): - l = l.replace("(", " ") - modules.append(l.split()[1]) - return modules - -# Properties that start with lexer. or fold. are automatically found but there are some -# older properties that don't follow this pattern so must be explicitly listed. -knownIrregularProperties = [ - "fold", - "styling.within.preprocessor", - "tab.timmy.whinge.level", - "asp.default.language", - "html.tags.case.sensitive", - "ps.level", - "ps.tokenize", - "sql.backslash.escapes", - "nsis.uservars", - "nsis.ignorecase" -] - -def FindProperties(lexFile): - properties = {} - with open(lexFile) as f: - for l in f.readlines(): - if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l: - l = l.strip() - if not l.startswith("//"): # Drop comments - propertyName = l.split("\"")[1] - if propertyName.lower() == propertyName: - # Only allow lower case property names - if propertyName in knownIrregularProperties or \ - propertyName.startswith("fold.") or \ - propertyName.startswith("lexer."): - properties[propertyName] = 1 - return properties - -def FindPropertyDocumentation(lexFile): - documents = {} - with open(lexFile) as f: - name = "" - for l in f.readlines(): - l = l.strip() - if "// property " in l: - propertyName = l.split()[2] - if propertyName.lower() == propertyName: - # Only allow lower case property names - name = propertyName - documents[name] = "" - elif "DefineProperty" in l and "\"" in l: - propertyName = l.split("\"")[1] - if propertyName.lower() == propertyName: - # Only allow lower case property names - name = propertyName - documents[name] = "" - elif name: - if l.startswith("//"): - if documents[name]: - documents[name] += " " - documents[name] += l[2:].strip() - elif l.startswith("\""): - l = l[1:].strip() - if l.endswith(";"): - l = l[:-1].strip() - if l.endswith(")"): - l = l[:-1].strip() - if l.endswith("\""): - l = l[:-1] - # Fix escaped double quotes - l = l.replace("\\\"", "\"") - documents[name] += l - else: - name = "" - for name in list(documents.keys()): - if documents[name] == "": - del documents[name] - return documents - -def ciCompare(a,b): - return cmp(a.lower(), b.lower()) - -def ciKey(a): - return a.lower() - -def sortListInsensitive(l): - try: # Try key function - l.sort(key=ciKey) - except TypeError: # Earlier version of Python, so use comparison function - l.sort(ciCompare) - -host = "prdownloads.sourceforge.net/" -def UpdateDownloadLinks(path, version): - lines = [] - with open(path, "r") as f: - for l in f.readlines(): - l = l.rstrip() - if host in l: - start, prd, rest = l.partition(host) - pth, dot, ending = rest.partition(".") - pthNew = pth[:-3] + version.rstrip() - lineWithNewVersion = start + prd +pthNew + dot + ending - lines.append(lineWithNewVersion) - else: - lines.append(l) - contents = lineEnd.join(lines) + lineEnd - UpdateFile(path, contents) - -def UpdateVersionNumbers(root): - with open(root + "scintilla/version.txt") as f: - version = f.read() - versionDotted = version[0] + '.' + version[1] + '.' + version[2] - versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0' - with open(root + "scintilla/doc/index.html") as f: - dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3] - # 20130602 - # index.html, SciTE.html - dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d") - yearModified = dateModified[0:4] - monthModified = dtModified.strftime("%B") - dayModified = "%d" % dtModified.day - mdyModified = monthModified + " " + dayModified + " " + yearModified - # May 22 2013 - # index.html, SciTE.html - dmyModified = dayModified + " " + monthModified + " " + yearModified - # 22 May 2013 - # ScintillaHistory.html -- only first should change - myModified = monthModified + " " + yearModified - # scite/src/SciTE.h - #define COPYRIGHT_DATES "December 1998-May 2013" - #define COPYRIGHT_YEARS "1998-2013" - dateLine = f.readlines() - - UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA", - "#define VERSION_SCINTILLA \"" + versionDotted + "\"") - UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS", - "#define VERSION_WORDS " + versionCommad) - UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro", +def UpdateVersionNumbers(sci, root): + UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_SCINTILLA", + "#define VERSION_SCINTILLA \"" + sci.versionDotted + "\"") + UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_WORDS", + "#define VERSION_WORDS " + sci.versionCommad) + UpdateLineInFile(root + "qt/ScintillaEditBase/ScintillaEditBase.pro", "VERSION =", - "VERSION = " + versionDotted) - UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro", + "VERSION = " + sci.versionDotted) + UpdateLineInFile(root + "qt/ScintillaEdit/ScintillaEdit.pro", "VERSION =", - "VERSION = " + versionDotted) - UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", " Release", - " Release " + versionDotted) - UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version) - UpdateLineInFile(root + "scintilla/doc/index.html", + "VERSION = " + sci.versionDotted) + UpdateLineInFile(root + "doc/ScintillaDownload.html", " Release", + " Release " + sci.versionDotted) + ReplaceREInFile(root + "doc/ScintillaDownload.html", + r"/scintilla/([a-zA-Z]+)\d\d\d", + r"/scintilla/\g<1>" + sci.version) + UpdateLineInFile(root + "doc/index.html", ' Release version', - ' Release version ' + versionDotted + '
') - UpdateLineInFile(root + "scintilla/doc/index.html", + ' Release version ' +\ + sci.versionDotted + '
') + UpdateLineInFile(root + "doc/index.html", ' Site last modified', - ' Site last modified ' + mdyModified + '') - UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html", + ' Site last modified ' + sci.mdyModified + '') + UpdateLineInFile(root + "doc/ScintillaHistory.html", ' Released ', - ' Released ' + dmyModified + '.') - - if os.path.exists(root + "scite"): - UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE", - "#define VERSION_SCITE \"" + versionDotted + "\"") - UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS", - "#define VERSION_WORDS " + versionCommad) - UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES", - '#define COPYRIGHT_DATES "December 1998-' + myModified + '"') - UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS", - '#define COPYRIGHT_YEARS "1998-' + yearModified + '"') - UpdateLineInFile(root + "scite/doc/SciTEDownload.html", " Release", - " Release " + versionDotted) - UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version) - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' Release version', - ' Release version ' + versionDotted + '
') - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' Site last modified', - ' Site last modified ' + mdyModified + '') - UpdateLineInFile(root + "scite/doc/SciTE.html", - ' ') - -def RegenerateAll(): - root="../../" - - # Find all the lexer source code files - lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx") - sortListInsensitive(lexFilePaths) - lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths] - print(lexFiles) - lexerModules = [] - lexerProperties = {} - propertyDocuments = {} - for lexFile in lexFilePaths: - lexerModules.extend(FindModules(lexFile)) - for k in FindProperties(lexFile).keys(): - lexerProperties[k] = 1 - documents = FindPropertyDocumentation(lexFile) - for k in documents.keys(): - if k not in propertyDocuments: - propertyDocuments[k] = documents[k] - sortListInsensitive(lexerModules) - lexerProperties = list(lexerProperties.keys()) - sortListInsensitive(lexerProperties) - - # Generate HTML to document each property - # This is done because tags can not be safely put inside comments in HTML - documentProperties = list(propertyDocuments.keys()) - sortListInsensitive(documentProperties) - propertiesHTML = [] - for k in documentProperties: - propertiesHTML.append("\t\n\t%s\n\t%s\n\t" % - (k, k, propertyDocuments[k])) + ' Released ' + sci.dmyModified + '.') - # Find all the SciTE properties files - otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"] - if os.path.exists(root + "scite"): - propFilePaths = glob.glob(root + "scite/src/*.properties") - sortListInsensitive(propFilePaths) - propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps] - sortListInsensitive(propFiles) - print(propFiles) +def RegenerateAll(root): + + sci = ScintillaData.ScintillaData(root) - Regenerate(root + "scintilla/src/Catalogue.cxx", "//", lexerModules) - Regenerate(root + "scintilla/win32/scintilla.mak", "#", lexFiles) - if os.path.exists(root + "scite"): - Regenerate(root + "scite/win32/makefile", "#", propFiles) - Regenerate(root + "scite/win32/scite.mak", "#", propFiles) - Regenerate(root + "scite/src/SciTEProps.cxx", "//", lexerProperties) - Regenerate(root + "scite/doc/SciTEDoc.html", "