Restructured LexGen.py by moving discovery code into ScintillaData.py and

SciTE regeneration into ../scite/scripts/RegenerateSource.py.
FileGenerator.py adds ReplaceREInFile function.

5 files changed, 243 insertions, 242 deletions

diff --git a/scripts/Face.py b/scripts/Face.py
index 855d6321a..17d161fd7 100644
--- a/scripts/Face.py
+++ b/scripts/Face.py
@@ -1,4 +1,7 @@
-# Module for reading and parsing Scintilla.iface file
+# Face.py - module for reading and parsing Scintilla.iface file
+# Implemented 2000 by Neil Hodgson neilh@scintilla.org
+# Released to the public domain.
+# Requires Python 2.5 or later
 
 def sanitiseLine(line):
 	if line[-1:] == '\n': line = line[:-1]
diff --git a/scripts/FileGenerator.py b/scripts/FileGenerator.py
index 601030cbb..01a79bf99 100644
--- a/scripts/FileGenerator.py
+++ b/scripts/FileGenerator.py
@@ -4,14 +4,16 @@
 
 # Generate or regenerate source files based on comments in those files.
 # May be modified in-place or a template may be generated into a complete file.
-# Requires Python 2.4 or later
+# Requires Python 2.5 or later
 # The files are copied to a string apart from sections between a
 # ++Autogenerated comment and a --Autogenerated comment which is
 # generated by the CopyWithInsertion function. After the whole string is 
 # instantiated, it is compared with the target file and if different the file 
 # is rewritten.
 
-import codecs, os, string, sys
+from __future__ import with_statement
+
+import codecs, os, re, string, sys
 
 lineEnd = "\r\n" if sys.platform == "win32" else "\n"
 
@@ -148,3 +150,9 @@ def UpdateLineInFile(path, linePrefix, lineReplace):
                 lines.append(l)
     contents = lineEnd.join(lines) + lineEnd
     UpdateFile(path, contents)
+
+def ReplaceREInFile(path, match, replace):
+    with codecs.open(path, "r", "utf-8") as f:
+        contents = f.read()
+    contents = re.sub(match, replace, contents)
+    UpdateFile(path, contents)
diff --git a/scripts/HFacer.py b/scripts/HFacer.py
index 27537fec3..05da04ee6 100755
--- a/scripts/HFacer.py
+++ b/scripts/HFacer.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface
 # definition file.
+# Implemented 2000 by Neil Hodgson neilh@scintilla.org
+# Requires Python 2.5 or later
 
 import sys
 import os
diff --git a/scripts/LexGen.py b/scripts/LexGen.py
index 4068ee485..ef29b8def 100755
--- a/scripts/LexGen.py
+++ b/scripts/LexGen.py
@@ -2,255 +2,50 @@
 # LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
 # Released to the public domain.
 
-# Regenerate the Scintilla and SciTE source files that list
-# all the lexers and all the properties files.
+# Regenerate the Scintilla source files that list all the lexers.
 # Should be run whenever a new lexer is added or removed.
-# Requires Python 2.4 or later
-# Most files are regenerated in place with templates stored in comments.
-# The VS .NET project file is generated into a different file as the
-# VS .NET environment will not retain comments when modifying the file.
+# Requires Python 2.5 or later
+# Files are regenerated in place with templates stored in comments.
 # The format of generation comments is documented in FileGenerator.py.
 
-import string
-import sys
-import os
-import glob
-import codecs
-import datetime
+from FileGenerator import Regenerate, UpdateLineInFile, ReplaceREInFile
+import ScintillaData
 
-from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd
-
-def FindModules(lexFile):
-    modules = []
-    with open(lexFile) as f:
-        for l in f.readlines():
-            if l.startswith("LexerModule"):
-                l = l.replace("(", " ")
-                modules.append(l.split()[1])
-    return modules
-
-# Properties that start with lexer. or fold. are automatically found but there are some
-# older properties that don't follow this pattern so must be explicitly listed.
-knownIrregularProperties = [
-    "fold",
-    "styling.within.preprocessor",
-    "tab.timmy.whinge.level",
-    "asp.default.language",
-    "html.tags.case.sensitive",
-    "ps.level",
-    "ps.tokenize",
-    "sql.backslash.escapes",
-    "nsis.uservars",
-    "nsis.ignorecase"
-]
-
-def FindProperties(lexFile):
-    properties = {}
-    with open(lexFile) as f:
-        for l in f.readlines():
-            if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l:
-                l = l.strip()
-                if not l.startswith("//"):	# Drop comments
-                    propertyName = l.split("\"")[1]
-                    if propertyName.lower() == propertyName:
-                        # Only allow lower case property names
-                        if propertyName in knownIrregularProperties or \
-                            propertyName.startswith("fold.") or \
-                            propertyName.startswith("lexer."):
-                            properties[propertyName] = 1
-    return properties
-
-def FindPropertyDocumentation(lexFile):
-    documents = {}
-    with open(lexFile) as f:
-        name = ""
-        for l in f.readlines():
-            l = l.strip()
-            if "// property " in l:
-                propertyName = l.split()[2]
-                if propertyName.lower() == propertyName:
-                    # Only allow lower case property names
-                    name = propertyName
-                    documents[name] = ""
-            elif "DefineProperty" in l and "\"" in l:
-                propertyName = l.split("\"")[1]
-                if propertyName.lower() == propertyName:
-                    # Only allow lower case property names
-                    name = propertyName
-                    documents[name] = ""
-            elif name:
-                if l.startswith("//"):
-                    if documents[name]:
-                        documents[name] += " "
-                    documents[name] += l[2:].strip()
-                elif l.startswith("\""):
-                    l = l[1:].strip()
-                    if l.endswith(";"):
-                        l = l[:-1].strip()
-                    if l.endswith(")"):
-                        l = l[:-1].strip()
-                    if l.endswith("\""):
-                        l = l[:-1]
-                    # Fix escaped double quotes
-                    l = l.replace("\\\"", "\"")
-                    documents[name] += l
-                else:
-                    name = ""
-    for name in list(documents.keys()):
-        if documents[name] == "":
-            del documents[name]
-    return documents
-
-def ciCompare(a,b):
-    return cmp(a.lower(), b.lower())
-
-def ciKey(a):
-    return a.lower()
-
-def sortListInsensitive(l):
-    try:    # Try key function
-        l.sort(key=ciKey)
-    except TypeError:    # Earlier version of Python, so use comparison function
-        l.sort(ciCompare)
-
-host = "prdownloads.sourceforge.net/"
-def UpdateDownloadLinks(path, version):
-    lines = []
-    with open(path, "r") as f:
-        for l in f.readlines():
-            l = l.rstrip()
-            if host in l:
-                start, prd, rest = l.partition(host)
-                pth, dot, ending = rest.partition(".")
-                pthNew = pth[:-3] + version.rstrip()
-                lineWithNewVersion = start + prd +pthNew + dot + ending
-                lines.append(lineWithNewVersion)
-            else:
-                lines.append(l)
-    contents = lineEnd.join(lines) + lineEnd
-    UpdateFile(path, contents)
-
-def UpdateVersionNumbers(root):
-    with open(root + "scintilla/version.txt") as f:
-        version = f.read()
-    versionDotted = version[0] + '.' + version[1] + '.' + version[2]
-    versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0'
-    with open(root + "scintilla/doc/index.html") as f:
-        dateModified = [l for l in f.readlines() if "Date.Modified" in l][0].split('\"')[3]
-        # 20130602
-        # index.html, SciTE.html
-        dtModified = datetime.datetime.strptime(dateModified, "%Y%m%d")
-        yearModified = dateModified[0:4]
-        monthModified = dtModified.strftime("%B")
-        dayModified = "%d" % dtModified.day
-        mdyModified = monthModified + " " + dayModified + " " + yearModified
-        # May 22 2013
-        # index.html, SciTE.html
-        dmyModified = dayModified + " " + monthModified + " " + yearModified
-        # 22 May 2013
-        # ScintillaHistory.html -- only first should change
-        myModified = monthModified + " " + yearModified
-        # scite/src/SciTE.h
-        #define COPYRIGHT_DATES "December 1998-May 2013"
-        #define COPYRIGHT_YEARS "1998-2013"
-        dateLine = f.readlines()
-
-    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA",
-        "#define VERSION_SCINTILLA \"" + versionDotted + "\"")
-    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS",
-        "#define VERSION_WORDS " + versionCommad)
-    UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro",
+def UpdateVersionNumbers(sci, root):
+    UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_SCINTILLA",
+        "#define VERSION_SCINTILLA \"" + sci.versionDotted + "\"")
+    UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_WORDS",
+        "#define VERSION_WORDS " + sci.versionCommad)
+    UpdateLineInFile(root + "qt/ScintillaEditBase/ScintillaEditBase.pro",
         "VERSION =",
-        "VERSION = " + versionDotted)
-    UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro",
+        "VERSION = " + sci.versionDotted)
+    UpdateLineInFile(root + "qt/ScintillaEdit/ScintillaEdit.pro",
         "VERSION =",
-        "VERSION = " + versionDotted)
-    UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", "       Release",
-        "       Release " + versionDotted)
-    UpdateDownloadLinks(root + "scintilla/doc/ScintillaDownload.html", version)
-    UpdateLineInFile(root + "scintilla/doc/index.html",
+        "VERSION = " + sci.versionDotted)
+    UpdateLineInFile(root + "doc/ScintillaDownload.html", "       Release",
+        "       Release " + sci.versionDotted)
+    ReplaceREInFile(root + "doc/ScintillaDownload.html",
+        r"/scintilla/([a-zA-Z]+)\d\d\d",
+        r"/scintilla/\g<1>" +  sci.version)
+    UpdateLineInFile(root + "doc/index.html",
         '          <font color="#FFCC99" size="3"> Release version',
-        '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
-    UpdateLineInFile(root + "scintilla/doc/index.html",
+        '          <font color="#FFCC99" size="3"> Release version ' +\
+        sci.versionDotted + '<br />')
+    UpdateLineInFile(root + "doc/index.html",
         '           Site last modified',
-        '           Site last modified ' + mdyModified + '</font>')
-    UpdateLineInFile(root + "scintilla/doc/ScintillaHistory.html",
+        '           Site last modified ' + sci.mdyModified + '</font>')
+    UpdateLineInFile(root + "doc/ScintillaHistory.html",
         '	Released ',
-        '	Released ' + dmyModified + '.')
-
-    if os.path.exists(root + "scite"):
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE",
-            "#define VERSION_SCITE \"" + versionDotted + "\"")
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS",
-            "#define VERSION_WORDS " + versionCommad)
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_DATES",
-            '#define COPYRIGHT_DATES "December 1998-' + myModified + '"')
-        UpdateLineInFile(root + "scite/src/SciTE.h", "#define COPYRIGHT_YEARS",
-            '#define COPYRIGHT_YEARS "1998-' + yearModified + '"')
-        UpdateLineInFile(root + "scite/doc/SciTEDownload.html", "       Release",
-            "       Release " + versionDotted)
-        UpdateDownloadLinks(root + "scite/doc/SciTEDownload.html", version)
-        UpdateLineInFile(root + "scite/doc/SciTE.html",
-            '          <font color="#FFCC99" size="3"> Release version',
-            '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
-        UpdateLineInFile(root + "scite/doc/SciTE.html",
-            '           Site last modified',
-            '           Site last modified ' + mdyModified + '</font>')
-        UpdateLineInFile(root + "scite/doc/SciTE.html",
-            '    <meta name="Date.Modified"',
-            '    <meta name="Date.Modified" content="' + dateModified + '" />')
-
-def RegenerateAll():
-    root="../../"
-
-    # Find all the lexer source code files
-    lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx")
-    sortListInsensitive(lexFilePaths)
-    lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
-    print(lexFiles)
-    lexerModules = []
-    lexerProperties = {}
-    propertyDocuments = {}
-    for lexFile in lexFilePaths:
-        lexerModules.extend(FindModules(lexFile))
-        for k in FindProperties(lexFile).keys():
-            lexerProperties[k] = 1
-        documents = FindPropertyDocumentation(lexFile)
-        for k in documents.keys():
-            if k not in propertyDocuments:
-                propertyDocuments[k] = documents[k]
-    sortListInsensitive(lexerModules)
-    lexerProperties = list(lexerProperties.keys())
-    sortListInsensitive(lexerProperties)
-
-    # Generate HTML to document each property
-    # This is done because tags can not be safely put inside comments in HTML
-    documentProperties = list(propertyDocuments.keys())
-    sortListInsensitive(documentProperties)
-    propertiesHTML = []
-    for k in documentProperties:
-        propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" %
-            (k, k, propertyDocuments[k]))
+        '	Released ' + sci.dmyModified + '.')
 
-    # Find all the SciTE properties files
-    otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
-    if os.path.exists(root + "scite"):
-        propFilePaths = glob.glob(root + "scite/src/*.properties")
-        sortListInsensitive(propFilePaths)
-        propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps]
-        sortListInsensitive(propFiles)
-        print(propFiles)
+def RegenerateAll(root):
+    
+    sci = ScintillaData.ScintillaData(root)
 
-    Regenerate(root + "scintilla/src/Catalogue.cxx", "//", lexerModules)
-    Regenerate(root + "scintilla/win32/scintilla.mak", "#", lexFiles)
-    if os.path.exists(root + "scite"):
-        Regenerate(root + "scite/win32/makefile", "#", propFiles)
-        Regenerate(root + "scite/win32/scite.mak", "#", propFiles)
-        Regenerate(root + "scite/src/SciTEProps.cxx", "//", lexerProperties)
-        Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", propertiesHTML)
-        Generate(root + "scite/boundscheck/vcproj.gen",
-         root + "scite/boundscheck/SciTE.vcproj", "#", lexFiles)
+    Regenerate(root + "src/Catalogue.cxx", "//", sci.lexerModules)
+    Regenerate(root + "win32/scintilla.mak", "#", sci.lexFiles)
 
-    UpdateVersionNumbers(root)
+    UpdateVersionNumbers(sci, root)
 
-RegenerateAll()
+if __name__=="__main__":
+    RegenerateAll("../")
diff --git a/scripts/ScintillaData.py b/scripts/ScintillaData.py
new file mode 100644
index 000000000..1c999a119
--- /dev/null
+++ b/scripts/ScintillaData.py
@@ -0,0 +1,193 @@
+# ScintillaData.py - implemented 2013 by Neil Hodgson neilh@scintilla.org
+# Released to the public domain.
+
+# Common code used by Scintilla and SciTE for source file regeneration.
+# The ScintillaData object exposes information about Scintilla as properties:
+# Version properties
+#     version
+#     versionDotted
+#     versionCommad
+#     
+# Date last modified
+#     dateModified
+#     yearModified
+#     mdyModified
+#     dmyModified
+#     myModified
+#     
+# Information about lexers and properties defined in lexers
+#     lexFiles
+#         sorted list of lexer files
+#     lexerModules
+#         sorted list of module names
+#     lexerProperties
+#         sorted list of lexer properties
+#     propertyDocuments
+#         dictionary of property documentation { name: document string }
+
+# This file can be run to see the data it provides.
+# Requires Python 2.5 or later
+
+from __future__ import with_statement
+
+import datetime, glob, os, textwrap
+
+import FileGenerator
+
+def FindModules(lexFile):
+    modules = []
+    with open(lexFile) as f:
+        for l in f.readlines():
+            if l.startswith("LexerModule"):
+                l = l.replace("(", " ")
+                modules.append(l.split()[1])
+    return modules
+
+# Properties that start with lexer. or fold. are automatically found but there are some
+# older properties that don't follow this pattern so must be explicitly listed.
+knownIrregularProperties = [
+    "fold",
+    "styling.within.preprocessor",
+    "tab.timmy.whinge.level",
+    "asp.default.language",
+    "html.tags.case.sensitive",
+    "ps.level",
+    "ps.tokenize",
+    "sql.backslash.escapes",
+    "nsis.uservars",
+    "nsis.ignorecase"
+]
+
+def FindProperties(lexFile):
+    properties = {}
+    with open(lexFile) as f:
+        for l in f.readlines():
+            if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l:
+                l = l.strip()
+                if not l.startswith("//"):	# Drop comments
+                    propertyName = l.split("\"")[1]
+                    if propertyName.lower() == propertyName:
+                        # Only allow lower case property names
+                        if propertyName in knownIrregularProperties or \
+                            propertyName.startswith("fold.") or \
+                            propertyName.startswith("lexer."):
+                            properties[propertyName] = 1
+    return properties
+
+def FindPropertyDocumentation(lexFile):
+    documents = {}
+    with open(lexFile) as f:
+        name = ""
+        for l in f.readlines():
+            l = l.strip()
+            if "// property " in l:
+                propertyName = l.split()[2]
+                if propertyName.lower() == propertyName:
+                    # Only allow lower case property names
+                    name = propertyName
+                    documents[name] = ""
+            elif "DefineProperty" in l and "\"" in l:
+                propertyName = l.split("\"")[1]
+                if propertyName.lower() == propertyName:
+                    # Only allow lower case property names
+                    name = propertyName
+                    documents[name] = ""
+            elif name:
+                if l.startswith("//"):
+                    if documents[name]:
+                        documents[name] += " "
+                    documents[name] += l[2:].strip()
+                elif l.startswith("\""):
+                    l = l[1:].strip()
+                    if l.endswith(";"):
+                        l = l[:-1].strip()
+                    if l.endswith(")"):
+                        l = l[:-1].strip()
+                    if l.endswith("\""):
+                        l = l[:-1]
+                    # Fix escaped double quotes
+                    l = l.replace("\\\"", "\"")
+                    documents[name] += l
+                else:
+                    name = ""
+    for name in list(documents.keys()):
+        if documents[name] == "":
+            del documents[name]
+    return documents
+
+def ciCompare(a,b):
+    return cmp(a.lower(), b.lower())
+
+def ciKey(a):
+    return a.lower()
+
+def SortListInsensitive(l):
+    try:    # Try key function
+        l.sort(key=ciKey)
+    except TypeError:    # Earlier version of Python, so use comparison function
+        l.sort(ciCompare)
+
+class ScintillaData:
+    def __init__(self, scintillaRoot):
+        # Discover verion information
+        with open(scintillaRoot + "version.txt") as f:
+            self.version = f.read().strip()
+        self.versionDotted = self.version[0] + '.' + self.version[1] + '.' + \
+            self.version[2]
+        self.versionCommad = self.version[0] + ', ' + self.version[1] + ', ' + \
+            self.version[2] + ', 0'
+
+        with open(scintillaRoot + "doc/index.html") as f:
+            self.dateModified = [l for l in f.readlines() if "Date.Modified" in l]\
+                [0].split('\"')[3]
+            # 20130602
+            # index.html, SciTE.html
+            dtModified = datetime.datetime.strptime(self.dateModified, "%Y%m%d")
+            self.yearModified = self.dateModified[0:4]
+            monthModified = dtModified.strftime("%B")
+            dayModified = "%d" % dtModified.day
+            self.mdyModified = monthModified + " " + dayModified + " " + self.yearModified
+            # May 22 2013
+            # index.html, SciTE.html
+            self.dmyModified = dayModified + " " + monthModified + " " + self.yearModified
+            # 22 May 2013
+            # ScintillaHistory.html -- only first should change
+            self.myModified = monthModified + " " + self.yearModified
+
+        # Find all the lexer source code files
+        lexFilePaths = glob.glob(scintillaRoot + "lexers/Lex*.cxx")
+        SortListInsensitive(lexFilePaths)
+        self.lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
+        self.lexerModules = []
+        lexerProperties = set()
+        self.propertyDocuments = {}
+        for lexFile in lexFilePaths:
+            self.lexerModules.extend(FindModules(lexFile))
+            for k in FindProperties(lexFile).keys():
+                lexerProperties.add(k)
+            documents = FindPropertyDocumentation(lexFile)
+            for k in documents.keys():
+                if k not in self.propertyDocuments:
+                    self.propertyDocuments[k] = documents[k]
+        SortListInsensitive(self.lexerModules)
+        self.lexerProperties = list(lexerProperties)
+        SortListInsensitive(self.lexerProperties)
+
+def printWrapped(text):
+    print(textwrap.fill(text, subsequent_indent="    "))
+
+if __name__=="__main__":
+    sci = ScintillaData("../")
+    print("Version   %s   %s   %s" % (sci.version, sci.versionDotted, sci.versionCommad))
+    print("Date last modified    %s   %s   %s   %s   %s" % (
+        sci.dateModified, sci.yearModified, sci.mdyModified, sci.dmyModified, sci.myModified))
+    printWrapped(str(len(sci.lexFiles)) + " lexer files: " + ", ".join(sci.lexFiles))
+    printWrapped(str(len(sci.lexerModules)) + " lexer modules: " + ", ".join(sci.lexerModules))
+    printWrapped("Lexer properties: " + ", ".join(sci.lexerProperties))
+    print("Lexer property documentation:")
+    documentProperties = list(sci.propertyDocuments.keys())
+    SortListInsensitive(documentProperties)
+    for k in documentProperties:
+        print("    " + k)
+        print(textwrap.fill(sci.propertyDocuments[k], initial_indent="        ",
+            subsequent_indent="        "))