Backport: Feature [feature-requests:#1259]. Add SCI_SETCHARACTERCATEGORYOPTIMIZATION API to optimize speed of character category features.

Backport of changeset 7392:2832adedd0f4, but with added includes for Sci::clamp().

1 files changed, 15 insertions, 5 deletions

diff --git a/scripts/GenerateCharacterCategory.py b/scripts/GenerateCharacterCategory.py
index 4596eec6a..ba6ac858d 100644
--- a/scripts/GenerateCharacterCategory.py
+++ b/scripts/GenerateCharacterCategory.py
@@ -17,17 +17,27 @@ def findCategories(filename):
 def updateCharacterCategory(filename):
     values = ["// Created with Python %s,  Unicode %s" % (
         platform.python_version(), unicodedata.unidata_version)]
-    category = unicodedata.category(chr(0))
+
     startRange = 0
+    category = unicodedata.category(chr(startRange))
+    table = []
     for ch in range(sys.maxunicode):
         uch = chr(ch)
-        if unicodedata.category(uch) != category:
+        current = unicodedata.category(uch)
+        if current != category:
             value = startRange * 32 + categories.index(category)
-            values.append("%d," % value)
-            category = unicodedata.category(uch)
+            table.append(value)
+            category = current
             startRange = ch
     value = startRange * 32 + categories.index(category)
-    values.append("%d," % value)
+    table.append(value)
+
+    # the sentinel value is used to simplify CharacterCategoryMap::Optimize()
+    category = 'Cn'
+    value = (sys.maxunicode + 1)*32 + categories.index(category)
+    table.append(value)
+
+    values.extend(["%d," % value for value in table])
 
     Regenerate(filename, "//", values)