The Python lexer recognizes identifiers more accurately when they include

non-ASCII characters.
Calls provided for determining whether characters are in the sets defined for
identifiers by the Unicode standard in UAX #31.

1 files changed, 4 insertions, 10 deletions

diff --git a/lexers/LexPython.cxx b/lexers/LexPython.cxx
index f667cb32c..62ed83c95 100644
--- a/lexers/LexPython.cxx
+++ b/lexers/LexPython.cxx
@@ -192,11 +192,8 @@ inline bool IsAWordChar(int ch, bool unicodeIdentifiers) {
 	if (!unicodeIdentifiers)
 		return false;
 
-	// Approximation, Python uses the XID_Continue set from unicode data
-	// see http://unicode.org/reports/tr31/
-	CharacterCategory c = CategoriseCharacter(ch);
-	return (c == ccLl || c == ccLu || c == ccLt || c == ccLm || c == ccLo
-		|| c == ccNl || c == ccMn || c == ccMc || c == ccNd || c == ccPc);
+	// Python uses the XID_Continue set from unicode data
+	return IsXidContinue(ch);
 }
 
 inline bool IsAWordStart(int ch, bool unicodeIdentifiers) {
@@ -206,11 +203,8 @@ inline bool IsAWordStart(int ch, bool unicodeIdentifiers) {
 	if (!unicodeIdentifiers)
 		return false;
 
-	// Approximation, Python uses the XID_Start set from unicode data
-	// see http://unicode.org/reports/tr31/
-	CharacterCategory c = CategoriseCharacter(ch);
-	return (c == ccLl || c == ccLu || c == ccLt || c == ccLm || c == ccLo
-		|| c == ccNl);
+	// Python uses the XID_Start set from unicode data
+	return IsXidStart(ch);
 }
 
 static bool IsFirstNonWhitespace(Sci_Position pos, Accessor &styler) {