Initial Unicode support code.

6 files changed, 100 insertions, 43 deletions

diff --git a/src/Document.cxx b/src/Document.cxx
index 7a30d7fd1..650c0ced2 100644
--- a/src/Document.cxx
+++ b/src/Document.cxx
@@ -202,18 +202,23 @@ bool Document::IsCrLf(int pos) {
 bool Document::IsDBCS(int pos) {
 #if PLAT_WIN
 	if (dbcsCodePage) {
-		// Anchor DBCS calculations at start of line because start of line can
-		// not be a DBCS trail byte.
-		int startLine = pos;
-		while (startLine > 0 && cb.CharAt(startLine) != '\r' && cb.CharAt(startLine) != '\n')
-			startLine--;
-		while (startLine <= pos) {
-			if (IsDBCSLeadByteEx(dbcsCodePage, cb.CharAt(startLine))) {
+		if (SC_CP_UTF8 == dbcsCodePage) {
+			unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
+			return ch >= 0x80;
+		} else {
+			// Anchor DBCS calculations at start of line because start of line can
+			// not be a DBCS trail byte.
+			int startLine = pos;
+			while (startLine > 0 && cb.CharAt(startLine) != '\r' && cb.CharAt(startLine) != '\n')
+				startLine--;
+			while (startLine <= pos) {
+				if (IsDBCSLeadByteEx(dbcsCodePage, cb.CharAt(startLine))) {
+					startLine++;
+					if (startLine >= pos)
+						return true;
+				}
 				startLine++;
-				if (startLine >= pos)
-					return true;
 			}
-			startLine++;
 		}
 	}
 	return false;
@@ -222,6 +227,28 @@ bool Document::IsDBCS(int pos) {
 #endif
 }
 
+int Document::LenChar(int pos) {
+	if (IsCrLf(pos)) {
+		return 2;
+	} else if (SC_CP_UTF8 == dbcsCodePage) {
+		unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
+		if (ch < 0x80)
+			return 1;
+		int len = 2;
+		if (ch >= (0x80+0x40+0x20))
+			len = 3;
+		int lengthDoc = Length();
+		if ((pos + len) > lengthDoc)
+			return lengthDoc-pos;
+		else 
+			return len;
+	} else if (IsDBCS(pos)) {
+		return 2;
+	} else {
+		return 1;
+	}
+}
+
 // Normalise a position so that it is not halfway through a two byte character.
 // This can occur in two situations -
 // When lines are terminated with \r\n pairs which should be treated as one character.
@@ -253,29 +280,41 @@ int Document::MovePositionOutsideChar(int pos, int moveDir, bool checkLineEnd) {
 
 #if PLAT_WIN
 	if (dbcsCodePage) {
-		// Anchor DBCS calculations at start of line because start of line can
-		// not be a DBCS trail byte.
-		int startLine = pos;
-		while (startLine > 0 && cb.CharAt(startLine) != '\r' && cb.CharAt(startLine) != '\n')
-			startLine--;
-		bool atLeadByte = false;
-		while (startLine < pos) {
-			if (atLeadByte)
-				atLeadByte = false;
-			else if (IsDBCSLeadByteEx(dbcsCodePage, cb.CharAt(startLine)))
-				atLeadByte = true;
-			else
-				atLeadByte = false;
-			startLine++;
-			//Platform::DebugPrintf("DBCS %s\n", atlead ? "D" : "-");
-		}
+		if (SC_CP_UTF8 == dbcsCodePage) {
+			unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
+			while ((pos > 0) && (pos < Length()) && (ch >= 0x80) && (ch < (0x80 + 0x40))) {
+				// ch is a trail byte
+				if (moveDir > 0)
+					pos++;
+				else 
+					pos--;
+				ch = static_cast<unsigned char>(cb.CharAt(pos));
+			}
+		} else {
+			// Anchor DBCS calculations at start of line because start of line can
+			// not be a DBCS trail byte.
+			int startLine = pos;
+			while (startLine > 0 && cb.CharAt(startLine) != '\r' && cb.CharAt(startLine) != '\n')
+				startLine--;
+			bool atLeadByte = false;
+			while (startLine < pos) {
+				if (atLeadByte)
+					atLeadByte = false;
+				else if (IsDBCSLeadByteEx(dbcsCodePage, cb.CharAt(startLine)))
+					atLeadByte = true;
+				else
+					atLeadByte = false;
+				startLine++;
+				//Platform::DebugPrintf("DBCS %s\n", atlead ? "D" : "-");
+			}
 
-		if (atLeadByte) {
-			// Position is between a lead byte and a trail byte
-			if (moveDir > 0)
-				return pos + 1;
-			else
-				return pos - 1;
+			if (atLeadByte) {
+				// Position is between a lead byte and a trail byte
+				if (moveDir > 0)
+					return pos + 1;
+				else
+					return pos - 1;
+			}
 		}
 	}
 #endif
@@ -440,13 +479,7 @@ void Document::ChangeChar(int pos, char ch) {
 }
 
 void Document::DelChar(int pos) {
-	if (IsCrLf(pos)) {
-		DeleteChars(pos, 2);
-	} else if (IsDBCS(pos)) {
-		DeleteChars(pos, 2);
-	} else if (pos < Length()) {
-		DeleteChars(pos, 1);
-	}
+	DeleteChars(pos, LenChar(pos));
 }
 
 int Document::DelCharBack(int pos) {
@@ -455,6 +488,10 @@ int Document::DelCharBack(int pos) {
 	} else if (IsCrLf(pos - 2)) {
 		DeleteChars(pos - 2, 2);
 		return pos - 2;
+	} else if (SC_CP_UTF8 == dbcsCodePage) {
+		int startChar = MovePositionOutsideChar(pos-1, -1, false);
+		DeleteChars(startChar, pos - startChar);
+		return startChar;
 	} else if (IsDBCS(pos - 1)) {
 		DeleteChars(pos - 2, 2);
 		return pos - 2;
@@ -529,6 +566,8 @@ void Document::ConvertLineEnds(int eolModeSet) {
 }
 
 bool Document::IsWordChar(unsigned char ch) {
+	if ((SC_CP_UTF8 == dbcsCodePage) && (ch >0x80))
+		return true;
 	return wordchars[ch];
 }
 
@@ -653,7 +692,7 @@ void Document::ChangeCase(Range r, bool makeUpperCase) {
 	for (int pos=r.start; pos<r.end; pos++) {
 		char ch = CharAt(pos);
 		if (dbcsCodePage && IsDBCS(pos)) {
-			pos++;
+			pos += LenChar(pos);
 		} else {
 			if (makeUpperCase) {
 				if (islower(ch)) {
diff --git a/src/Document.h b/src/Document.h
index 7ab187573..ae25d69f4 100644
--- a/src/Document.h
+++ b/src/Document.h
@@ -87,6 +87,7 @@ public:
 	int stylingBitsMask;
 	
 	int eolMode;
+	// dbcsCodePage can also be SC_CP_UTF8 to enable UTF-8 mode
 	int dbcsCodePage;
 	int tabInChars;
 	
@@ -99,6 +100,7 @@ public:
 	int LineFromPosition(int pos);
 	int ClampPositionIntoDocument(int pos);
 	bool IsCrLf(int pos);
+	int LenChar(int pos);
 	int MovePositionOutsideChar(int pos, int moveDir, bool checkLineEnd=true);
 
 	// Gateways to modifying document
diff --git a/src/DocumentAccessor.cxx b/src/DocumentAccessor.cxx
index dd156fa24..420a359d6 100644
--- a/src/DocumentAccessor.cxx
+++ b/src/DocumentAccessor.cxx
@@ -22,7 +22,12 @@ bool DocumentAccessor::InternalIsLeadByte(char ch) {
 	// TODO: support DBCS under GTK+
 	return false;
 #elif PLAT_WIN 
-	return IsDBCSLeadByteEx(codePage, ch);
+	if (SC_CP_UTF8 == codePage)
+		// For lexing, all characters >= 0x80 are treated the
+		// same so none is considered a lead byte.
+		return false;	
+	else
+		return IsDBCSLeadByteEx(codePage, ch);
 #elif PLAT_WX 
 	return false;
 #endif 
diff --git a/src/Editor.cxx b/src/Editor.cxx
index 1566ffa6e..111142d90 100644
--- a/src/Editor.cxx
+++ b/src/Editor.cxx
@@ -210,6 +210,7 @@ Point Editor::LocationFromPosition(unsigned int pos) {
 	//Platform::DebugPrintf("line=%d\n", line);
 	Surface surface;
 	surface.Init();
+	surface.SetUnicodeMode(SC_CP_UTF8 == pdoc->dbcsCodePage);
 	Point pt;
 	pt.y = (lineVisible - topLine) * vs.lineHeight;  	// + half a lineheight?
 	unsigned int posLineStart = pdoc->LineStart(line);
@@ -252,6 +253,7 @@ int Editor::PositionFromLocation(Point pt) {
 //Platform::DebugPrintf("Position of (%d,%d) line = %d top=%d\n", pt.x, pt.y, line, topLine);
 	Surface surface;
 	surface.Init();
+	surface.SetUnicodeMode(SC_CP_UTF8 == pdoc->dbcsCodePage);
 	unsigned int posLineStart = pdoc->LineStart(line);
 
 	LineLayout ll;
@@ -273,6 +275,7 @@ int Editor::PositionFromLineX(int line, int x) {
 	//Platform::DebugPrintf("Position of (%d,%d) line = %d top=%d\n", pt.x, pt.y, line, topLine);
 	Surface surface;
 	surface.Init();
+	surface.SetUnicodeMode(SC_CP_UTF8 == pdoc->dbcsCodePage);
 	unsigned int posLineStart = pdoc->LineStart(line);
 
 	LineLayout ll;
@@ -1024,6 +1027,7 @@ void Editor::Paint(Surface *surfaceWindow, PRectangle rcArea) {
 		} else {
 			surface = surfaceWindow;
 		}
+		surface->SetUnicodeMode(SC_CP_UTF8 == pdoc->dbcsCodePage);
 
 		int visibleLine = topLine + screenLinePaintFirst;
 		int line = cs.DocFromDisplay(visibleLine);
@@ -1164,8 +1168,10 @@ long Editor::FormatRange(bool draw, FORMATRANGE *pfr) {
 
 	Surface *surface = new Surface();
 	surface->Init(pfr->hdc);
+	surface->SetUnicodeMode(SC_CP_UTF8 == pdoc->dbcsCodePage);
 	Surface *surfaceMeasure = new Surface();
 	surfaceMeasure->Init(pfr->hdcTarget);
+	surfaceMeasure->SetUnicodeMode(SC_CP_UTF8 == pdoc->dbcsCodePage);
 	
 	ViewStyle vsPrint(vs);
 	
diff --git a/src/LexPython.cxx b/src/LexPython.cxx
index 86597e863..1864412a6 100644
--- a/src/LexPython.cxx
+++ b/src/LexPython.cxx
@@ -71,9 +71,9 @@ static void ColourisePyDoc(unsigned int startPos, int length, int initStyle,
 	for (int i = startPos; i <= lengthDoc; i++) {
 	
 		if (atStartLine) {
-			char chFlags;
 			char chBad = static_cast<char>(64);
 			char chGood = static_cast<char>(0);
+			char chFlags = chGood;
 			if (whingeLevel == 1) {
 				chFlags = (spaceFlags & wsInconsistent) ? chBad : chGood;
 			} else if (whingeLevel == 2) {
diff --git a/src/WindowAccessor.cxx b/src/WindowAccessor.cxx
index d04c0142d..f0ad45914 100644
--- a/src/WindowAccessor.cxx
+++ b/src/WindowAccessor.cxx
@@ -19,7 +19,12 @@ bool WindowAccessor::InternalIsLeadByte(char ch) {
 	// TODO: support DBCS under GTK+
 	return false;
 #elif PLAT_WIN 
-	return IsDBCSLeadByteEx(codePage, ch);
+	if (SC_CP_UTF8 == codePage)
+		// For lexing, all characters >= 0x80 are treated the
+		// same so none is considered a lead byte.
+		return false;	
+	else
+		return IsDBCSLeadByteEx(codePage, ch);
 #elif PLAT_WX 
 	return false;
 #endif