Added support for //TRANSLIT option to iconv to allow approximate

conversions, such as when pasting UTF-8 text and the buffer is set to an
encoding that does not support all of the characters on the clipboard.

3 files changed, 22 insertions, 35 deletions

diff --git a/gtk/Converter.h b/gtk/Converter.h
index 09f213a66..7dc2e6875 100644
--- a/gtk/Converter.h
+++ b/gtk/Converter.h
@@ -27,9 +27,9 @@ public:
 	Converter() {
 		iconvh = iconvhBad;
 	}
-	Converter(const char *charSetDestination, const char *charSetSource) {
+	Converter(const char *charSetDestination, const char *charSetSource, bool transliterations) {
 		iconvh = iconvhBad;
-	    	Open(charSetDestination, charSetSource);
+	    	Open(charSetDestination, charSetSource, transliterations);
 	}
 	~Converter() {
 		Close();
@@ -37,13 +37,18 @@ public:
 	operator bool() const {
 		return iconvh != iconvhBad;
 	}
-	void Open(const char *charSetDestination, const char *charSetSource) {
+	void Open(const char *charSetDestination, const char *charSetSource, bool transliterations=true) {
 		Close();
 		if (*charSetSource) {
+			char fullDest[200];
+			strcpy(fullDest, charSetDestination);
+			if (transliterations) {
+				strcat(fullDest, "//TRANSLIT");
+			}
 #if GTK_MAJOR_VERSION >= 2
-			iconvh = g_iconv_open(charSetDestination, charSetSource);
+			iconvh = g_iconv_open(fullDest, charSetSource);
 #else
-			iconvh = iconv_open(charSetDestination, charSetSource);
+			iconvh = iconv_open(fullDest, charSetSource);
 #endif
 		}
 	}
diff --git a/gtk/PlatGTK.cxx b/gtk/PlatGTK.cxx
index c7a85fea2..a520b63bf 100644
--- a/gtk/PlatGTK.cxx
+++ b/gtk/PlatGTK.cxx
@@ -794,7 +794,7 @@ const char *CharacterSetID(int characterSet) {
 void SurfaceImpl::SetConverter(int characterSet_) {
 	if (characterSet != characterSet_) {
 		characterSet = characterSet_;
-		conv.Open("UTF-8", CharacterSetID(characterSet));
+		conv.Open("UTF-8", CharacterSetID(characterSet), false);
 	}
 }
 #endif
@@ -1400,7 +1400,7 @@ void SurfaceImpl::MeasureWidths(Font &font_, const char *s, int len, int *positi
 						// Convert to UTF-8 so can ask Pango for widths, then
 						// Loop through UTF-8 and DBCS forms, taking account of different
 						// character byte lengths.
-						Converter convMeasure("UCS-2", CharacterSetID(characterSet));
+						Converter convMeasure("UCS-2", CharacterSetID(characterSet), false);
 						pango_layout_set_text(layout, utfForm, strlen(utfForm));
 						int i = 0;
 						int utfIndex = 0;
@@ -2632,7 +2632,6 @@ bool Platform::IsDBCSLeadByte(int /* codePage */, char /* ch */) {
 	return false;
 }
 
-#if GTK_MAJOR_VERSION < 2
 int Platform::DBCSCharLength(int, const char *s) {
 	int bytes = mblen(s, MB_CUR_MAX);
 	if (bytes >= 1)
@@ -2640,24 +2639,6 @@ int Platform::DBCSCharLength(int, const char *s) {
 	else
 		return 1;
 }
-#else
-int Platform::DBCSCharLength(int codePage, const char *s) {
-	if (codePage == 999932) {
-		// Experimental and disabled code - change 999932 to 932 above to
-		// enable locale avoiding but expensive character length determination.
-		// Avoid locale with explicit use of iconv
-		Converter convMeasure("UCS-2", CharacterSetID(SC_CHARSET_SHIFTJIS));
-		size_t lenChar = MultiByteLenFromIconv(convMeasure, s, strlen(s));
-		return lenChar;
-	} else {
-		int bytes = mblen(s, MB_CUR_MAX);
-		if (bytes >= 1)
-			return bytes;
-		else
-			return 1;
-	}
-}
-#endif
 
 int Platform::DBCSCharMaxLength() {
 	return MB_CUR_MAX;
diff --git a/gtk/ScintillaGTK.cxx b/gtk/ScintillaGTK.cxx
index fafc12b99..dfc403dc5 100644
--- a/gtk/ScintillaGTK.cxx
+++ b/gtk/ScintillaGTK.cxx
@@ -797,10 +797,11 @@ void ScintillaGTK::StartDrag() {
 }
 
 #ifdef USE_CONVERTER
-static char *ConvertText(int *lenResult, char *s, size_t len, const char *charSetDest, const char *charSetSource) {
+static char *ConvertText(int *lenResult, char *s, size_t len, const char *charSetDest, 
+	const char *charSetSource, bool transliterations) {
 	*lenResult = 0;
 	char *destForm = 0;
-	Converter conv(charSetDest, charSetSource);
+	Converter conv(charSetDest, charSetSource, transliterations);
 	if (conv) {
 		destForm = new char[len*3+1];
 		char *pin = s;
@@ -848,7 +849,7 @@ int ScintillaGTK::TargetAsUTF8(char *text) {
 				pdoc->GetCharRange(s, targetStart, targetLength);
 //~ fprintf(stderr, "    \"%s\"\n", s);
 				if (text) {
-					char *tmputf = ConvertText(&targetLength, s, targetLength, "UTF-8", charSetBuffer);
+					char *tmputf = ConvertText(&targetLength, s, targetLength, "UTF-8", charSetBuffer, false);
 					memcpy(text, tmputf, targetLength);
 					delete []tmputf;
 //~ fprintf(stderr, "    \"%s\"\n", text);
@@ -885,7 +886,7 @@ int ScintillaGTK::EncodedFromUTF8(char *utf8, char *encoded) {
 		if (*charSetBuffer) {
 //~ fprintf(stderr, "Encode %s %d\n", charSetBuffer, inputLength);
 			int outLength = 0;
-			char *tmpEncoded = ConvertText(&outLength, utf8, inputLength, charSetBuffer, "UTF-8");
+			char *tmpEncoded = ConvertText(&outLength, utf8, inputLength, charSetBuffer, "UTF-8", true);
 			if (tmpEncoded) {
 //~ fprintf(stderr, "    \"%s\"\n", tmpEncoded);
 				if (encoded) {
@@ -1411,7 +1412,7 @@ void ScintillaGTK::GetGtkSelectionText(GtkSelectionData *selectionData, Selectio
 		if (!IsUnicodeMode() && *charSetBuffer) {
 //fprintf(stderr, "Convert to locale %s\n", CharacterSetID());
 				// Convert to locale
-				dest = ConvertText(&len, selText.s, selText.len, charSetBuffer, "UTF-8");
+				dest = ConvertText(&len, selText.s, selText.len, charSetBuffer, "UTF-8", true);
 				selText.Set(dest, len, pdoc->dbcsCodePage,
 					vs.styles[STYLE_DEFAULT].characterSet, selText.rectangular);
 		}
@@ -1496,7 +1497,7 @@ void ScintillaGTK::GetSelection(GtkSelectionData *selection_data, guint info, Se
 		const char *charSet = ::CharacterSetID(text->characterSet);
 		if (*charSet) {
 			int new_len;
-			char* tmputf = ConvertText(&new_len, text->s, text->len, "UTF-8", charSet);
+			char* tmputf = ConvertText(&new_len, text->s, text->len, "UTF-8", charSet, false);
 			converted = new SelectionText();
 			converted->Set(tmputf, new_len, SC_CP_UTF8, 0, text->rectangular);
 			text = converted;
@@ -1539,14 +1540,14 @@ void ScintillaGTK::GetSelection(GtkSelectionData *selection_data, guint info, Se
 			if (text->codePage != SC_CP_UTF8) {
 				// Convert to UTF-8
 	//fprintf(stderr, "Convert to UTF-8 from %s\n", charSetBuffer);
-				tmputf = ConvertText(&len, selBuffer, len, "UTF-8", charSetBuffer);
+				tmputf = ConvertText(&len, selBuffer, len, "UTF-8", charSetBuffer, false);
 				selBuffer = tmputf;
 			}
 		} else if (info == TARGET_STRING) {
 			if (text->codePage == SC_CP_UTF8) {
 	//fprintf(stderr, "Convert to locale %s\n", charSetBuffer);
 				// Convert to locale
-				tmputf = ConvertText(&len, selBuffer, len, charSetBuffer, "UTF-8");
+				tmputf = ConvertText(&len, selBuffer, len, charSetBuffer, "UTF-8", true);
 				selBuffer = tmputf;
 			}
 		}
@@ -2037,7 +2038,7 @@ void ScintillaGTK::CommitThis(char *utfVal) {
 	} else {
 		const char *source = CharacterSetID();
 		if (*source) {
-			Converter conv(source, "UTF-8");
+			Converter conv(source, "UTF-8", true);
 			if (conv) {
 				char localeVal[4]="\0\0\0";
 				char *pin = utfVal;