Modifications to selection handling to make copying and

pasting from/to various encodings work.

4 files changed, 100 insertions, 87 deletions

diff --git a/gtk/PlatGTK.cxx b/gtk/PlatGTK.cxx
index 330810883..eaecb8de0 100644
--- a/gtk/PlatGTK.cxx
+++ b/gtk/PlatGTK.cxx
@@ -1026,8 +1026,25 @@ static size_t UTF8Len(char ch) {
 		return 3;
 }
 
+char *UTF8FromLatin1(const char *s, int &len) {
+	char *utfForm = new char[len*2+1];
+	size_t lenU = 0;
+	for (int i=0;i<len;i++) {
+		unsigned int uch = static_cast<unsigned char>(s[i]);
+		if (uch < 0x80) {
+			utfForm[lenU++] = uch;
+		} else {
+			utfForm[lenU++] = static_cast<char>(0xC0 | (uch >> 6));
+			utfForm[lenU++] = static_cast<char>(0x80 | (uch & 0x3f));
+		}
+	}
+	utfForm[lenU] = '\0';
+	len = lenU;
+	return utfForm;
+}
+
 #ifdef USE_PANGO
-static char *UTF8FromIconv(const Converter &conv, const char *s, int len) {
+static char *UTF8FromIconv(const Converter &conv, const char *s, int &len) {
 	if (conv) {
 		char *utfForm = new char[len*3+1];
 		char *pin = const_cast<char *>(s);
@@ -1037,6 +1054,7 @@ static char *UTF8FromIconv(const Converter &conv, const char *s, int len) {
 		size_t conversions = conv.Convert(&pin, &inLeft, &pout, &outLeft);
 		if (conversions != ((size_t)(-1))) {
 			*pout = '\0';
+			len = pout - utfForm;
 			return utfForm;
 		}
 		delete []utfForm;
@@ -1061,22 +1079,6 @@ static size_t MultiByteLenFromIconv(const Converter &conv, const char *s, size_t
 	return 1;
 }
 
-static char *UTF8FromLatin1(const char *s, int len) {
-	char *utfForm = new char[len*2+1];
-	size_t lenU = 0;
-	for (int i=0;i<len;i++) {
-		unsigned int uch = static_cast<unsigned char>(s[i]);
-		if (uch < 0x80) {
-			utfForm[lenU++] = uch;
-		} else {
-			utfForm[lenU++] = static_cast<char>(0xC0 | (uch >> 6));
-			utfForm[lenU++] = static_cast<char>(0x80 | (uch & 0x3f));
-		}
-	}
-	utfForm[lenU] = '\0';
-	return utfForm;
-}
-
 static char *UTF8FromGdkWChar(GdkWChar *wctext, int wclen) {
 	char *utfForm = new char[wclen*3+1];	// Maximum of 3 UTF-8 bytes per character
 	size_t lenU = 0;
@@ -1097,7 +1099,7 @@ static char *UTF8FromGdkWChar(GdkWChar *wctext, int wclen) {
 	return utfForm;
 }
 
-static char *UTF8FromDBCS(const char *s, int len) {
+static char *UTF8FromDBCS(const char *s, int &len) {
 	GdkWChar *wctext = new GdkWChar[len + 1];
 	GdkWChar *wcp = wctext;
 	int wclen = gdk_mbstowcs(wcp, s, len);
@@ -1110,6 +1112,7 @@ static char *UTF8FromDBCS(const char *s, int len) {
 
 	char *utfForm = UTF8FromGdkWChar(wctext, wclen);
 	delete []wctext;
+	len = strlen(utfForm);
 	return utfForm;
 }
 
@@ -1156,7 +1159,7 @@ void SurfaceImpl::DrawTextBase(PRectangle rc, Font &font_, int ybase, const char
 				if (!utfForm) {	// iconv and DBCS failed so treat as Latin1
 					utfForm = UTF8FromLatin1(s, len);
 				}
-				pango_layout_set_text(layout, utfForm, strlen(utfForm));
+				pango_layout_set_text(layout, utfForm, len);
 			}
 			pango_layout_set_font_description(layout, PFont(font_)->pfd);
 			PangoLayoutLine *pll = pango_layout_get_line(layout,0);
@@ -1311,7 +1314,7 @@ void SurfaceImpl::MeasureWidths(Font &font_, const char *s, int len, int *positi
 					if (!utfForm) {
 						utfForm = UTF8FromLatin1(s, len);
 					}
-					pango_layout_set_text(layout, utfForm, strlen(utfForm));
+					pango_layout_set_text(layout, utfForm, len);
 					int i = 0;
 					PangoLayoutIter *iter = pango_layout_get_iter (layout);
 					while (pango_layout_iter_next_cluster (iter)) {
@@ -1412,8 +1415,7 @@ int SurfaceImpl::WidthText(Font &font_, const char *s, int len) {
 				if (!utfForm) {	// g_locale_to_utf8 failed so treat as Latin1
 					utfForm = UTF8FromLatin1(s, len);
 				}
-				pango_layout_set_text(layout, utfForm, strlen(utfForm));
-				len = strlen(utfForm);
+				pango_layout_set_text(layout, utfForm, len);
 			}
 			PangoLayoutLine *pangoLine = pango_layout_get_line(layout, 0);
 			pango_layout_line_get_extents(pangoLine, NULL, &pos);
diff --git a/gtk/ScintillaGTK.cxx b/gtk/ScintillaGTK.cxx
index 8a9dc283d..5194ec101 100644
--- a/gtk/ScintillaGTK.cxx
+++ b/gtk/ScintillaGTK.cxx
@@ -72,6 +72,8 @@
 #pragma warning(disable: 4505)
 #endif
 
+extern char *UTF8FromLatin1(const char *s, int &len);
+
 class ScintillaGTK : public ScintillaBase {
 	_ScintillaObject *sci;
 	Window wText;
@@ -727,9 +729,10 @@ void ScintillaGTK::DisplayCursor(Window::Cursor c) {
 void ScintillaGTK::StartDrag() {
 	dragWasDropped = false;
 	static const GtkTargetEntry targets[] = {
+	    { "UTF8_STRING", 0, TARGET_UTF8_STRING },
 	    { "STRING", 0, TARGET_STRING },
-	    { "TEXT", 0, TARGET_TEXT },
-	    { "COMPOUND_TEXT", 0, TARGET_COMPOUND_TEXT },
+	    // { "TEXT", 0, TARGET_TEXT },
+	    // { "COMPOUND_TEXT", 0, TARGET_COMPOUND_TEXT },
 	};
 	static const gint n_targets = sizeof(targets) / sizeof(targets[0]);
 	GtkTargetList *tl = gtk_target_list_new(targets, n_targets);
@@ -1134,7 +1137,7 @@ int ScintillaGTK::KeyDefault(int key, int modifiers) {
 
 void ScintillaGTK::CopyToClipboard(const SelectionText &selectedText) {
 #if GTK_MAJOR_VERSION == 1
-	copyText.Copy(selectedText.s, selectedText.len, selectedText.characterSet);
+	copyText.Copy(selectedText);
 	gtk_selection_owner_set(GTK_WIDGET(PWidget(wMain)),
 				atomClipboard,
 				GDK_CURRENT_TIME);
@@ -1145,7 +1148,7 @@ void ScintillaGTK::CopyToClipboard(const SelectionText &selectedText) {
 		return;
 
 	SelectionText *clipText = new SelectionText();
-	clipText->Copy(selectedText.s, selectedText.len, selectedText.characterSet);
+	clipText->Copy(selectedText);
 
 	gtk_clipboard_set_with_data(clipBoard, clipboardTargets, nClipboardTargets,
 				    ClipboardGetSelection, ClipboardClearSelection, clipText);
@@ -1246,19 +1249,19 @@ void ScintillaGTK::ClaimSelection() {
 		primarySelection = true;
 		gtk_selection_owner_set(GTK_WIDGET(PWidget(wMain)),
 		                        GDK_SELECTION_PRIMARY, GDK_CURRENT_TIME);
-		primary.Set(0, 0, 0);
+		primary.Free();
 	} else if (OwnPrimarySelection()) {
 		primarySelection = true;
 		if (primary.s == NULL)
 			gtk_selection_owner_set(NULL, GDK_SELECTION_PRIMARY, GDK_CURRENT_TIME);
 	} else {
 		primarySelection = false;
-		primary.Set(0, 0, 0);
+		primary.Free();
 	}
 }
 
 #ifdef USE_CONVERTER
-static char *ConvertText(size_t *lenResult, char *s, size_t len, const char *charSetDest, const char *charSetSource) {
+static char *ConvertText(int *lenResult, char *s, size_t len, const char *charSetDest, const char *charSetSource) {
 	*lenResult = 0;
 	char *destForm = 0;
 	Converter conv(charSetDest, charSetSource);
@@ -1270,7 +1273,7 @@ static char *ConvertText(size_t *lenResult, char *s, size_t len, const char *cha
 		size_t outLeft = len*3+1;
 		size_t conversions = conv.Convert(&pin, &inLeft, &pout, &outLeft);
 		if (conversions == ((size_t)(-1))) {
-fprintf(stderr, "iconv failed for %s\n", static_cast<char *>(s));
+fprintf(stderr, "iconv %s->%s failed for %s\n", charSetSource, charSetDest, static_cast<char *>(s));
 			delete []destForm;
 			destForm = 0;
 		} else {
@@ -1292,7 +1295,7 @@ fprintf(stderr, "iconv failed for %s\n", static_cast<char *>(s));
 
 // Convert line endings for a piece of text to a particular mode.
 // Stop at len or when a NUL is found.
-char *ConvertLineEnds(size_t *pLenOut, const char *s, size_t len, int eolMode) {
+char *ConvertLineEnds(int *pLenOut, const char *s, size_t len, int eolMode) {
 	char *dest = new char[2 * len + 1];
 	const char *sptr = s;
 	char *dptr = dest;
@@ -1323,14 +1326,14 @@ char *ConvertLineEnds(size_t *pLenOut, const char *s, size_t len, int eolMode) {
 // Detect rectangular text, convert line ends to current mode, convert from or to UTF-8
 void ScintillaGTK::GetGtkSelectionText(GtkSelectionData *selectionData, SelectionText &selText) {
 	char *data = reinterpret_cast<char *>(selectionData->data);
-	size_t len = selectionData->length;
+	int len = selectionData->length;
 	GdkAtom selectionType = selectionData->type;
 
 	// Return empty string if selection is not a string
 	if ((selectionType != GDK_TARGET_STRING) && (selectionType != atomUTF8)) {
 		char *empty = new char[1];
 		empty[0] = '\0';
-		selText.Set(empty, 0, SC_CP_UTF8);
+		selText.Set(empty, 0, SC_CP_UTF8, 0, false);
 		return;
 	}
 
@@ -1342,48 +1345,34 @@ void ScintillaGTK::GetGtkSelectionText(GtkSelectionData *selectionData, Selectio
 	isRectangular = ((len > 2) && (data[len - 1] == 0 && data[len - 2] == '\n'));
 #endif
 
-#if GTK_MAJOR_VERSION >= 2
 	char *dest;
-	char *utf8 = reinterpret_cast<char*>(gtk_selection_data_get_text(selectionData));
-	if (utf8 != NULL) {
-		dest = ConvertLineEnds(&len, utf8, strlen(utf8), pdoc->eolMode);
-		g_free(utf8);
-	}
-	else {
-		dest = new char[1];
-		dest[0] = '\0';
-		len = 0;
-	}
-	selText.Set(dest, len, SC_CP_UTF8, isRectangular);
-#else
-	char *dest = ConvertLineEnds(&len, data, len, pdoc->eolMode);
-	if (selectionType == GDK_TARGET_STRING)
-		selText.Set(dest, len, pdoc->dbcsCodePage, isRectangular);
-	else
-		selText.Set(dest, len, SC_CP_UTF8, isRectangular);
-#endif
-
-#ifdef USE_CONVERTER
-	// Possible character set conversion
-	const char *charSetBuffer = CharacterSetID();
-	if (*charSetBuffer) {
+	if (selectionType == GDK_TARGET_STRING) {
+		dest = ConvertLineEnds(&len, data, len, pdoc->eolMode);
 		if (IsUnicodeMode()) {
-			if (selText.characterSet != SC_CP_UTF8) {
-				// Convert to UTF-8
-//fprintf(stderr, "Convert to UTF-8 from %s\n", charSetBuffer);
-				dest = ConvertText(&len, selText.s, selText.len, "UTF-8", charSetBuffer);
-				selText.Set(dest, len, SC_CP_UTF8, selText.rectangular);
-			}
+			// Unknown encoding so assume in Latin1
+			char *destPrevious = dest;
+			dest = UTF8FromLatin1(dest, len);
+			selText.Set(dest, len, SC_CP_UTF8, 0, selText.rectangular);
+			delete []destPrevious;
 		} else {
-			if (selText.characterSet == SC_CP_UTF8) {
-//fprintf(stderr, "Convert to locale %s\n", charSetBuffer);
+			// Assume buffer is in same encoding as selection
+			selText.Set(dest, len, pdoc->dbcsCodePage,
+				vs.styles[STYLE_DEFAULT].characterSet, isRectangular);
+		}
+	} else {	// UTF-8
+		dest = ConvertLineEnds(&len, data, len, pdoc->eolMode);
+		selText.Set(dest, len, SC_CP_UTF8, 0, isRectangular);
+#ifdef USE_CONVERTER
+		const char *charSetBuffer = CharacterSetID();
+		if (!IsUnicodeMode() && *charSetBuffer) {
+//fprintf(stderr, "Convert to locale %s\n", CharacterSetID());
 				// Convert to locale
 				dest = ConvertText(&len, selText.s, selText.len, charSetBuffer, "UTF-8");
-				selText.Set(dest, len, pdoc->dbcsCodePage, selText.rectangular);
-			}
+				selText.Set(dest, len, pdoc->dbcsCodePage,
+					vs.styles[STYLE_DEFAULT].characterSet, selText.rectangular);
 		}
-	}
 #endif
+	}
 }
 
 void ScintillaGTK::ReceivedSelection(GtkSelectionData *selection_data) {
@@ -1437,14 +1426,16 @@ void ScintillaGTK::ReceivedDrop(GtkSelectionData *selection_data) {
 
 void ScintillaGTK::GetSelection(GtkSelectionData *selection_data, guint info, SelectionText *text) {
 #if GTK_MAJOR_VERSION >= 2
-	(void)info;	// Parameter unused on GTK+ 2
 	// Convert text to utf8 if it isn't already
-	if (text->characterSet != SC_CP_UTF8) {
+	SelectionText *converted = 0;
+	if ((text->codePage != SC_CP_UTF8) && (info == TARGET_UTF8_STRING)) {
 		const char *charSet = ::CharacterSetID(text->characterSet);
 		if (*charSet) {
-			size_t new_len;
+			int new_len;
 			char* tmputf = ConvertText(&new_len, text->s, text->len, "UTF-8", charSet);
-			text->Set(tmputf, new_len, SC_CP_UTF8, text->rectangular);
+			converted = new SelectionText();
+			converted->Set(tmputf, new_len, SC_CP_UTF8, 0, text->rectangular);
+			text = converted;
 		}
 	}
 
@@ -1461,7 +1452,14 @@ void ScintillaGTK::GetSelection(GtkSelectionData *selection_data, guint info, Se
 		len++;
 #endif
 
-	gtk_selection_data_set_text(selection_data, text->s, len);
+	if (info == TARGET_UTF8_STRING) {
+		gtk_selection_data_set_text(selection_data, text->s, len);
+	} else {
+		gtk_selection_data_set(selection_data,
+			static_cast<GdkAtom>(GDK_SELECTION_TYPE_STRING),
+			8, reinterpret_cast<unsigned char *>(text->s), len);
+	}
+	delete converted;
 
 #else /* Gtk 1 */
 	char *selBuffer = text->s;
@@ -1471,26 +1469,26 @@ void ScintillaGTK::GetSelection(GtkSelectionData *selection_data, guint info, Se
 	// Many native win32 programs require \n line endings,
 	 // so make a copy of the clip text now with newlines converted
 
-	size_t new_len;
+	int new_len;
 	char *tmpstr = ConvertLineEnds(&new_len, selBuffer, text->len, SC_EOL_LF);
 	selBuffer = tmpstr;
 #endif
 	char *tmputf = 0;
 	if ((info == TARGET_UTF8_STRING) || (info == TARGET_STRING)) {
-		size_t len = strlen(selBuffer);
+		int len = strlen(selBuffer);
 #ifdef USE_CONVERTER
 		// Possible character set conversion
 		const char *charSetBuffer = ::CharacterSetID(text->characterSet);
 		if (info == TARGET_UTF8_STRING) {
 			//fprintf(stderr, "Copy to clipboard as UTF-8\n");
-			if (text->characterSet != SC_CP_UTF8) {
+			if (text->codePage != SC_CP_UTF8) {
 				// Convert to UTF-8
 	//fprintf(stderr, "Convert to UTF-8 from %s\n", charSetBuffer);
 				tmputf = ConvertText(&len, selBuffer, len, "UTF-8", charSetBuffer);
 				selBuffer = tmputf;
 			}
 		} else if (info == TARGET_STRING) {
-			if (text->characterSet == SC_CP_UTF8) {
+			if (text->codePage == SC_CP_UTF8) {
 	//fprintf(stderr, "Convert to locale %s\n", charSetBuffer);
 				// Convert to locale
 				tmputf = ConvertText(&len, selBuffer, len, charSetBuffer, "UTF-8");
@@ -1550,7 +1548,7 @@ void ScintillaGTK::UnclaimSelection(GdkEventSelection *selection_event) {
 	if (selection_event->selection == GDK_SELECTION_PRIMARY) {
 		//Platform::DebugPrintf("UnclaimPrimarySelection\n");
 		if (!OwnPrimarySelection()) {
-			primary.Set(0, 0, 0);
+			primary.Free();
 			primarySelection = false;
 			FullPaint();
 		}
diff --git a/src/Editor.cxx b/src/Editor.cxx
index 4b1c6691d..7bafb3fff 100644
--- a/src/Editor.cxx
+++ b/src/Editor.cxx
@@ -4631,7 +4631,8 @@ char *Editor::CopyRange(int start, int end) {
 }
 
 void Editor::CopySelectionFromRange(SelectionText *ss, int start, int end) {
-	ss->Set(CopyRange(start, end), end - start + 1, pdoc->dbcsCodePage, false);
+	ss->Set(CopyRange(start, end), end - start + 1, 
+		pdoc->dbcsCodePage, vs.styles[STYLE_DEFAULT].characterSet, false);
 }
 
 void Editor::CopySelectionRange(SelectionText *ss) {
@@ -4673,7 +4674,8 @@ void Editor::CopySelectionRange(SelectionText *ss) {
 				text[size] = '\0';
 			}
 		}
- 		ss->Set(text, size + 1, pdoc->dbcsCodePage, selType == selRectangle);
+ 		ss->Set(text, size + 1, pdoc->dbcsCodePage, 
+			vs.styles[STYLE_DEFAULT].characterSet, selType == selRectangle);
 	}
 }
 
@@ -4681,13 +4683,15 @@ void Editor::CopyRangeToClipboard(int start, int end) {
 	start = pdoc->ClampPositionIntoDocument(start);
 	end = pdoc->ClampPositionIntoDocument(end);
 	SelectionText selectedText;
-	selectedText.Set(CopyRange(start, end), end - start + 1, pdoc->dbcsCodePage);
+	selectedText.Set(CopyRange(start, end), end - start + 1, 
+		pdoc->dbcsCodePage, vs.styles[STYLE_DEFAULT].characterSet, false);
 	CopyToClipboard(selectedText);
 }
 
 void Editor::CopyText(int length, const char *text) {
 	SelectionText selectedText;
-	selectedText.Copy(text, length, pdoc->dbcsCodePage);
+	selectedText.Copy(text, length, 
+		pdoc->dbcsCodePage, vs.styles[STYLE_DEFAULT].characterSet, false);
 	CopyToClipboard(selectedText);
 }
 
@@ -5161,7 +5165,7 @@ void Editor::ButtonUp(Point pt, unsigned int curTime, bool ctrl) {
 					} else {
 						SetEmptySelection(newPos);
 					}
-					drag.Set(0, 0, 0);
+					drag.Free();
 				}
 				selectionType = selChar;
 			}
diff --git a/src/Editor.h b/src/Editor.h
index 76635e658..9e9af0ec1 100644
--- a/src/Editor.h
+++ b/src/Editor.h
@@ -134,22 +134,27 @@ public:
 	char *s;
 	int len;
 	bool rectangular;
+	int codePage;
 	int characterSet;
-	SelectionText() : s(0), len(0), rectangular(false), characterSet(0) {}
+	SelectionText() : s(0), len(0), rectangular(false), codePage(0), characterSet(0) {}
 	~SelectionText() {
-		Set(0, 0, 0);
+		Free();
 	}
-	void Set(char *s_, int len_, int characterSet_, bool rectangular_=false) {
+	void Free() {
+		Set(0, 0, 0, 0, false);
+	}
+	void Set(char *s_, int len_, int codePage_, int characterSet_, bool rectangular_) {
 		delete []s;
 		s = s_;
 		if (s)
 			len = len_;
 		else
 			len = 0;
+		codePage = codePage_;
 		characterSet = characterSet_;
 		rectangular = rectangular_;
 	}
-	void Copy(const char *s_, int len_, int characterSet_, bool rectangular_=false) {
+	void Copy(const char *s_, int len_, int codePage_, int characterSet_, bool rectangular_) {
 		delete []s;
 		s = new char[len_];
 		if (s) {
@@ -160,9 +165,13 @@ public:
 		} else {
 			len = 0;
 		}
+		codePage = codePage_;
 		characterSet = characterSet_;
 		rectangular = rectangular_;
 	}
+	void Copy(const SelectionText &other) {
+		Copy(other.s, other.len, other.codePage, other.characterSet, other.rectangular);
+	}
 };
 
 /**