Converting between UTF-8 and current encoding.

Allows platform-independent code to be written just to handle Unicode.

8 files changed, 105 insertions, 0 deletions

diff --git a/cocoa/ScintillaCocoa.h b/cocoa/ScintillaCocoa.h
index b94a30aa2..e75efffef 100644
--- a/cocoa/ScintillaCocoa.h
+++ b/cocoa/ScintillaCocoa.h
@@ -182,6 +182,9 @@ public:
 
 	NSPoint GetCaretPosition();
 
+	std::string UTF8FromEncoded(std::string_view encoded) const override;
+	std::string EncodedFromUTF8(std::string_view utf8) const override;
+
 	static sptr_t DirectFunction(sptr_t ptr, unsigned int iMessage, uptr_t wParam, sptr_t lParam);
 
 	NSTimer *timers[tickPlatform+1];
diff --git a/cocoa/ScintillaCocoa.mm b/cocoa/ScintillaCocoa.mm
index 657651908..e1b11c16a 100644
--- a/cocoa/ScintillaCocoa.mm
+++ b/cocoa/ScintillaCocoa.mm
@@ -1195,6 +1195,38 @@ NSPoint ScintillaCocoa::GetCaretPosition() {
 
 // -------------------------------------------------------------------------------------------------
 
+std::string ScintillaCocoa::UTF8FromEncoded(std::string_view encoded) const {
+	if (IsUnicodeMode()) {
+		return std::string(encoded);
+	} else {
+		CFStringEncoding encoding = EncodingFromCharacterSet(IsUnicodeMode(),
+					    vs.styles[STYLE_DEFAULT].characterSet);
+		CFStringRef cfsVal = CFStringFromString(encoded.data(), encoded.length(), encoding);
+		std::string utf = EncodedBytesString(cfsVal, kCFStringEncodingUTF8);
+		if (cfsVal)
+			CFRelease(cfsVal);
+		return utf;
+	}
+}
+
+// -------------------------------------------------------------------------------------------------
+
+std::string ScintillaCocoa::EncodedFromUTF8(std::string_view utf8) const {
+	if (IsUnicodeMode()) {
+		return std::string(utf8);
+	} else {
+		CFStringEncoding encoding = EncodingFromCharacterSet(IsUnicodeMode(),
+					    vs.styles[STYLE_DEFAULT].characterSet);
+		CFStringRef cfsVal = CFStringFromString(utf8.data(), utf8.length(), kCFStringEncodingUTF8);
+		const std::string sEncoded = EncodedBytesString(cfsVal, encoding);
+		if (cfsVal)
+			CFRelease(cfsVal);
+		return sEncoded;
+	}
+}
+
+// -------------------------------------------------------------------------------------------------
+
 #pragma mark Drag
 
 /**
diff --git a/gtk/ScintillaGTK.cxx b/gtk/ScintillaGTK.cxx
index 61ce04ad0..e3148e9a8 100755
--- a/gtk/ScintillaGTK.cxx
+++ b/gtk/ScintillaGTK.cxx
@@ -816,6 +816,24 @@ bool ScintillaGTK::ValidCodePage(int codePage) const {
 	       || codePage == 1361;
 }
 
+std::string ScintillaGTK::UTF8FromEncoded(std::string_view encoded) const {
+	if (IsUnicodeMode()) {
+		return std::string(encoded);
+	} else {
+		const char *charSetBuffer = CharacterSetID();
+		return ConvertText(encoded.data(), encoded.length(), "UTF-8", charSetBuffer, true);
+	}
+}
+
+std::string ScintillaGTK::EncodedFromUTF8(std::string_view utf8) const {
+	if (IsUnicodeMode()) {
+		return std::string(utf8);
+	} else {
+		const char *charSetBuffer = CharacterSetID();
+		return ConvertText(utf8.data(), utf8.length(), charSetBuffer, "UTF-8", true);
+	}
+}
+
 sptr_t ScintillaGTK::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 	try {
 		switch (iMessage) {
diff --git a/gtk/ScintillaGTK.h b/gtk/ScintillaGTK.h
index a671f5b7a..100abd56b 100755
--- a/gtk/ScintillaGTK.h
+++ b/gtk/ScintillaGTK.h
@@ -92,6 +92,8 @@ private:
 	Sci::Position TargetAsUTF8(char *text) const;
 	Sci::Position EncodedFromUTF8(const char *utf8, char *encoded) const;
 	bool ValidCodePage(int codePage) const override;
+	std::string UTF8FromEncoded(std::string_view encoded) const override;
+	std::string EncodedFromUTF8(std::string_view utf8) const override;
 public: 	// Public for scintilla_send_message
 	sptr_t WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) override;
 private:
diff --git a/qt/ScintillaEditBase/ScintillaQt.cpp b/qt/ScintillaEditBase/ScintillaQt.cpp
index 374027bd0..761ff78bb 100644
--- a/qt/ScintillaEditBase/ScintillaQt.cpp
+++ b/qt/ScintillaEditBase/ScintillaQt.cpp
@@ -250,6 +250,28 @@ bool ScintillaQt::ValidCodePage(int codePage) const
 	|| codePage == 1361;
 }
 
+std::string ScintillaQt::UTF8FromEncoded(std::string_view encoded) const {
+	if (IsUnicodeMode()) {
+		return std::string(encoded);
+	} else {
+		QTextCodec *codec = QTextCodec::codecForName(
+				CharacterSetID(CharacterSetOfDocument()));
+		QString text = codec->toUnicode(encoded.data(), static_cast<int>(encoded.length()));
+		return text.toStdString();
+	}
+}
+
+std::string ScintillaQt::EncodedFromUTF8(std::string_view utf8) const {
+	if (IsUnicodeMode()) {
+		return std::string(utf8);
+	} else {
+		QString text = QString::fromUtf8(utf8.data(), static_cast<int>(utf8.length()));
+		QTextCodec *codec = QTextCodec::codecForName(
+				CharacterSetID(CharacterSetOfDocument()));
+		QByteArray ba = codec->fromUnicode(text);
+		return std::string(ba.data(), ba.length());
+	}
+}
 
 void ScintillaQt::ScrollText(Sci::Line linesToMove)
 {
diff --git a/qt/ScintillaEditBase/ScintillaQt.h b/qt/ScintillaEditBase/ScintillaQt.h
index 5a6f9efa1..64268aa75 100644
--- a/qt/ScintillaEditBase/ScintillaQt.h
+++ b/qt/ScintillaEditBase/ScintillaQt.h
@@ -104,6 +104,8 @@ private:
 	void Finalise() override;
 	bool DragThreshold(Point ptStart, Point ptNow) override;
 	bool ValidCodePage(int codePage) const override;
+	std::string UTF8FromEncoded(std::string_view encoded) const override;
+	std::string EncodedFromUTF8(std::string_view utf8) const override;
 
 private:
 	void ScrollText(Sci::Line linesToMove) override;
diff --git a/src/Editor.h b/src/Editor.h
index 4d01cb16a..60742b403 100644
--- a/src/Editor.h
+++ b/src/Editor.h
@@ -570,6 +570,9 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 
 	int CodePage() const noexcept;
 	virtual bool ValidCodePage(int /* codePage */) const { return true; }
+	virtual std::string UTF8FromEncoded(std::string_view encoded) const = 0;
+	virtual std::string EncodedFromUTF8(std::string_view utf8) const = 0;
+
 	Sci::Line WrapCount(Sci::Line line);
 	void AddStyledText(const char *buffer, Sci::Position appendLength);
 
diff --git a/win32/ScintillaWin.cxx b/win32/ScintillaWin.cxx
index d96199170..bde9ef66b 100644
--- a/win32/ScintillaWin.cxx
+++ b/win32/ScintillaWin.cxx
@@ -403,6 +403,9 @@ class ScintillaWin :
 
 	UINT CodePageOfDocument() const noexcept;
 	bool ValidCodePage(int codePage) const override;
+	std::string UTF8FromEncoded(std::string_view encoded) const override;
+	std::string EncodedFromUTF8(std::string_view utf8) const override;
+
 	std::string EncodeWString(std::wstring_view wsv);
 	sptr_t DefWndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) override;
 	void IdleWork() override;
@@ -2053,6 +2056,26 @@ bool ScintillaWin::ValidCodePage(int codePage) const {
 	       codePage == 950 || codePage == 1361;
 }
 
+std::string ScintillaWin::UTF8FromEncoded(std::string_view encoded) const {
+	if (IsUnicodeMode()) {
+		return std::string(encoded);
+	} else {
+		// Pivot through wide string
+		std::wstring ws = StringDecode(encoded, CodePageOfDocument());
+		return StringEncode(ws, SC_CP_UTF8);
+	}
+}
+
+std::string ScintillaWin::EncodedFromUTF8(std::string_view utf8) const {
+	if (IsUnicodeMode()) {
+		return std::string(utf8);
+	} else {
+		// Pivot through wide string
+		std::wstring ws = StringDecode(utf8, SC_CP_UTF8);
+		return StringEncode(ws, CodePageOfDocument());
+	}
+}
+
 sptr_t ScintillaWin::DefWndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 	return ::DefWindowProc(MainHWND(), iMessage, wParam, lParam);
 }