From 14c9bcc7dc7d5a8577136baad5b43c6bc311312d Mon Sep 17 00:00:00 2001
From: Neil <nyamatongwe@gmail.com>
Date: Thu, 2 Mar 2017 09:41:01 +1100
Subject: Fix potential problems with IME on Cocoa when document contains
 invalid UTF-8.

---
 cocoa/ScintillaCocoa.h    |  1 +
 cocoa/ScintillaView.mm    |  1 +
 doc/ScintillaHistory.html |  4 ++++
 src/UniConversion.cxx     | 23 +++++++++++++++++++++++
 src/UniConversion.h       |  1 +
 5 files changed, 30 insertions(+)

diff --git a/cocoa/ScintillaCocoa.h b/cocoa/ScintillaCocoa.h
index 4d5d5c023..1defaf220 100644
--- a/cocoa/ScintillaCocoa.h
+++ b/cocoa/ScintillaCocoa.h
@@ -48,6 +48,7 @@
 #include "CaseFolder.h"
 #include "Document.h"
 #include "CaseConvert.h"
+#include "UniConversion.h"
 #include "Selection.h"
 #include "PositionCache.h"
 #include "EditModel.h"
diff --git a/cocoa/ScintillaView.mm b/cocoa/ScintillaView.mm
index 107c421c3..61c471d6a 100644
--- a/cocoa/ScintillaView.mm
+++ b/cocoa/ScintillaView.mm
@@ -437,6 +437,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
   [mOwner message: SCI_SETTARGETRANGE wParam: posRange.location lParam: NSMaxRange(posRange)];
   std::string text([mOwner message: SCI_TARGETASUTF8] + 1, 0);
   [mOwner message: SCI_TARGETASUTF8 wParam: 0 lParam: reinterpret_cast<sptr_t>(&text[0])];
+  text = FixInvalidUTF8(text);
   NSString *result = [NSString stringWithUTF8String: text.c_str()];
   NSMutableAttributedString *asResult = [[[NSMutableAttributedString alloc] initWithString:result] autorelease];
 
diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html
index 47966f3c1..c63563c7f 100644
--- a/doc/ScintillaHistory.html
+++ b/doc/ScintillaHistory.html
@@ -537,6 +537,10 @@
 	<a href="http://sourceforge.net/p/scintilla/bugs/1881/">Bug #1881</a>.
 	</li>
 	<li>
+	Fix potential problems with IME on Cocoa when document contains invalid
+	UTF-8.
+	</li>
+	<li>
 	Fix crash on Cocoa with OS X 10.9 due to accessibility API not available.
 	<a href="http://sourceforge.net/p/scintilla/bugs/1915/">Bug #1915</a>.
 	</li>
diff --git a/src/UniConversion.cxx b/src/UniConversion.cxx
index 4da9e102a..d0028d65e 100644
--- a/src/UniConversion.cxx
+++ b/src/UniConversion.cxx
@@ -8,6 +8,7 @@
 #include <stdlib.h>
 
 #include <stdexcept>
+#include <string>
 
 #include "UniConversion.h"
 
@@ -304,6 +305,28 @@ int UTF8DrawBytes(const unsigned char *us, int len) {
 	return (utf8StatusNext & UTF8MaskInvalid) ? 1 : (utf8StatusNext & UTF8MaskWidth);
 }
 
+// Replace invalid bytes in UTF-8 with the replacement character
+std::string FixInvalidUTF8(const std::string &text) {
+	std::string result;
+	const unsigned char *us = reinterpret_cast<const unsigned char *>(text.c_str());
+	size_t remaining = text.size();
+	while (remaining > 0) {
+		const int utf8Status = UTF8Classify(us, static_cast<int>(remaining));
+		if (utf8Status & UTF8MaskInvalid) {
+			// Replacement character 0xFFFD = UTF8:"efbfbd".
+			result.append("\xef\xbf\xbd");
+			us++;
+			remaining--;
+		} else {
+			const int len = utf8Status&UTF8MaskWidth;
+			result.append(reinterpret_cast<const char *>(us), len);
+			us += len;
+			remaining -= len;
+		}
+	}
+	return result;
+}
+
 #ifdef SCI_NAMESPACE
 }
 #endif
diff --git a/src/UniConversion.h b/src/UniConversion.h
index aeb13f0c2..c5867a664 100644
--- a/src/UniConversion.h
+++ b/src/UniConversion.h
@@ -23,6 +23,7 @@ size_t UTF16Length(const char *s, size_t len);
 size_t UTF16FromUTF8(const char *s, size_t len, wchar_t *tbuf, size_t tlen);
 unsigned int UTF32FromUTF8(const char *s, unsigned int len, unsigned int *tbuf, unsigned int tlen);
 unsigned int UTF16FromUTF32Character(unsigned int val, wchar_t *tbuf);
+std::string FixInvalidUTF8(const std::string &text);
 
 extern int UTF8BytesOfLead[256];
 void UTF8BytesOfLeadInitialise();
-- 
cgit v1.2.3