Implement additional methods from the NSTextInputClient protocol so that more

features of the IME work. attributedSubstringForProposedRange:actualRange: and
characterIndexForPoint: now have full implementations.
This required using UTF-16 document indexes in many places as that is what
Cocoa wants. Tentative undo is used for the composition text instead of turning
off undo as that is safer and similar to IME code on other platforms.

4 files changed, 230 insertions, 129 deletions

diff --git a/cocoa/ScintillaCocoa.h b/cocoa/ScintillaCocoa.h
index 45d04e564..0941ac174 100644
--- a/cocoa/ScintillaCocoa.h
+++ b/cocoa/ScintillaCocoa.h
@@ -197,7 +197,14 @@ public:
   virtual void IdleWork();
   virtual void QueueIdleWork(WorkNeeded::workItems items, int upTo);
   int InsertText(NSString* input);
+  NSRange PositionsFromCharacters(NSRange range) const;
+  NSRange CharactersFromPositions(NSRange range) const;
   void SelectOnlyMainSelection();
+  void ConvertSelectionVirtualSpace();
+  bool ClearAllSelections();
+  void CompositionStart();
+  void CompositionCommit();
+  void CompositionUndo();
   virtual void SetDocPointer(Document *document);
 
   bool KeyboardInput(NSEvent* event);
diff --git a/cocoa/ScintillaCocoa.mm b/cocoa/ScintillaCocoa.mm
index f248f3bb0..ced7be3d6 100644
--- a/cocoa/ScintillaCocoa.mm
+++ b/cocoa/ScintillaCocoa.mm
@@ -2102,22 +2102,42 @@ bool ScintillaCocoa::KeyboardInput(NSEvent* event)
 int ScintillaCocoa::InsertText(NSString* input)
 {
   CFStringEncoding encoding = EncodingFromCharacterSet(IsUnicodeMode(),
-                                                         vs.styles[STYLE_DEFAULT].characterSet);
-  CFRange rangeAll = {0, static_cast<CFIndex>([input length])};
-  CFIndex usedLen = 0;
-  CFStringGetBytes((CFStringRef)input, rangeAll, encoding, '?',
-                   false, NULL, 0, &usedLen);
-
-  if (usedLen > 0)
+                                                       vs.styles[STYLE_DEFAULT].characterSet);
+  std::string encoded = EncodedBytesString((CFStringRef)input, encoding);
+  
+  if (encoded.length() > 0)
   {
-    std::vector<UInt8> buffer(usedLen);
+    AddCharUTF((char*) encoded.c_str(), static_cast<unsigned int>(encoded.length()), false);
+  }
+  return static_cast<int>(encoded.length());
+}
 
-    CFStringGetBytes((CFStringRef)input, rangeAll, encoding, '?',
-                       false, buffer.data(),usedLen, NULL);
+//--------------------------------------------------------------------------------------------------
 
-    AddCharUTF((char*) buffer.data(), static_cast<unsigned int>(usedLen), false);
-  }
-  return static_cast<int>(usedLen);
+/**
+ * Convert from a range of characters to a range of bytes.
+ */
+NSRange ScintillaCocoa::PositionsFromCharacters(NSRange range) const
+{
+  long start = pdoc->GetRelativePositionUTF16(0, range.location);
+  if (start == INVALID_POSITION)
+    start = pdoc->Length();
+  long end = pdoc->GetRelativePositionUTF16(start, range.length);
+  if (end == INVALID_POSITION)
+    end = pdoc->Length();
+  return NSMakeRange(start, end - start);
+}
+
+//--------------------------------------------------------------------------------------------------
+
+/**
+ * Convert from a range of characters from a range of bytes.
+ */
+NSRange ScintillaCocoa::CharactersFromPositions(NSRange range) const
+{
+  const long start = pdoc->CountUTF16(0, range.location);
+  const long len = pdoc->CountUTF16(range.location, NSMaxRange(range));
+  return NSMakeRange(start, len);
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -2125,17 +2145,72 @@ int ScintillaCocoa::InsertText(NSString* input)
 /**
  * Used to ensure that only one selection is active for input composition as composition
  * does not support multi-typing.
- * Also drop virtual space as that is not supported by composition.
  */
 void ScintillaCocoa::SelectOnlyMainSelection()
 {
-  SelectionRange mainSel = sel.RangeMain();
-  mainSel.ClearVirtualSpace();
-  sel.SetSelection(mainSel);
+  sel.SetSelection(sel.RangeMain());
   Redraw();
 }
 
 //--------------------------------------------------------------------------------------------------
+
+/**
+ * Convert virtual space before selection into real space.
+ */
+void ScintillaCocoa::ConvertSelectionVirtualSpace()
+{
+  FillVirtualSpace();
+}
+
+//--------------------------------------------------------------------------------------------------
+
+/**
+ * Erase all selected text and return whether the selection is now empty.
+ * The selection may not be empty if the selection contained protected text.
+ */
+bool ScintillaCocoa::ClearAllSelections()
+{
+  ClearSelection(true);
+  return sel.Empty();
+}
+
+//--------------------------------------------------------------------------------------------------
+
+/**
+ * Start composing for IME.
+ */
+void ScintillaCocoa::CompositionStart()
+{
+  if (!sel.Empty())
+  {
+    NSLog(@"Selection not empty when starting composition");
+  }
+  pdoc->TentativeStart();
+}
+
+//--------------------------------------------------------------------------------------------------
+
+/**
+ * Commit the IME text.
+ */
+void ScintillaCocoa::CompositionCommit()
+{
+  pdoc->TentativeCommit();
+  pdoc->decorations.SetCurrentIndicator(INDIC_IME);
+  pdoc->DecorationFillRange(0, 0, pdoc->Length());
+}
+
+//--------------------------------------------------------------------------------------------------
+
+/**
+ * Remove the IME text.
+ */
+void ScintillaCocoa::CompositionUndo()
+{
+  pdoc->TentativeUndo();
+}
+
+//--------------------------------------------------------------------------------------------------
 /**
  * When switching documents discard any incomplete character composition state as otherwise tries to
  * act on the new document.
diff --git a/cocoa/ScintillaView.h b/cocoa/ScintillaView.h
index 102bf3b60..e328646a5 100644
--- a/cocoa/ScintillaView.h
+++ b/cocoa/ScintillaView.h
@@ -79,12 +79,10 @@ extern NSString *const SCIUpdateUINotification;
 
   // Set when we are in composition mode and partial input is displayed.
   NSRange mMarkedTextRange;
-  BOOL undoCollectionWasActive;
 }
 
 @property (nonatomic, assign) ScintillaView* owner;
 
-- (void) removeMarkedText;
 - (void) setCursor: (int) cursor;
 
 - (BOOL) canUndo;
diff --git a/cocoa/ScintillaView.mm b/cocoa/ScintillaView.mm
index 43ceb8b48..d5f658755 100644
--- a/cocoa/ScintillaView.mm
+++ b/cocoa/ScintillaView.mm
@@ -19,9 +19,6 @@ using namespace Scintilla;
 static NSCursor* reverseArrowCursor;
 static NSCursor* waitCursor;
 
-// The scintilla indicator used for keyboard input.
-#define INPUT_INDICATOR INDIC_MAX - 1
-
 NSString *const SCIUpdateUINotification = @"SCIUpdateUI";
 
 /**
@@ -347,14 +344,52 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 
 - (NSAttributedString *)attributedSubstringForProposedRange:(NSRange)aRange actualRange:(NSRangePointer)actualRange
 {
-  return nil;
+  const NSRange posRange = mOwner.backend->PositionsFromCharacters(aRange);
+  // The backend validated aRange and may have removed characters beyond the end of the document.
+  const NSRange charRange = mOwner.backend->CharactersFromPositions(posRange);
+  if (!NSEqualRanges(aRange, charRange))
+  {
+    *actualRange = charRange;
+  }
+
+  [mOwner message: SCI_SETTARGETRANGE wParam: posRange.location lParam: NSMaxRange(posRange)];
+  std::string text([mOwner message: SCI_TARGETASUTF8] + 1, 0);
+  [mOwner message: SCI_TARGETASUTF8 wParam: 0 lParam: reinterpret_cast<sptr_t>(&text[0])];
+  NSString *result = [NSString stringWithUTF8String: text.c_str()];
+  NSMutableAttributedString *asResult = [[[NSMutableAttributedString alloc] initWithString:result] autorelease];
+
+  const NSRange rangeAS = NSMakeRange(0, [asResult length]);
+  const long style = [mOwner message: SCI_GETSTYLEAT wParam:posRange.location];
+  std::string fontName([mOwner message: SCI_STYLEGETFONT wParam:style lParam:0] + 1, 0);
+  [mOwner message: SCI_STYLEGETFONT wParam:style lParam:(sptr_t)&fontName[0]];
+  const CGFloat fontSize = [mOwner message: SCI_STYLEGETSIZEFRACTIONAL wParam:style] / 100.0f;
+  NSString *sFontName = [NSString stringWithUTF8String: fontName.c_str()];
+  NSFont *font = [NSFont fontWithName:sFontName size:fontSize];
+  [asResult addAttribute:NSFontAttributeName value:font range:rangeAS];
+
+  return asResult;
 }
 
 //--------------------------------------------------------------------------------------------------
 
 - (NSUInteger) characterIndexForPoint: (NSPoint) point
 {
-  return NSNotFound;
+  const NSRect rectPoint = {point, NSZeroSize};
+  const NSRect rectInWindow = [self.window convertRectFromScreen:rectPoint];
+  const NSRect rectLocal = [[[self superview] superview] convertRect:rectInWindow fromView:nil];
+
+  const long position = [mOwner message: SCI_CHARPOSITIONFROMPOINT
+                                 wParam: rectLocal.origin.x
+                                 lParam: rectLocal.origin.y];
+  if (position == INVALID_POSITION)
+  {
+    return NSNotFound;
+  }
+  else
+  {
+    const NSRange index = mOwner.backend->CharactersFromPositions(NSMakeRange(position, 0));
+    return index.location;
+  }
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -369,32 +404,19 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 
 - (NSRect) firstRectForCharacterRange: (NSRange) aRange actualRange: (NSRangePointer) actualRange
 {
+  const NSRange posRange = mOwner.backend->PositionsFromCharacters(aRange);
+
   NSRect rect;
-  rect.origin.x = [ScintillaView directCall: mOwner
-				    message: SCI_POINTXFROMPOSITION
-				     wParam: 0
-				     lParam: aRange.location];
-  rect.origin.y = [ScintillaView directCall: mOwner
-				    message: SCI_POINTYFROMPOSITION
-				     wParam: 0
-				     lParam: aRange.location];
-  NSUInteger rangeEnd = aRange.location + aRange.length;
-  rect.size.width = [ScintillaView directCall: mOwner
-				      message: SCI_POINTXFROMPOSITION
-				       wParam: 0
-				       lParam: rangeEnd] - rect.origin.x;
-  rect.size.height = [ScintillaView directCall: mOwner
-				       message: SCI_POINTYFROMPOSITION
-					wParam: 0
-					lParam: rangeEnd] - rect.origin.y;
-  rect.size.height += [ScintillaView directCall: mOwner
-					message: SCI_TEXTHEIGHT
-					 wParam: 0
-					 lParam: 0];
-  rect = [[[self superview] superview] convertRect:rect toView:nil];
-  rect = [self.window convertRectToScreen:rect];
-
-  return rect;
+  rect.origin.x = [mOwner message: SCI_POINTXFROMPOSITION wParam: 0 lParam: posRange.location];
+  rect.origin.y = [mOwner message: SCI_POINTYFROMPOSITION wParam: 0 lParam: posRange.location];
+  const NSUInteger rangeEnd = NSMaxRange(posRange);
+  rect.size.width = [mOwner message: SCI_POINTXFROMPOSITION wParam: 0 lParam: rangeEnd] - rect.origin.x;
+  rect.size.height = [mOwner message: SCI_POINTYFROMPOSITION wParam: 0 lParam: rangeEnd] - rect.origin.y;
+  rect.size.height += [mOwner message: SCI_TEXTHEIGHT wParam: 0 lParam: 0];
+  const NSRect rectInWindow = [[[self superview] superview] convertRect:rect toView:nil];
+  const NSRect rectScreen = [self.window convertRectToScreen:rectInWindow];
+
+  return rectScreen;
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -419,14 +441,27 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 	}
 
 	// Remove any previously marked text first.
-	[self removeMarkedText];
+	mOwner.backend->CompositionUndo();
+	if (mMarkedTextRange.location != NSNotFound)
+	{
+		const NSRange posRangeMark = mOwner.backend->PositionsFromCharacters(mMarkedTextRange);
+		[mOwner message: SCI_SETEMPTYSELECTION wParam: posRangeMark.location];
+	}
+	mMarkedTextRange = NSMakeRange(NSNotFound, 0);
 
 	if (replacementRange.location == (NSNotFound-1))
 		// This occurs when the accent popup is visible and menu selected.
 		// Its replacing a non-existent position so do nothing.
 		return;
 
-	[mOwner deleteRange: replacementRange];
+	if (replacementRange.location != NSNotFound)
+	{
+		const NSRange posRangeReplacement = mOwner.backend->PositionsFromCharacters(replacementRange);
+		[mOwner message: SCI_DELETERANGE
+			 wParam: posRangeReplacement.location
+			 lParam: posRangeReplacement.length];
+		[mOwner message: SCI_SETEMPTYSELECTION wParam: posRangeReplacement.location];
+	}
 
 	NSString* newText = @"";
 	if ([aString isKindOfClass:[NSString class]])
@@ -448,9 +483,10 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 
 - (NSRange) selectedRange
 {
-  long begin = [mOwner getGeneralProperty: SCI_GETSELECTIONSTART parameter: 0];
-  long end = [mOwner getGeneralProperty: SCI_GETSELECTIONEND parameter: 0];
-  return NSMakeRange(begin, end - begin);
+  const long positionBegin = [mOwner message: SCI_GETSELECTIONSTART];
+  const long positionEnd = [mOwner message: SCI_GETSELECTIONEND];
+  NSRange posRangeSel = NSMakeRange(positionBegin, positionEnd-positionBegin);
+  return mOwner.backend->CharactersFromPositions(posRangeSel);
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -473,86 +509,85 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
     if ([aString isKindOfClass:[NSAttributedString class]])
       newText = (NSString*) [aString string];
 
-  long currentPosition;
-
   // Replace marked text if there is one.
   if (mMarkedTextRange.length > 0)
   {
+    mOwner.backend->CompositionUndo();
     if (replacementRange.location != NSNotFound)
     {
       // This situation makes no sense and has not occurred in practice.
-      // Should the marked range remain marked in addition to the new text
-      // or should it be removed first?
       NSLog(@"Can not handle a replacement range when there is also a marked range");
     }
-
-    [mOwner deleteRange: mMarkedTextRange];
-    currentPosition = mMarkedTextRange.location;
+    else
+    {
+      replacementRange = mMarkedTextRange;
+      const NSRange posRangeMark = mOwner.backend->PositionsFromCharacters(mMarkedTextRange);
+      [mOwner message: SCI_SETEMPTYSELECTION wParam: posRangeMark.location];
+    }
   }
   else
   {
     // Must perform deletion before entering composition mode or else
     // both document and undo history will not contain the deleted text
     // leading to an inaccurate and unusable undo history.
+    
+    // Convert selection virtual space into real space
+    mOwner.backend->ConvertSelectionVirtualSpace();
 
     if (replacementRange.location != NSNotFound)
     {
-      [mOwner deleteRange: replacementRange];
-      currentPosition = replacementRange.location;
+      const NSRange posRangeReplacement = mOwner.backend->PositionsFromCharacters(replacementRange);
+      [mOwner message: SCI_DELETERANGE
+               wParam: posRangeReplacement.location
+               lParam: posRangeReplacement.length];
     }
     else  // No marked or replacement range, so replace selection
     {
-      // Ensure only a single selection
-      mOwner.backend->SelectOnlyMainSelection();
-  
-      NSRange selectionRangeCurrent = [self selectedRange];
-      if (selectionRangeCurrent.length > 0)
-      {
-        [mOwner deleteRange: selectionRangeCurrent];
+      if (!mOwner.backend->ScintillaCocoa::ClearAllSelections()) {
+        // Some of the selection is protected so can not perform composition here
+        return;
       }
-      currentPosition = selectionRangeCurrent.location;
+      // Ensure only a single selection.
+      mOwner.backend->SelectOnlyMainSelection();
+      replacementRange = [self selectedRange];
     }
-    
-    // Switching into composition so remember if collecting undo.
-    undoCollectionWasActive = [mOwner getGeneralProperty: SCI_GETUNDOCOLLECTION] != 0;
-
-    // Keep Scintilla from collecting undo actions for the composition task.
-    [mOwner setGeneralProperty: SCI_SETUNDOCOLLECTION value: 0];
   }
 
-  [mOwner message: SCI_SETEMPTYSELECTION wParam: currentPosition];
-  // Note: Scintilla internally works almost always with bytes instead chars, so we need to take
-  //       this into account when determining selection ranges and such.
-  int lengthInserted = mOwner.backend->InsertText(newText);
+  // To support IME input to multiple selections, the following code would
+  // need to insert newText at each selection, mark each piece of new text and then
+  // select range relative to each insertion.
 
-  if (lengthInserted > 0)
+  if ([newText length])
   {
-    mMarkedTextRange = NSMakeRange(currentPosition, lengthInserted);
+    // Switching into composition.
+    mOwner.backend->CompositionStart();
+    
+    NSRange posRangeCurrent = mOwner.backend->PositionsFromCharacters(NSMakeRange(replacementRange.location, 0));
+    // Note: Scintilla internally works almost always with bytes instead chars, so we need to take
+    //       this into account when determining selection ranges and such.
+    int lengthInserted = mOwner.backend->InsertText(newText);
+    posRangeCurrent.length = lengthInserted;
+    mMarkedTextRange = mOwner.backend->CharactersFromPositions(posRangeCurrent);
     // Mark the just inserted text. Keep the marked range for later reset.
-    [mOwner setGeneralProperty: SCI_SETINDICATORCURRENT value: INPUT_INDICATOR];
+    [mOwner setGeneralProperty: SCI_SETINDICATORCURRENT value: INDIC_IME];
     [mOwner setGeneralProperty: SCI_INDICATORFILLRANGE
-                     parameter: mMarkedTextRange.location
-                         value: mMarkedTextRange.length];
+                     parameter: posRangeCurrent.location
+                         value: posRangeCurrent.length];
   }
   else
   {
     mMarkedTextRange = NSMakeRange(NSNotFound, 0);
     // Re-enable undo action collection if composition ended (indicated by an empty mark string).
-    if (undoCollectionWasActive)
-      [mOwner setGeneralProperty: SCI_SETUNDOCOLLECTION value: range.length == 0];
+    mOwner.backend->CompositionCommit();
   }
 
   // Select the part which is indicated in the given range. It does not scroll the caret into view.
   if (range.length > 0)
   {
     // range is in characters so convert to bytes for selection.
-    long rangeStart = currentPosition;
-    for (size_t characterInComposition=0; characterInComposition<range.location; characterInComposition++)
-      rangeStart = [mOwner getGeneralProperty: SCI_POSITIONAFTER parameter: rangeStart];
-    long rangeEnd = rangeStart;
-    for (size_t characterInRange=0; characterInRange<range.length; characterInRange++)
-      rangeEnd = [mOwner getGeneralProperty: SCI_POSITIONAFTER parameter: rangeEnd];
-    [mOwner setGeneralProperty: SCI_SETSELECTION parameter: rangeEnd value: rangeStart];
+    range.location += replacementRange.location;
+    NSRange posRangeSelect = mOwner.backend->PositionsFromCharacters(range);
+    [mOwner setGeneralProperty: SCI_SETSELECTION parameter: NSMaxRange(posRangeSelect) value: posRangeSelect.location];
   }
 }
 
@@ -562,35 +597,8 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 {
   if (mMarkedTextRange.length > 0)
   {
-    [mOwner setGeneralProperty: SCI_SETINDICATORCURRENT value: INPUT_INDICATOR];
-    [mOwner setGeneralProperty: SCI_INDICATORCLEARRANGE
-                     parameter: mMarkedTextRange.location
-                         value: mMarkedTextRange.length];
-    mMarkedTextRange = NSMakeRange(NSNotFound, 0);
-
-    // Reenable undo action collection, after we are done with text composition.
-    if (undoCollectionWasActive)
-      [mOwner setGeneralProperty: SCI_SETUNDOCOLLECTION value: 1];
-  }
-}
-
-//--------------------------------------------------------------------------------------------------
-
-/**
- * Removes any currently marked text.
- */
-- (void) removeMarkedText
-{
-  if (mMarkedTextRange.length > 0)
-  {
-    // We have already marked text. Replace that.
-    [mOwner deleteRange: mMarkedTextRange];
-
+    mOwner.backend->CompositionCommit();
     mMarkedTextRange = NSMakeRange(NSNotFound, 0);
-
-    // Reenable undo action collection, after we are done with text composition.
-    if (undoCollectionWasActive)
-      [mOwner setGeneralProperty: SCI_SETUNDOCOLLECTION value: 1];
   }
 }
 
@@ -831,12 +839,24 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 - (void) paste: (id) sender
 {
 #pragma unused(sender)
+  if (mMarkedTextRange.location != NSNotFound)
+  {
+    [[NSTextInputContext currentInputContext] discardMarkedText];
+    mOwner.backend->CompositionCommit();
+    mMarkedTextRange = NSMakeRange(NSNotFound, 0);
+  }
   mOwner.backend->Paste();
 }
 
 - (void) undo: (id) sender
 {
 #pragma unused(sender)
+  if (mMarkedTextRange.location != NSNotFound)
+  {
+    [[NSTextInputContext currentInputContext] discardMarkedText];
+    mOwner.backend->CompositionCommit();
+    mMarkedTextRange = NSMakeRange(NSNotFound, 0);
+  }
   mOwner.backend->Undo();
 }
 
@@ -848,7 +868,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 
 - (BOOL) canUndo
 {
-  return mOwner.backend->CanUndo();
+  return mOwner.backend->CanUndo() && (mMarkedTextRange.location == NSNotFound);
 }
 
 - (BOOL) canRedo
@@ -1141,10 +1161,10 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 
     // Setup a special indicator used in the editor to provide visual feedback for
     // input composition, depending on language, keyboard etc.
-    [self setColorProperty: SCI_INDICSETFORE parameter: INPUT_INDICATOR fromHTML: @"#FF0000"];
-    [self setGeneralProperty: SCI_INDICSETUNDER parameter: INPUT_INDICATOR value: 1];
-    [self setGeneralProperty: SCI_INDICSETSTYLE parameter: INPUT_INDICATOR value: INDIC_PLAIN];
-    [self setGeneralProperty: SCI_INDICSETALPHA parameter: INPUT_INDICATOR value: 100];
+    [self setColorProperty: SCI_INDICSETFORE parameter: INDIC_IME fromHTML: @"#FF0000"];
+    [self setGeneralProperty: SCI_INDICSETUNDER parameter: INDIC_IME value: 1];
+    [self setGeneralProperty: SCI_INDICSETSTYLE parameter: INDIC_IME value: INDIC_PLAIN];
+    [self setGeneralProperty: SCI_INDICSETALPHA parameter: INDIC_IME value: 100];
 
     NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
     [center addObserver:self
@@ -1318,7 +1338,8 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor)
 {
   if (aRange.length > 0)
   {
-    [self message: SCI_DELETERANGE wParam: aRange.location lParam: aRange.length];
+    NSRange posRange = mBackend->PositionsFromCharacters(aRange);
+    [self message: SCI_DELETERANGE wParam: posRange.location lParam: posRange.length];
   }
 }