Add APIs for setting appearance (traditional blob or plain text) and colour of

representations and support setting a representation for the "\r\n" line end
sequence.

4 files changed, 155 insertions, 66 deletions

diff --git a/src/EditView.cxx b/src/EditView.cxx
index 2774d5f2e..80edd31c6 100644
--- a/src/EditView.cxx
+++ b/src/EditView.cxx
@@ -474,9 +474,11 @@ void EditView::LayoutLine(const EditModel &model, Surface *surface, const ViewSt
 					} else {
 						if (representationWidth <= 0.0) {
 							XYPOSITION positionsRepr[256];	// Should expand when needed
-							posCache.MeasureWidths(surface, vstyle, StyleControlChar, ts.representation->stringRep.c_str(),
-								static_cast<unsigned int>(ts.representation->stringRep.length()), positionsRepr, model.pdoc);
-							representationWidth = positionsRepr[ts.representation->stringRep.length() - 1] + vstyle.ctrlCharPadding;
+							surface->MeasureWidthsUTF8(vstyle.styles[StyleControlChar].font.get(), ts.representation->stringRep, positionsRepr);
+							representationWidth = positionsRepr[ts.representation->stringRep.length() - 1];
+							if (FlagSet(ts.representation->appearance, RepresentationAppearance::Blob)) {
+								representationWidth += vstyle.ctrlCharPadding;
+							}
 						}
 					}
 					for (int ii = 0; ii < ts.length; ii++)
@@ -607,7 +609,7 @@ void EditView::UpdateBidiData(const EditModel &model, const ViewStyle &vstyle, L
 
 		for (int charsInLine = 0; charsInLine < ll->numCharsInLine; charsInLine++) {
 			const int charWidth = UTF8DrawBytes(reinterpret_cast<unsigned char *>(&ll->chars[charsInLine]), ll->numCharsInLine - charsInLine);
-			const Representation *repr = model.reprs.RepresentationFromCharacter(&ll->chars[charsInLine], charWidth);
+			const Representation *repr = model.reprs.RepresentationFromCharacter(std::string_view(&ll->chars[charsInLine], charWidth));
 
 			ll->bidiData->widthReprs[charsInLine] = 0.0f;
 			if (repr && ll->chars[charsInLine] != '\t') {
@@ -911,7 +913,7 @@ static void DrawTextBlob(Surface *surface, const ViewStyle &vsDraw, PRectangle r
 	PRectangle rcChar = rcCChar;
 	rcChar.left++;
 	rcChar.right--;
-	surface->DrawTextClipped(rcChar, ctrlCharsFont,
+	surface->DrawTextClippedUTF8(rcChar, ctrlCharsFont,
 		rcSegment.top + vsDraw.maxAscent, text,
 		textBack, textFore);
 }
@@ -1000,29 +1002,41 @@ void EditView::DrawEOL(Surface *surface, const EditModel &model, const ViewStyle
 	// Draw the [CR], [LF], or [CR][LF] blobs if visible line ends are on
 	XYPOSITION blobsWidth = 0;
 	if (lastSubLine) {
-		for (Sci::Position eolPos = ll->numCharsBeforeEOL; eolPos<ll->numCharsInLine; eolPos++) {
-			rcSegment.left = xStart + ll->positions[eolPos] - static_cast<XYPOSITION>(subLineStart)+virtualSpace;
-			rcSegment.right = xStart + ll->positions[eolPos + 1] - static_cast<XYPOSITION>(subLineStart)+virtualSpace;
-			blobsWidth += rcSegment.Width();
-			char hexits[4] = "";
-			const char *ctrlChar;
-			const unsigned char chEOL = ll->chars[eolPos];
+		for (Sci::Position eolPos = ll->numCharsBeforeEOL; eolPos<ll->numCharsInLine;) {
 			const int styleMain = ll->styles[eolPos];
-			const ColourRGBA textBack = TextBackground(model, vsDraw, ll, background, eolInSelection, false, styleMain, eolPos);
-			if (UTF8IsAscii(chEOL)) {
-				ctrlChar = ControlCharacterString(chEOL);
+			const std::optional<ColourRGBA> selectionFore = SelectionForeground(model, vsDraw, eolInSelection);
+			ColourRGBA textFore = selectionFore.value_or(vsDraw.styles[styleMain].fore);
+			char hexits[4] = "";
+			std::string_view ctrlChar;
+			Sci::Position widthBytes = 1;
+			RepresentationAppearance appearance = RepresentationAppearance::Blob;
+			const Representation *repr = model.reprs.RepresentationFromCharacter(std::string_view(&ll->chars[eolPos], ll->numCharsInLine - eolPos));
+			if (repr) {
+				// Representation of whole text
+				widthBytes = ll->numCharsInLine - eolPos;
+			} else {
+				repr = model.reprs.RepresentationFromCharacter(std::string_view(&ll->chars[eolPos], 1));
+			}
+			if (repr) {
+				ctrlChar = repr->stringRep;
+				appearance = repr->appearance;
+				if (FlagSet(appearance, RepresentationAppearance::Colour)) {
+					textFore = repr->colour;
+				}
 			} else {
-				const Representation *repr = model.reprs.RepresentationFromCharacter(&ll->chars[eolPos], ll->numCharsInLine - eolPos);
-				if (repr) {
-					ctrlChar = repr->stringRep.c_str();
-					eolPos = ll->numCharsInLine;
+				const unsigned char chEOL = ll->chars[eolPos];
+				if (UTF8IsAscii(chEOL)) {
+					ctrlChar = ControlCharacterString(chEOL);
 				} else {
 					sprintf(hexits, "x%2X", chEOL);
 					ctrlChar = hexits;
 				}
 			}
-			const std::optional<ColourRGBA> selectionFore = SelectionForeground(model, vsDraw, eolInSelection);
-			const ColourRGBA textFore = selectionFore.value_or(vsDraw.styles[styleMain].fore);
+
+			rcSegment.left = xStart + ll->positions[eolPos] - static_cast<XYPOSITION>(subLineStart)+virtualSpace;
+			rcSegment.right = xStart + ll->positions[eolPos + widthBytes] - static_cast<XYPOSITION>(subLineStart)+virtualSpace;
+			blobsWidth += rcSegment.Width();
+			const ColourRGBA textBack = TextBackground(model, vsDraw, ll, background, eolInSelection, false, styleMain, eolPos);
 			if (eolInSelection && (line < model.pdoc->LinesTotal() - 1)) {
 				if (vsDraw.selection.layer == Layer::Base) {
 					surface->FillRectangleAligned(rcSegment, Fill(selectionBack.Opaque()));
@@ -1038,10 +1052,16 @@ void EditView::DrawEOL(Surface *surface, const EditModel &model, const ViewStyle
 				surface->FillRectangleAligned(rcSegment, selectionBack);
 				blobText = textBack.MixedWith(selectionBack, selectionBack.GetAlphaComponent());
 			}
-			DrawTextBlob(surface, vsDraw, rcSegment, ctrlChar, blobText, textFore, phasesDraw == PhasesDraw::One);
+			if (FlagSet(appearance, RepresentationAppearance::Blob)) {
+				DrawTextBlob(surface, vsDraw, rcSegment, ctrlChar, blobText, textFore, phasesDraw == PhasesDraw::One);
+			} else {
+				surface->DrawTextTransparentUTF8(rcSegment, vsDraw.styles[StyleControlChar].font.get(),
+					rcSegment.top + vsDraw.maxAscent, ctrlChar, textFore);
+			}
 			if (drawEOLSelection && (vsDraw.selection.layer == Layer::OverText)) {
 				surface->FillRectangleAligned(rcSegment, selectionBack);
 			}
+			eolPos += widthBytes;
 		}
 	}
 
@@ -2019,8 +2039,16 @@ void EditView::DrawForeground(Surface *surface, const EditModel &model, const Vi
 							rcSegment.top + vsDraw.maxAscent,
 							cc, textBack, textFore);
 					} else {
-						DrawTextBlob(surface, vsDraw, rcSegment, ts.representation->stringRep,
-							textBack, textFore, phasesDraw == PhasesDraw::One);
+						if (FlagSet(ts.representation->appearance, RepresentationAppearance::Colour)) {
+							textFore = ts.representation->colour;
+						}
+						if (FlagSet(ts.representation->appearance, RepresentationAppearance::Blob)) {
+							DrawTextBlob(surface, vsDraw, rcSegment, ts.representation->stringRep,
+								textBack, textFore, phasesDraw == PhasesDraw::One);
+						} else {
+							surface->DrawTextTransparentUTF8(rcSegment, vsDraw.styles[StyleControlChar].font.get(),
+								rcSegment.top + vsDraw.maxAscent, ts.representation->stringRep, textFore);
+						}
 					}
 				}
 			} else {
diff --git a/src/Editor.cxx b/src/Editor.cxx
index f59eb5785..6860e153f 100644
--- a/src/Editor.cxx
+++ b/src/Editor.cxx
@@ -222,7 +222,7 @@ void Editor::SetRepresentations() {
 	};
 	for (size_t j=0; j < std::size(reps); j++) {
 		const char c[2] = { static_cast<char>(j), 0 };
-		reprs.SetRepresentation(c, reps[j]);
+		reprs.SetRepresentation(std::string_view(c, 1), reps[j]);
 	}
 	reprs.SetRepresentation("\x7f", "DEL");
 
@@ -8106,7 +8106,7 @@ sptr_t Editor::WndProc(Message iMessage, uptr_t wParam, sptr_t lParam) {
 
 	case Message::GetRepresentation: {
 			const Representation *repr = reprs.RepresentationFromCharacter(
-				ConstCharPtrFromUPtr(wParam), UTF8MaxBytes);
+				ConstCharPtrFromUPtr(wParam));
 			if (repr) {
 				return StringResult(lParam, repr->stringRep.c_str());
 			}
@@ -8117,6 +8117,35 @@ sptr_t Editor::WndProc(Message iMessage, uptr_t wParam, sptr_t lParam) {
 		reprs.ClearRepresentation(ConstCharPtrFromUPtr(wParam));
 		break;
 
+	case Message::ClearAllRepresentations:
+		SetRepresentations();
+		break;
+
+	case Message::SetRepresentationAppearance:
+		reprs.SetRepresentationAppearance(ConstCharPtrFromUPtr(wParam), static_cast<RepresentationAppearance>(lParam));
+		break;
+
+	case Message::GetRepresentationAppearance: {
+			const Representation *repr = reprs.RepresentationFromCharacter(
+				ConstCharPtrFromUPtr(wParam));
+			if (repr) {
+				return static_cast<sptr_t>(repr->appearance);
+			}
+			return 0;
+		}
+	case Message::SetRepresentationColour:
+		reprs.SetRepresentationColour(ConstCharPtrFromUPtr(wParam), ColourRGBA(static_cast<int>(lParam)));
+		break;
+
+	case Message::GetRepresentationColour: {
+			const Representation *repr = reprs.RepresentationFromCharacter(
+				ConstCharPtrFromUPtr(wParam));
+			if (repr) {
+				return repr->colour.AsInteger();
+			}
+			return 0;
+		}
+
 	case Message::StartRecord:
 		recordingMacro = true;
 		return 0;
diff --git a/src/PositionCache.cxx b/src/PositionCache.cxx
index fb3d8fa01..8123552bd 100644
--- a/src/PositionCache.cxx
+++ b/src/PositionCache.cxx
@@ -534,10 +534,10 @@ std::shared_ptr<LineLayout> LineLayoutCache::Retrieve(Sci::Line lineNumber, Sci:
 }
 
 // Simply pack the (maximum 4) character bytes into an int
-static unsigned int KeyFromString(const char *charBytes, size_t len) noexcept {
-	PLATFORM_ASSERT(len <= 4);
+static unsigned int KeyFromString(std::string_view charBytes) noexcept {
+	PLATFORM_ASSERT(charBytes.length() <= 4);
 	unsigned int k=0;
-	for (size_t i=0; i<len && charBytes[i]; i++) {
+	for (size_t i=0; i < charBytes.length(); i++) {
 		k = k * 0x100;
 		const unsigned char uc = charBytes[i];
 		k += uc;
@@ -545,49 +545,74 @@ static unsigned int KeyFromString(const char *charBytes, size_t len) noexcept {
 	return k;
 }
 
-SpecialRepresentations::SpecialRepresentations() {
-	constexpr short none = 0;
-	std::fill(startByteHasReprs, std::end(startByteHasReprs), none);
+void SpecialRepresentations::SetRepresentation(std::string_view charBytes, std::string_view value) {
+	if (charBytes.length() <= 4) {
+		const unsigned int key = KeyFromString(charBytes);
+		MapRepresentation::iterator it = mapReprs.find(key);
+		if (it == mapReprs.end()) {
+			// New entry so increment for first byte
+			const unsigned char ucStart = charBytes.empty() ? 0 : charBytes[0];
+			startByteHasReprs[ucStart]++;
+		}
+		mapReprs[key] = Representation(value);
+	}
+}
+
+void SpecialRepresentations::SetRepresentationAppearance(std::string_view charBytes, RepresentationAppearance appearance) {
+	if (charBytes.length() <= 4) {
+		const unsigned int key = KeyFromString(charBytes);
+		MapRepresentation::iterator it = mapReprs.find(key);
+		if (it == mapReprs.end()) {
+			// Not present so fail
+			return;
+		}
+		mapReprs[key].appearance = appearance;
+	}
 }
 
-void SpecialRepresentations::SetRepresentation(const char *charBytes, const char *value) {
-	const unsigned int key = KeyFromString(charBytes, UTF8MaxBytes);
-	MapRepresentation::iterator it = mapReprs.find(key);
-	if (it == mapReprs.end()) {
-		// New entry so increment for first byte
-		const unsigned char ucStart = charBytes[0];
-		startByteHasReprs[ucStart]++;
+void SpecialRepresentations::SetRepresentationColour(std::string_view charBytes, ColourRGBA colour) {
+	if (charBytes.length() <= 4) {
+		const unsigned int key = KeyFromString(charBytes);
+		MapRepresentation::iterator it = mapReprs.find(key);
+		if (it == mapReprs.end()) {
+			// Not present so fail
+			return;
+		}
+		mapReprs[key].appearance = mapReprs[key].appearance | RepresentationAppearance::Colour;
+		mapReprs[key].colour = colour;
 	}
-	mapReprs[key] = Representation(value);
 }
 
-void SpecialRepresentations::ClearRepresentation(const char *charBytes) {
-	MapRepresentation::iterator it = mapReprs.find(KeyFromString(charBytes, UTF8MaxBytes));
-	if (it != mapReprs.end()) {
-		mapReprs.erase(it);
-		const unsigned char ucStart = charBytes[0];
-		startByteHasReprs[ucStart]--;
+void SpecialRepresentations::ClearRepresentation(std::string_view charBytes) {
+	if (charBytes.length() <= 4) {
+		MapRepresentation::iterator it = mapReprs.find(KeyFromString(charBytes));
+		if (it != mapReprs.end()) {
+			mapReprs.erase(it);
+			const unsigned char ucStart = charBytes.empty() ? 0 : charBytes[0];
+			startByteHasReprs[ucStart]--;
+		}
 	}
 }
 
-const Representation *SpecialRepresentations::RepresentationFromCharacter(const char *charBytes, size_t len) const {
-	PLATFORM_ASSERT(len <= 4);
-	const unsigned char ucStart = charBytes[0];
-	if (!startByteHasReprs[ucStart])
-		return nullptr;
-	MapRepresentation::const_iterator it = mapReprs.find(KeyFromString(charBytes, len));
-	if (it != mapReprs.end()) {
-		return &(it->second);
+const Representation *SpecialRepresentations::RepresentationFromCharacter(std::string_view charBytes) const {
+	if (charBytes.length() <= 4) {
+		const unsigned char ucStart = charBytes.empty() ? 0 : charBytes[0];
+		if (!startByteHasReprs[ucStart])
+			return nullptr;
+		MapRepresentation::const_iterator it = mapReprs.find(KeyFromString(charBytes));
+		if (it != mapReprs.end()) {
+			return &(it->second);
+		}
 	}
 	return nullptr;
 }
 
-bool SpecialRepresentations::Contains(const char *charBytes, size_t len) const {
-	PLATFORM_ASSERT(len <= 4);
-	const unsigned char ucStart = charBytes[0];
+bool SpecialRepresentations::Contains(std::string_view charBytes) const {
+	PLATFORM_ASSERT(charBytes.length() <= 4);
+	const unsigned char ucStart = charBytes.empty() ? 0 : charBytes[0];
 	if (!startByteHasReprs[ucStart])
 		return false;
-	MapRepresentation::const_iterator it = mapReprs.find(KeyFromString(charBytes, len));
+	MapRepresentation::const_iterator it = mapReprs.find(KeyFromString(charBytes));
 	return it != mapReprs.end();
 }
 
@@ -675,7 +700,10 @@ TextSegment BreakFinder::Next() {
 			else if (encodingFamily == EncodingFamily::dbcs)
 				charWidth = pdoc->DBCSDrawBytes(
 					std::string_view(&ll->chars[nextBreak], lineRange.end - nextBreak));
-			const Representation *repr = preprs->RepresentationFromCharacter(&ll->chars[nextBreak], charWidth);
+			// Special case \r\n line ends if there is a representation
+			if (preprs->Contains("\r\n") && ll->chars[nextBreak] == '\r' && ll->chars[nextBreak + 1] == '\n')
+				charWidth = 2;
+			const Representation *repr = preprs->RepresentationFromCharacter(std::string_view(&ll->chars[nextBreak], charWidth));
 			if (((nextBreak > 0) && (ll->styles[nextBreak] != ll->styles[nextBreak - 1])) ||
 					repr ||
 					(nextBreak == saeNext)) {
diff --git a/src/PositionCache.h b/src/PositionCache.h
index 16ac42ef8..b3e87bcd5 100644
--- a/src/PositionCache.h
+++ b/src/PositionCache.h
@@ -199,7 +199,10 @@ public:
 class Representation {
 public:
 	std::string stringRep;
-	explicit Representation(const char *value="") : stringRep(value) {
+	RepresentationAppearance appearance;
+	ColourRGBA colour;
+	explicit Representation(std::string_view value="", RepresentationAppearance appearance_= RepresentationAppearance::Blob) :
+		stringRep(value), appearance(appearance_) {
 	}
 };
 
@@ -207,13 +210,14 @@ typedef std::map<unsigned int, Representation> MapRepresentation;
 
 class SpecialRepresentations {
 	MapRepresentation mapReprs;
-	short startByteHasReprs[0x100];
+	short startByteHasReprs[0x100] {};
 public:
-	SpecialRepresentations();
-	void SetRepresentation(const char *charBytes, const char *value);
-	void ClearRepresentation(const char *charBytes);
-	const Representation *RepresentationFromCharacter(const char *charBytes, size_t len) const;
-	bool Contains(const char *charBytes, size_t len) const;
+	void SetRepresentation(std::string_view charBytes, std::string_view value);
+	void SetRepresentationAppearance(std::string_view charBytes, RepresentationAppearance appearance);
+	void SetRepresentationColour(std::string_view charBytes, ColourRGBA colour);
+	void ClearRepresentation(std::string_view charBytes);
+	const Representation *RepresentationFromCharacter(std::string_view charBytes) const;
+	bool Contains(std::string_view charBytes) const;
 	void Clear();
 };