Allow UTF-8 entries to coexist in the position cache alongside the document

encoding. Elements like character representations may be in UTF-8 while the
document is in a different encoding. UTF-8 will be used for additional features
and may later be the only encoding for the position cache.
This change does not change performance or the proportion of strings cached -
its purpose is to clean up callers and prepare for greater UTF-8 use.

3 files changed, 44 insertions, 56 deletions

diff --git a/src/EditView.cxx b/src/EditView.cxx
index 78d38f110..79c4206b2 100644
--- a/src/EditView.cxx
+++ b/src/EditView.cxx
@@ -339,10 +339,6 @@ inline char CaseForce(Style::CaseForce caseForce, char chDoc, char chPrevious) n
 	}
 }
 
-bool ViewIsASCII(std::string_view text) {
-	return std::all_of(text.cbegin(), text.cend(), IsASCII);
-}
-
 void LayoutSegments(IPositionCache *pCache,
 	Surface *surface,
 	const ViewStyle &vstyle,
@@ -357,58 +353,44 @@ void LayoutSegments(IPositionCache *pCache,
 			break;
 		}
 		const TextSegment &ts = segments[i];
-		if (vstyle.styles[ll->styles[ts.start]].visible) {
+		const unsigned int styleSegment = ll->styles[ts.start];
+		XYPOSITION *positions = &ll->positions[ts.start + 1];
+		if (vstyle.styles[styleSegment].visible) {
 			if (ts.representation) {
-				XYPOSITION representationWidth = vstyle.controlCharWidth;
-				if (ll->chars[ts.start] == '\t') {
-					// Tab is a special case of representation, taking a variable amount of space
-					// which will be filled in later.
-					representationWidth = 0;
-				} else {
+				XYPOSITION representationWidth = 0.0;
+				// Tab is a special case of representation, taking a variable amount of space
+				// which will be filled in later.
+				if (ll->chars[ts.start] != '\t') {
+					representationWidth = vstyle.controlCharWidth;
 					if (representationWidth <= 0.0) {
 						assert(ts.representation->stringRep.length() <= Representation::maxLength);
 						XYPOSITION positionsRepr[Representation::maxLength + 1];
-						// ts.representation->stringRep is UTF-8 which only matches cache if document is UTF-8
-						// or it only contains ASCII which is a subset of all currently supported encodings.
-						if (textUnicode || ViewIsASCII(ts.representation->stringRep)) {
-							pCache->MeasureWidths(surface, vstyle, StyleControlChar, ts.representation->stringRep,
-								positionsRepr, multiThreaded);
-						} else {
-							surface->MeasureWidthsUTF8(vstyle.styles[StyleControlChar].font.get(), ts.representation->stringRep, positionsRepr);
-						}
+						// ts.representation->stringRep is UTF-8.
+						pCache->MeasureWidths(surface, vstyle, StyleControlChar, true, ts.representation->stringRep,
+							positionsRepr, multiThreaded);
 						representationWidth = positionsRepr[ts.representation->stringRep.length() - 1];
 						if (FlagSet(ts.representation->appearance, RepresentationAppearance::Blob)) {
 							representationWidth += vstyle.ctrlCharPadding;
 						}
 					}
 				}
-				for (int ii = 0; ii < ts.length; ii++) {
-					ll->positions[ts.start + 1 + ii] = representationWidth;
-				}
+				std::fill(positions, positions + ts.length, representationWidth);
 			} else {
 				if ((ts.length == 1) && (' ' == ll->chars[ts.start])) {
 					// Over half the segments are single characters and of these about half are space characters.
-					ll->positions[ts.start + 1] = vstyle.styles[ll->styles[ts.start]].spaceWidth;
+					positions[0] = vstyle.styles[styleSegment].spaceWidth;
 				} else {
-					pCache->MeasureWidths(surface, vstyle, ll->styles[ts.start],
-						std::string_view(&ll->chars[ts.start], ts.length), &ll->positions[ts.start + 1], multiThreaded);
+					pCache->MeasureWidths(surface, vstyle, styleSegment, textUnicode,
+						std::string_view(&ll->chars[ts.start], ts.length), positions, multiThreaded);
 				}
 			}
-		} else if (vstyle.styles[ll->styles[ts.start]].invisibleRepresentation[0]) {
-			const int styleInvisible = ll->styles[ts.start];
-			const std::string_view text = vstyle.styles[styleInvisible].invisibleRepresentation;
+		} else if (vstyle.styles[styleSegment].invisibleRepresentation[0]) {
+			const std::string_view text = vstyle.styles[styleSegment].invisibleRepresentation;
 			XYPOSITION positionsRepr[Representation::maxLength + 1];
-			// invisibleRepresentation is UTF-8 which only matches cache if document is UTF-8
-			// or it only contains ASCII which is a subset of all currently supported encodings.
-			if (textUnicode || ViewIsASCII(text)) {
-				pCache->MeasureWidths(surface, vstyle, styleInvisible, text, positionsRepr, multiThreaded);
-			} else {
-				surface->MeasureWidthsUTF8(vstyle.styles[styleInvisible].font.get(), text, positionsRepr);
-			}
+			// invisibleRepresentation is UTF-8.
+			pCache->MeasureWidths(surface, vstyle, styleSegment, true, text, positionsRepr, multiThreaded);
 			const XYPOSITION representationWidth = positionsRepr[text.length() - 1];
-			for (int ii = 0; ii < ts.length; ii++) {
-				ll->positions[ts.start + 1 + ii] = representationWidth;
-			}
+			std::fill(positions, positions + ts.length, representationWidth);
 		}
 	}
 }
diff --git a/src/PositionCache.cxx b/src/PositionCache.cxx
index c4bcbc411..ce932521a 100644
--- a/src/PositionCache.cxx
+++ b/src/PositionCache.cxx
@@ -826,6 +826,7 @@ class PositionCacheEntry {
 	uint16_t styleNumber;
 	uint16_t len;
 	uint16_t clock;
+	bool unicode;
 	std::unique_ptr<XYPOSITION[]> positions;
 public:
 	PositionCacheEntry() noexcept;
@@ -836,10 +837,10 @@ public:
 	void operator=(const PositionCacheEntry &) = delete;
 	void operator=(PositionCacheEntry &&) = delete;
 	~PositionCacheEntry();
-	void Set(unsigned int styleNumber_, std::string_view sv, const XYPOSITION *positions_, uint16_t clock_);
+	void Set(unsigned int styleNumber_, bool unicode_, std::string_view sv, const XYPOSITION *positions_, uint16_t clock_);
 	void Clear() noexcept;
-	bool Retrieve(unsigned int styleNumber_, std::string_view sv, XYPOSITION *positions_) const noexcept;
-	static size_t Hash(unsigned int styleNumber_, std::string_view sv) noexcept;
+	bool Retrieve(unsigned int styleNumber_, bool unicode_, std::string_view sv, XYPOSITION *positions_) const noexcept;
+	static size_t Hash(unsigned int styleNumber_, bool unicode_, std::string_view sv) noexcept;
 	bool NewerThan(const PositionCacheEntry &other) const noexcept;
 	void ResetClock() noexcept;
 };
@@ -862,16 +863,16 @@ public:
 	void SetSize(size_t size_) override;
 	size_t GetSize() const noexcept override;
 	void MeasureWidths(Surface *surface, const ViewStyle &vstyle, unsigned int styleNumber,
-		std::string_view sv, XYPOSITION *positions, bool needsLocking) override;
+		bool unicode, std::string_view sv, XYPOSITION *positions, bool needsLocking) override;
 };
 
 PositionCacheEntry::PositionCacheEntry() noexcept :
-	styleNumber(0), len(0), clock(0) {
+	styleNumber(0), len(0), clock(0), unicode(false) {
 }
 
 // Copy constructor not currently used, but needed for being element in std::vector.
 PositionCacheEntry::PositionCacheEntry(const PositionCacheEntry &other) :
-	styleNumber(other.styleNumber), len(other.len), clock(other.clock) {
+	styleNumber(other.styleNumber), len(other.len), clock(other.clock), unicode(other.unicode) {
 	if (other.positions) {
 		const size_t lenData = len + (len / sizeof(XYPOSITION)) + 1;
 		positions = std::make_unique<XYPOSITION[]>(lenData);
@@ -879,12 +880,13 @@ PositionCacheEntry::PositionCacheEntry(const PositionCacheEntry &other) :
 	}
 }
 
-void PositionCacheEntry::Set(unsigned int styleNumber_, std::string_view sv,
+void PositionCacheEntry::Set(unsigned int styleNumber_, bool unicode_, std::string_view sv,
 	const XYPOSITION *positions_, uint16_t clock_) {
 	Clear();
 	styleNumber = static_cast<uint16_t>(styleNumber_);
 	len = static_cast<uint16_t>(sv.length());
 	clock = clock_;
+	unicode = unicode_;
 	if (sv.data() && positions_) {
 		positions = std::make_unique<XYPOSITION[]>(len + (len / sizeof(XYPOSITION)) + 1);
 		for (unsigned int i=0; i<len; i++) {
@@ -905,8 +907,8 @@ void PositionCacheEntry::Clear() noexcept {
 	clock = 0;
 }
 
-bool PositionCacheEntry::Retrieve(unsigned int styleNumber_, std::string_view sv, XYPOSITION *positions_) const noexcept {
-	if ((styleNumber == styleNumber_) && (len == sv.length()) &&
+bool PositionCacheEntry::Retrieve(unsigned int styleNumber_, bool unicode_, std::string_view sv, XYPOSITION *positions_) const noexcept {
+	if ((styleNumber == styleNumber_) && (unicode == unicode_) && (len == sv.length()) &&
 		(memcmp(&positions[len], sv.data(), sv.length())== 0)) {
 		for (unsigned int i=0; i<len; i++) {
 			positions_[i] = positions[i];
@@ -917,10 +919,10 @@ bool PositionCacheEntry::Retrieve(unsigned int styleNumber_, std::string_view sv
 	}
 }
 
-size_t PositionCacheEntry::Hash(unsigned int styleNumber_, std::string_view sv) noexcept {
+size_t PositionCacheEntry::Hash(unsigned int styleNumber_, bool unicode_, std::string_view sv) noexcept {
 	const size_t h1 = std::hash<std::string_view>{}(sv);
 	const size_t h2 = std::hash<unsigned int>{}(styleNumber_);
-	return h1 ^ (h2 << 1);
+	return h1 ^ (h2 << 1) ^ static_cast<size_t>(unicode_);
 }
 
 bool PositionCacheEntry::NewerThan(const PositionCacheEntry &other) const noexcept {
@@ -959,7 +961,7 @@ size_t PositionCache::GetSize() const noexcept {
 }
 
 void PositionCache::MeasureWidths(Surface *surface, const ViewStyle &vstyle, unsigned int styleNumber,
-	std::string_view sv, XYPOSITION *positions, bool needsLocking) {
+	bool unicode, std::string_view sv, XYPOSITION *positions, bool needsLocking) {
 	const Style &style = vstyle.styles[styleNumber];
 	if (style.monospaceASCII) {
 		if (AllGraphicASCII(sv)) {
@@ -977,17 +979,17 @@ void PositionCache::MeasureWidths(Surface *surface, const ViewStyle &vstyle, uns
 		// long comments with only a single comment.
 
 		// Two way associative: try two probe positions.
-		const size_t hashValue = PositionCacheEntry::Hash(styleNumber, sv);
+		const size_t hashValue = PositionCacheEntry::Hash(styleNumber, unicode, sv);
 		probe = hashValue % pces.size();
 		std::unique_lock<std::mutex> guard(mutex, std::defer_lock);
 		if (needsLocking) {
 			guard.lock();
 		}
-		if (pces[probe].Retrieve(styleNumber, sv, positions)) {
+		if (pces[probe].Retrieve(styleNumber, unicode, sv, positions)) {
 			return;
 		}
 		const size_t probe2 = (hashValue * 37) % pces.size();
-		if (pces[probe2].Retrieve(styleNumber, sv, positions)) {
+		if (pces[probe2].Retrieve(styleNumber, unicode, sv, positions)) {
 			return;
 		}
 		// Not found. Choose the oldest of the two slots to replace
@@ -997,7 +999,11 @@ void PositionCache::MeasureWidths(Surface *surface, const ViewStyle &vstyle, uns
 	}
 
 	const Font *fontStyle = style.font.get();
-	surface->MeasureWidths(fontStyle, sv, positions);
+	if (unicode) {
+		surface->MeasureWidthsUTF8(fontStyle, sv, positions);
+	} else {
+		surface->MeasureWidths(fontStyle, sv, positions);
+	}
 	if (probe < pces.size()) {
 		// Store into cache
 		std::unique_lock<std::mutex> guard(mutex, std::defer_lock);
@@ -1014,7 +1020,7 @@ void PositionCache::MeasureWidths(Surface *surface, const ViewStyle &vstyle, uns
 			clock = 2;
 		}
 		allClear = false;
-		pces[probe].Set(styleNumber, sv, positions, clock);
+		pces[probe].Set(styleNumber, unicode, sv, positions, clock);
 	}
 }
 
diff --git a/src/PositionCache.h b/src/PositionCache.h
index 834472e75..9a4babae7 100644
--- a/src/PositionCache.h
+++ b/src/PositionCache.h
@@ -257,7 +257,7 @@ public:
 	virtual void SetSize(size_t size_) = 0;
 	virtual size_t GetSize() const noexcept = 0;
 	virtual void MeasureWidths(Surface *surface, const ViewStyle &vstyle, unsigned int styleNumber,
-		std::string_view sv, XYPOSITION *positions, bool needsLocking) = 0;
+		bool unicode, std::string_view sv, XYPOSITION *positions, bool needsLocking) = 0;
 };
 
 std::unique_ptr<IPositionCache> CreatePositionCache();