revised and improved the Unicode glyph-to-byte conversion heuristics

Previously almost all glyph-to-byte offset conversions consulted
Scintilla's line index and counted characters on the resulting line.
For instance a simple expression like `.+1J` would scan the same line
twice completely, which would be very slow on pathologically long lines.
Even insertions did that due to having to update the ^Y ranges.
If you repeat such an operation over all characters as in `<.+1:J;>`
you would have complexity O(n^2) for n = line length.
Only commands with an explicit relative nature like `C` and `A` would
use teco_view_glyph2bytes_relative() which scans beginning at dot
as long as the relative movement is less than 1024 glyphs.

Wit the new heuristics almost all glyph-to-byte and byte-to-glyph
conversions can make use of that optimization.
This requires that dot must at all times be known in glyphs as well -
the byte position is managed by Scintilla (SCI_GETCURRENTPOS).
We therefore introduced teco_current_doc_set_dot() and
teco_current_doc_get_dot() to update dot in the current buffer or
Q-Register -- it cannot be stored along with the view since
Q-Registers share a single view.
A number of auxiliary functions have been introduced for
converting relative to a known (glyphs,bytes) offset pair
and for converting absolute and relative positions with regard
to the current doc and SCI_GETCURRENTPOS position.
Of course this is error-prone since the glyph and dot positions
are interdependant - they must always be kept in sync.

With these new optimizations even pathologically long lines can
(usually) be managed even in UTF-8 documents.
It does not address slow-downs in Scintilla's line layout, yet.
grosciteco.tes for instance runs twice as fast now.

1 files changed, 35 insertions, 14 deletions

diff --git a/src/search.c b/src/search.c
index 1ab925c..accee69 100644
--- a/src/search.c
+++ b/src/search.c
@@ -131,12 +131,12 @@ teco_state_search_initial(teco_machine_main_t *ctx, GError **error)
 		v1 = teco_expressions_pop_num(0);
 		if (v1 <= v2) {
 			teco_search_parameters.count = 1;
-			teco_search_parameters.from = teco_interface_glyphs2bytes(v1);
-			teco_search_parameters.to = teco_interface_glyphs2bytes(v2);
+			teco_search_parameters.from = teco_interface_glyphs2bytes_absdot(v1);
+			teco_search_parameters.to = teco_interface_glyphs2bytes_absdot(v2);
 		} else {
 			teco_search_parameters.count = -1;
-			teco_search_parameters.from = teco_interface_glyphs2bytes(v2);
-			teco_search_parameters.to = teco_interface_glyphs2bytes(v1);
+			teco_search_parameters.from = teco_interface_glyphs2bytes_absdot(v2);
+			teco_search_parameters.to = teco_interface_glyphs2bytes_absdot(v1);
 		}
 
 		if (teco_search_parameters.from < 0 ||
@@ -890,18 +890,21 @@ teco_do_search(regex_t *re, gsize from, gsize to, gint *count, GError **error)
 		/* match success */
 		g_assert(teco_ranges_count > 0);
 
-		teco_interface_ssm(SCI_SETSEL, teco_ranges[0].from, teco_ranges[0].to);
+		gsize from = teco_ranges[0].from, to = teco_ranges[0].to;
 
 		/*
 		 * teco_get_ranges() returned byte positions,
 		 * while everything else expects glyph offsets.
 		 * They are fixed up only now to avoid unnecessary
-		 * possibly costly teco_interface_bytes2glyphs() calls.
+		 * possibly costly teco_interface_bytes2glyphs_absdot() calls.
 		 */
 		for (guint i = 0; i < teco_ranges_count; i++) {
-			teco_ranges[i].from = teco_interface_bytes2glyphs(teco_ranges[i].from);
-			teco_ranges[i].to = teco_interface_bytes2glyphs(teco_ranges[i].to);
+			teco_ranges[i].from = teco_interface_bytes2glyphs_absdot(teco_ranges[i].from);
+			teco_ranges[i].to = teco_interface_bytes2glyphs_absdot(teco_ranges[i].to);
 		}
+
+		teco_interface_ssm(SCI_SETSEL, from, to);
+		teco_current_doc_set_dot(teco_ranges[0].to);
 	}
 
 	return TRUE;
@@ -940,6 +943,7 @@ teco_state_search_process(teco_machine_main_t *ctx, teco_string_t str, gsize new
 		/*
 		 * While SciTECO code is always guaranteed to be in valid UTF-8,
 		 * the result of string building may not (eg. if ^EQq inserts garbage).
+		 * terex does not currently reliably detect invalid UTF-8 sequences.
 		 */
 		g_set_error_literal(error, TECO_ERROR, TECO_ERROR_CODEPOINT,
 		                    "Invalid UTF-8 byte sequence in search pattern");
@@ -1333,8 +1337,8 @@ teco_state_search_kill_done(teco_machine_main_t *ctx, teco_string_t str, GError
 	if (teco_search_parameters.dot < dot) {
 		/* kill forwards */
 		sptr_t anchor = teco_interface_ssm(SCI_GETANCHOR, 0, 0);
-		teco_int_t len_glyphs = teco_interface_bytes2glyphs(anchor) -
-		                        teco_interface_bytes2glyphs(teco_search_parameters.dot);
+		teco_int_t len_glyphs = teco_interface_bytes2glyphs_absdot(anchor) -
+		                        teco_interface_bytes2glyphs_absdot(teco_search_parameters.dot);
 
 		if (teco_current_doc_must_undo())
 			undo__teco_interface_ssm(SCI_GOTOPOS, dot, 0);
@@ -1353,8 +1357,13 @@ teco_state_search_kill_done(teco_machine_main_t *ctx, teco_string_t str, GError
 			teco_ranges[i].from -= len_glyphs;
 			teco_ranges[i].to -= len_glyphs;
 		}
+
+		g_assert(teco_ranges_count > 0);
+		teco_current_doc_set_dot(teco_ranges[0].from);
 	} else {
 		/* kill backwards */
+		teco_current_doc_set_dot(teco_interface_bytes2glyphs_absdot(dot));
+
 		teco_interface_ssm(SCI_DELETERANGE, dot, teco_search_parameters.dot - dot);
 
 		/* NOTE: An undo action is not always created. */
@@ -1409,6 +1418,9 @@ teco_state_search_delete_done(teco_machine_main_t *ctx, teco_string_t str, GErro
 		return NULL;
 
 	if (teco_is_success(search_state)) {
+		sptr_t anchor = teco_interface_ssm(SCI_GETANCHOR, 0, 0);
+		teco_current_doc_set_dot(teco_interface_bytes2glyphs_absdot(anchor));
+
 		teco_interface_ssm(SCI_BEGINUNDOACTION, 0, 0);
 		teco_interface_ssm(SCI_REPLACESEL, 0, (sptr_t)"");
 		teco_interface_ssm(SCI_ENDUNDOACTION, 0, 0);
@@ -1450,8 +1462,8 @@ teco_state_replace_insert_initial(teco_machine_main_t *ctx, GError **error)
 	 * FIXME: Wastes undo tokens in teco_do_search().
 	 * Perhaps make this configurable in the state.
 	 */
-	sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-	teco_undo_int(teco_ranges[0].from) = teco_interface_bytes2glyphs(pos);
+	teco_undo_int(teco_ranges[0].from) = teco_current_doc_get_dot();
+	teco_undo_int(teco_ranges[0].to) = teco_ranges[0].from;
 	teco_undo_guint(teco_ranges_count) = 1;
 
 	/*
@@ -1531,7 +1543,8 @@ teco_state_replace_default_insert_done(teco_machine_main_t *ctx, teco_string_t s
 	teco_qreg_t *replace_reg = teco_qreg_table_find(&teco_qreg_table_globals, "-", 1);
 	g_assert(replace_reg != NULL);
 
-	if (str.len > 0) {
+	gsize len = str.len;
+	if (len > 0) {
 		if (!replace_reg->vtable->undo_set_string(replace_reg, error) ||
 		    !replace_reg->vtable->set_string(replace_reg, str.data, str.len,
 		                                     teco_default_codepage(), error))
@@ -1542,10 +1555,18 @@ teco_state_replace_default_insert_done(teco_machine_main_t *ctx, teco_string_t s
 		                                     NULL, error) ||
 		    (replace_str.len > 0 && !teco_state_insert_process(ctx, replace_str, replace_str.len, error)))
 			return NULL;
+		len = replace_str.len;
 	}
 
+	/*
+	 * NOTE: teco_interface_bytes2glyphs_reldot() would count relative to
+	 * dot, which is still invalid.
+	 */
 	sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-	teco_undo_int(teco_ranges[0].to) = teco_interface_bytes2glyphs(pos);
+	teco_int_t to = teco_interface_bytes2glyphs_rel(teco_ranges[0].to, pos-len, len);
+	teco_undo_int(teco_ranges[0].to) = to;
+	teco_current_doc_set_dot(to);
+
 	return &teco_state_start;
 }