revised and improved the Unicode glyph-to-byte conversion heuristics

Previously almost all glyph-to-byte offset conversions consulted
Scintilla's line index and counted characters on the resulting line.
For instance a simple expression like `.+1J` would scan the same line
twice completely, which would be very slow on pathologically long lines.
Even insertions did that due to having to update the ^Y ranges.
If you repeat such an operation over all characters as in `<.+1:J;>`
you would have complexity O(n^2) for n = line length.
Only commands with an explicit relative nature like `C` and `A` would
use teco_view_glyph2bytes_relative() which scans beginning at dot
as long as the relative movement is less than 1024 glyphs.

Wit the new heuristics almost all glyph-to-byte and byte-to-glyph
conversions can make use of that optimization.
This requires that dot must at all times be known in glyphs as well -
the byte position is managed by Scintilla (SCI_GETCURRENTPOS).
We therefore introduced teco_current_doc_set_dot() and
teco_current_doc_get_dot() to update dot in the current buffer or
Q-Register -- it cannot be stored along with the view since
Q-Registers share a single view.
A number of auxiliary functions have been introduced for
converting relative to a known (glyphs,bytes) offset pair
and for converting absolute and relative positions with regard
to the current doc and SCI_GETCURRENTPOS position.
Of course this is error-prone since the glyph and dot positions
are interdependant - they must always be kept in sync.

With these new optimizations even pathologically long lines can
(usually) be managed even in UTF-8 documents.
It does not address slow-downs in Scintilla's line layout, yet.
grosciteco.tes for instance runs twice as fast now.

1 files changed, 62 insertions, 27 deletions

diff --git a/src/core-commands.c b/src/core-commands.c
index 81d5869..94501b6 100644
--- a/src/core-commands.c
+++ b/src/core-commands.c
@@ -72,7 +72,8 @@ static teco_state_t *teco_state_ctlc_control_input(teco_machine_main_t *ctx, gun
  * But it needs to discern between invalid ranges and other errors.
  */
 gboolean
-teco_get_range_args(const gchar *cmd, gsize *from_ret, gsize *len_ret, GError **error)
+teco_get_range_args(const gchar *cmd, teco_int_t *from_glyphs_ret, gsize *from_ret,
+                    teco_int_t *len_glyphs_ret, gsize *len_ret, GError **error)
 {
 	gssize from, len; /* in bytes */
 
@@ -99,15 +100,27 @@ teco_get_range_args(const gchar *cmd, gsize *from_ret, gsize *len_ret, GError **
 			from += len;
 			len *= -1;
 		}
+
+		if (from_glyphs_ret)
+			*from_glyphs_ret = teco_interface_bytes2glyphs_absdot(from);
+		if (len_glyphs_ret)
+			*len_glyphs_ret = teco_interface_bytes2glyphs_absdot(from+len);
 	} else {
-		gssize to = teco_interface_glyphs2bytes(teco_expressions_pop_num(0));
-		from = teco_interface_glyphs2bytes(teco_expressions_pop_num(0));
+		teco_int_t to_glyphs = teco_expressions_pop_num(0);
+		gssize to = teco_interface_glyphs2bytes_absdot(to_glyphs);
+		teco_int_t from_glyphs = teco_expressions_pop_num(0);
+		from = teco_interface_glyphs2bytes_absdot(from_glyphs);
 		len = to - from;
 
 		if (len < 0 || from < 0 || to < 0) {
 			teco_error_range_set(error, cmd);
 			return FALSE;
 		}
+
+		if (from_glyphs_ret)
+			*from_glyphs_ret = from_glyphs;
+		if (len_glyphs_ret)
+			*len_glyphs_ret = to_glyphs - from_glyphs;
 	}
 
 	*from_ret = from;
@@ -199,8 +212,7 @@ teco_state_start_dot(teco_machine_main_t *ctx, GError **error)
 {
 	if (!teco_expressions_eval(FALSE, error))
 		return;
-	sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-	teco_expressions_push(teco_interface_bytes2glyphs(pos));
+	teco_expressions_push(teco_current_doc_get_dot());
 }
 
 /*$ Z size
@@ -217,7 +229,7 @@ teco_state_start_zed(teco_machine_main_t *ctx, GError **error)
 	if (!teco_expressions_eval(FALSE, error))
 		return;
 	sptr_t pos = teco_interface_ssm(SCI_GETLENGTH, 0, 0);
-	teco_expressions_push(teco_interface_bytes2glyphs(pos));
+	teco_expressions_push(teco_interface_bytes2glyphs_absdot(pos));
 }
 
 /*$ H
@@ -235,7 +247,7 @@ teco_state_start_range(teco_machine_main_t *ctx, GError **error)
 		return;
 	teco_expressions_push(0);
 	sptr_t pos = teco_interface_ssm(SCI_GETLENGTH, 0, 0);
-	teco_expressions_push(teco_interface_bytes2glyphs(pos));
+	teco_expressions_push(teco_interface_bytes2glyphs_absdot(pos));
 }
 
 /*$ \[rs]
@@ -267,8 +279,7 @@ teco_state_start_backslash(teco_machine_main_t *ctx, GError **error)
 		g_assert(*str != '\0');
 		gsize len = strlen(str);
 
-		sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-		teco_undo_int(teco_ranges[0].from) = teco_interface_bytes2glyphs(pos);
+		teco_undo_int(teco_ranges[0].from) = teco_current_doc_get_dot();
 		/*
 		 * We can assume that `len` is already in glyphs,
 		 * i.e. formatted numbers will never use multi-byte/Unicode characters.
@@ -276,6 +287,8 @@ teco_state_start_backslash(teco_machine_main_t *ctx, GError **error)
 		teco_undo_int(teco_ranges[0].to) = teco_ranges[0].from + len;
 		teco_undo_guint(teco_ranges_count) = 1;
 
+		teco_current_doc_set_dot(teco_ranges[0].to);
+
 		teco_interface_ssm(SCI_BEGINUNDOACTION, 0, 0);
 		teco_interface_ssm(SCI_ADDTEXT, len, (sptr_t)str);
 		teco_interface_ssm(SCI_ENDUNDOACTION, 0, 0);
@@ -563,6 +576,12 @@ teco_state_start_cmdline_push(teco_machine_main_t *ctx, GError **error)
 	 * A undo action should always have been generated.
 	 */
 	undo__teco_interface_ssm(SCI_UNDO, 0, 0);
+
+	/*
+	 * FIXME: If we would maintain dot in the command line,
+	 * it could simply be copied.
+	 */
+	teco_current_doc_set_dot(teco_interface_bytes2glyphs_rel(0, 0, teco_cmdline.pc));
 }
 
 static void
@@ -617,8 +636,7 @@ teco_state_start_get(teco_machine_main_t *ctx, GError **error)
 	if (!teco_expressions_pop_num_calc(&v, teco_num_sign, error))
 		return;
 
-	sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-	gssize get_pos = teco_interface_glyphs2bytes_relative(pos, v);
+	gssize get_pos = teco_interface_glyphs2bytes_reldot(v);
 	sptr_t len = teco_interface_ssm(SCI_GETLENGTH, 0, 0);
 
 	teco_expressions_push(get_pos < 0 || get_pos == len
@@ -1444,14 +1462,9 @@ teco_state_control_glyphs2bytes(teco_machine_main_t *ctx, GError **error)
 		res = teco_interface_ssm(colon_modified ? SCI_GETLENGTH : SCI_GETCURRENTPOS, 0, 0);
 	} else {
 		teco_int_t pos = teco_expressions_pop_num(0);
-		if (colon_modified) {
-			/* teco_interface_bytes2glyphs() does not check addresses */
-			res = 0 <= pos && pos <= teco_interface_ssm(SCI_GETLENGTH, 0, 0)
-				? teco_interface_bytes2glyphs(pos) : -1;
-		} else {
-			/* negative values for invalid indexes are passed down. */
-			res = teco_interface_glyphs2bytes(pos);
-		}
+		/* negative values for invalid indexes are passed down. */
+		res = colon_modified ? teco_interface_bytes2glyphs_absdot(pos)
+		                     : teco_interface_glyphs2bytes_absdot(pos);
 	}
 
 	teco_expressions_push(res);
@@ -2671,7 +2684,7 @@ teco_state_ecommand_encoding(teco_machine_main_t *ctx, GError **error)
 	teco_int_t dot_glyphs = 0;
 	if (colon_modified) {
 		sptr_t dot_bytes = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-		dot_glyphs = teco_interface_bytes2glyphs(dot_bytes);
+		dot_glyphs = teco_current_doc_get_dot();
 
 		/*
 		 * Convert buffer to new codepage.
@@ -2719,7 +2732,7 @@ teco_state_ecommand_encoding(teco_machine_main_t *ctx, GError **error)
 		teco_interface_ssm(SCI_SETCODEPAGE, SC_CP_UTF8, 0);
 		/*
 		 * UTF-8 documents strictly require the line character index.
-		 * See teco_view_glyphs2bytes() and teco_view_bytes2glyphs().
+		 * See teco_view_glyphs2bytes_unicode() and teco_view_bytes2glyphs_unicode().
 		 */
 		g_assert(!(teco_interface_ssm(SCI_GETLINECHARACTERINDEX, 0, 0)
 						& SC_LINECHARACTERINDEX_UTF32));
@@ -2763,13 +2776,18 @@ teco_state_ecommand_encoding(teco_machine_main_t *ctx, GError **error)
 		teco_interface_ssm(SCI_SETCODEPAGE, 0, 0);
 	}
 
-	if (colon_modified)
+	if (colon_modified) {
 		/*
 		 * Only now, it will be safe to recalculate dot in the new encoding.
 		 * If the new codepage is UTF-8, the line character index will be
 		 * ready only now.
 		 */
-		teco_interface_ssm(SCI_GOTOPOS, teco_interface_glyphs2bytes(dot_glyphs), 0);
+		teco_interface_ssm(SCI_GOTOPOS, teco_interface_glyphs2bytes_rel(0, 0, dot_glyphs), 0);
+	} else {
+		/* recalculate dot */
+		sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
+		teco_current_doc_set_dot(teco_interface_bytes2glyphs_rel(0, 0, pos));
+	}
 }
 
 /*$ EO version
@@ -2928,8 +2946,7 @@ teco_state_insert_initial(teco_machine_main_t *ctx, GError **error)
 	if (ctx->flags.mode > TECO_MODE_NORMAL)
 		return TRUE;
 
-	sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-	teco_undo_int(teco_ranges[0].from) = teco_interface_bytes2glyphs(pos);
+	teco_undo_int(teco_ranges[0].from) = teco_current_doc_get_dot();
 	teco_undo_guint(teco_ranges_count) = 1;
 
 	/*
@@ -2942,8 +2959,10 @@ teco_state_insert_initial(teco_machine_main_t *ctx, GError **error)
 	if (!teco_expressions_eval(FALSE, error))
 		return FALSE;
 	guint args = teco_expressions_args();
-	if (!args)
+	if (!args) {
+		teco_undo_int(teco_ranges[0].to) = teco_ranges[0].from;
 		return TRUE;
+	}
 
 	if (teco_interface_ssm(SCI_GETCODEPAGE, 0, 0) == SC_CP_UTF8) {
 		/* detect possible errors before introducing side effects */
@@ -2986,6 +3005,9 @@ teco_state_insert_initial(teco_machine_main_t *ctx, GError **error)
 	for (gint i = 0; i < args; i++)
 		teco_expressions_pop_num(0);
 
+	/* this way we remember how many numeric arguments were given */
+	teco_undo_int(teco_ranges[0].to) = teco_ranges[0].from + args;
+
 	return TRUE;
 }
 
@@ -3013,8 +3035,15 @@ teco_state_insert_done(teco_machine_main_t *ctx, teco_string_t str, GError **err
 	if (ctx->flags.mode > TECO_MODE_NORMAL)
 		return &teco_state_start;
 
+	/*
+	 * NOTE: teco_interface_bytes2glyphs_reldot() would count relative to
+	 * dot, which is still invalid.
+	 */
 	sptr_t pos = teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0);
-	teco_undo_int(teco_ranges[0].to) = teco_interface_bytes2glyphs(pos);
+	teco_int_t to = teco_interface_bytes2glyphs_rel(teco_ranges[0].to, pos-str.len, str.len);
+	teco_undo_int(teco_ranges[0].to) = to;
+	teco_current_doc_set_dot(to);
+
 	return &teco_state_start;
 }
 
@@ -3053,12 +3082,18 @@ teco_state_insert_indent_initial(teco_machine_main_t *ctx, GError **error)
 	teco_interface_ssm(SCI_BEGINUNDOACTION, 0, 0);
 	if (teco_interface_ssm(SCI_GETUSETABS, 0, 0)) {
 		teco_interface_ssm(SCI_ADDTEXT, 1, (sptr_t)"\t");
+
+		/* teco_state_insert_done() expects this */
+		teco_undo_int(teco_ranges[0].to)++;
 	} else {
 		gint len = teco_interface_ssm(SCI_GETTABWIDTH, 0, 0);
 
 		len -= teco_interface_ssm(SCI_GETCOLUMN,
 		                          teco_interface_ssm(SCI_GETCURRENTPOS, 0, 0), 0) % len;
 
+		/* teco_state_insert_done() expects this */
+		teco_undo_int(teco_ranges[0].to) += len;
+
 		gchar space = ' ';
 		while (len-- > 0)
 			teco_interface_ssm(SCI_ADDTEXT, 1, (sptr_t)&space);