numbers are separate parser states and there are syntactic octal and hexadecimal constants nowHEADmaster-fmsbw-cimaster

* Instead of every digit being a stack-modifying standalone
  command, there is a single continuous parser state (teco_state_number)
  now - even though it has certain unique characteristics.
  They are consequently documented in the ARITHMETICS AND
  EXPRESSIONS chapter of sciteco(7).
* A number will therefore be rubbed out entirely by CTRL+W,
  including a possible @-modifier.
* (!) Spaces within constants are no longer ignored, but will push
  separate numbers to the stack instead (`1 2 3` pushes 3 numbers).
  The significance of the `,` operator is thus further reduced.
* (!) You can no longer "append" to numbers on the stack as in
  `Qa001` - this was never actually used and caused more confusion.
* (!) Digits outside the current radix are no longer accepted.
  E.g. digit 2 after `2^R`. This feature was also never used.
* (!) Octal constants are supported with leading zeros as in C.
  This works regardless of the current radix set via `8^R` or `^O`.
  It remains to see whether this is actually useful or is rather
  annoying (since every constant beginning with 0 is automatically
  octal, you cannot left-pad arbitrary numbers with 0).
* Hexadecimal constants are supported by at-modifying them as in
  `@0ff`. They must still begin with a digit 0-9 and since A-F are
  valid SciTECO commands, you may need additional spaces to separate
  hexadecimal constants from regular commands as in `@0ff D`.
  During syntax highlighting the number style is extended back to the
  at-sign.
  Note: Hexadecimal constants couldn't have been realized via
  `16^R` (or some shortcut) since that depends on runtime state, but
  we must know during parsing whether symbols A-F are part of the
  number of regular SciTECO commands.
* The new hexadecimal constants are used in the color schemes now.
  Thanks to them, we no longer need workarounds like the "color.rgb"
  macro.
* Since numbers don't have a dedicated terminator character, but any
  letter may terminate it, the number state must behave similar to a
  start state. Consequently, hexadecimal constants will currently be
  auto-case-folded if you enabled this feature.
  I recommend not to write hex constants in upper-case, though.
  Perhaps this should be worked around with a custom process_edit_cmd()
  callback.
* Test suite updated

1 files changed, 37 insertions, 26 deletions

diff --git a/src/lexer.c b/src/lexer.c
index 25ea8f2..2eae8c4 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -29,30 +29,30 @@
 #include "core-commands.h"
 #include "lexer.h"
 
+/**
+ * Get style after parsing character.
+ *
+ * By default, this is the parser state's style \b before
+ * inputting the character.
+ */
 static teco_style_t
 teco_lexer_getstyle(teco_view_t *view, teco_machine_main_t *machine,
                     gunichar chr)
 {
-	teco_style_t style = machine->parent.current->style;
-
-	/*
-	 * FIXME: At least this special workaround for numbers might be
-	 * unnecessary once we get a special parser state for parsing numbers.
-	 */
-	if (machine->parent.current->keymacro_mask & TECO_KEYMACRO_MASK_START &&
-	    chr <= 0xFF) {
-		if (g_ascii_isdigit(chr))
-			style = SCE_SCITECO_NUMBER;
-		else if (strchr(",+-*/#&()", chr))
-			style = SCE_SCITECO_OPERATOR;
-	} else if (machine->parent.current == &teco_state_control) {
-		/*
-		 * Two-character operators must always begin with caret
-		 * They get a separate style, so we can extend it back to
-		 * the caret in teco_lexter_step.
-		 */
-		if (strchr("*/#", chr))
-			style = SCE_SCITECO_OPERATOR2;
+	teco_style_t old_style = machine->parent.current->style;
+
+	if (chr <= 0xFF) {
+		if (machine->parent.current->keymacro_mask & TECO_KEYMACRO_MASK_START &&
+		    strchr(",+-*/#&()", chr))
+			old_style = SCE_SCITECO_OPERATOR;
+		else if (machine->parent.current == &teco_state_control &&
+		         strchr("*/#", chr))
+			/*
+			 * Two-character operators must always begin with caret
+			 * They get a separate style, so we can extend it back to
+			 * the caret in teco_lexer_step().
+			 */
+			old_style = SCE_SCITECO_OPERATOR2;
 	}
 
 	/*
@@ -81,16 +81,23 @@ teco_lexer_getstyle(teco_view_t *view, teco_machine_main_t *machine,
 	 * True comments also begin with `!`, so make sure they are highlighted
 	 * already from the second character.
 	 * This is then extended back by one character in teco_lexer_step().
+	 * Numbers must also be highlighted from the first character.
 	 */
 	switch (machine->parent.current->style) {
 	case SCE_SCITECO_COMMENT:
 	case SCE_SCITECO_LABEL:
+	case SCE_SCITECO_NUMBER:
 		return machine->parent.current->style;
 	default:
 		break;
 	}
 
-	return style;
+	/*
+	 * All non-digits automatically terminate the teco_state_number state,
+	 * so always report the new style (so that a command after a number is
+	 * highlighted properly).
+	 */
+	return old_style == SCE_SCITECO_NUMBER ? SCE_SCITECO_COMMAND : old_style;
 }
 
 static void
@@ -113,6 +120,7 @@ teco_lexer_step(teco_view_t *view, teco_machine_main_t *machine,
 	gssize old_pc = machine->macro_pc;
 
 	teco_style_t style = SCE_SCITECO_DEFAULT;
+	guint radix = 10; /* if style == SCE_SCITECO_NUMBER */
 
 	/*
 	 * g_utf8_get_char_validated() sometimes(?) returns -2 for "\0".
@@ -136,6 +144,7 @@ teco_lexer_step(teco_view_t *view, teco_machine_main_t *machine,
 		                   (escape_char == '{' ? 1 : 0);
 
 		style = teco_lexer_getstyle(view, machine, chr);
+		radix = machine->number.radix;
 
 		/*
 		 * Apply folding. This currently folds only {...} string arguments
@@ -171,8 +180,10 @@ teco_lexer_step(teco_view_t *view, teco_machine_main_t *machine,
 		 * rewrite the lexer against the ILexer5 interface, which requires C++.
 		 */
 		if ((escape_char == '{' || machine->expectstring.machine.escape_char == '{') &&
-		    teco_view_ssm(view, SCI_GETPROPERTYINT, (uptr_t)"lexer.sciteco.macrodef", TRUE))
+		    teco_view_ssm(view, SCI_GETPROPERTYINT, (uptr_t)"lexer.sciteco.macrodef", TRUE)) {
 			style = teco_lexer_getstyle(view, macrodef_machine, chr);
+			radix = macrodef_machine->number.radix;
+		}
 	}
 
 	*cur_col += machine->macro_pc - old_pc;
@@ -180,9 +191,10 @@ teco_lexer_step(teco_view_t *view, teco_machine_main_t *machine,
 	/*
 	 * True comments begin with `!*` or `!!`, but only the second character gets
 	 * the correct style by default, so we extend it backwards.
-	 * The same is true for two-letter operators.
+	 * The same is true for two-letter operators and hexadecimal constants.
 	 */
-	if (style == SCE_SCITECO_COMMENT || style == SCE_SCITECO_OPERATOR2)
+	if (style == SCE_SCITECO_COMMENT || style == SCE_SCITECO_OPERATOR2 ||
+	    (style == SCE_SCITECO_NUMBER && radix == 16))
 		old_pc--;
 
 	teco_view_ssm(view, SCI_STARTSTYLING, start+old_pc, 0);
@@ -196,8 +208,7 @@ teco_lexer_step(teco_view_t *view, teco_machine_main_t *machine,
 		*safe_col = -1; /* no clean state by default */
 	}
 
-	if (style != SCE_SCITECO_INVALID &&
-	    machine->parent.current->keymacro_mask & TECO_KEYMACRO_MASK_START &&
+	if (style != SCE_SCITECO_INVALID && machine->parent.current->is_start &&
 	    !machine->flags.modifier_at)
 		/* clean parser state */
 		*safe_col = *cur_col;