Curses: support arbitrary RGB colorsHEADmaster-fmsbw-cimaster

* Instead of supporting only 16 predefined RGB placeholder
  values in Scintilla messages and styles, you can now use
  arbitrary RGB values and colors are allocated via the terminal
  on the fly.
  You no longer need to call 3EJ to change the default color
  palette.
* The placeholder RGB values are still available.
  Since you will usually want exact RGB values when using
  anything outside of the range of 16 default colors
  and the RGB placeholders will not always exactly correspond
  to their RGB value, you can now call `0,3EJ` to ignore
  the default palette and allocate all colors dynamically.
* Allows for more than 16 colors on the screen simultaneously.
  Also simplifies the solarized.tes color scheme.
  Since both Scinterm and SciTECO try not to touch the 16
  default colors, you also no longer have to deal with
  restoring the palette after program termination
  (which was never reliable anyway).
* Color schemes with non-default colors (solarized.tes)
  may now be broken on TERM=linux-16color (Linux VT)
  since Scinterm will get only 8 colors, but solarized.tes
  needs 16.

1 files changed, 98 insertions, 195 deletions

diff --git a/src/interface-curses/interface.c b/src/interface-curses/interface.c
index b30e5dc..21e728f 100644
--- a/src/interface-curses/interface.c
+++ b/src/interface-curses/interface.c
@@ -140,6 +140,16 @@ static gint teco_xterm_version(void) G_GNUC_UNUSED;
 static gint teco_interface_blocking_getch(void);
 static inline gboolean teco_interface_check_key(gint key);
 
+#define MAX_COLORS (MIN(COLORS, G_MAXSHORT)/2)
+/**
+ * Maximum effective number of color pairs.
+ * Scinterm uses shorts for color pairs, so G_MAXSHORT is the maximum.
+ * Some Curses implementations (NetBSD, PDCurses) may support less,
+ * but set COLOR_PAIRS accordingly.
+ * Half of the range is reserved to Scinterm.
+ */
+#define MAX_PAIRS (MIN(COLOR_PAIRS, G_MAXSHORT)/2)
+
 /**
  * Get bright variant of one of the 8 standard
  * curses colors.
@@ -174,30 +184,22 @@ static inline gboolean teco_interface_check_key(gint key);
 
 static struct {
 	/**
+	 * Mapping of RGB codes (encoded into a pointer)
+	 * to a curses color number.
+	 */
+	GHashTable *color_table;
+	/**
 	 * Mapping of foreground and background curses color tuples
 	 * (encoded into a pointer) to a color pair number.
 	 */
 	GHashTable *pair_table;
-
 	/**
-	 * Mapping of the first 16 curses color codes (that may or may not
-	 * correspond with the standard terminal color codes) to
-	 * Scintilla-compatible RGB values (red is LSB) to initialize after
-	 * Curses startup.
-	 * Negative values mean no color redefinition (keep the original
-	 * palette entry).
+	 * Whether to map the 16 standard palettized colors
+	 * in color_table.
+	 * Should be disabled when using arbitrary RGB values,
+	 * so every color is initialized to exact RGB values.
 	 */
-	gint32 color_table[16];
-
-	/**
-	 * Mapping of the first 16 curses color codes to their
-	 * original values for restoring them on shutdown.
-	 * Unfortunately, this may not be supported on all
-	 * curses ports, so this array may be unused.
-	 */
-	struct {
-		gshort r, g, b;
-	} orig_color_table[16];
+	gboolean use_palette;
 
 	int stdin_orig, stdout_orig, stderr_orig;
 	SCREEN *screen;
@@ -272,7 +274,7 @@ teco_color_pair(attr_t *attr, gshort fg, gshort bg)
 	gpointer value = g_hash_table_lookup(teco_interface.pair_table, key);
 	if (G_LIKELY(value != NULL))
 		return GPOINTER_TO_UINT(value);
-	if (G_UNLIKELY(next_pair >= COLOR_PAIRS || next_pair > G_MAXSHORT))
+	if (G_UNLIKELY(next_pair >= MAX_PAIRS))
 		return 0;
 	init_pair(next_pair, fg, bg);
 	g_hash_table_insert(teco_interface.pair_table, key, GUINT_TO_POINTER(next_pair));
@@ -300,38 +302,60 @@ teco_rgb2curses_triple(guint32 rgb, gshort *r, gshort *g, gshort *b)
 	*b = ((rgb & 0xFF0000) >> 16)*1000/0xFF;
 }
 
+static inline void
+teco_interface_insert_color(guint32 rgb, gshort color)
+{
+	/*
+	 * Color ids are stored one-offset, so that COLOR_BLACK
+	 * can be discerned from a missing entry.
+	 */
+	g_hash_table_insert(teco_interface.color_table,
+	                    GUINT_TO_POINTER(rgb), GUINT_TO_POINTER(color)+1);
+}
+
 /**
  * Convert a Scintilla-compatible RGB color value
  * (0xBBGGRR) to a Curses color code (e.g. COLOR_BLACK).
- * This does not work with arbitrary RGB values but
- * only the 16 RGB color values defined by Scinterm
- * corresponding to the 16 terminal colors.
- * It is equivalent to Scinterm's internal `term_color`
- * function.
+ *
+ * This will automatically initialize new curses colors
+ * when necessary.
+ * Unless the palette is disabled with
+ * teco_interface_disable_palette(), this will return
+ * the 16 "standard" colors for a predefined list of
+ * RGB values (just like defined by Scinterm).
  */
 static gshort
 teco_rgb2curses(guint32 rgb)
 {
-	switch (rgb) {
-	case 0x000000: return COLOR_BLACK;
-	case 0x000080: return COLOR_RED;
-	case 0x008000: return COLOR_GREEN;
-	case 0x008080: return COLOR_YELLOW;
-	case 0x800000: return COLOR_BLUE;
-	case 0x800080: return COLOR_MAGENTA;
-	case 0x808000: return COLOR_CYAN;
-	case 0xC0C0C0: return COLOR_WHITE;
-	case 0x404040: return COLOR_LBLACK;
-	case 0x0000FF: return COLOR_LRED;
-	case 0x00FF00: return COLOR_LGREEN;
-	case 0x00FFFF: return COLOR_LYELLOW;
-	case 0xFF0000: return COLOR_LBLUE;
-	case 0xFF00FF: return COLOR_LMAGENTA;
-	case 0xFFFF00: return COLOR_LCYAN;
-	case 0xFFFFFF: return COLOR_LWHITE;
-	}
+	static guint colors = 0;
+
+	if (!has_colors())
+		return COLOR_WHITE;
+
+	gpointer value = g_hash_table_lookup(teco_interface.color_table,
+	                                     GUINT_TO_POINTER(rgb));
+	if (G_LIKELY(value != NULL))
+		return GPOINTER_TO_UINT(value)-1;
+
+	/*
+	 * Even when initializing new colors, we will __try__ not to
+	 * touch the 16 palettized colors.
+	 * First of all, we might still need them and secondly,
+	 * this ensures we don't have to reset them on exit
+	 * (which is not always possible anyway).
+	 * There can still be 16 color terminals with
+	 * can_change_color() == TRUE, though.
+	 */
+	guint next_color = colors + (COLORS > 16 ? 16 : 0);
 
-	return COLOR_WHITE;
+	if (G_UNLIKELY(next_color >= MAX_COLORS) || !can_change_color())
+		return COLOR_WHITE;
+	gshort r, g, b;
+	teco_rgb2curses_triple(rgb, &r, &g, &b);
+	init_color(next_color, r, g, b);
+	teco_interface_insert_color(rgb, next_color);
+	colors++;
+	return next_color;
 }
 
 static gint
@@ -422,9 +446,6 @@ teco_view_free(teco_view_t *ctx)
 	scintilla_delete(ctx);
 }
 
-static void teco_interface_init_color_safe(guint color, guint32 rgb);
-static void teco_interface_restore_colors(void);
-
 static void teco_interface_init_screen(void);
 static gboolean teco_interface_init_interactive(GError **error);
 static void teco_interface_restore_batch(void);
@@ -439,10 +460,7 @@ static void teco_interface_draw_info(void);
 void
 teco_interface_init(gint argc, gchar **argv)
 {
-	for (guint i = 0; i < G_N_ELEMENTS(teco_interface.color_table); i++)
-		teco_interface.color_table[i] = -1;
-	for (guint i = 0; i < G_N_ELEMENTS(teco_interface.orig_color_table); i++)
-		teco_interface.orig_color_table[i].r = -1;
+	teco_interface.use_palette = TRUE;
 
 	teco_interface.stdin_orig = teco_interface.stdout_orig = teco_interface.stderr_orig = -1;
 
@@ -484,133 +502,11 @@ teco_interface_get_options(void)
 	return NULL;
 }
 
-static void
-teco_interface_init_color_safe(guint color, guint32 rgb)
-{
-#if defined(__PDCURSES__) && !defined(PDCURSES_GUI)
-	if (teco_interface.orig_color_table[color].r < 0) {
-		color_content((short)color,
-		              &teco_interface.orig_color_table[color].r,
-		              &teco_interface.orig_color_table[color].g,
-		              &teco_interface.orig_color_table[color].b);
-	}
-#endif
-
-	gshort r, g, b;
-	teco_rgb2curses_triple(rgb, &r, &g, &b);
-	init_color((short)color, r, g, b);
-}
-
-#if defined(__PDCURSES__) && !defined(PDCURSES_GUI)
-
-/*
- * On PDCurses/WinCon, color_content() will actually return
- * the real console color palette - or at least the default
- * palette when the console started.
- */
-static void
-teco_interface_restore_colors(void)
-{
-	if (!can_change_color())
-		return;
-
-	for (guint i = 0; i < G_N_ELEMENTS(teco_interface.orig_color_table); i++) {
-		if (teco_interface.orig_color_table[i].r < 0)
-			continue;
-
-		init_color((short)i,
-		           teco_interface.orig_color_table[i].r,
-		           teco_interface.orig_color_table[i].g,
-		           teco_interface.orig_color_table[i].b);
-	}
-}
-
-#elif defined(CURSES_TTY)
-
-/*
- * FIXME: On UNIX/ncurses teco_interface_init_color_safe() __may__
- * change the terminal's palette permanently and there does not
- * appear to be any portable way of restoring the original one.
- * Curses has color_content(), but there is actually no terminal
- * that allows querying the current palette and so color_content()
- * will return bogus "default" values and only for the first 8 colors.
- * It would do more damage to restore the palette returned by
- * color_content() than it helps.
- * xterm has the escape sequence "\e]104\a" which restores
- * the palette from Xdefaults but not all terminal emulators
- * claiming to be "xterm" via $TERM support this escape sequence.
- * lxterminal for instance will print gibberish instead.
- * So we try to look whether $XTERM_VERSION is set.
- * There are hardly any other terminal emulators that support palette
- * resets.
- * The only emulator I'm aware of which can be identified reliably
- * by $TERM supporting a palette reset is the Linux console
- * (see console_codes(4)). The escape sequence "\e]R" is already
- * part of its terminfo description (orig_colors capability)
- * which is apparently sent by endwin(), so the palette is
- * already properly restored on endwin().
- * Welcome in Curses hell.
- */
-static void
-teco_interface_restore_colors(void)
-{
-	if (teco_xterm_version() < 0)
-		return;
-
-	/*
-	 * Looks like a real XTerm
-	 */
-	fputs("\e]104\a", teco_interface.screen_tty);
-	fflush(teco_interface.screen_tty);
-}
-
-#else /* (!__PDCURSES__ || PDCURSES_GUI) && !CURSES_TTY */
-
-static void
-teco_interface_restore_colors(void)
-{
-	/*
-	 * No way to restore the palette, or it's
-	 * unnecessary (e.g. XCurses)
-	 */
-}
-
-#endif
-
 void
-teco_interface_init_color(guint color, guint32 rgb)
+teco_interface_disable_palette(void)
 {
-	if (color >= G_N_ELEMENTS(teco_interface.color_table))
-		return;
-
-#if defined(__PDCURSES__) && !defined(PDC_RGB)
-	/*
-	 * PDCurses will usually number color codes differently
-	 * (least significant bit is the blue component) while
-	 * SciTECO macros will assume a standard terminal color
-	 * code numbering with red as the LSB.
-	 * Therefore we have to swap the bit order of the least
-	 * significant 3 bits here.
-	 */
-	color = (color & ~0x5) |
-	        ((color & 0x1) << 2) | ((color & 0x4) >> 2);
-#endif
-
-	if (teco_interface.input_pad) {
-		/* interactive mode */
-		if (!can_change_color())
-			return;
-
-		teco_interface_init_color_safe(color, rgb);
-	} else {
-		/*
-		 * batch mode: store colors,
-		 * they can only be initialized after start_color()
-		 * which is called by Scinterm when interactive
-		 * mode is initialized
-		 */
-		teco_interface.color_table[color] = (gint32)rgb;
-	}
+	teco_interface.use_palette = FALSE;
+	scintilla_disable_color_palette();
 }
 
 #ifdef CURSES_TTY
@@ -752,14 +648,33 @@ teco_interface_init_interactive(GError **error)
 
 	teco_interface_init_screen();
 
+	teco_interface.color_table = g_hash_table_new(g_direct_hash, g_direct_equal);
 	teco_interface.pair_table = g_hash_table_new(g_direct_hash, g_direct_equal);
 	start_color();
-	/*
-	 * Scinterm uses shorts for color pairs, so G_MAXSHORT is the maximum.
-	 * Some Curses implementations (NetBSD, PDCurses) may support less,
-	 * but set COLOR_PAIRS accordingly.
-	 */
-	scintilla_set_color_offsets(0, MIN(COLOR_PAIRS, G_MAXSHORT)/2);
+	scintilla_set_color_offsets(MAX_COLORS, MAX_PAIRS);
+
+	if (teco_interface.use_palette && has_colors()) {
+		teco_interface_insert_color(0x000000, COLOR_BLACK);
+		teco_interface_insert_color(0x000080, COLOR_RED);
+		teco_interface_insert_color(0x008000, COLOR_GREEN);
+		teco_interface_insert_color(0x008080, COLOR_YELLOW);
+		teco_interface_insert_color(0x800000, COLOR_BLUE);
+		teco_interface_insert_color(0x800080, COLOR_MAGENTA);
+		teco_interface_insert_color(0x808000, COLOR_CYAN);
+		teco_interface_insert_color(0xC0C0C0, COLOR_WHITE);
+		/*
+		 * On 8-color terminals, we still support the
+		 * higher (light) colors, but map them to the lower codes as well.
+		 */
+		teco_interface_insert_color(0x404040, COLOR_LBLACK);
+		teco_interface_insert_color(0x0000FF, COLOR_LRED);
+		teco_interface_insert_color(0x00FF00, COLOR_LGREEN);
+		teco_interface_insert_color(0x00FFFF, COLOR_LYELLOW);
+		teco_interface_insert_color(0xFF0000, COLOR_LBLUE);
+		teco_interface_insert_color(0xFF00FF, COLOR_LMAGENTA);
+		teco_interface_insert_color(0xFFFF00, COLOR_LCYAN);
+		teco_interface_insert_color(0xFFFFFF, COLOR_LWHITE);
+	}
 
 	/*
 	 * On UNIX terminals, the escape key is usually
@@ -841,19 +756,6 @@ teco_interface_init_interactive(GError **error)
 		teco_interface_show_view(teco_interface_current_view);
 
 	/*
-	 * Only now it's safe to redefine the 16 default colors.
-	 */
-	if (can_change_color()) {
-		for (guint i = 0; i < G_N_ELEMENTS(teco_interface.color_table); i++) {
-			/*
-			 * init_color() may still fail if COLORS < 16
-			 */
-			if (teco_interface.color_table[i] >= 0)
-				teco_interface_init_color_safe(i, (guint32)teco_interface.color_table[i]);
-		}
-	}
-
-	/*
 	 * Only now (in interactive mode), it's safe to initialize
 	 * the clipboard Q-Registers on ncurses with a compatible terminal
 	 * emulator since clipboard operations will no longer interfer
@@ -885,7 +787,6 @@ teco_interface_restore_batch(void)
 	 * (i.e. return to batch mode)
 	 */
 	endwin();
-	teco_interface_restore_colors();
 
 	/*
 	 * Restore stdin, stdout and stderr, so output goes to
@@ -2409,6 +2310,8 @@ teco_interface_cleanup(void)
 	if (teco_interface.stdout_orig >= 0)
 		close(teco_interface.stdout_orig);
 
+	if (teco_interface.color_table)
+		g_hash_table_destroy(teco_interface.color_table);
 	if (teco_interface.pair_table)
 		g_hash_table_destroy(teco_interface.pair_table);