Rust: Highlight byte-string literals.

This adds support for two new string literals and one new string literal,
corresponding to the existing versions of these. Compared to the originals, the
new literals have slightly different escapes and are ASCII only. I've decided
to simply add flags to the existing scanners to handle them. New styles had to
be added to handle the line spanning string literals. The byte character style
was added for consistency.

3 files changed, 48 insertions, 22 deletions

diff --git a/include/SciLexer.h b/include/SciLexer.h
index 297843e29..673acb55d 100644
--- a/include/SciLexer.h
+++ b/include/SciLexer.h
@@ -1693,6 +1693,9 @@
 #define SCE_RUST_LIFETIME 18
 #define SCE_RUST_MACRO 19
 #define SCE_RUST_LEXERROR 20
+#define SCE_RUST_BYTESTRING 21
+#define SCE_RUST_BYTESTRINGR 22
+#define SCE_RUST_BYTECHARACTER 23
 #define SCE_DMAP_DEFAULT 0
 #define SCE_DMAP_COMMENT 1
 #define SCE_DMAP_NUMBER 2
diff --git a/include/Scintilla.iface b/include/Scintilla.iface
index 51e9c4735..cfbc7bc9f 100644
--- a/include/Scintilla.iface
+++ b/include/Scintilla.iface
@@ -4458,6 +4458,9 @@ val SCE_RUST_IDENTIFIER=17
 val SCE_RUST_LIFETIME=18
 val SCE_RUST_MACRO=19
 val SCE_RUST_LEXERROR=20
+val SCE_RUST_BYTESTRING=21
+val SCE_RUST_BYTESTRINGR=22
+val SCE_RUST_BYTECHARACTER=23
 # Lexical states for SCLEX_DMAP
 lex DMAP=SCLEX_DMAP SCE_DMAP_
 val SCE_DMAP_DEFAULT=0
diff --git a/lexers/LexRust.cxx b/lexers/LexRust.cxx
index d0e499533..43544ea36 100644
--- a/lexers/LexRust.cxx
+++ b/lexers/LexRust.cxx
@@ -373,12 +373,12 @@ static bool ScanNumericEscape(Accessor &styler, int& pos, int num_digits, bool s
 
 /* This is overly permissive for character literals in order to accept UTF-8 encoded
  * character literals. */
-static void ScanCharacterLiteralOrLifetime(Accessor &styler, int& pos) {
+static void ScanCharacterLiteralOrLifetime(Accessor &styler, int& pos, bool ascii_only) {
 	pos++;
 	int c = styler.SafeGetCharAt(pos, '\0');
 	int n = styler.SafeGetCharAt(pos + 1, '\0');
 	bool done = false;
-	bool valid_lifetime = IsIdentifierStart(c);
+	bool valid_lifetime = !ascii_only && IsIdentifierStart(c);
 	bool valid_char = true;
 	bool first = true;
 	while (!done) {
@@ -390,10 +390,10 @@ static void ScanCharacterLiteralOrLifetime(Accessor &styler, int& pos) {
 				} else if (n == 'x') {
 					pos += 2;
 					valid_char = ScanNumericEscape(styler, pos, 2, false);
-				} else if (n == 'u') {
+				} else if (n == 'u' && !ascii_only) {
 					pos += 2;
 					valid_char = ScanNumericEscape(styler, pos, 4, false);
-				} else if (n == 'U') {
+				} else if (n == 'U' && !ascii_only) {
 					pos += 2;
 					valid_char = ScanNumericEscape(styler, pos, 8, false);
 				} else {
@@ -412,7 +412,10 @@ static void ScanCharacterLiteralOrLifetime(Accessor &styler, int& pos) {
 				done = true;
 				break;
 			default:
-				if (!IsIdentifierContinue(c) && !first) {
+				if (ascii_only && !IsASCII((char)c)) {
+					done = true;
+					valid_char = false;
+				} else if (!IsIdentifierContinue(c) && !first) {
 					done = true;
 				} else {
 					pos++;
@@ -433,7 +436,7 @@ static void ScanCharacterLiteralOrLifetime(Accessor &styler, int& pos) {
 		styler.ColourTo(pos - 1, SCE_RUST_LIFETIME);
 	} else if (valid_char) {
 		pos++;
-		styler.ColourTo(pos - 1, SCE_RUST_CHARACTER);
+		styler.ColourTo(pos - 1, ascii_only ? SCE_RUST_BYTECHARACTER : SCE_RUST_CHARACTER);
 	} else {
 		styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 	}
@@ -542,7 +545,7 @@ static void ScanComments(Accessor &styler, int& pos, int max) {
 		ResumeBlockComment(styler, pos, max, UnknownComment, 1);
 }
 
-static void ResumeString(Accessor &styler, int& pos, int max) {
+static void ResumeString(Accessor &styler, int& pos, int max, bool ascii_only) {
 	int c = styler.SafeGetCharAt(pos, '\0');
 	bool error = false;
 	while (c != '"' && !error) {
@@ -559,10 +562,10 @@ static void ResumeString(Accessor &styler, int& pos, int max) {
 			} else if (n == 'x') {
 				pos += 2;
 				error = !ScanNumericEscape(styler, pos, 2, true);
-			} else if (n == 'u') {
+			} else if (n == 'u' && !ascii_only) {
 				pos += 2;
 				error = !ScanNumericEscape(styler, pos, 4, true);
-			} else if (n == 'U') {
+			} else if (n == 'U' && !ascii_only) {
 				pos += 2;
 				error = !ScanNumericEscape(styler, pos, 8, true);
 			} else {
@@ -570,16 +573,19 @@ static void ResumeString(Accessor &styler, int& pos, int max) {
 				error = true;
 			}
 		} else {
-			pos++;
+			if (ascii_only && !IsASCII((char)c))
+				error = true;
+			else
+				pos++;
 		}
 		c = styler.SafeGetCharAt(pos, '\0');
 	}
 	if (!error)
 		pos++;
-	styler.ColourTo(pos - 1, SCE_RUST_STRING);
+	styler.ColourTo(pos - 1, ascii_only ? SCE_RUST_BYTESTRING : SCE_RUST_STRING);
 }
 
-static void ResumeRawString(Accessor &styler, int& pos, int max, int num_hashes) {
+static void ResumeRawString(Accessor &styler, int& pos, int max, int num_hashes, bool ascii_only) {
 	for (;;) {
 		if (pos == styler.LineEnd(styler.GetLine(pos)))
 			styler.SetLineState(styler.GetLine(pos), num_hashes);
@@ -594,19 +600,20 @@ static void ResumeRawString(Accessor &styler, int& pos, int max, int num_hashes)
 			}
 			if (trailing_num_hashes == num_hashes) {
 				styler.SetLineState(styler.GetLine(pos), 0);
-				styler.ColourTo(pos - 1, SCE_RUST_STRINGR);
 				break;
 			}
 		} else if (pos >= max) {
-			styler.ColourTo(pos - 1, SCE_RUST_STRINGR);
 			break;
-		} else {		
+		} else {
+			if (ascii_only && !IsASCII((char)c)) 
+				break;
 			pos++;
 		}
 	}
+	styler.ColourTo(pos - 1, ascii_only ? SCE_RUST_BYTESTRINGR : SCE_RUST_STRINGR);
 }
 
-static void ScanRawString(Accessor &styler, int& pos, int max) {
+static void ScanRawString(Accessor &styler, int& pos, int max, bool ascii_only) {
 	pos++;
 	int num_hashes = 0;
 	while (styler.SafeGetCharAt(pos, '\0') == '#') {
@@ -617,7 +624,7 @@ static void ScanRawString(Accessor &styler, int& pos, int max) {
 		styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 	} else {
 		pos++;
-		ResumeRawString(styler, pos, max, num_hashes);
+		ResumeRawString(styler, pos, max, num_hashes, ascii_only);
 	}
 }
 
@@ -635,9 +642,13 @@ void SCI_METHOD LexerRust::Lex(unsigned int startPos, int length, int initStyle,
 	} else if (initStyle == SCE_RUST_COMMENTLINE || initStyle == SCE_RUST_COMMENTLINEDOC) {
 		ResumeLineComment(styler, pos, max, initStyle == SCE_RUST_COMMENTLINEDOC ? DocComment : NotDocComment);
 	} else if (initStyle == SCE_RUST_STRING) {
-		ResumeString(styler, pos, max);
+		ResumeString(styler, pos, max, false);
+	} else if (initStyle == SCE_RUST_BYTESTRING) {
+		ResumeString(styler, pos, max, true);
 	} else if (initStyle == SCE_RUST_STRINGR) {
-		ResumeRawString(styler, pos, max, styler.GetLineState(styler.GetLine(pos) - 1));
+		ResumeRawString(styler, pos, max, styler.GetLineState(styler.GetLine(pos) - 1), false);
+	} else if (initStyle == SCE_RUST_BYTESTRINGR) {
+		ResumeRawString(styler, pos, max, styler.GetLineState(styler.GetLine(pos) - 1), true);
 	}
 
 	while (pos < max) {
@@ -653,7 +664,16 @@ void SCI_METHOD LexerRust::Lex(unsigned int startPos, int length, int initStyle,
 		} else if (c == '/' && (n == '/' || n == '*')) {
 			ScanComments(styler, pos, max);
 		} else if (c == 'r' && (n == '#' || n == '"')) {
-			ScanRawString(styler, pos, max);
+			ScanRawString(styler, pos, max, false);
+		} else if (c == 'b' && n == 'r' && (n2 == '#' || n2 == '"')) {
+			pos++;
+			ScanRawString(styler, pos, max, true);
+		} else if (c == 'b' && n == '"') {
+			pos += 2;
+			ResumeString(styler, pos, max, true);
+		} else if (c == 'b' && n == '\'') {
+			pos++;
+			ScanCharacterLiteralOrLifetime(styler, pos, true);
 		} else if (IsIdentifierStart(c)) {
 			ScanIdentifier(styler, pos, keywords);
 		} else if (IsADigit(c)) {
@@ -668,10 +688,10 @@ void SCI_METHOD LexerRust::Lex(unsigned int startPos, int length, int initStyle,
 			pos++;
 			styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 		} else if (c == '\'') {
-			ScanCharacterLiteralOrLifetime(styler, pos);
+			ScanCharacterLiteralOrLifetime(styler, pos, false);
 		} else if (c == '"') {
 			pos++;
-			ResumeString(styler, pos, max);
+			ResumeString(styler, pos, max, false);
 		} else {
 			pos++;
 			styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);