Handles BOM on input files by removing.

Fixed for UTF-8 input.
Added D test.

3 files changed, 110 insertions, 9 deletions

diff --git a/test/examples/x.d b/test/examples/x.d
new file mode 100644
index 000000000..617aa38a1
--- /dev/null
+++ b/test/examples/x.d
@@ -0,0 +1,47 @@
+$
+// /++ +/ doccomments are not yet supported
+/* */
+/** */
+/// drdr
+/+ /+ +/ +/
+//keyword test
+keyword1
+keyword2
+keyword4
+keyword5
+keyword6
+keyword7
+//unicode identifier test
+вапёasdÓΘΣαԷԸՑהכ拉麺とひシマイ단결을
+//strings test
+'s
+'
+w's'w
+"multiline
+			string"w
+e"zz"e
+r"asd\"e
+r"multiline
+			string"c
+r`asd\`e
+`multiline
+			string`d
+x"023 abc"e
+x"023
+	abc"w
+//numbers test
+a[3..4]=3
+2.stringof
+2.0.stringof
+2.
+2.2e+2
+2.2e-2
+.2e+2
+.2
+2e+2
+0x2e+2
+0x2ep+10
+,.2.stringof,
+
+end
+ 	  	 
diff --git a/test/examples/x.d.styled b/test/examples/x.d.styled
new file mode 100644
index 000000000..ef90ffb6b
--- /dev/null
+++ b/test/examples/x.d.styled
@@ -0,0 +1,47 @@
+{14}${0}
+{2}// /++ +/ doccomments are not yet supported
+{1}/* */{0}
+{3}/** */{0}
+{15}/// drdr
+{4}/+ /+ +/ +/{0}
+{2}//keyword test
+{6}keyword1{0}
+{7}keyword2{0}
+{9}keyword4{0}
+{8}keyword5{0}
+{20}keyword6{0}
+{21}keyword7{0}
+{2}//unicode identifier test
+{14}вапёasdÓΘΣαԷԸՑהכ拉麺とひシマイ단결을{0}
+{2}//strings test
+{11}'s
+'
+{14}w{12}'s'{14}w{0}
+{10}"multiline
+			string"w{0}
+{14}e{10}"zz"{14}e{0}
+{19}r"asd\"{14}e{0}
+{19}r"multiline
+			string"c{0}
+{14}r{18}`asd\`{14}e{0}
+{18}`multiline
+			string`d{0}
+{19}x"023 abc"{14}e{0}
+{19}x"023
+	abc"w{0}
+{2}//numbers test
+{14}a{13}[{5}3{13}..{5}4{13}]={5}3{0}
+{5}2.stringof{0}
+{5}2.0{13}.{14}stringof{0}
+{5}2.{0}
+{5}2.2e+2{0}
+{5}2.2e-2{0}
+{5}.2e+2{0}
+{5}.2{0}
+{5}2e+2{0}
+{5}0x2e{13}+{5}2{0}
+{5}0x2ep+10{0}
+{13},{5}.2{13}.{14}stringof{13},{0}
+
+{14}end{0}
+ 	  	 
diff --git a/test/lexTests.py b/test/lexTests.py
index b4caa95f6..c90f22e9b 100644
--- a/test/lexTests.py
+++ b/test/lexTests.py
@@ -24,18 +24,17 @@ class TestLexers(unittest.TestCase):
 		self.ed.EmptyUndoBuffer()
 
 	def AsStyled(self):
-		data = io.StringIO()
-		len = 0
+		text = self.ed.Contents()
+		data = io.BytesIO()
 		prevStyle = -1
 		for o in range(self.ed.Length):
-			styleNow = self.ed.GetStyleAt(len)
+			styleNow = self.ed.GetStyleAt(o)
 			if styleNow != prevStyle:
 				styleBuf = "{%0d}" % styleNow
-				data.write(styleBuf)
+				data.write(styleBuf.encode('utf-8'))
 				prevStyle = styleNow
-			data.write(chr(self.ed.GetCharAt(len)))
-			len += 1
-		return data.getvalue( )
+			data.write(text[o:o+1])
+		return data.getvalue()
 
 	def LexExample(self, name, lexerName, keywords=None):
 		if keywords is None:
@@ -52,13 +51,16 @@ class TestLexers(unittest.TestCase):
 		nameNew = nameExample +".new"
 		with open(nameExample, "rb") as f:
 			prog = f.read()
+		BOM = b"\xEF\xBB\xBF"
+		if prog.startswith(BOM):
+			prog = prog[len(BOM):]
 		lenDocument = len(prog)
 		self.ed.AddText(lenDocument, prog)
 		self.ed.Colourise(0, lenDocument)
 		self.assertEquals(self.ed.EndStyled, lenDocument)
 		with open(namePrevious, "rb") as f:
 			prevStyled = f.read()
-		progStyled = self.AsStyled().encode('utf-8')
+		progStyled = self.AsStyled()
 		if progStyled != prevStyled:
 			with open(nameNew, "wb") as f:
 				f.write(progStyled)
@@ -75,7 +77,7 @@ class TestLexers(unittest.TestCase):
 			self.ed.StartStyling(lineStart, mask)
 			self.assertEquals(self.ed.EndStyled, lineStart)
 			self.ed.Colourise(0, lenDocument)
-			progStyled = self.AsStyled().encode('utf-8')
+			progStyled = self.AsStyled()
 			if progStyled != prevStyled:
 				with open(nameNew, "wb") as f:
 					f.write(progStyled)
@@ -102,5 +104,10 @@ class TestLexers(unittest.TestCase):
 	def testVB(self):
 		self.LexExample("x.vb", b"vb", [b"as dim or string"])
 
+	def testD(self):
+		self.LexExample("x.d", b"d", 
+			[b"keyword1", b"keyword2", b"", b"keyword4", b"keyword5",
+			b"keyword6", b"keyword7"])
+
 if __name__ == '__main__':
 	XiteWin.main("lexTests")