Bug [#1793]. Add support for Perl 5.22.

Double-diamond operator <<>>
Hexadecimal floating point literals
Repetition in list assignment
Added example file to the lexing tests and changed lexing tests to allow larger
files and to standardise on Unix line ends when testing these larger files to
avoid spurious failures.

1 files changed, 65 insertions, 21 deletions

diff --git a/test/lexTests.py b/test/lexTests.py
index a53db7eb8..421c6393b 100644
--- a/test/lexTests.py
+++ b/test/lexTests.py
@@ -15,6 +15,37 @@ b"function",
 b"sub"
 ]
 
+keywordsPerl = [
+b"NULL __FILE__ __LINE__ __PACKAGE__ __DATA__ __END__ AUTOLOAD "
+b"BEGIN CORE DESTROY END EQ GE GT INIT LE LT NE CHECK abs accept "
+b"alarm and atan2 bind binmode bless caller chdir chmod chomp chop "
+b"chown chr chroot close closedir cmp connect continue cos crypt "
+b"dbmclose dbmopen defined delete die do dump each else elsif endgrent "
+b"endhostent endnetent endprotoent endpwent endservent eof eq eval "
+b"exec exists exit exp fcntl fileno flock for foreach fork format "
+b"formline ge getc getgrent getgrgid getgrnam gethostbyaddr gethostbyname "
+b"gethostent getlogin getnetbyaddr getnetbyname getnetent getpeername "
+b"getpgrp getppid getpriority getprotobyname getprotobynumber getprotoent "
+b"getpwent getpwnam getpwuid getservbyname getservbyport getservent "
+b"getsockname getsockopt glob gmtime goto grep gt hex if index "
+b"int ioctl join keys kill last lc lcfirst le length link listen "
+b"local localtime lock log lstat lt map mkdir msgctl msgget msgrcv "
+b"msgsnd my ne next no not oct open opendir or ord our pack package "
+b"pipe pop pos print printf prototype push quotemeta qu "
+b"rand read readdir readline readlink readpipe recv redo "
+b"ref rename require reset return reverse rewinddir rindex rmdir "
+b"scalar seek seekdir select semctl semget semop send setgrent "
+b"sethostent setnetent setpgrp setpriority setprotoent setpwent "
+b"setservent setsockopt shift shmctl shmget shmread shmwrite shutdown "
+b"sin sleep socket socketpair sort splice split sprintf sqrt srand "
+b"stat study sub substr symlink syscall sysopen sysread sysseek "
+b"system syswrite tell telldir tie tied time times truncate "
+b"uc ucfirst umask undef unless unlink unpack unshift untie until "
+b"use utime values vec wait waitpid wantarray warn while write "
+b"xor "
+b"given when default break say state UNITCHECK __SUB__ fc"
+]
+
 class TestLexers(unittest.TestCase):
 
 	def setUp(self):
@@ -23,7 +54,7 @@ class TestLexers(unittest.TestCase):
 		self.ed.ClearAll()
 		self.ed.EmptyUndoBuffer()
 
-	def AsStyled(self):
+	def AsStyled(self, withWindowsLineEnds):
 		text = self.ed.Contents()
 		data = io.BytesIO()
 		prevStyle = -1
@@ -34,11 +65,14 @@ class TestLexers(unittest.TestCase):
 				data.write(styleBuf.encode('utf-8'))
 				prevStyle = styleNow
 			data.write(text[o:o+1])
-		return data.getvalue()
+		if withWindowsLineEnds:
+			return data.getvalue().replace(b"\n", b"\r\n")
+		else:
+			return data.getvalue()
 
-	def LexExample(self, name, lexerName, keywords=None):
-		if keywords is None:
-			keywords = []
+	def LexExample(self, name, lexerName, keywords, fileMode="b"):
+		self.ed.ClearAll()
+		self.ed.EmptyUndoBuffer()
 		self.ed.SetCodePage(65001)
 		self.ed.LexerLanguage = lexerName
 		bits = self.ed.StyleBitsNeeded
@@ -52,6 +86,8 @@ class TestLexers(unittest.TestCase):
 		nameNew = nameExample +".new"
 		with open(nameExample, "rb") as f:
 			prog = f.read()
+		if fileMode == "t" and sys.platform == "win32":
+			prog = prog.replace(b"\r\n", b"\n")
 		BOM = b"\xEF\xBB\xBF"
 		if prog.startswith(BOM):
 			prog = prog[len(BOM):]
@@ -62,12 +98,15 @@ class TestLexers(unittest.TestCase):
 		try:
 			with open(namePrevious, "rb") as f:
 				prevStyled = f.read()
-		except FileNotFoundError:
+			if fileMode == "t" and sys.platform == "win32":
+				prog = prog.replace(b"\r\n", b"\n")
+		except EnvironmentError:
 			prevStyled = ""
-		progStyled = self.AsStyled()
+		progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32")
 		if progStyled != prevStyled:
 			with open(nameNew, "wb") as f:
 				f.write(progStyled)
+			print("Incorrect lex for " + name)
 			print(progStyled)
 			print(prevStyled)
 			self.assertEquals(progStyled, prevStyled)
@@ -75,19 +114,21 @@ class TestLexers(unittest.TestCase):
 			# as that is likely to fail many times.
 			return
 
-		# Try partial lexes from the start of every line which should all be identical.
-		for line in range(self.ed.LineCount):
-			lineStart = self.ed.PositionFromLine(line)
-			self.ed.StartStyling(lineStart, mask)
-			self.assertEquals(self.ed.EndStyled, lineStart)
-			self.ed.Colourise(lineStart, lenDocument)
-			progStyled = self.AsStyled()
-			if progStyled != prevStyled:
-				with open(nameNew, "wb") as f:
-					f.write(progStyled)
-				self.assertEquals(progStyled, prevStyled)
-				# Give up after one failure
-				return
+		if fileMode == "b":	# "t" files are large and this is a quadratic check
+			# Try partial lexes from the start of every line which should all be identical.
+			for line in range(self.ed.LineCount):
+				lineStart = self.ed.PositionFromLine(line)
+				self.ed.StartStyling(lineStart, mask)
+				self.assertEquals(self.ed.EndStyled, lineStart)
+				self.ed.Colourise(lineStart, lenDocument)
+				progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32")
+				if progStyled != prevStyled:
+					print("Incorrect partial lex for " + name + " at line " + line)
+					with open(nameNew, "wb") as f:
+						f.write(progStyled)
+					self.assertEquals(progStyled, prevStyled)
+					# Give up after one failure
+					return
 
 	def testCXX(self):
 		self.LexExample("x.cxx", b"cpp", [b"int"])
@@ -115,7 +156,10 @@ class TestLexers(unittest.TestCase):
 		self.LexExample("x.rb", b"ruby", [b"class def end"])
 
 	def testPerl(self):
-		self.LexExample("x.pl", b"perl", [b"printf sleep use while"])
+		self.LexExample("x.pl", b"perl", keywordsPerl)
+
+	def testPerlCases(self):
+		self.LexExample("perl-test-5220delta.pl", b"perl", keywordsPerl, "t")
 
 	def testD(self):
 		self.LexExample("x.d", b"d",