modify LexVerilog.cxx to support coloring of inactive code due to preprocessor commands

1 files changed, 652 insertions, 142 deletions

diff --git a/lexers/LexVerilog.cxx b/lexers/LexVerilog.cxx
index 8062c09cf..dae650bf8 100644
--- a/lexers/LexVerilog.cxx
+++ b/lexers/LexVerilog.cxx
@@ -24,10 +24,303 @@
 #include "CharacterSet.h"
 #include "LexerModule.h"
 
+#include <string>
+#include <vector>
+#include <map>
+
+#include "OptionSet.h"
+#include "SubStyles.h"
+
 #ifdef SCI_NAMESPACE
 using namespace Scintilla;
 #endif
 
+namespace {
+	// Use an unnamed namespace to protect the functions and classes from name conflicts
+
+struct PPDefinition {
+	int line;
+	std::string key;
+	std::string value;
+	bool isUndef;
+	std::string arguments;
+	PPDefinition(int line_, const std::string &key_, const std::string &value_, bool isUndef_ = false, std::string arguments_="") :
+		line(line_), key(key_), value(value_), isUndef(isUndef_), arguments(arguments_) {
+	}
+};
+
+class LinePPState {
+	int state;
+	int ifTaken;
+	int level;
+	bool ValidLevel() const {
+		return level >= 0 && level < 32;
+	}
+	int maskLevel() const {
+		return 1 << level;
+	}
+public:
+	LinePPState() : state(0), ifTaken(0), level(-1) {
+	}
+	bool IsInactive() const {
+		return state != 0;
+	}
+	bool CurrentIfTaken() const {
+		return (ifTaken & maskLevel()) != 0;
+	}
+	void StartSection(bool on) {
+		level++;
+		if (ValidLevel()) {
+			if (on) {
+				state &= ~maskLevel();
+				ifTaken |= maskLevel();
+			} else {
+				state |= maskLevel();
+				ifTaken &= ~maskLevel();
+			}
+		}
+	}
+	void EndSection() {
+		if (ValidLevel()) {
+			state &= ~maskLevel();
+			ifTaken &= ~maskLevel();
+		}
+		level--;
+	}
+	void InvertCurrentLevel() {
+		if (ValidLevel()) {
+			state ^= maskLevel();
+			ifTaken |= maskLevel();
+		}
+	}
+};
+
+// Hold the preprocessor state for each line seen.
+// Currently one entry per line but could become sparse with just one entry per preprocessor line.
+class PPStates {
+	std::vector<LinePPState> vlls;
+public:
+	LinePPState ForLine(int line) const {
+		if ((line > 0) && (vlls.size() > static_cast<size_t>(line))) {
+			return vlls[line];
+		} else {
+			return LinePPState();
+		}
+	}
+	void Add(int line, LinePPState lls) {
+		vlls.resize(line+1);
+		vlls[line] = lls;
+	}
+};
+
+// Options used for LexerVerilog
+struct OptionsVerilog {
+	bool foldComment;
+	bool foldPreprocessor;
+	bool foldCompact;
+	bool foldAtElse;
+	bool foldAtModule;
+	bool trackPreprocessor;
+	bool updatePreprocessor;
+	OptionsVerilog() {
+		foldComment = false;
+		foldPreprocessor = false;
+		foldCompact = false;
+		foldAtElse = false;
+		foldAtModule = false;
+		// for backwards compatibility, preprocessor functionality is disabled by default
+		trackPreprocessor = false;
+		updatePreprocessor = false;
+	}
+};
+
+struct OptionSetVerilog : public OptionSet<OptionsVerilog> {
+	OptionSetVerilog() {
+		DefineProperty("fold.comment", &OptionsVerilog::foldComment,
+			"This option enables folding multi-line comments when using the Verilog lexer.");
+		DefineProperty("fold.preprocessor", &OptionsVerilog::foldPreprocessor,
+			"This option enables folding preprocessor directives when using the Verilog lexer.");
+		DefineProperty("fold.compact", &OptionsVerilog::foldCompact);
+		DefineProperty("fold.at.else", &OptionsVerilog::foldAtElse,
+			"This option enables folding on the else line of an if statement.");
+		DefineProperty("fold.verilog.flags", &OptionsVerilog::foldAtModule,
+			"This option enables folding module definitions. Typically source files "
+			"contain only one module definition so this option is somewhat useless.");
+		DefineProperty("lexer.verilog.track.preprocessor", &OptionsVerilog::trackPreprocessor,
+			"Set to 1 to interpret `if/`else/`endif to grey out code that is not active.");
+		DefineProperty("lexer.verilog.update.preprocessor", &OptionsVerilog::updatePreprocessor,
+			"Set to 1 to update preprocessor definitions when `define, `undef, or `undefineall found.");
+	}
+};
+
+const char styleSubable[] = {0};
+
+}
+
+class LexerVerilog : public ILexerWithSubStyles {
+	CharacterSet setWord;
+	WordList keywords;
+	WordList keywords2;
+	WordList keywords3;
+	WordList keywords4;
+	WordList keywords5;
+	WordList ppDefinitions;
+	PPStates vlls;
+	std::vector<PPDefinition> ppDefineHistory;
+	struct SymbolValue {
+		std::string value;
+		std::string arguments;
+		SymbolValue(const std::string &value_="", const std::string &arguments_="") : value(value_), arguments(arguments_) {
+		}
+		SymbolValue &operator = (const std::string &value_) {
+			value = value_;
+			arguments.clear();
+			return *this;
+		}
+		bool IsMacro() const {
+			return !arguments.empty();
+		}
+	};
+	typedef std::map<std::string, SymbolValue> SymbolTable;
+	SymbolTable preprocessorDefinitionsStart;
+	OptionsVerilog options;
+	OptionSetVerilog osVerilog;
+	enum { activeFlag = 0x40 };
+	SubStyles subStyles;
+public:
+	LexerVerilog() :
+		setWord(CharacterSet::setAlphaNum, "._", 0x80, true),
+		subStyles(styleSubable, 0x80, 0x40, activeFlag) {
+		}
+	virtual ~LexerVerilog() {}
+	int SCI_METHOD Version() const {
+		return lvSubStyles;
+	}
+	void SCI_METHOD Release() {
+		delete this;
+	}
+	const char* SCI_METHOD PropertyNames() {
+		return osVerilog.PropertyNames();
+	}
+	int SCI_METHOD PropertyType(const char* name) {
+		return osVerilog.PropertyType(name);
+	}
+	const char* SCI_METHOD DescribeProperty(const char* name) {
+		return osVerilog.DescribeProperty(name);
+	}
+	int SCI_METHOD PropertySet(const char* key, const char* val) {
+	    return osVerilog.PropertySet(&options, key, val);
+	}
+	const char* SCI_METHOD DescribeWordListSets() {
+		return osVerilog.DescribeWordListSets();
+	}
+	int SCI_METHOD WordListSet(int n, const char* wl);
+	void SCI_METHOD Lex(unsigned int startPos, int length, int initStyle, IDocument *pAccess);
+	void SCI_METHOD Fold(unsigned int startPos, int length, int initStyle, IDocument *pAccess);
+	void* SCI_METHOD PrivateCall(int, void*) {
+		return 0;
+	}
+	int SCI_METHOD LineEndTypesSupported() {
+		return SC_LINE_END_TYPE_UNICODE;
+	}
+	int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) {
+		return subStyles.Allocate(styleBase, numberStyles);
+	}
+	int SCI_METHOD SubStylesStart(int styleBase) {
+		return subStyles.Start(styleBase);
+	}
+	int SCI_METHOD SubStylesLength(int styleBase) {
+		return subStyles.Length(styleBase);
+	}
+	int SCI_METHOD StyleFromSubStyle(int subStyle) {
+		int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
+		int active = subStyle & activeFlag;
+		return styleBase | active;
+	}
+	int SCI_METHOD PrimaryStyleFromStyle(int style) {
+		return MaskActive(style);
+ 	}
+	void SCI_METHOD FreeSubStyles() {
+		subStyles.Free();
+	}
+	void SCI_METHOD SetIdentifiers(int style, const char *identifiers) {
+		subStyles.SetIdentifiers(style, identifiers);
+	}
+	int SCI_METHOD DistanceToSecondaryStyles() {
+		return activeFlag;
+	}
+	const char * SCI_METHOD GetSubStyleBases() {
+		return styleSubable;
+	}
+	static ILexer* LexerFactoryVerilog() {
+		return new LexerVerilog();
+	}
+	static int MaskActive(int style) {
+		return style & ~activeFlag;
+	}
+	std::vector<std::string> Tokenize(const std::string &expr) const;
+};
+
+int SCI_METHOD LexerVerilog::WordListSet(int n, const char *wl) {
+	WordList *wordListN = 0;
+	switch (n) {
+	case 0:
+		wordListN = &keywords;
+		break;
+	case 1:
+		wordListN = &keywords2;
+		break;
+	case 2:
+		wordListN = &keywords3;
+		break;
+	case 3:
+		wordListN = &keywords4;
+		break;
+	case 4:
+		wordListN = &keywords5;
+		break;
+	case 5:
+		wordListN = &ppDefinitions;
+		break;
+	}
+	int firstModification = -1;
+	if (wordListN) {
+		WordList wlNew;
+		wlNew.Set(wl);
+		if (*wordListN != wlNew) {
+			wordListN->Set(wl);
+			firstModification = 0;
+			if (n == 5) {
+				// Rebuild preprocessorDefinitions
+				preprocessorDefinitionsStart.clear();
+				for (int nDefinition = 0; nDefinition < ppDefinitions.Length(); nDefinition++) {
+					const char *cpDefinition = ppDefinitions.WordAt(nDefinition);
+					const char *cpEquals = strchr(cpDefinition, '=');
+					if (cpEquals) {
+						std::string name(cpDefinition, cpEquals - cpDefinition);
+						std::string val(cpEquals+1);
+						size_t bracket = name.find('(');
+						size_t bracketEnd = name.find(')');
+						if ((bracket != std::string::npos) && (bracketEnd != std::string::npos)) {
+							// Macro
+							std::string args = name.substr(bracket + 1, bracketEnd - bracket - 1);
+							name = name.substr(0, bracket);
+							preprocessorDefinitionsStart[name] = SymbolValue(val, args);
+						} else {
+							preprocessorDefinitionsStart[name] = val;
+						}
+					} else {
+						std::string name(cpDefinition);
+						std::string val("1");
+						preprocessorDefinitionsStart[name] = val;
+					}
+				}
+			}
+		}
+	}
+	return firstModification;
+}
+
 static inline bool IsAWordChar(const int ch) {
 	return (ch < 0x80) && (isalnum(ch) || ch == '.' || ch == '_' || ch == '\''|| ch == '$');
 }
@@ -44,17 +337,43 @@ static inline bool AllUpperCase(const char *a) {
 	return true;
 }
 
-static void ColouriseVerilogDoc(unsigned int startPos, int length, int initStyle, WordList *keywordlists[],
-                            Accessor &styler) {
+// Functor used to truncate history
+struct After {
+	int line;
+	explicit After(int line_) : line(line_) {}
+	bool operator()(PPDefinition &p) const {
+		return p.line > line;
+	}
+};
+
+static std::string GetRestOfLine(LexAccessor &styler, int start, bool allowSpace) {
+	std::string restOfLine;
+	int i =0;
+	char ch = styler.SafeGetCharAt(start, '\n');
+	int endLine = styler.LineEnd(styler.GetLine(start));
+	while (((start+i) < endLine) && (ch != '\r')) {
+		char chNext = styler.SafeGetCharAt(start + i + 1, '\n');
+		if (ch == '/' && (chNext == '/' || chNext == '*'))
+			break;
+		if (allowSpace || (ch != ' '))
+			restOfLine += ch;
+		i++;
+		ch = chNext;
+	}
+	return restOfLine;
+}
+
+static bool IsSpaceOrTab(int ch) {
+	return ch == ' ' || ch == '\t';
+}
+
+void SCI_METHOD LexerVerilog::Lex(unsigned int startPos, int length, int initStyle, IDocument *pAccess)
+{
+	LexAccessor styler(pAccess);
 
 	const int kwOther=0, kwDot=0x100, kwInput=0x200, kwOutput=0x300, kwInout=0x400;
 	int lineState = kwOther;
-
-	WordList &keywords = *keywordlists[0];
-	WordList &keywords2 = *keywordlists[1];
-	WordList &keywords3 = *keywordlists[2];
-	WordList &keywords4 = *keywordlists[3];
-	WordList &keywords5 = *keywordlists[4];
+	bool continuationLine = false;
 
 	int curLine = styler.GetLine(startPos);
 	if (curLine > 0) lineState = styler.GetLineState(curLine - 1);
@@ -63,158 +382,336 @@ static void ColouriseVerilogDoc(unsigned int startPos, int length, int initStyle
 	if (initStyle == SCE_V_STRINGEOL)
 		initStyle = SCE_V_DEFAULT;
 
+	if ((MaskActive(initStyle) == SCE_V_PREPROCESSOR) ||
+			(MaskActive(initStyle) == SCE_V_COMMENTLINE) ||
+			(MaskActive(initStyle) == SCE_V_COMMENTLINEBANG)) {
+		// Set continuationLine if last character of previous line is '\'
+		if (curLine > 0) {
+			int endLinePrevious = styler.LineEnd(curLine - 1);
+			if (endLinePrevious > 0) {
+				continuationLine = styler.SafeGetCharAt(endLinePrevious-1) == '\\';
+			}
+		}
+	}
+
 	StyleContext sc(startPos, length, initStyle, styler);
+	LinePPState preproc = vlls.ForLine(curLine);
+
+	bool definitionsChanged = false;
+
+	// Truncate ppDefineHistory before current line
+
+	if (!options.updatePreprocessor)
+		ppDefineHistory.clear();
+
+	std::vector<PPDefinition>::iterator itInvalid = std::find_if(ppDefineHistory.begin(), ppDefineHistory.end(), After(curLine-1));
+	if (itInvalid != ppDefineHistory.end()) {
+		ppDefineHistory.erase(itInvalid, ppDefineHistory.end());
+		definitionsChanged = true;
+	}
+
+	SymbolTable preprocessorDefinitions = preprocessorDefinitionsStart;
+	for (std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef) {
+		if (itDef->isUndef)
+			preprocessorDefinitions.erase(itDef->key);
+		else
+			preprocessorDefinitions[itDef->key] = SymbolValue(itDef->value, itDef->arguments);
+	}
+
+	int activitySet = preproc.IsInactive() ? activeFlag : 0;
+	int lineEndNext = styler.LineEnd(curLine);
 
 	for (; sc.More(); sc.Forward()) {
-		curLine = styler.GetLine(sc.currentPos);
+		if (sc.atLineStart) {
+			if (sc.state == SCE_V_STRING) {
+				// Prevent SCE_V_STRINGEOL from leaking back to previous line
+				sc.SetState(SCE_V_STRING);
+			}
+			if ((MaskActive(sc.state) == SCE_V_PREPROCESSOR) && (!continuationLine)) {
+				sc.SetState(SCE_V_DEFAULT|activitySet);
+			}
+			if (preproc.IsInactive()) {
+				activitySet = activeFlag;
+				sc.SetState(sc.state | activitySet);
+			}
+		}
 
 		if (sc.atLineEnd) {
+			curLine++;
+			lineEndNext = styler.LineEnd(curLine);
+			vlls.Add(curLine, preproc);
 			// Update the line state, so it can be seen by next line
 			styler.SetLineState(curLine, lineState);
 		}
 
-		if (sc.atLineStart && (sc.state == SCE_V_STRING)) {
-			// Prevent SCE_V_STRINGEOL from leaking back to previous line
-			sc.SetState(SCE_V_STRING);
-		}
-
 		// Handle line continuation generically.
 		if (sc.ch == '\\') {
-			if (sc.chNext == '\n' || sc.chNext == '\r') {
+			if (static_cast<int>((sc.currentPos+1)) >= lineEndNext) {
+				curLine++;
+				lineEndNext = styler.LineEnd(curLine);
+				vlls.Add(curLine, preproc);
+				// Update the line state, so it can be seen by next line
+				styler.SetLineState(curLine, lineState);
 				sc.Forward();
 				if (sc.ch == '\r' && sc.chNext == '\n') {
+					// Even in UTF-8, \r and \n are separate
 					sc.Forward();
 				}
+				continuationLine = true;
+				sc.Forward();
 				continue;
 			}
 		}
 
-		// for comment keyword
-		if (sc.state == SCE_V_COMMENT_WORD && !IsAWordChar(sc.ch)) {
-			char s[100];
-			int state = lineState & 0xff;
-			sc.GetCurrent(s, sizeof(s));
-			if (keywords5.InList(s)) {
-				sc.ChangeState(SCE_V_COMMENT_WORD);
-			} else {
-				sc.ChangeState(state);
-			}
-			sc.SetState(state);
-		}
+		const bool atLineEndBeforeSwitch = sc.atLineEnd;
+
 		// Determine if the current state should terminate.
-		if (sc.state == SCE_V_OPERATOR) {
-			sc.SetState(SCE_V_DEFAULT);
-		} else if (sc.state == SCE_V_NUMBER) {
-			if (!(IsAWordChar(sc.ch) || (sc.ch == '?'))) {
-				sc.SetState(SCE_V_DEFAULT);
-			}
-		} else if (sc.state == SCE_V_IDENTIFIER) {
-			if (!IsAWordChar(sc.ch) || (sc.ch == '.')) {
-				char s[100];
-				lineState &= 0xff00;
-
-				sc.GetCurrent(s, sizeof(s));
-
-				if (strcmp(s, "input") == 0) {
-					lineState = kwInput;
-					sc.ChangeState(SCE_V_INPUT);
-				} else if (strcmp(s, "output") == 0) {
-					lineState = kwOutput;
-					sc.ChangeState(SCE_V_OUTPUT);
-				} else if (strcmp(s, "inout") == 0) {
-					lineState = kwInout;
-					sc.ChangeState(SCE_V_INOUT);
-
-				} else if (lineState == kwInput) {
-					sc.ChangeState(SCE_V_INPUT);
-				} else if (lineState == kwOutput) {
-					sc.ChangeState(SCE_V_OUTPUT);
-				} else if (lineState == kwInout) {
-					sc.ChangeState(SCE_V_INOUT);
-				} else if (lineState == kwDot) {
-					lineState = kwOther;
-					sc.ChangeState(SCE_V_PORT_CONNECT);
-
-				} else if (keywords.InList(s)) {
-					sc.ChangeState(SCE_V_WORD);
-				} else if (keywords2.InList(s)) {
-					sc.ChangeState(SCE_V_WORD2);
-				} else if (keywords3.InList(s)) {
-					sc.ChangeState(SCE_V_WORD3);
-				} else if (keywords4.InList(s)) {
-					sc.ChangeState(SCE_V_USER);
-
-				} else if (AllUpperCase(s)) {
-					sc.ChangeState(SCE_V_USER);
+		switch (MaskActive(sc.state)) {
+			case SCE_V_COMMENT_WORD:
+				// for comment keyword
+				if (!IsAWordChar(sc.ch)) {
+					char s[100];
+					int state = lineState & 0xff;
+					sc.GetCurrent(s, sizeof(s));
+					if (keywords5.InList(s)) {
+						sc.ChangeState(SCE_V_COMMENT_WORD|activitySet);
+					} else {
+						sc.ChangeState(state|activitySet);
+					}
+				    sc.SetState(state|activitySet);
 				}
-
-				sc.SetState(SCE_V_DEFAULT);
-			}
-		} else if (sc.state == SCE_V_PREPROCESSOR) {
-			if (!IsAWordChar(sc.ch)) {
-				sc.SetState(SCE_V_DEFAULT);
-			}
-		} else if (sc.state == SCE_V_COMMENT) {
-			if (sc.Match('*', '/')) {
-				sc.Forward();
-				sc.ForwardSetState(SCE_V_DEFAULT);
-			} else if (IsAWordStart(sc.ch)) {
-				lineState = sc.state | (lineState & 0xff00);
-				sc.SetState(SCE_V_COMMENT_WORD);
-			}
-		} else if (sc.state == SCE_V_COMMENTLINE || sc.state == SCE_V_COMMENTLINEBANG) {
-			if (sc.atLineStart) {
-				sc.SetState(SCE_V_DEFAULT);
-			} else if (IsAWordStart(sc.ch)) {
-				lineState = sc.state | (lineState & 0xff00);
-				sc.SetState(SCE_V_COMMENT_WORD);
-			}
-		} else if (sc.state == SCE_V_STRING) {
-			if (sc.ch == '\\') {
-				if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
+				break;
+			case SCE_V_OPERATOR:
+				sc.SetState(SCE_V_DEFAULT|activitySet);
+				break;
+			case SCE_V_NUMBER:
+				if (!(IsAWordChar(sc.ch) || (sc.ch == '?'))) {
+					sc.SetState(SCE_V_DEFAULT|activitySet);
+				}
+				break;
+			case SCE_V_IDENTIFIER:
+				if (!IsAWordChar(sc.ch) || (sc.ch == '.')) {
+					char s[100];
+					lineState &= 0xff00;
+					sc.GetCurrent(s, sizeof(s));
+					if (strcmp(s, "input") == 0) {
+						lineState = kwInput;
+						sc.ChangeState(SCE_V_INPUT|activitySet);
+					} else if (strcmp(s, "output") == 0) {
+						lineState = kwOutput;
+						sc.ChangeState(SCE_V_OUTPUT|activitySet);
+					} else if (strcmp(s, "inout") == 0) {
+						lineState = kwInout;
+						sc.ChangeState(SCE_V_INOUT|activitySet);
+					} else if (lineState == kwInput) {
+						sc.ChangeState(SCE_V_INPUT|activitySet);
+					} else if (lineState == kwOutput) {
+						sc.ChangeState(SCE_V_OUTPUT|activitySet);
+					} else if (lineState == kwInout) {
+						sc.ChangeState(SCE_V_INOUT|activitySet);
+					} else if (lineState == kwDot) {
+						lineState = kwOther;
+						sc.ChangeState(SCE_V_PORT_CONNECT|activitySet);
+					} else if (keywords.InList(s)) {
+						sc.ChangeState(SCE_V_WORD|activitySet);
+					} else if (keywords2.InList(s)) {
+						sc.ChangeState(SCE_V_WORD2|activitySet);
+					} else if (keywords3.InList(s)) {
+						sc.ChangeState(SCE_V_WORD3|activitySet);
+					} else if (keywords4.InList(s)) {
+						sc.ChangeState(SCE_V_USER|activitySet);
+					} else if (AllUpperCase(s)) {
+						sc.ChangeState(SCE_V_USER|activitySet);
+					}
+					sc.SetState(SCE_V_DEFAULT|activitySet);
+				}
+				break;
+			case SCE_V_PREPROCESSOR:
+				if (!IsAWordChar(sc.ch) && !sc.atLineEnd) {
+					sc.SetState(SCE_V_DEFAULT|activitySet);
+				}
+				break;
+			case SCE_V_COMMENT:
+				if (sc.Match('*', '/')) {
 					sc.Forward();
+					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
+				} else if (IsAWordStart(sc.ch)) {
+					lineState = sc.state | (lineState & 0xff00);
+					sc.SetState(SCE_V_COMMENT_WORD|activitySet);
 				}
-			} else if (sc.ch == '\"') {
-				sc.ForwardSetState(SCE_V_DEFAULT);
-			} else if (sc.atLineEnd) {
-				sc.ChangeState(SCE_V_STRINGEOL);
-				sc.ForwardSetState(SCE_V_DEFAULT);
-			}
+				break;
+			case SCE_V_COMMENTLINE:
+			case SCE_V_COMMENTLINEBANG:
+				if (sc.atLineStart) {
+					sc.SetState(SCE_V_DEFAULT|activitySet);
+				} else if (IsAWordStart(sc.ch)) {
+					lineState = sc.state | (lineState & 0xff00);
+					sc.SetState(SCE_V_COMMENT_WORD|activitySet);
+				}
+				break;
+			case SCE_V_STRING:
+				if (sc.ch == '\\') {
+					if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
+						sc.Forward();
+					}
+				} else if (sc.ch == '\"') {
+					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
+				} else if (sc.atLineEnd) {
+					sc.ChangeState(SCE_V_STRINGEOL|activitySet);
+					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
+				}
+				break;
+		}
+
+		if (sc.atLineEnd && !atLineEndBeforeSwitch) {
+			// State exit processing consumed characters up to end of line.
+			curLine++;
+			lineEndNext = styler.LineEnd(curLine);
+			vlls.Add(curLine, preproc);
+			// Update the line state, so it can be seen by next line
+			styler.SetLineState(curLine, lineState);
 		}
 
 		// Determine if a new state should be entered.
-		if (sc.state == SCE_V_DEFAULT) {
+		if (MaskActive(sc.state) == SCE_V_DEFAULT) {
 			if (IsADigit(sc.ch) || (sc.ch == '\'') || (sc.ch == '.' && IsADigit(sc.chNext))) {
-				sc.SetState(SCE_V_NUMBER);
+				sc.SetState(SCE_V_NUMBER|activitySet);
 			} else if (IsAWordStart(sc.ch)) {
-				sc.SetState(SCE_V_IDENTIFIER);
+				sc.SetState(SCE_V_IDENTIFIER|activitySet);
 			} else if (sc.Match('/', '*')) {
-				sc.SetState(SCE_V_COMMENT);
+				sc.SetState(SCE_V_COMMENT|activitySet);
 				sc.Forward();	// Eat the * so it isn't used for the end of the comment
 			} else if (sc.Match('/', '/')) {
 				if (sc.Match("//!"))	// Nice to have a different comment style
-					sc.SetState(SCE_V_COMMENTLINEBANG);
+					sc.SetState(SCE_V_COMMENTLINEBANG|activitySet);
 				else
-					sc.SetState(SCE_V_COMMENTLINE);
+					sc.SetState(SCE_V_COMMENTLINE|activitySet);
 			} else if (sc.ch == '\"') {
-				sc.SetState(SCE_V_STRING);
+				sc.SetState(SCE_V_STRING|activitySet);
 			} else if (sc.ch == '`') {
-				sc.SetState(SCE_V_PREPROCESSOR);
+				sc.SetState(SCE_V_PREPROCESSOR|activitySet);
 				// Skip whitespace between ` and preprocessor word
 				do {
 					sc.Forward();
 				} while ((sc.ch == ' ' || sc.ch == '\t') && sc.More());
 				if (sc.atLineEnd) {
-					sc.SetState(SCE_V_DEFAULT);
+					sc.SetState(SCE_V_DEFAULT|activitySet);
 					styler.SetLineState(curLine, lineState);
+				} else {
+					if (options.trackPreprocessor) {
+						if (sc.Match("ifdef") || sc.Match("ifndef")) {
+							bool isIfDef = sc.Match("ifdef");
+							int i = isIfDef ? 5 : 6;
+							std::string restOfLine = GetRestOfLine(styler, sc.currentPos + i + 1, false);
+							bool foundDef = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
+							preproc.StartSection(isIfDef == foundDef);
+						} else if (sc.Match("else")) {
+							if (!preproc.CurrentIfTaken()) {
+								preproc.InvertCurrentLevel();
+								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								if (!activitySet) {
+									sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
+								}
+							} else if (!preproc.IsInactive()) {
+								preproc.InvertCurrentLevel();
+								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								if (!activitySet) {
+									sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
+								}
+							}
+						} else if (sc.Match("elsif")) {
+							// Ensure only one chosen out of `if .. `elsif .. `elsif .. `else .. `endif
+							if (!preproc.CurrentIfTaken()) {
+								// Similar to `ifdef
+								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
+								bool ifGood = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
+								if (ifGood) {
+									preproc.InvertCurrentLevel();
+									activitySet = preproc.IsInactive() ? activeFlag : 0;
+									if (!activitySet)
+										sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
+								}
+							} else if (!preproc.IsInactive()) {
+								preproc.InvertCurrentLevel();
+								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								if (!activitySet)
+									sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
+							}
+						} else if (sc.Match("endif")) {
+							preproc.EndSection();
+							activitySet = preproc.IsInactive() ? activeFlag : 0;
+							sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
+						} else if (sc.Match("define")) {
+							if (options.updatePreprocessor && !preproc.IsInactive()) {
+								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
+								size_t startName = 0;
+								while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
+									startName++;
+								size_t endName = startName;
+								while ((endName < restOfLine.length()) && setWord.Contains(static_cast<unsigned char>(restOfLine[endName])))
+									endName++;
+								std::string key = restOfLine.substr(startName, endName-startName);
+								if ((endName < restOfLine.length()) && (restOfLine.at(endName) == '(')) {
+									// Macro
+									size_t endArgs = endName;
+									while ((endArgs < restOfLine.length()) && (restOfLine[endArgs] != ')'))
+										endArgs++;
+									std::string args = restOfLine.substr(endName + 1, endArgs - endName - 1);
+									size_t startValue = endArgs+1;
+									while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
+										startValue++;
+									std::string value;
+									if (startValue < restOfLine.length())
+										value = restOfLine.substr(startValue);
+									preprocessorDefinitions[key] = SymbolValue(value, args);
+									ppDefineHistory.push_back(PPDefinition(curLine, key, value, false, args));
+									definitionsChanged = true;
+								} else {
+									// Value
+									size_t startValue = endName;
+									while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
+										startValue++;
+									std::string value = restOfLine.substr(startValue);
+									preprocessorDefinitions[key] = value;
+									ppDefineHistory.push_back(PPDefinition(curLine, key, value));
+									definitionsChanged = true;
+								}
+							}
+						} else if (sc.Match("undefineall")) {
+							if (options.updatePreprocessor && !preproc.IsInactive()) {
+								// remove all preprocessor definitions
+								std::map<std::string, SymbolValue>::iterator itDef;
+								for(itDef = preprocessorDefinitions.begin(); itDef != preprocessorDefinitions.end(); ++itDef) {
+									ppDefineHistory.push_back(PPDefinition(curLine, itDef->first, "", true));
+								}
+								preprocessorDefinitions.clear();
+								definitionsChanged = true;
+							}
+						} else if (sc.Match("undef")) {
+							if (options.updatePreprocessor && !preproc.IsInactive()) {
+								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, true);
+								std::vector<std::string> tokens = Tokenize(restOfLine);
+								std::string key;
+								if (tokens.size() >= 1) {
+									key = tokens[0];
+									preprocessorDefinitions.erase(key);
+									ppDefineHistory.push_back(PPDefinition(curLine, key, "", true));
+									definitionsChanged = true;
+								}
+							}
+						}
+					}
 				}
 			} else if (isoperator(static_cast<char>(sc.ch)) || sc.ch == '@' || sc.ch == '#') {
-				sc.SetState(SCE_V_OPERATOR);
+				sc.SetState(SCE_V_OPERATOR|activitySet);
 				if (sc.ch == '.') lineState = kwDot;
 				if (sc.ch == ';') lineState = kwOther;
 			}
 		}
 	}
+	if (definitionsChanged) {
+		styler.ChangeLexerState(startPos, startPos + length);
+	}
 	sc.Complete();
 }
 
@@ -222,7 +719,7 @@ static bool IsStreamCommentStyle(int style) {
 	return style == SCE_V_COMMENT;
 }
 
-static bool IsCommentLine(int line, Accessor &styler) {
+static bool IsCommentLine(int line, LexAccessor &styler) {
 	int pos = styler.LineStart(line);
 	int eolPos = styler.LineStart(line + 1) - 1;
 	for (int i = pos; i < eolPos; i++) {
@@ -238,22 +735,13 @@ static bool IsCommentLine(int line, Accessor &styler) {
 	}
 	return false;
 }
+
 // Store both the current line's fold level and the next lines in the
 // level store to make it easy to pick up with each increment
 // and to make it possible to fiddle the current level for "} else {".
-static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle,
-                            Accessor &styler) {
-	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
-	bool foldPreprocessor = styler.GetPropertyInt("fold.preprocessor") != 0;
-	bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
-	bool foldAtElse = styler.GetPropertyInt("fold.at.else", 0) != 0;
-	// Verilog specific folding options:
-	// fold_at_module -
-	//      Generally used methodology in verilog code is
-	//      one module per file, so folding at module definition is useless.
-	// fold_at_brace/parenthese -
-	//      Folding of long port lists can be convenient.
-	bool foldAtModule = styler.GetPropertyInt("fold.verilog.flags", 0) != 0;
+void SCI_METHOD LexerVerilog::Fold(unsigned int startPos, int length, int initStyle, IDocument *pAccess)
+{
+	LexAccessor styler(pAccess);
 	bool foldAtBrace  = 1;
 	bool foldAtParenthese  = 1;
 
@@ -266,16 +754,16 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 	int levelMinCurrent = levelCurrent;
 	int levelNext = levelCurrent;
 	char chNext = styler[startPos];
-	int styleNext = styler.StyleAt(startPos);
-	int style = initStyle;
+	int styleNext = MaskActive(styler.StyleAt(startPos));
+	int style = MaskActive(initStyle);
 	for (unsigned int i = startPos; i < endPos; i++) {
 		char ch = chNext;
 		chNext = styler.SafeGetCharAt(i + 1);
 		int stylePrev = style;
 		style = styleNext;
-		styleNext = styler.StyleAt(i + 1);
+		styleNext = MaskActive(styler.StyleAt(i + 1));
 		bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
-		if (foldComment && IsStreamCommentStyle(style)) {
+		if (options.foldComment && IsStreamCommentStyle(style)) {
 			if (!IsStreamCommentStyle(stylePrev)) {
 				levelNext++;
 			} else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
@@ -283,7 +771,7 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 				levelNext--;
 			}
 		}
-		if (foldComment && atEOL && IsCommentLine(lineCurrent, styler))
+		if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler))
 		{
 			if (!IsCommentLine(lineCurrent - 1, styler)
 			    && IsCommentLine(lineCurrent + 1, styler))
@@ -292,7 +780,7 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 			         && !IsCommentLine(lineCurrent+1, styler))
 				levelNext--;
 		}
-		if (foldComment && (style == SCE_V_COMMENTLINE)) {
+		if (options.foldComment && (style == SCE_V_COMMENTLINE)) {
 			if ((ch == '/') && (chNext == '/')) {
 				char chNext2 = styler.SafeGetCharAt(i + 2);
 				if (chNext2 == '{') {
@@ -302,7 +790,7 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 				}
 			}
 		}
-		if (foldPreprocessor && (style == SCE_V_PREPROCESSOR)) {
+		if (options.foldPreprocessor && (style == SCE_V_PREPROCESSOR)) {
 			if (ch == '`') {
 				unsigned int j = i + 1;
 				while ((j < endPos) && IsASpaceOrTab(styler.SafeGetCharAt(j))) {
@@ -350,7 +838,7 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 				styler.Match(j, "table") ||
 				styler.Match(j, "task") ||
 				styler.Match(j, "fork") ||
-				(styler.Match(j, "module") && foldAtModule) ||
+				(styler.Match(j, "module") && options.foldAtModule) ||
 				styler.Match(j, "begin")) {
 				levelNext++;
 			} else if (styler.Match(j, "endcase") ||
@@ -368,18 +856,18 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 				styler.Match(j, "join") ||
 				styler.Match(j, "join_any") ||
 				styler.Match(j, "join_none") ||
-				(styler.Match(j, "endmodule") && foldAtModule) ||
+				(styler.Match(j, "endmodule") && options.foldAtModule) ||
 				(styler.Match(j, "end") && !IsAWordChar(styler.SafeGetCharAt(j + 3)))) {
 				levelNext--;
 			}
 		}
 		if (atEOL) {
 			int levelUse = levelCurrent;
-			if (foldAtElse) {
+			if (options.foldAtElse) {
 				levelUse = levelMinCurrent;
 			}
 			int lev = levelUse | levelNext << 16;
-			if (visibleChars == 0 && foldCompact)
+			if (visibleChars == 0 && options.foldCompact)
 				lev |= SC_FOLDLEVELWHITEFLAG;
 			if (levelUse < levelNext)
 				lev |= SC_FOLDLEVELHEADERFLAG;
@@ -396,9 +884,31 @@ static void FoldNoBoxVerilogDoc(unsigned int startPos, int length, int initStyle
 	}
 }
 
-static void FoldVerilogDoc(unsigned int startPos, int length, int initStyle, WordList *[],
-                       Accessor &styler) {
-	FoldNoBoxVerilogDoc(startPos, length, initStyle, styler);
+std::vector<std::string> LexerVerilog::Tokenize(const std::string &expr) const {
+	// Break into tokens
+	std::vector<std::string> tokens;
+	const char *cp = expr.c_str();
+	while (*cp) {
+		std::string word;
+		if (setWord.Contains(static_cast<unsigned char>(*cp))) {
+			// Identifiers and numbers
+			while (setWord.Contains(static_cast<unsigned char>(*cp))) {
+				word += *cp;
+				cp++;
+			}
+		} else if (IsSpaceOrTab(*cp)) {
+			while (IsSpaceOrTab(*cp)) {
+				cp++;
+			}
+			continue;
+		} else {
+			// Should handle strings, characters, and comments here
+			word += *cp;
+			cp++;
+		}
+		tokens.push_back(word);
+	}
+	return tokens;
 }
 
 static const char * const verilogWordLists[] = {
@@ -407,8 +917,8 @@ static const char * const verilogWordLists[] = {
             "System Tasks",
             "User defined tasks and identifiers",
             "Documentation comment keywords",
+            "Preprocessor definitions",
             0,
         };
 
-
-LexerModule lmVerilog(SCLEX_VERILOG, ColouriseVerilogDoc, "verilog", FoldVerilogDoc, verilogWordLists);
+LexerModule lmVerilog(SCLEX_VERILOG, LexerVerilog::LexerFactoryVerilog, "verilog", verilogWordLists);