path: root/src/LexPerl.cpp

// Scintilla source code edit control
/** @file LexPerl.cxx
 ** Lexer for subset of Perl.
 **/
// Copyright 1998-2005 by Neil Hodgson <[email protected]>
// Lexical analysis fixes by Kein-Hong Man <[email protected]>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <stdio.h>
#include <stdarg.h>

#include "Platform.h"

#include "PropSet.h"
#include "Accessor.h"
#include "KeyWords.h"
#include "Scintilla.h"
#include "SciLexer.h"

#define PERLNUM_BINARY 1    // order is significant: 1-4 cannot have a dot
#define PERLNUM_HEX 2
#define PERLNUM_OCTAL 3
#define PERLNUM_FLOAT 4     // actually exponent part
#define PERLNUM_DECIMAL 5   // 1-5 are numbers; 6-7 are strings
#define PERLNUM_VECTOR 6
#define PERLNUM_V_VECTOR 7
#define PERLNUM_BAD 8

#define BACK_NONE 0         // lookback state for bareword disambiguation:
#define BACK_OPERATOR 1     // whitespace/comments are insignificant
#define BACK_KEYWORD 2      // operators/keywords are needed for disambiguation

#define HERE_DELIM_MAX 256

static inline bool isEOLChar(char ch) {
	return (ch == '\r') || (ch == '\n');
}

static bool isSingleCharOp(char ch) {
	char strCharSet[2];
	strCharSet[0] = ch;
	strCharSet[1] = '\0';
	return (NULL != strstr("rwxoRWXOezsfdlpSbctugkTBMAC", strCharSet));
}

static inline bool isPerlOperator(char ch) {
	if (ch == '^' || ch == '&' || ch == '\\' ||
	        ch == '(' || ch == ')' || ch == '-' || ch == '+' ||
	        ch == '=' || ch == '|' || ch == '{' || ch == '}' ||
	        ch == '[' || ch == ']' || ch == ':' || ch == ';' ||
	        ch == '>' || ch == ',' ||
	        ch == '?' || ch == '!' || ch == '.' || ch == '~')
		return true;
	// these chars are already tested before this call
	// ch == '%' || ch == '*' || ch == '<' || ch == '/' ||
	return false;
}

static bool isPerlKeyword(unsigned int start, unsigned int end, WordList &keywords, Accessor &styler) {
	char s[100];
    unsigned int i, len = end - start;
    if (len > 30) { len = 30; }
	for (i = 0; i < len; i++, start++) s[i] = styler[start];
    s[i] = '\0';
	return keywords.InList(s);
}

// Note: as lexer uses chars, UTF-8 bytes are considered as <0 values
// Note: iswordchar() was used in only one place in LexPerl, it is
// unnecessary as '.' is processed as the concatenation operator, so
// only isWordStart() is used in LexPerl

static inline bool isWordStart(char ch) {
	return !isascii(ch) || isalnum(ch) || ch == '_';
}

static inline bool isEndVar(char ch) {
	return isascii(ch) && !isalnum(ch) && ch != '#' && ch != '$' &&
	       ch != '_' && ch != '\'';
}

static inline bool isNonQuote(char ch) {
	return !isascii(ch) || isalnum(ch) || ch == '_';
}

static inline char actualNumStyle(int numberStyle) {
    if (numberStyle == PERLNUM_VECTOR || numberStyle == PERLNUM_V_VECTOR) {
        return SCE_PL_STRING;
    } else if (numberStyle == PERLNUM_BAD) {
        return SCE_PL_ERROR;
    }
    return SCE_PL_NUMBER;
}

static bool isMatch(Accessor &styler, int lengthDoc, int pos, const char *val) {
	if ((pos + static_cast<int>(strlen(val))) >= lengthDoc) {
		return false;
	}
	while (*val) {
		if (*val != styler[pos++]) {
			return false;
		}
		val++;
	}
	return true;
}

static char opposite(char ch) {
	if (ch == '(')
		return ')';
	if (ch == '[')
		return ']';
	if (ch == '{')
		return '}';
	if (ch == '<')
		return '>';
	return ch;
}

static void ColourisePerlDoc(unsigned int startPos, int length, int initStyle,
                             WordList *keywordlists[], Accessor &styler) {

	// Lexer for perl often has to backtrack to start of current style to determine
	// which characters are being used as quotes, how deeply nested is the
	// start position and what the termination string is for here documents

	WordList &keywords = *keywordlists[0];

	class HereDocCls {
	public:
		int State;		// 0: '<<' encountered
		// 1: collect the delimiter
		// 2: here doc text (lines after the delimiter)
		char Quote;		// the char after '<<'
		bool Quoted;		// true if Quote in ('\'','"','`')
		int DelimiterLength;	// strlen(Delimiter)
		char *Delimiter;	// the Delimiter, 256: sizeof PL_tokenbuf
		HereDocCls() {
			State = 0;
            Quote = 0;
            Quoted = false;
			DelimiterLength = 0;
			Delimiter = new char[HERE_DELIM_MAX];
			Delimiter[0] = '\0';
		}
		~HereDocCls() {
			delete []Delimiter;
		}
	};
	HereDocCls HereDoc;	// TODO: FIFO for stacked here-docs

	class QuoteCls {
		public:
		int  Rep;
		int  Count;
		char Up;
		char Down;
		QuoteCls() {
			this->New(1);
		}
		void New(int r) {
			Rep   = r;
			Count = 0;
			Up    = '\0';
			Down  = '\0';
		}
		void Open(char u) {
			Count++;
			Up    = u;
			Down  = opposite(Up);
		}
	};
	QuoteCls Quote;

	int state = initStyle;
	char numState = PERLNUM_DECIMAL;
	int dotCount = 0;
	unsigned int lengthDoc = startPos + length;
	//int sookedpos = 0; // these have no apparent use, see POD state
	//char sooked[100];
	//sooked[sookedpos] = '\0';

	// If in a long distance lexical state, seek to the beginning to find quote characters
	// Perl strings can be multi-line with embedded newlines, so backtrack.
	// Perl numbers have additional state during lexing, so backtrack too.
	if (state == SCE_PL_HERE_Q || state == SCE_PL_HERE_QQ || state == SCE_PL_HERE_QX) {
		while ((startPos > 1) && (styler.StyleAt(startPos) != SCE_PL_HERE_DELIM)) {
			startPos--;
		}
		startPos = styler.LineStart(styler.GetLine(startPos));
		state = styler.StyleAt(startPos - 1);
	}
	if ( state == SCE_PL_STRING_Q
	|| state == SCE_PL_STRING_QQ
	|| state == SCE_PL_STRING_QX
	|| state == SCE_PL_STRING_QR
	|| state == SCE_PL_STRING_QW
	|| state == SCE_PL_REGEX
	|| state == SCE_PL_REGSUBST
	|| state == SCE_PL_STRING
	|| state == SCE_PL_BACKTICKS
	|| state == SCE_PL_CHARACTER
	|| state == SCE_PL_NUMBER
	|| state == SCE_PL_IDENTIFIER
    || state == SCE_PL_ERROR
	) {
		while ((startPos > 1) && (styler.StyleAt(startPos - 1) == state)) {
			startPos--;
		}
		state = SCE_PL_DEFAULT;
	}

    // lookback at start of lexing to set proper state for backflag
    // after this, they are updated when elements are lexed
    int backflag = BACK_NONE;
    unsigned int backPos = startPos;
    if (backPos > 0) {
        backPos--;
        int sty = SCE_PL_DEFAULT;
        while ((backPos > 0) && (sty = styler.StyleAt(backPos),
               sty == SCE_PL_DEFAULT || sty == SCE_PL_COMMENTLINE))
            backPos--;
        if (sty == SCE_PL_OPERATOR)
            backflag = BACK_OPERATOR;
        else if (sty == SCE_PL_WORD)
            backflag = BACK_KEYWORD;
    }

	styler.StartAt(startPos);
	char chPrev = styler.SafeGetCharAt(startPos - 1);
	if (startPos == 0)
		chPrev = '\n';
	char chNext = styler[startPos];
	styler.StartSegment(startPos);

	for (unsigned int i = startPos; i < lengthDoc; i++) {
		char ch = chNext;
		// if the current character is not consumed due to the completion of an
		// earlier style, lexing can be restarted via a simple goto
	restartLexer:
		chNext = styler.SafeGetCharAt(i + 1);
		char chNext2 = styler.SafeGetCharAt(i + 2);

		if (styler.IsLeadByte(ch)) {
			chNext = styler.SafeGetCharAt(i + 2);
			chPrev = ' ';
			i += 1;
			continue;
		}
		if ((chPrev == '\r' && ch == '\n')) {	// skip on DOS/Windows
			styler.ColourTo(i, state);
			chPrev = ch;
			continue;
		}

		if (HereDoc.State == 1 && isEOLChar(ch)) {
			// Begin of here-doc (the line after the here-doc delimiter):
			// Lexically, the here-doc starts from the next line after the >>, but the
			// first line of here-doc seem to follow the style of the last EOL sequence
			HereDoc.State = 2;
			if (HereDoc.Quoted) {
				if (state == SCE_PL_HERE_DELIM) {
					// Missing quote at end of string! We are stricter than perl.
					// Colour here-doc anyway while marking this bit as an error.
					state = SCE_PL_ERROR;
				}
				styler.ColourTo(i - 1, state);
				switch (HereDoc.Quote) {
				case '\'':
					state = SCE_PL_HERE_Q ;
					break;
				case '"':
					state = SCE_PL_HERE_QQ;
					break;
				case '`':
					state = SCE_PL_HERE_QX;
					break;
				}
			} else {
				styler.ColourTo(i - 1, state);
				switch (HereDoc.Quote) {
				case '\\':
					state = SCE_PL_HERE_Q ;
					break;
				default :
					state = SCE_PL_HERE_QQ;
				}
			}
		}

		if (state == SCE_PL_DEFAULT) {
			if ((isascii(ch) && isdigit(ch)) || (isascii(chNext) && isdigit(chNext) &&
				(ch == '.' || ch == 'v'))) {
				state = SCE_PL_NUMBER;
                backflag = BACK_NONE;
				numState = PERLNUM_DECIMAL;
				dotCount = 0;
				if (ch == '0') {	// hex,bin,octal
					if (chNext == 'x') {
						numState = PERLNUM_HEX;
					} else if (chNext == 'b') {
                        numState = PERLNUM_BINARY;
                    } else if (isascii(chNext) && isdigit(chNext)) {
                        numState = PERLNUM_OCTAL;
                    }
                    if (numState != PERLNUM_DECIMAL) {
						i++;
						ch = chNext;
						chNext = chNext2;
                    }
				} else if (ch == 'v') {	// vector
					numState = PERLNUM_V_VECTOR;
				}
			} else if (isWordStart(ch)) {
                // if immediately prefixed by '::', always a bareword
                state = SCE_PL_WORD;
                if (chPrev == ':' && styler.SafeGetCharAt(i - 2) == ':') {
                    state = SCE_PL_IDENTIFIER;
                }
                unsigned int kw = i + 1;
                // first check for possible quote-like delimiter
				if (ch == 's' && !isNonQuote(chNext)) {
					state = SCE_PL_REGSUBST;
					Quote.New(2);
				} else if (ch == 'm' && !isNonQuote(chNext)) {
					state = SCE_PL_REGEX;
					Quote.New(1);
				} else if (ch == 'q' && !isNonQuote(chNext)) {
					state = SCE_PL_STRING_Q;
					Quote.New(1);
				} else if (ch == 'y' && !isNonQuote(chNext)) {
					state = SCE_PL_REGSUBST;
					Quote.New(2);
				} else if (ch == 't' && chNext == 'r' && !isNonQuote(chNext2)) {
					state = SCE_PL_REGSUBST;
					Quote.New(2);
                    kw++;
				} else if (ch == 'q' && (chNext == 'q' || chNext == 'r' || chNext == 'w' || chNext == 'x') && !isNonQuote(chNext2)) {
					if      (chNext == 'q') state = SCE_PL_STRING_QQ;
					else if (chNext == 'x') state = SCE_PL_STRING_QX;
					else if (chNext == 'r') state = SCE_PL_STRING_QR;
					else if (chNext == 'w') state = SCE_PL_STRING_QW;
					Quote.New(1);
                    kw++;
				} else if (ch == 'x' && (chNext == '=' ||	// repetition
                           !isWordStart(chNext) ||
                           (isdigit(chPrev) && isdigit(chNext)))) {
                    state = SCE_PL_OPERATOR;
                }
                // if potentially a keyword, scan forward and grab word, then check
                // if it's really one; if yes, disambiguation test is performed
                // otherwise it is always a bareword and we skip a lot of scanning
                // note: keywords assumed to be limited to [_a-zA-Z] only
                if (state == SCE_PL_WORD) {
                    while (isWordStart(styler.SafeGetCharAt(kw))) kw++;
                    if (!isPerlKeyword(styler.GetStartSegment(), kw, keywords, styler)) {
                        state = SCE_PL_IDENTIFIER;
                    }
                }
                // if already SCE_PL_IDENTIFIER, then no ambiguity, skip this
                // for quote-like delimiters/keywords, attempt to disambiguate
                // to select for bareword, change state -> SCE_PL_IDENTIFIER
                if (state != SCE_PL_IDENTIFIER && i > 0) {
                    unsigned int j = i;
                    bool moreback = false;      // true if passed newline/comments
                    bool brace = false;         // true if opening brace found
                    char ch2;
                    // first look backwards past whitespace/comments for EOLs
                    // if BACK_NONE, neither operator nor keyword, so skip test
                    if (backflag != BACK_NONE) {
                        while (--j > backPos) {
                            if (isEOLChar(styler.SafeGetCharAt(j)))
                                moreback = true;
                        }
                        ch2 = styler.SafeGetCharAt(j);
                        if (ch2 == '{' && !moreback) {
                            // {bareword: possible variable spec
                            brace = true;
                        } else if ((ch2 == '&' && styler.SafeGetCharAt(j - 1) != '&')
                                // &bareword: subroutine call
                                || (ch2 == '>' && styler.SafeGetCharAt(j - 1) == '-')
                                // ->bareword: part of variable spec
                                || (ch2 == 'b' && styler.Match(j - 2, "su"))) {
                                // sub bareword: subroutine declaration
                                // (implied BACK_KEYWORD, no keywords end in 'sub'!)
                            state = SCE_PL_IDENTIFIER;
                        }
                        // if status still ambiguous, look forward after word past
                        // tabs/spaces only; if ch2 isn't one of '[{(,' it can never
                        // match anything, so skip the whole thing
                        j = kw;
                        if (state != SCE_PL_IDENTIFIER
                            && (ch2 == '{' || ch2 == '(' || ch2 == '['|| ch2 == ',')
                            && kw < lengthDoc) {
                            while (ch2 = styler.SafeGetCharAt(j),
                                   (ch2 == ' ' || ch2 == '\t') && j < lengthDoc) {
                                j++;
                            }
                            if ((ch2 == '}' && brace)
                             // {bareword}: variable spec
                             || (ch2 == '=' && styler.SafeGetCharAt(j + 1) == '>')) {
                             // [{(, bareword=>: hash literal
                                state = SCE_PL_IDENTIFIER;
                            }
                        }
                    }
                }
                backflag = BACK_NONE;
                // an identifier or bareword
                if (state == SCE_PL_IDENTIFIER) {
                    if ((!isWordStart(chNext) && chNext != '\'')
                        || (chNext == '.' && chNext2 == '.')) {
                        // We need that if length of word == 1!
                        // This test is copied from the SCE_PL_WORD handler.
                        styler.ColourTo(i, SCE_PL_IDENTIFIER);
                        state = SCE_PL_DEFAULT;
                    }
                // a keyword
                } else if (state == SCE_PL_WORD) {
                    i = kw - 1;
                    if (ch == '_' && chNext == '_' &&
                        (isMatch(styler, lengthDoc, styler.GetStartSegment(), "__DATA__")
                      || isMatch(styler, lengthDoc, styler.GetStartSegment(), "__END__"))) {
                        styler.ColourTo(i, SCE_PL_DATASECTION);
                        state = SCE_PL_DATASECTION;
                    } else {
                        styler.ColourTo(i, SCE_PL_WORD);
                        state = SCE_PL_DEFAULT;
                        backflag = BACK_KEYWORD;
                        backPos = i;
                    }
                    ch = styler.SafeGetCharAt(i);
                    chNext = styler.SafeGetCharAt(i + 1);
                // a repetition operator 'x'
                } else if (state == SCE_PL_OPERATOR) {
                    styler.ColourTo(i, SCE_PL_OPERATOR);
                    state = SCE_PL_DEFAULT;
                // quote-like delimiter, skip one char if double-char delimiter
                } else {
                    i = kw - 1;
                    chNext = styler.SafeGetCharAt(i + 1);
                }
			} else if (ch == '#') {
				state = SCE_PL_COMMENTLINE;
			} else if (ch == '\"') {
				state = SCE_PL_STRING;
				Quote.New(1);
				Quote.Open(ch);
                backflag = BACK_NONE;
			} else if (ch == '\'') {
				if (chPrev == '&') {
					// Archaic call
					styler.ColourTo(i, state);
				} else {
					state = SCE_PL_CHARACTER;
					Quote.New(1);
					Quote.Open(ch);
				}
                backflag = BACK_NONE;
			} else if (ch == '`') {
				state = SCE_PL_BACKTICKS;
				Quote.New(1);
				Quote.Open(ch);
                backflag = BACK_NONE;
			} else if (ch == '$') {
				if ((chNext == '{') || isspacechar(chNext)) {
					styler.ColourTo(i, SCE_PL_SCALAR);
				} else {
					state = SCE_PL_SCALAR;
					if ((chNext == '`' && chNext2 == '`')
                     || (chNext == ':' && chNext2 == ':')) {
						i += 2;
						ch = styler.SafeGetCharAt(i);
						chNext = styler.SafeGetCharAt(i + 1);
					} else {
						i++;
						ch = chNext;
						chNext = chNext2;
					}
				}
                backflag = BACK_NONE;
			} else if (ch == '@') {
				if (!isascii(chNext) || isalpha(chNext) || chNext == '#' || chNext == '$'
					|| chNext == '_' || chNext == '+' || chNext == '-') {
					state = SCE_PL_ARRAY;
                } else if (chNext == ':' && chNext2 == ':') {
                    state = SCE_PL_ARRAY;
                    i += 2;
                    ch = styler.SafeGetCharAt(i);
                    chNext = styler.SafeGetCharAt(i + 1);
				} else if (chNext != '{' && chNext != '[') {
					styler.ColourTo(i, SCE_PL_ARRAY);
				} else {
					styler.ColourTo(i, SCE_PL_ARRAY);
				}
                backflag = BACK_NONE;
			} else if (ch == '%') {
				if (!isascii(chNext) || isalpha(chNext) || chNext == '#' || chNext == '$'
                    || chNext == '_' || chNext == '!' || chNext == '^') {
					state = SCE_PL_HASH;
                    i++;
                    ch = chNext;
                    chNext = chNext2;
                } else if (chNext == ':' && chNext2 == ':') {
                    state = SCE_PL_HASH;
                    i += 2;
                    ch = styler.SafeGetCharAt(i);
                    chNext = styler.SafeGetCharAt(i + 1);
				} else if (chNext == '{') {
					styler.ColourTo(i, SCE_PL_HASH);
				} else {
					styler.ColourTo(i, SCE_PL_OPERATOR);
				}
                backflag = BACK_NONE;
			} else if (ch == '*') {
                char strch[2];
                strch[0] = chNext;
                strch[1] = '\0';
                if (chNext == ':' && chNext2 == ':') {
                    state = SCE_PL_SYMBOLTABLE;
                    i += 2;
                    ch = styler.SafeGetCharAt(i);
                    chNext = styler.SafeGetCharAt(i + 1);
				} else if (!isascii(chNext) || isalpha(chNext) || chNext == '_'
                        || NULL != strstr("^/|,\\\";#%^:?<>)[]", strch)) {
					state = SCE_PL_SYMBOLTABLE;
                    i++;
                    ch = chNext;
                    chNext = chNext2;
				} else if (chNext == '{') {
					styler.ColourTo(i, SCE_PL_SYMBOLTABLE);
				} else {
					if (chNext == '*') {	// exponentiation
						i++;
						ch = chNext;
						chNext = chNext2;
					}
					styler.ColourTo(i, SCE_PL_OPERATOR);
				}
                backflag = BACK_NONE;
			} else if (ch == '/' || (ch == '<' && chNext == '<')) {
				// Explicit backward peeking to set a consistent preferRE for
				// any slash found, so no longer need to track preferRE state.
				// Find first previous significant lexed element and interpret.
                // Test for HERE doc start '<<' shares this code, helps to
                // determine if it should be an operator.
				bool preferRE = false;
                bool isHereDoc = (ch == '<');
                bool hereDocSpace = false;      // these are for corner case:
                bool hereDocScalar = false;     // SCALAR [whitespace] '<<'
				unsigned int bk = (i > 0)? i - 1: 0;
				char bkch;
				styler.Flush();
                if (styler.StyleAt(bk) == SCE_PL_DEFAULT)
                    hereDocSpace = true;
				while ((bk > 0) && (styler.StyleAt(bk) == SCE_PL_DEFAULT ||
					styler.StyleAt(bk) == SCE_PL_COMMENTLINE)) {
					bk--;
				}
				if (bk == 0) {
					// position 0 won't really be checked; rarely happens
					// hard to fix due to an unsigned index i
					preferRE = true;
				} else {
					int bkstyle = styler.StyleAt(bk);
					bkch = styler.SafeGetCharAt(bk);
					switch(bkstyle) {
					case SCE_PL_OPERATOR:
						preferRE = true;
						if (bkch == ')' || bkch == ']') {
							preferRE = false;
						} else if (bkch == '}') {
							// backtrack further, count balanced brace pairs
							// if a brace pair found, see if it's a variable
							int braceCount = 1;
							while (--bk > 0) {
								bkstyle = styler.StyleAt(bk);
								if (bkstyle == SCE_PL_OPERATOR) {
									bkch = styler.SafeGetCharAt(bk);
									if (bkch == ';') {	// early out
										break;
									} else if (bkch == '}') {
										braceCount++;
									} else if (bkch == '{') {
										if (--braceCount == 0)
											break;
									}
								}
							}
							if (bk == 0) {
								// at beginning, true
							} else if (braceCount == 0) {
								// balanced { found, bk>0, skip more whitespace
								if (styler.StyleAt(--bk) == SCE_PL_DEFAULT) {
									while (bk > 0) {
										bkstyle = styler.StyleAt(--bk);
										if (bkstyle != SCE_PL_DEFAULT)
											break;
									}
								}
								bkstyle = styler.StyleAt(bk);
								if (bkstyle == SCE_PL_SCALAR
								 || bkstyle == SCE_PL_ARRAY
								 || bkstyle == SCE_PL_HASH
								 || bkstyle == SCE_PL_SYMBOLTABLE
								 || bkstyle == SCE_PL_OPERATOR) {
									preferRE = false;
								}
							}
						}
						break;
					case SCE_PL_IDENTIFIER:
						preferRE = true;
						if (bkch == '>') {	// inputsymbol
							preferRE = false;
							break;
						}
						// backtrack to find "->" or "::" before identifier
						while (bk > 0 && styler.StyleAt(bk) == SCE_PL_IDENTIFIER) {
							bk--;
						}
						while (bk > 0) {
							bkstyle = styler.StyleAt(bk);
							if (bkstyle == SCE_PL_DEFAULT ||
							    bkstyle == SCE_PL_COMMENTLINE) {
							} else if (bkstyle == SCE_PL_OPERATOR) {
								// gcc 3.2.3 bloats if more compact form used
								bkch = styler.SafeGetCharAt(bk);
								if (bkch == '>') { // "->"
									if (styler.SafeGetCharAt(bk - 1) == '-') {
										preferRE = false;
										break;
									}
								} else if (bkch == ':') { // "::"
									if (styler.SafeGetCharAt(bk - 1) == ':') {
										preferRE = false;
										break;
									}
								}
							} else {// bare identifier, usually a function call but Perl
								// optimizes them as pseudo-constants, then the next
								// '/' will be a divide; favour divide over regex
								// if there is a whitespace after the '/'
								if (isspacechar(chNext)) {
									preferRE = false;
								}
								break;
							}
							bk--;
						}
						break;
                    case SCE_PL_SCALAR:     // for $var<< case
                        hereDocScalar = true;
                        break;
					// other styles uses the default, preferRE=false
					case SCE_PL_WORD:
					case SCE_PL_POD:
					case SCE_PL_POD_VERB:
					case SCE_PL_HERE_Q:
					case SCE_PL_HERE_QQ:
					case SCE_PL_HERE_QX:
						preferRE = true;
						break;
					}
				}
                if (isHereDoc) {    // handle HERE doc
                    // if SCALAR whitespace '<<', *always* a HERE doc
                    if (preferRE || (hereDocSpace && hereDocScalar)) {
                        state = SCE_PL_HERE_DELIM;
                        HereDoc.State = 0;
                    } else {        // << operator
						i++;
						ch = chNext;
						chNext = chNext2;
                        styler.ColourTo(i, SCE_PL_OPERATOR);
                    }
                } else {            // handle regexp
                    if (preferRE) {
                        state = SCE_PL_REGEX;
                        Quote.New(1);
                        Quote.Open(ch);
                    } else {        // / operator
                        styler.ColourTo(i, SCE_PL_OPERATOR);
                    }
                }
                backflag = BACK_NONE;
			} else if (ch == '<') {
				// looks forward for matching > on same line
				unsigned int fw = i + 1;
				while (fw < lengthDoc) {
					char fwch = styler.SafeGetCharAt(fw);
					if (fwch == ' ') {
						if (styler.SafeGetCharAt(fw-1) != '\\' ||
						    styler.SafeGetCharAt(fw-2) != '\\')
						break;
					} else if (isEOLChar(fwch) || isspacechar(fwch)) {
						break;
					} else if (fwch == '>') {
						if ((fw - i) == 2 &&	// '<=>' case
						    styler.SafeGetCharAt(fw-1) == '=') {
							styler.ColourTo(fw, SCE_PL_OPERATOR);
						} else {
							styler.ColourTo(fw, SCE_PL_IDENTIFIER);
						}
						i = fw;
						ch = fwch;
						chNext = styler.SafeGetCharAt(i+1);
					}
					fw++;
				}
				styler.ColourTo(i, SCE_PL_OPERATOR);
                backflag = BACK_NONE;
			} else if (ch == '='	// POD
			           && isalpha(chNext)
			           && (isEOLChar(chPrev))) {
				state = SCE_PL_POD;
                backflag = BACK_NONE;
				//sookedpos = 0;
				//sooked[sookedpos] = '\0';
			} else if (ch == '-'	// file test operators
			           && isSingleCharOp(chNext)
			           && !isalnum((chNext2 = styler.SafeGetCharAt(i+2)))) {
				styler.ColourTo(i + 1, SCE_PL_WORD);
				state = SCE_PL_DEFAULT;
				i++;
				ch = chNext;
				chNext = chNext2;
                backflag = BACK_NONE;
			} else if (isPerlOperator(ch)) {
				if (ch == '.' && chNext == '.') { // .. and ...
					i++;
					if (chNext2 == '.') { i++; }
					state = SCE_PL_DEFAULT;
					ch = styler.SafeGetCharAt(i);
					chNext = styler.SafeGetCharAt(i + 1);
				}
				styler.ColourTo(i, SCE_PL_OPERATOR);
                backflag = BACK_OPERATOR;
                backPos = i;
			} else {
				// keep colouring defaults to make restart easier
				styler.ColourTo(i, SCE_PL_DEFAULT);
			}
		} else if (state == SCE_PL_NUMBER) {
			if (ch == '.') {
				if (chNext == '.') {
					// double dot is always an operator
					goto numAtEnd;
				} else if (numState <= PERLNUM_FLOAT) {
					// non-decimal number or float exponent, consume next dot
					styler.ColourTo(i - 1, SCE_PL_NUMBER);
					styler.ColourTo(i, SCE_PL_OPERATOR);
					state = SCE_PL_DEFAULT;
				} else { // decimal or vectors allows dots
					dotCount++;
					if (numState == PERLNUM_DECIMAL) {
						if (dotCount > 1) {
							if (isdigit(chNext)) { // really a vector
								numState = PERLNUM_VECTOR;
							} else	// number then dot
								goto numAtEnd;
						}
					} else { // vectors
						if (!isdigit(chNext))	// vector then dot
							goto numAtEnd;
					}
				}
			} else if (ch == '_' && numState == PERLNUM_DECIMAL) {
				if (!isdigit(chNext)) {
					goto numAtEnd;
				}
			} else if (!isascii(ch) || isalnum(ch)) {
				if (numState == PERLNUM_VECTOR || numState == PERLNUM_V_VECTOR) {
					if (!isascii(ch) || isalpha(ch)) {
						if (dotCount == 0) { // change to word
							state = SCE_PL_IDENTIFIER;
						} else { // vector then word
							goto numAtEnd;
						}
					}
				} else if (numState == PERLNUM_DECIMAL) {
					if (ch == 'E' || ch == 'e') { // exponent
						numState = PERLNUM_FLOAT;
						if (chNext == '+' || chNext == '-') {
							i++;
							ch = chNext;
							chNext = chNext2;
						}
					} else if (!isascii(ch) || !isdigit(ch)) { // number then word
						goto numAtEnd;
					}
				} else if (numState == PERLNUM_FLOAT) {
					if (!isdigit(ch)) { // float then word
						goto numAtEnd;
					}
				} else if (numState == PERLNUM_OCTAL) {
                    if (!isdigit(ch))
                        goto numAtEnd;
                    else if (ch > '7')
                        numState = PERLNUM_BAD;
                } else if (numState == PERLNUM_BINARY) {
                    if (!isdigit(ch))
                        goto numAtEnd;
                    else if (ch > '1')
                        numState = PERLNUM_BAD;
                } else if (numState == PERLNUM_HEX) {
                    int ch2 = toupper(ch);
                    if (!isdigit(ch) && !(ch2 >= 'A' && ch2 <= 'F'))
                        goto numAtEnd;
				} else {//(numState == PERLNUM_BAD) {
                    if (!isdigit(ch))
                        goto numAtEnd;
                }
			} else {
				// complete current number or vector
			numAtEnd:
				styler.ColourTo(i - 1, actualNumStyle(numState));
				state = SCE_PL_DEFAULT;
				goto restartLexer;
			}
		} else if (state == SCE_PL_IDENTIFIER) {
			if (!isWordStart(chNext) && chNext != '\'') {
				styler.ColourTo(i, SCE_PL_IDENTIFIER);
				state = SCE_PL_DEFAULT;
				ch = ' ';
			}
		} else {
			if (state == SCE_PL_COMMENTLINE) {
				if (isEOLChar(ch)) {
					styler.ColourTo(i - 1, state);
					state = SCE_PL_DEFAULT;
					goto restartLexer;
				} else if (isEOLChar(chNext)) {
					styler.ColourTo(i, state);
					state = SCE_PL_DEFAULT;
				}
			} else if (state == SCE_PL_HERE_DELIM) {
				//
				// From perldata.pod:
				// ------------------
				// A line-oriented form of quoting is based on the shell ``here-doc''
				// syntax.
				// Following a << you specify a string to terminate the quoted material,
				// and all lines following the current line down to the terminating
				// string are the value of the item.
				// The terminating string may be either an identifier (a word),
				// or some quoted text.
				// If quoted, the type of quotes you use determines the treatment of
				// the text, just as in regular quoting.
				// An unquoted identifier works like double quotes.
				// There must be no space between the << and the identifier.
				// (If you put a space it will be treated as a null identifier,
				// which is valid, and matches the first empty line.)
				// (This is deprecated, -w warns of this syntax)
				// The terminating string must appear by itself (unquoted and with no
				// surrounding whitespace) on the terminating line.
				//
				// From Bash info:
				// ---------------
				// Specifier format is: <<[-]WORD
				// Optional '-' is for removal of leading tabs from here-doc.
				// Whitespace acceptable after <<[-] operator.
				//
				if (HereDoc.State == 0) { // '<<' encountered
                    bool gotspace = false;
                    unsigned int oldi = i;
                    if (chNext == ' ' || chNext == '\t') {
                        // skip whitespace; legal for quoted delimiters
                        gotspace = true;
                        do {
                            i++;
                            chNext = styler.SafeGetCharAt(i + 1);
                        } while ((i + 1 < lengthDoc) && (chNext == ' ' || chNext == '\t'));
                        chNext2 = styler.SafeGetCharAt(i + 2);
                    }
					HereDoc.State = 1;
					HereDoc.Quote = chNext;
					HereDoc.Quoted = false;
					HereDoc.DelimiterLength = 0;
					HereDoc.Delimiter[HereDoc.DelimiterLength] = '\0';
					if (chNext == '\'' || chNext == '"' || chNext == '`') {
                        // a quoted here-doc delimiter
						i++;
						ch = chNext;
						chNext = chNext2;
						HereDoc.Quoted = true;
					} else if (isspacechar(chNext) || isdigit(chNext) || chNext == '\\'
						|| chNext == '=' || chNext == '$' || chNext == '@'
                        || ((isalpha(chNext) || chNext == '_') && gotspace)) {
						// left shift << or <<= operator cases
                        // restore position if operator
                        i = oldi;
						styler.ColourTo(i, SCE_PL_OPERATOR);
						state = SCE_PL_DEFAULT;
						HereDoc.State = 0;
                        goto restartLexer;
					} else {
						// an unquoted here-doc delimiter, no special handling
                        // (cannot be prefixed by spaces/tabs), or
						// symbols terminates; deprecated zero-length delimiter
					}

				} else if (HereDoc.State == 1) { // collect the delimiter
                    backflag = BACK_NONE;
					if (HereDoc.Quoted) { // a quoted here-doc delimiter
						if (ch == HereDoc.Quote) { // closing quote => end of delimiter
							styler.ColourTo(i, state);
							state = SCE_PL_DEFAULT;
						} else {
							if (ch == '\\' && chNext == HereDoc.Quote) { // escaped quote
								i++;
								ch = chNext;
								chNext = chNext2;
							}
							HereDoc.Delimiter[HereDoc.DelimiterLength++] = ch;
							HereDoc.Delimiter[HereDoc.DelimiterLength] = '\0';
						}
					} else { // an unquoted here-doc delimiter
						if (isalnum(ch) || ch == '_') {
							HereDoc.Delimiter[HereDoc.DelimiterLength++] = ch;
							HereDoc.Delimiter[HereDoc.DelimiterLength] = '\0';
						} else {
							styler.ColourTo(i - 1, state);
							state = SCE_PL_DEFAULT;
							goto restartLexer;
						}
					}
					if (HereDoc.DelimiterLength >= HERE_DELIM_MAX - 1) {
						styler.ColourTo(i - 1, state);
						state = SCE_PL_ERROR;
						goto restartLexer;
					}
				}
			} else if (HereDoc.State == 2) {
				// state == SCE_PL_HERE_Q || state == SCE_PL_HERE_QQ || state == SCE_PL_HERE_QX
				if (isEOLChar(chPrev) && isMatch(styler, lengthDoc, i, HereDoc.Delimiter)) {
					i += HereDoc.DelimiterLength;
					chPrev = styler.SafeGetCharAt(i - 1);
					ch = styler.SafeGetCharAt(i);
					if (isEOLChar(ch)) {
						styler.ColourTo(i - 1, state);
						state = SCE_PL_DEFAULT;
                        backflag = BACK_NONE;
						HereDoc.State = 0;
						goto restartLexer;
					}
					chNext = styler.SafeGetCharAt(i + 1);
				}
			} else if (state == SCE_PL_POD
				|| state == SCE_PL_POD_VERB) {
				if (isEOLChar(chPrev)) {
					if (ch == ' ' || ch == '\t') {
						styler.ColourTo(i - 1, state);
						state = SCE_PL_POD_VERB;
					} else {
						styler.ColourTo(i - 1, state);
						state = SCE_PL_POD;
						if (ch == '=') {
							if (isMatch(styler, lengthDoc, i, "=cut")) {
								styler.ColourTo(i - 1 + 4, state);
								i += 4;
								state = SCE_PL_DEFAULT;
								ch = styler.SafeGetCharAt(i);
								//chNext = styler.SafeGetCharAt(i + 1);
								goto restartLexer;
							}
						}
					}
				}
			} else if (state == SCE_PL_SCALAR	// variable names
				|| state == SCE_PL_ARRAY
				|| state == SCE_PL_HASH
				|| state == SCE_PL_SYMBOLTABLE) {
				if (ch == ':' && chNext == ':') {	// skip ::
					i++;
					ch = chNext;
					chNext = chNext2;
				}
				else if (isEndVar(ch)) {
					if (i == (styler.GetStartSegment() + 1)) {
						// Special variable: $(, $_ etc.
						styler.ColourTo(i, state);
						state = SCE_PL_DEFAULT;
					} else {
						styler.ColourTo(i - 1, state);
						state = SCE_PL_DEFAULT;
						goto restartLexer;
					}
				}
			} else if (state == SCE_PL_REGEX
				|| state == SCE_PL_STRING_QR
				) {
				if (!Quote.Up && !isspacechar(ch)) {
					Quote.Open(ch);
				} else if (ch == '\\' && Quote.Up != '\\') {
					// SG: Is it save to skip *every* escaped char?
					i++;
					ch = chNext;
					chNext = styler.SafeGetCharAt(i + 1);
				} else {
					if (ch == Quote.Down /*&& chPrev != '\\'*/) {
						Quote.Count--;
						if (Quote.Count == 0) {
							Quote.Rep--;
							if (Quote.Up == Quote.Down) {
								Quote.Count++;
							}
						}
						if (!isalpha(chNext)) {
							if (Quote.Rep <= 0) {
								styler.ColourTo(i, state);
								state = SCE_PL_DEFAULT;
								ch = ' ';
							}
						}
					} else if (ch == Quote.Up /*&& chPrev != '\\'*/) {
						Quote.Count++;
					} else if (!isascii(chNext) || !isalpha(chNext)) {
						if (Quote.Rep <= 0) {
							styler.ColourTo(i, state);
							state = SCE_PL_DEFAULT;
							ch = ' ';
						}
					}
				}
			} else if (state == SCE_PL_REGSUBST) {
				if (!Quote.Up && !isspacechar(ch)) {
					Quote.Open(ch);
				} else if (ch == '\\' && Quote.Up != '\\') {
					// SG: Is it save to skip *every* escaped char?
					i++;
					ch = chNext;
					chNext = styler.SafeGetCharAt(i + 1);
				} else {
					if (Quote.Count == 0 && Quote.Rep == 1) {
						/* We matched something like s(...) or tr{...}
						* and are looking for the next matcher characters,
						* which could be either bracketed ({...}) or non-bracketed
						* (/.../).
						*
						* Number-signs are problematic.  If they occur after
						* the close of the first part, treat them like
						* a Quote.Up char, even if they actually start comments.
						*
						* If we find an alnum, we end the regsubst, and punt.
						*
						* Eric Promislow   [email protected]  Aug 9,2000
						*/
						if (isspacechar(ch)) {
							// Keep going
						}
						else if (!isascii(ch) || isalnum(ch)) {
							styler.ColourTo(i, state);
							state = SCE_PL_DEFAULT;
							ch = ' ';
						} else {
							Quote.Open(ch);
						}
					} else if (ch == Quote.Down /*&& chPrev != '\\'*/) {
						Quote.Count--;
						if (Quote.Count == 0) {
							Quote.Rep--;
						}
						if (!isascii(chNext) || !isalpha(chNext)) {
							if (Quote.Rep <= 0) {
								styler.ColourTo(i, state);
								state = SCE_PL_DEFAULT;
								ch = ' ';
							}
						}
						if (Quote.Up == Quote.Down) {
							Quote.Count++;
						}
					} else if (ch == Quote.Up /*&& chPrev != '\\'*/) {
						Quote.Count++;
					} else if (!isascii(chNext) || !isalpha(chNext)) {
						if (Quote.Rep <= 0) {
							styler.ColourTo(i, state);
							state = SCE_PL_DEFAULT;
							ch = ' ';
						}
					}
				}
			} else if (state == SCE_PL_STRING_Q
				|| state == SCE_PL_STRING_QQ
				|| state == SCE_PL_STRING_QX
				|| state == SCE_PL_STRING_QW
				|| state == SCE_PL_STRING
				|| state == SCE_PL_CHARACTER
				|| state == SCE_PL_BACKTICKS
				) {
				if (!Quote.Down && !isspacechar(ch)) {
					Quote.Open(ch);
				} else if (ch == '\\' && Quote.Up != '\\') {
					i++;
					ch = chNext;
					chNext = styler.SafeGetCharAt(i + 1);
				} else if (ch == Quote.Down) {
					Quote.Count--;
					if (Quote.Count == 0) {
						Quote.Rep--;
						if (Quote.Rep <= 0) {
							styler.ColourTo(i, state);
							state = SCE_PL_DEFAULT;
							ch = ' ';
						}
						if (Quote.Up == Quote.Down) {
							Quote.Count++;
						}
					}
				} else if (ch == Quote.Up) {
					Quote.Count++;
				}
			}
		}
		if (state == SCE_PL_ERROR) {
			break;
		}
		chPrev = ch;
	}
	styler.ColourTo(lengthDoc - 1, state);
}

static bool IsCommentLine(int line, Accessor &styler) {
	int pos = styler.LineStart(line);
	int eol_pos = styler.LineStart(line + 1) - 1;
	for (int i = pos; i < eol_pos; i++) {
		char ch = styler[i];
        int style = styler.StyleAt(i);
		if (ch == '#' && style == SCE_PL_COMMENTLINE)
			return true;
		else if (ch != ' ' && ch != '\t')
			return false;
	}
	return false;
}

static void FoldPerlDoc(unsigned int startPos, int length, int, WordList *[],
                            Accessor &styler) {
	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
	bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
	// Custom folding of POD and packages
	bool foldPOD = styler.GetPropertyInt("fold.perl.pod", 1) != 0;
	bool foldPackage = styler.GetPropertyInt("fold.perl.package", 1) != 0;
	unsigned int endPos = startPos + length;
	int visibleChars = 0;
	int lineCurrent = styler.GetLine(startPos);
	int levelPrev = SC_FOLDLEVELBASE;
	if (lineCurrent > 0)
		levelPrev = styler.LevelAt(lineCurrent - 1) >> 16;
	int levelCurrent = levelPrev;
	char chNext = styler[startPos];
	char chPrev = styler.SafeGetCharAt(startPos - 1);
	int styleNext = styler.StyleAt(startPos);
	// Used at end of line to determine if the line was a package definition
	bool isPackageLine = false;
	bool isPodHeading = false;
	for (unsigned int i = startPos; i < endPos; i++) {
		char ch = chNext;
		chNext = styler.SafeGetCharAt(i + 1);
		int style = styleNext;
		styleNext = styler.StyleAt(i + 1);
		bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
		bool atLineStart = isEOLChar(chPrev) || i == 0;
        // Comment folding
		if (foldComment && atEOL && IsCommentLine(lineCurrent, styler))
        {
            if (!IsCommentLine(lineCurrent - 1, styler)
                && IsCommentLine(lineCurrent + 1, styler))
                levelCurrent++;
            else if (IsCommentLine(lineCurrent - 1, styler)
                     && !IsCommentLine(lineCurrent+1, styler))
                levelCurrent--;
        }
		if (style == SCE_C_OPERATOR) {
			if (ch == '{') {
				levelCurrent++;
			} else if (ch == '}') {
				levelCurrent--;
			}
		}
		// Custom POD folding
		if (foldPOD && atLineStart) {
			int stylePrevCh = (i) ? styler.StyleAt(i - 1):SCE_PL_DEFAULT;
			if (style == SCE_PL_POD) {
				if (stylePrevCh != SCE_PL_POD && stylePrevCh != SCE_PL_POD_VERB)
					levelCurrent++;
				else if (styler.Match(i, "=cut"))
					levelCurrent--;
				else if (styler.Match(i, "=head"))
					isPodHeading = true;
			} else if (style == SCE_PL_DATASECTION) {
                if (ch == '=' && isalpha(chNext) && levelCurrent == SC_FOLDLEVELBASE)
                    levelCurrent++;
                else if (styler.Match(i, "=cut") && levelCurrent > SC_FOLDLEVELBASE)
                    levelCurrent--;
                else if (styler.Match(i, "=head"))
					isPodHeading = true;
                // if package used or unclosed brace, level > SC_FOLDLEVELBASE!
                // reset needed as level test is vs. SC_FOLDLEVELBASE
                else if (styler.Match(i, "__END__"))
                    levelCurrent = SC_FOLDLEVELBASE;
            }
		}
		// Custom package folding
		if (foldPackage && atLineStart) {
			if (style == SCE_PL_WORD && styler.Match(i, "package")) {
				isPackageLine = true;
			}
		}

		if (atEOL) {
			int lev = levelPrev;
			if (isPodHeading) {
                lev = levelPrev - 1;
                lev |= SC_FOLDLEVELHEADERFLAG;
                isPodHeading = false;
			}
			// Check if line was a package declaration
			// because packages need "special" treatment
			if (isPackageLine) {
				lev = SC_FOLDLEVELBASE | SC_FOLDLEVELHEADERFLAG;
				levelCurrent = SC_FOLDLEVELBASE + 1;
				isPackageLine = false;
			}
            lev |= levelCurrent << 16;
			if (visibleChars == 0 && foldCompact)
				lev |= SC_FOLDLEVELWHITEFLAG;
			if ((levelCurrent > levelPrev) && (visibleChars > 0))
				lev |= SC_FOLDLEVELHEADERFLAG;
			if (lev != styler.LevelAt(lineCurrent)) {
				styler.SetLevel(lineCurrent, lev);
			}
			lineCurrent++;
			levelPrev = levelCurrent;
			visibleChars = 0;
		}
		if (!isspacechar(ch))
			visibleChars++;
		chPrev = ch;
	}
	// Fill in the real level of the next line, keeping the current flags as they will be filled in later
	int flagsNext = styler.LevelAt(lineCurrent) & ~SC_FOLDLEVELNUMBERMASK;
	styler.SetLevel(lineCurrent, levelPrev | flagsNext);
}

static const char * const perlWordListDesc[] = {
	"Keywords",
	0
};

LexerModule lmPerl(SCLEX_PERL, ColourisePerlDoc, "perl", FoldPerlDoc, perlWordListDesc);