public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(_startSymbol, false))
{
return(null);
}
source.Position += _startSymbol.Length;
while (!source.EOF())
{
int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
return(_isLineComment
? TokenAst.Create(this, context, source.TokenStart, source.GetLexeme())
: LRParser.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
}
source.Position = firstCharPos;
if (source.MatchSymbol(_endSymbol, false))
{
source.Position += _endSymbol.Length;
return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
}
source.Position++;
}
throw new NotSupportedException();
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(_startSymbol, false)) return null;
source.Position += _startSymbol.Length;
while (!source.EOF())
{
int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
if (_isLineComment)
return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
else
return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
}
source.Position = firstCharPos;
if (source.MatchSymbol(_endSymbol, false))
{
source.Position += _endSymbol.Length;
return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
}
source.Position++;
}
throw new NotSupportedException();
}
private bool BeginMatch(ParsingContext context, ISourceStream source)
{
//Check starting symbol
if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
{
return(false);
}
source.PreviewPosition += 1;
return(true);
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
bool ignoreCase = !Grammar.CaseSensitive;
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, ignoreCase))
{
return(null);
}
//Find end symbol
source.Position += StartSymbol.Length;
while (!source.EOF())
{
int firstCharPos;
if (EndSymbols.Count == 1)
{
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
}
else
{
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
}
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
{
return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
}
else
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
}
}
//We found a character that might start an end symbol; let's see if it is true.
source.Position = firstCharPos;
foreach (string endSymbol in EndSymbols)
{
if (source.MatchSymbol(endSymbol, ignoreCase))
{
//We found end symbol
source.Position += endSymbol.Length;
return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
}//if
}
source.Position++; //move to the next char and try again
} //while
return(null); //never happens
} //method
private bool BeginMatch(ParsingContext context, ISourceStream source)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol)) return false;
source.PreviewPosition += StartSymbol.Length;
return true;
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var text = source.Text;
foreach (var entry in Constants)
{
source.PreviewPosition = source.Position;
var constant = entry.Key;
if (source.PreviewPosition + constant.Length > text.Length)
{
continue;
}
if (source.MatchSymbol(constant))
{
source.PreviewPosition += constant.Length;
if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
{
continue; //make sure it is delimiter
}
return(source.CreateToken(OutputTerminal, entry.Value));
}
}
return(null);
}
} //method
protected virtual void ReadSuffix(ISourceStream source, ScanDetails details)
{
if (_suffixesFirsts.IndexOf(source.CurrentChar) < 0)
{
return;
}
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
foreach (string sfx in Suffixes)
{
if (!source.MatchSymbol(sfx, ignoreCase))
{
continue;
}
//We found suffix
details.Suffix = sfx;
source.Position += sfx.Length;
//Set TypeCode from suffix
TypeCode[] codes;
if (!string.IsNullOrEmpty(details.Suffix) && SuffixTypeCodes.TryGetValue(details.Suffix, out codes))
{
details.TypeCodes = codes;
}
return;
} //foreach
} //method
protected virtual void ReadPrefix(ISourceStream source, ScanDetails details)
{
if (_prefixesFirsts.IndexOf(source.CurrentChar) < 0)
{
return;
}
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
foreach (string pfx in Prefixes)
{
if (!source.MatchSymbol(pfx, ignoreCase))
{
continue;
}
//We found prefix
details.Prefix = pfx;
source.Position += pfx.Length;
//Set numeric base flag from prefix
ScanFlags pfxFlags;
if (!string.IsNullOrEmpty(details.Prefix) && PrefixFlags.TryGetValue(details.Prefix, out pfxFlags))
{
details.Flags |= pfxFlags;
}
return;
} //foreach
} //method
protected override bool ReadBody(ISourceStream source, ScanDetails details)
{
if (!ReadStartSymbol(source, details))
{
return(false);
}
bool escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
int start = source.Position;
string startS = details.ControlSymbol;
string startS2 = startS + startS; //doubled start symbol
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);
while (!source.EOF())
{
int endPos = source.Text.IndexOf(startS, source.Position);
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed)
{
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0)
{
endPos = nlPos;
}
if (endPos > 0)
{
source.Position = endPos + 1;
}
details.Error = "Mal-formed string literal - cannot find termination symbol.";
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && source.Text[endPos - 1] == EscapeChar)
{
source.Position = endPos + startS.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.Position = endPos;
if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase))
{
source.Position = endPos + startS.Length * 2;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.Text.Substring(start, endPos - start);
source.Position = endPos + startS.Length;
return(true); //if we come here it means we're done - we found string end.
} //end of loop to find string end;
return(false);
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
bool ignoreCase = !Grammar.CaseSensitive;
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
//Find end symbol
source.Position += StartSymbol.Length;
while(!source.EOF()) {
int firstCharPos;
if (EndSymbols.Count == 1)
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
else
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
if (firstCharPos < 0) {
source.Position = source.Text.Length;
if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
return Token.Create(this, context, source.TokenStart, source.GetLexeme());
else
return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
}
//We found a character that might start an end symbol; let's see if it is true.
source.Position = firstCharPos;
foreach (string endSymbol in EndSymbols)
if (source.MatchSymbol(endSymbol, ignoreCase)) {
//We found end symbol
source.Position += endSymbol.Length;
return Token.Create(this, context, source.TokenStart, source.GetLexeme());
}//if
source.Position++; //move to the next char and try again
}//while
return null; //never happens
}
private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte commentLevel)
{
if (!source.MatchSymbol(StartSymbol, !base.Grammar.CaseSensitive))
{
return(false);
}
int num = source.PreviewPosition + StartSymbol.Length;
int length = source.Text.Length;
if (length > num + 1 && source.Text[num] == '[')
{
byte b = 1;
int num2 = num + 255;
num++;
while (num < num2 && length > num)
{
if (source.Text[num] == '=')
{
b += 1;
num++;
}
else
{
if (source.Text[num] == '[')
{
commentLevel = b;
break;
}
break;
}
}
}
source.PreviewPosition += StartSymbol.Length + (int)commentLevel;
return(true);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
if (isHeadingOrList)
{
bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
if (!isAfterNewLine)
{
return(null);
}
}
if (!source.MatchSymbol(OpenTag, true))
{
return(null);
}
source.PreviewPosition += OpenTag.Length;
//For headings and lists require space after
if (TermType == WikiTermType.Heading || TermType == WikiTermType.List)
{
const string whitespaces = " \t\r\n\v";
if (!whitespaces.Contains(source.PreviewChar))
{
return(null);
}
}
var token = source.CreateToken(this.OutputTerminal);
return(token);
}
protected override string ReadBody(ParsingContext context, ISourceStream source) {
if (!source.MatchSymbol(StartSymbol)) return null; //this will result in null returned from TryMatch, no token
var start = source.Location.Position + StartSymbol.Length;
var end = source.Text.IndexOf(EndSymbol, start);
if (end < 0) return null;
var body = source.Text.Substring(start, end - start);
source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
return body;
}
private bool TryMatchPrefixes(ParsingContext context, ISourceStream source) {
if (Firsts.Count == 0)
return true;
foreach (var first in Firsts)
if (source.MatchSymbol(first)) {
source.PreviewPosition += first.Length;
return true;
}
return false;
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(Symbol, false))
return null;
source.Position += Symbol.Length;
TokenAst tokenAst = TokenAst.Create(this, context, source.TokenStart, Symbol);
return tokenAst;
}
private bool BeginMatch(CompilerContext context, ISourceStream source)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, !OwnerGrammar.CaseSensitive))
{
return(false);
}
source.Position += StartSymbol.Length;
return(true);
}
private bool CheckEscape(ISourceStream source, ref string tokenText) {
foreach (var dictEntry in Escapes) {
if (source.MatchSymbol(dictEntry.Key, !Grammar.CaseSensitive)) {
source.PreviewPosition += dictEntry.Key.Length;
tokenText += dictEntry.Value;
return true;
}
}//foreach
return false;
}
private bool BeginMatch(ParsingContext context, ISourceStream source)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol))
{
return(false);
}
source.PreviewPosition += StartSymbol.Length;
return(true);
}
private bool BeginMatch(ISourceStream source, int startFrom, char lookAhead) {
foreach (var startSymbol in StartSymbols.Skip(startFrom)) {
if (startSymbol[0] != lookAhead)
continue;
if (source.MatchSymbol(startSymbol, !Grammar.CaseSensitive)) {
source.PreviewPosition += startSymbol.Length;
return true;
}
}
return false;
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(_symbol, !OwnerGrammar.CaseSensitive))
{
return(null);
}
source.Position += _symbol.Length;
Token tkn = new Token(this, source.TokenStart, Symbol, null);
return(tkn);
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(Symbol, false))
{
return(null);
}
source.Position += Symbol.Length;
TokenAst tokenAst = TokenAst.Create(this, context, source.TokenStart, Symbol);
return(tokenAst);
}
Token Match_compile_end_tag(Terminal terminal, ParsingContext context, ISourceStream source)
{
if (source.MatchSymbol("__End"))
{
var p = source.Location.Position;
var lineEnd = source.Text.IndexOf("\n", p);
var text = source.Text.Substring(p, lineEnd - p + 1);
return(new Token(_comment, source.Location, text, text));
}
//otherwise return null
return(null);
}
private bool CheckEscape(ISourceStream source, StringBuilder tokenText)
{
foreach (var dictEntry in Escapes)
{
if (source.MatchSymbol(dictEntry.Key))
{
source.PreviewPosition += dictEntry.Key.Length;
tokenText.Append(dictEntry.Value);
return(true);
}
}//foreach
return(false);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source)
{
//Find end symbol
while (!source.EOF())
{
var firstCharPos = source.Text.IndexOf('{', source.PreviewPosition);
if (firstCharPos < 0)
{
source.PreviewPosition = source.Text.Length;
return(source.CreateToken(this.OutputTerminal));
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
{
return(source.CreateToken(this.OutputTerminal));
}
source.PreviewPosition++; //move to the next char and try again
}
return(source.CreateToken(this.OutputTerminal));
}
private bool CheckEscape(ISourceStream source, ref string tokenText)
{
foreach (var dictEntry in Escapes)
{
if (source.MatchSymbol(dictEntry.Key, !Grammar.CaseSensitive))
{
source.PreviewPosition += dictEntry.Key.Length;
tokenText += dictEntry.Value;
return(true);
}
}//foreach
return(false);
}
private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
{
if (!source.MatchSymbol(StartSymbol, !base.Grammar.CaseSensitive))
{
return(false);
}
Match match = Regex.Match(source.Text.Substring(source.PreviewPosition + StartSymbol.Length), "^(=*)\\[");
if (match.Value != string.Empty)
{
level = (byte)match.Groups[1].Value.Length;
return(true);
}
return(false);
}
private void MoveSourcePositionAfterTerminator(ISourceStream source)
{
while (!source.EOF())
{
while (source.PreviewChar != Terminator[0])
{
source.PreviewPosition++;
}
if (source.MatchSymbol(Terminator, !Grammar.CaseSensitive))
{
source.PreviewPosition += Terminator.Length;
return;
} //if
} //while
} //method
public override Token TryMatch(ParsingContext context, ISourceStream source) {
if (!source.MatchSymbol(OpenTag, true)) return null;
source.PreviewPosition += OpenTag.Length;
var endPos = source.Text.IndexOf(CloseTag, source.PreviewPosition);
string content;
if(endPos > 0) {
content = source.Text.Substring(source.PreviewPosition, endPos - source.PreviewPosition);
source.PreviewPosition = endPos + CloseTag.Length;
} else {
content = source.Text.Substring(source.PreviewPosition, source.Text.Length - source.PreviewPosition);
source.PreviewPosition = source.Text.Length;
}
var token = source.CreateToken(this.OutputTerminal, content);
return token;
}
private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
{
if (!source.MatchSymbol(StartSymbol))
return false;
string text = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
var match = Regex.Match(text, @"^(=*)\[");
if(match.Value != string.Empty)
{
level = (byte)match.Groups[1].Value.Length;
return true;
}
return false;
}
private bool BeginMatch(ISourceStream source, int startFrom, char lookAhead)
{
foreach (var startSymbol in StartSymbols.Skip(startFrom))
{
if (startSymbol[0] != lookAhead)
{
continue;
}
if (source.MatchSymbol(startSymbol))
{
source.PreviewPosition += startSymbol.Length;
return(true);
}
}
return(false);
}
private bool TryMatchPrefixes(ParsingContext context, ISourceStream source)
{
if (Firsts.Count == 0)
{
return(true);
}
foreach (var first in Firsts)
{
if (source.MatchSymbol(first))
{
source.PreviewPosition += first.Length;
return(true);
}
}
return(false);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
if(isHeadingOrList) {
bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
if(!isAfterNewLine) return null;
}
if(!source.MatchSymbol(OpenTag)) return null;
source.PreviewPosition += OpenTag.Length;
//For headings and lists require space after
if(TermType == WikiTermType.Heading || TermType == WikiTermType.List) {
const string whitespaces = " \t\r\n\v";
if (!whitespaces.Contains(source.PreviewChar)) return null;
}
var token = source.CreateToken(this.OutputTerminal);
return token;
}
private void MoveSourcePositionAfterTerminator(ISourceStream source)
{
while (!source.EOF)
{
while (source.PreviewChar != Terminator[0])
{
source.PreviewPosition++;
}
if (source.MatchSymbol(Terminator))
{
source.PreviewPosition += Terminator.Length;
return;
}
}
}
private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, !Grammar.CaseSensitive))
return false;
//Found starting --, now determine whether this is a long comment.
string text = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
var match = Regex.Match(text, @"^(=*)\[");
if(match.Value != string.Empty)
{
level = (byte)match.Groups[1].Value.Length;
return true;
}
return false;
}
protected override string ReadBody(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(StartSymbol))
{
return(null); //this will result in null returned from TryMatch, no token
}
var start = source.Location.Position + StartSymbol.Length;
var end = source.IndexOf(EndSymbol, start);
if (end < 0)
{
return(null);
}
var body = source.GetText(start, end - start);
source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
return(body);
}
private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte level)
{
if (!source.MatchSymbol(StartSymbol))
{
return(false);
}
string text = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
var match = Regex.Match(text, @"^(=*)\[");
if (match.Value != string.Empty)
{
level = (byte)match.Groups[1].Value.Length;
return(true);
}
return(false);
}
private bool BeginMatch(ParsingContext context, ISourceStream source, out int numberOfEquals)
{
numberOfEquals = 0;
//Check starting symbol
if (!source.MatchSymbol("--["))
{
return(false);
}
//Do our regex magic
var match = counterRegex.Match(source.Text, source.PreviewPosition);
numberOfEquals = match.Groups[1].Length;
source.PreviewPosition += match.Length;
return(true);
}
private bool CheckTerminators(ISourceStream source, ref string tokenText)
{
foreach (var term in Terminators)
{
if (source.MatchSymbol(term, !Grammar.CaseSensitive))
{
if (IsSet(FreeTextOptions.IncludeTerminator))
{
tokenText += term;
}
if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
{
source.PreviewPosition += term.Length;
}
return(true);
}
}
return(false);
}
private bool CheckTerminators(ISourceStream source, StringBuilder tokenText)
{
foreach (var term in Terminators)
{
if (source.MatchSymbol(term))
{
if (IsSet(FreeTextOptions.IncludeTerminator))
{
tokenText.Append(term);
}
if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
{
source.PreviewPosition += term.Length;
}
return(true);
}
}
return(false);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
string text = source.Text;
foreach (var entry in Constants)
{
var constant = entry.Key;
if (source.PreviewPosition + constant.Length > text.Length)
{
continue;
}
if (source.MatchSymbol(constant, !Grammar.CaseSensitive))
{
source.PreviewPosition += constant.Length;
return(source.CreateToken(this.OutputTerminal, entry.Value));
}
}
return(null);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source)
{
//Find end symbol
while (!source.EOF)
{
int firstCharPos;
if (EndSymbols.Count == 1)
{
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition, StringComparison.InvariantCulture);
}
else
{
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
}
if (firstCharPos < 0)
{
source.PreviewPosition = source.Text.Length;
return(null); //indicating error
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
foreach (var endSymbol in EndSymbols)
{
if (!source.MatchSymbol(endSymbol))
{
continue;
}
//We found end symbol; eat end symbol only if it is not line comment.
// For line comment, leave LF symbol there, it might be important to have a separate LF token
if (!_isLineComment)
{
source.PreviewPosition += endSymbol.Length;
}
return(source.CreateToken(OutputTerminal));
}
source.PreviewPosition++; //move to the next char and try again
}
return(null); //might happen if we found a start char of end symbol, but not the full endSymbol
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
string text = source.Text;
foreach (string lexeme in Table.Keys)
{
if (source.Position + lexeme.Length > text.Length)
{
continue;
}
if (!source.MatchSymbol(lexeme, !OwnerGrammar.CaseSensitive))
{
continue;
}
Token tkn = new Token(this, source.TokenStart, lexeme, Table[lexeme]);
source.Position += lexeme.Length;
return(tkn);
}
return(null);
}
private bool ReadStartSymbol(ISourceStream source, ScanDetails details)
{
if (_startEndFirsts.IndexOf(source.CurrentChar) < 0)
return false;
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
foreach (string startEnd in _startEndSymbols) {
if (!source.MatchSymbol(startEnd, ignoreCase))
continue;
//We found start symbol
details.ControlSymbol = startEnd;
details.Flags |= StartEndSymbolTable[startEnd];
source.Position += startEnd.Length;
return true;
}//foreach
return false;
}
protected override bool ReadBody(ISourceStream source, ScanDetails details)
{
if (!ReadStartSymbol(source, details)) return false;
bool escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
int start = source.Position;
string startS = details.ControlSymbol;
string startS2 = startS + startS; //doubled start symbol
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);
while (!source.EOF()) {
int endPos = source.Text.IndexOf(startS, source.Position);
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed) {
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0) endPos = nlPos;
if (endPos > 0) source.Position = endPos + 1;
details.Error = "Mal-formed string literal - cannot find termination symbol.";
return true;
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && source.Text[endPos - 1] == EscapeChar) {
source.Position = endPos + startS.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.Position = endPos;
if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase)) {
source.Position = endPos + startS.Length * 2;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.Text.Substring(start, endPos - start);
source.Position = endPos + startS.Length;
return true; //if we come here it means we're done - we found string end.
} //end of loop to find string end;
return false;
}
private bool BeginMatch(CompilerContext context, ISourceStream source)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, !Grammar.CaseSensitive)) return false;
source.Position += StartSymbol.Length;
return true;
}
private bool CheckEscape(ISourceStream source, StringBuilder tokenText) {
foreach (var dictEntry in Escapes) {
if (source.MatchSymbol(dictEntry.Key)) {
source.PreviewPosition += dictEntry.Key.Length;
tokenText.Append(dictEntry.Value);
return true;
}
}//foreach
return false;
}
private bool CheckTerminators(ISourceStream source, StringBuilder tokenText) {
foreach(var term in Terminators)
if(source.MatchSymbol(term)) {
if (IsSet(FreeTextOptions.IncludeTerminator))
tokenText.Append(term);
if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
source.PreviewPosition += term.Length;
return true;
}
return false;
}
private void MoveSourcePositionAfterTerminator(ISourceStream source) {
while(!source.EOF()) {
while(source.PreviewChar != Terminator[0])
source.PreviewPosition++;
if(source.MatchSymbol(Terminator, !Grammar.CaseSensitive)) {
source.PreviewPosition += Terminator.Length;
return;
}//if
}//while
}//method
Token Match_compile_end_tag(Terminal terminal, ParsingContext context, ISourceStream source) {
if (source.MatchSymbol("__End")) {
var p = source.Location.Position;
var lineEnd = source.Text.IndexOf("\n", p);
var text = source.Text.Substring(p, lineEnd - p + 1);
return new Token(_comment, source.Location, text, text);
}
//otherwise return null
return null;
}
private Token CompleteMatch(ParsingContext context, ISourceStream source) {
//Find end symbol
while (!source.EOF()) {
int firstCharPos;
if (EndSymbols.Count == 1)
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
else
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
if (firstCharPos < 0) {
source.PreviewPosition = source.Text.Length;
return null; //indicating error
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
foreach (string endSymbol in EndSymbols) {
if (source.MatchSymbol(endSymbol)) {
//We found end symbol; eat end symbol only if it is not line comment.
// For line comment, leave LF symbol there, it might be important to have a separate LF token
if (!_isLineComment)
source.PreviewPosition += endSymbol.Length;
return source.CreateToken(this.OutputTerminal);
}//if
}//foreach endSymbol
source.PreviewPosition++; //move to the next char and try again
}//while
return null; //might happen if we found a start char of end symbol, but not the full endSymbol
}//method
private bool CheckTerminators(ISourceStream source, ref string tokenText) {
foreach(var term in Terminators)
if(source.MatchSymbol(term, !Grammar.CaseSensitive)) {
if (IsSet(FreeTextOptions.IncludeTerminator))
tokenText += term;
if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
source.PreviewPosition += term.Length;
return true;
}
return false;
}