private Token MatchQuoted(ParsingContext context, ISourceStream source)
{
char quoteChar = source.PreviewChar;
if ((quoteChar != '\'') && (quoteChar != '"'))
{
return null;
}
source.PreviewPosition++;
while (!source.EOF())
{
if (source.PreviewChar == quoteChar)
{
source.PreviewPosition++;
return source.CreateToken(this.OutputTerminal);
}
// Escaped?
if (source.PreviewChar == '\\')
{
// Consume next
++source.PreviewPosition;
}
source.PreviewPosition++;
}
return context.CreateErrorToken("Unbalanced quoted string");
}
private Token CreateIncompleteToken(ParsingContext context, ISourceStream source) {
source.PreviewPosition = source.Text.Length;
Token result = source.CreateToken(this.OutputTerminal);
result.Flags |= TokenFlags.IsIncomplete;
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
return result;
}
private Token CompleteMatch(ISourceStream source) {
if (source.EOF())
return null;
do {
// Match NewLine
var lookAhead = source.PreviewChar;
if (LineTerminators.IndexOf(lookAhead) >= 0)
{
source.PreviewPosition++;
// Treat \r\n as single NewLine
if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
source.PreviewPosition++;
break;
}
// Eat up whitespace
if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
{
source.PreviewPosition++;
continue;
}
// Fail on anything else
return null;
}
while (!source.EOF());
// Create output token
return source.CreateToken(this.OutputTerminal);
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(_startSymbol, false)) return null;
source.Position += _startSymbol.Length;
while (!source.EOF())
{
int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
if (_isLineComment)
return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
else
return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
}
source.Position = firstCharPos;
if (source.MatchSymbol(_endSymbol, false))
{
source.Position += _endSymbol.Length;
return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
}
source.Position++;
}
throw new NotSupportedException();
}
private bool BeginMatch(ParsingContext context, ISourceStream source)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol)) return false;
source.PreviewPosition += StartSymbol.Length;
return true;
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach(Match match in matches)
{
if (match.Groups[1].Value.Length == (int)level)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return new Token(this, tokenStart, lexeme, null);
}
else
{
return source.CreateToken(this.OutputTerminal);
}
}
}
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
context.VsLineScanState.TokenSubType = level;
return null;
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
Token result;
if (context.VsLineScanState.Value != 0)
{
byte level = context.VsLineScanState.TokenSubType;
result = CompleteMatch(context, source, level);
}
else
{
//we are starting from scratch
byte level = 0;
if (!BeginMatch(context, source, ref level))
return null;
result = CompleteMatch(context, source, level);
}
if (result != null)
return result;
if (context.Mode == ParseMode.VsLineScan)
return CreateIncompleteToken(context, source);
return source.CreateErrorToken("Unclosed comment block");
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
Match m = _expression.Match(source.Text, source.PreviewPosition);
if (!m.Success || m.Index != source.PreviewPosition)
return null;
source.PreviewPosition += m.Length;
return source.CreateToken(this.OutputTerminal);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
string tokenText = string.Empty;
while (true) {
//Find next position
var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
if(newPos == -1) {
if(IsSet(FreeTextOptions.AllowEof)) {
source.PreviewPosition = source.Text.Length;
return source.CreateToken(this.OutputTerminal);
} else
return null;
}
if (newPos == source.PreviewPosition) // DC
{
context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {});
break; // DC
}
tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
source.PreviewPosition = newPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, ref tokenText))
continue;
//check terminators
if (CheckTerminators(source, ref tokenText))
break; //from while (true)
}
return source.CreateToken(this.OutputTerminal, tokenText);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source) {
//Find end symbol
while (!source.EOF()) {
int firstCharPos;
if (EndSymbols.Count == 1)
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
else
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
if (firstCharPos < 0) {
source.PreviewPosition = source.Text.Length;
return null; //indicating error
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
foreach (string endSymbol in EndSymbols) {
if (source.MatchSymbol(endSymbol)) {
//We found end symbol; eat end symbol only if it is not line comment.
// For line comment, leave LF symbol there, it might be important to have a separate LF token
if (!_isLineComment)
source.PreviewPosition += endSymbol.Length;
return source.CreateToken(this.OutputTerminal);
}//if
}//foreach endSymbol
source.PreviewPosition++; //move to the next char and try again
}//while
return null; //might happen if we found a start char of end symbol, but not the full endSymbol
}//method
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
bool ignoreCase = !Grammar.CaseSensitive;
//Check starting symbol
if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
//Find end symbol
source.Position += StartSymbol.Length;
while(!source.EOF()) {
int firstCharPos;
if (EndSymbols.Count == 1)
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
else
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
if (firstCharPos < 0) {
source.Position = source.Text.Length;
if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
return Token.Create(this, context, source.TokenStart, source.GetLexeme());
else
return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
}
//We found a character that might start an end symbol; let's see if it is true.
source.Position = firstCharPos;
foreach (string endSymbol in EndSymbols)
if (source.MatchSymbol(endSymbol, ignoreCase)) {
//We found end symbol
source.Position += endSymbol.Length;
return Token.Create(this, context, source.TokenStart, source.GetLexeme());
}//if
source.Position++; //move to the next char and try again
}//while
return null; //never happens
}
private static string ReadQuotedBody(ParsingContext context, ISourceStream source)
{
const char dQuoute = '"';
StringBuilder sb = null;
var from = source.Location.Position + 1; //skip initial double quote
while(true) {
var until = source.Text.IndexOf(dQuoute, from);
if (until < 0)
throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value."
source.PreviewPosition = until; //now points at double-quote
var piece = source.Text.Substring(from, until - from);
source.PreviewPosition++; //move after double quote
if (source.PreviewChar != dQuoute && sb == null)
return piece; //quick path - if sb (string builder) was not created yet, we are looking at the very first segment;
// and if we found a standalone dquote, then we are done - the "piece" is the result.
if (sb == null)
sb = new StringBuilder(100);
sb.Append(piece);
if (source.PreviewChar != dQuoute)
return sb.ToString();
//we have doubled double-quote; add a single double-quoute char to the result and move over both symbols
sb.Append(dQuoute);
from = source.PreviewPosition + 1;
}
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
Token result;
if (context.VsLineScanState.Value != 0)
{
byte commentLevel = context.VsLineScanState.TokenSubType;
result = CompleteMatch(context, source, commentLevel);
}
else
{
byte commentLevel = 0;
if (!BeginMatch(context, source, ref commentLevel))
return null;
result = CompleteMatch(context, source, commentLevel);
}
if (result != null)
return result;
if (context.Mode == ParseMode.VsLineScan)
return CreateIncompleteToken(context, source);
return context.CreateErrorToken("unclosed comment");
}
public void Prepare(CompilerContext context, ISourceStream source)
{
_context = context;
_source = source;
_currentToken = null;
_bufferedTokens.Clear();
ResetSource();
}
public override Token TryMatch(CompilerContext context, ISourceStream source) {
Match m = _expression.Match(source.Text, source.Position);
if (!m.Success || m.Index != source.Position)
return null;
source.Position += m.Length;
string text = source.GetLexeme();
return new Token(this, source.TokenStart, text, null);
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
Match result = Expression.Match(source.Text, source.Position);
if (!result.Success)
return null;
source.Position += result.Length;
return CreateToken(context, source);
}
public void SetPartialSource(string text, int offset) {
if (_source == null)
_source = new SourceStream(text, offset);
else {
_source.Text = text;
_source.Position = offset;
}
_nextNewLinePosition = _source.Text.IndexOf('\n', offset);
}
protected override TokenAst CreateToken(CompilerContext context, ISourceStream source)
{
TokenAst token = base.CreateToken(context, source);
if (Keywords.Contains(token.Text))
token.IsKeyword = true;
return token;
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
bool isVerbatim = false;
int start = source.Position;
if (source.CurrentChar == '@')
{
isVerbatim = true;
source.Position++;
start++;
}
if (IsCurrentQuote(source))
{
source.Position++;
start++;
}
else
return null;
while (!source.EOF())
{
if (!isVerbatim)
{
if (source.CurrentChar == '\\')
{
//TODO: Escape processing
source.Position += 2;
continue;
}
else
//Single line string ends incorrectly
if (ParserData.LineTerminators.IndexOf(source.CurrentChar) >= 0)
return null;
}
if (IsCurrentQuote(source)) break;
source.Position++;
}
if (IsCurrentQuote(source))
source.Position++;
else
return null;
string lexeme = source.GetLexeme();
string body = source.Text.Substring(start, source.Position - start - 1);
//TODO: handle this in escape processing
if (!isVerbatim)
body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");
TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);
return token;
//return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Match m = _expression.Match(source.Text, source.Position);
if (!m.Success)
return null;
source.Position += m.Length + 1;
string text = source.GetLexeme();
return Token.Create(this, context, source.TokenStart, text);
}
private string ReadNotQuotedBody(ParsingContext context, ISourceStream source) {
var startPos = source.Location.Position;
var sepPos = source.Text.IndexOfAny(_terminators, startPos);
if (sepPos < 0)
sepPos = source.Text.Length;
source.PreviewPosition = sepPos;
var valueText = source.Text.Substring(startPos, sepPos - startPos);
return valueText;
}
protected override string ReadBody(ParsingContext context, ISourceStream source) {
if (!source.MatchSymbol(StartSymbol)) return null; //this will result in null returned from TryMatch, no token
var start = source.Location.Position + StartSymbol.Length;
var end = source.Text.IndexOf(EndSymbol, start);
if (end < 0) return null;
var body = source.Text.Substring(start, end - start);
source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
return body;
}
private bool CheckEscape(ISourceStream source, ref string tokenText) {
foreach (var dictEntry in Escapes) {
if (source.MatchSymbol(dictEntry.Key, !Grammar.CaseSensitive)) {
source.PreviewPosition += dictEntry.Key.Length;
tokenText += dictEntry.Value;
return true;
}
}//foreach
return false;
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(Symbol, false))
return null;
source.Position += Symbol.Length;
TokenAst tokenAst = TokenAst.Create(this, context, source.TokenStart, Symbol);
return tokenAst;
}
private bool TryMatchPrefixes(ParsingContext context, ISourceStream source) {
if (Firsts.Count == 0)
return true;
foreach (var first in Firsts)
if (source.MatchSymbol(first)) {
source.PreviewPosition += first.Length;
return true;
}
return false;
}
protected override string ReadBody(ParsingContext context, ISourceStream source) {
string body;
if (source.PreviewChar == '"')
body = ReadQuotedBody(context, source);
else
body = ReadNotQuotedBody(context, source);
if (ConsumeTerminator && Terminator != null)
MoveSourcePositionAfterTerminator(source);
return body;
}
private Token MatchUnquoted(ParsingContext context, ISourceStream source)
{
if (source.PreviewChar == '{')
{
// Member names can't start with {
if (IsMemberName) return null;
// Check for special {} at start of token indicating that this is a STRING token.
if (source.NextPreviewChar != '}') return null;
source.PreviewPosition +=2;
}
var runningBraceTotal = 0;
// This variable tracks the position of the last non whitespace (or significant whitespace).
var lastNonWhitespacePosition = source.PreviewPosition;
while (!source.EOF())
{
bool isWhiteSpace = false;
switch (source.PreviewChar)
{
case '{':
runningBraceTotal++;
break;
case '}':
if (--runningBraceTotal < 0)
return CreateToken(source, lastNonWhitespacePosition);
break;
case ',':
if (runningBraceTotal == 0)
return CreateToken(source, lastNonWhitespacePosition);
break;
case '=':
if (runningBraceTotal == 0)
{
// Equal sign. Only allowed after MemberNames.
return IsMemberName
? CreateToken(source, lastNonWhitespacePosition)
: null;
}
break;
case '\\':
source.PreviewPosition++;
break;
default:
isWhiteSpace = Char.IsWhiteSpace(source.PreviewChar);
break;
}
source.PreviewPosition++;
if (!isWhiteSpace)
lastNonWhitespacePosition = source.PreviewPosition;
}
return context.CreateErrorToken("Unterminated string terminal");
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
char current = source.PreviewChar;
if (!LineTerminators.Contains(current)) return null;
//Treat \r\n as a single terminator
bool doExtraShift = (current == '\r' && source.NextPreviewChar == '\n');
source.PreviewPosition++; //main shift
if (doExtraShift)
source.PreviewPosition++;
Token result = source.CreateToken(this.OutputTerminal);
return result;
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
try {
var textValue = ReadBody(context, source);
if (textValue == null) return null;
var value = ConvertValue(context, textValue);
return source.CreateToken(this.OutputTerminal, value);
} catch(Exception ex) {
//we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
return source.CreateErrorToken(ex.Message);
}
}//method
public override Token TryMatch(CompilerContext context, ISourceStream source) {
char current = source.CurrentChar;
if (!LineTerminators.Contains(current)) return null;
//Treat \r\n as a single terminator
bool doExtraShift = (current == '\r' && source.NextChar == '\n');
source.Position++; //main shift
if (doExtraShift)
source.Position++;
Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);
return result;
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var current = source.PreviewChar;
if (!LineTerminators.Contains(current))
{
return(null);
}
//Treat \r\n as a single terminator
var doExtraShift = (current == '\r' && source.NextPreviewChar == '\n');
source.PreviewPosition++; //main shift
if (doExtraShift)
{
source.PreviewPosition++;
}
var result = source.CreateToken(OutputTerminal);
return(result);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
try
{
var previewPosition = source.Location.Position + 36;
if (source.Text.Length >= previewPosition)
{
source.PreviewPosition = previewPosition;
var body = source.Text.Substring(source.Location.Position, 36);
if (Guid.TryParse(body, out Guid value))
{
return(source.CreateToken(this.OutputTerminal, value));
}
}
return(null);
}
catch (Exception ex)
{
return(context.CreateErrorToken(ex.Message));
}
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(Text, !Grammar.CaseSensitive))
{
return(null);
}
source.PreviewPosition += Text.Length;
//In case of keywords, check that it is not followed by letter or digit
if (this.FlagIsSet(TermFlags.IsKeyword) && !AllowAlphaAfterKeyword)
{
var previewChar = source.PreviewChar;
if (char.IsLetterOrDigit(previewChar) || previewChar == '_')
{
return(null); //reject
}
}
var token = source.CreateToken(this.OutputTerminal, Text);
token.Symbol = this.Symbol;
return(token);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
while (!source.EOF())
{
switch (source.PreviewChar)
{
case '\n':
case '\r':
case ' ':
case '}':
if (source.PreviewPosition > source.Position)
{
return(source.CreateToken(this.OutputTerminal));
}
return(context.CreateErrorToken(Name + " was expected"));
}
source.PreviewPosition++;
}
return(context.CreateErrorToken("Unbalanced " + Name));
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
string tokenText = string.Empty;
while (true)
{
//Find next position
var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
if (newPos == -1)
{
if (IsSet(FreeTextOptions.AllowEof))
{
source.PreviewPosition = source.Text.Length;
return(source.CreateToken(this.OutputTerminal));
}
else
{
return(null);
}
}
if (newPos == source.PreviewPosition) // DC
{
context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {});
break; // DC
}
tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
source.PreviewPosition = newPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, ref tokenText))
{
continue;
}
//check terminators
if (CheckTerminators(source, ref tokenText))
{
break; //from while (true)
}
}
return(source.CreateToken(this.OutputTerminal, tokenText));
}
private bool ReadStartSymbol(ISourceStream source, ScanDetails details)
{
if (_startEndFirsts.IndexOf(source.CurrentChar) < 0)
{
return(false);
}
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
foreach (string startEnd in _startEndSymbols)
{
if (!source.MatchSymbol(startEnd, ignoreCase))
{
continue;
}
//We found start symbol
details.ControlSymbol = startEnd;
details.Flags |= StartEndSymbolTable[startEnd];
source.Position += startEnd.Length;
return(true);
} //foreach
return(false);
} //method
protected override Token QuickParse(CompilerContext context, ISourceStream source)
{
if (AllFirstChars.IndexOf(source.CurrentChar) < 0)
{
return(null);
}
source.Position++;
while (AllChars.IndexOf(source.CurrentChar) >= 0 && !source.EOF())
{
source.Position++;
}
//if it is not a terminator then cancel; we need to go through full algorithm
if (_terminators.IndexOf(source.CurrentChar) < 0)
{
return(null);
}
string text = source.GetLexeme();
Token token = new Token(this, source.TokenStart, text, text);
CheckReservedWord(token);
return(token);
}
private bool BeginMatch(ParsingContext context, ISourceStream source, ref byte commentLevel)
{
//Check starting symbol
if (!source.MatchSymbol(StartSymbol))
{
return(false);
}
//Found starting --, now determine whether this is a long comment.
string text = source.Text.Substring(source.PreviewPosition + StartSymbol.Length);
var match = Regex.Match(text, @"^\[(=*)\[");
if (match.Value != string.Empty)
{
commentLevel = (byte)(match.Groups[1].Value.Length + 1);
}
//Increment position of comment so we don't rescan the same text.
source.PreviewPosition += StartSymbol.Length + commentLevel;
return(true);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source)
{
//Find end symbol
while (!source.EOF())
{
var firstCharPos = source.Text.IndexOf('{', source.PreviewPosition);
if (firstCharPos < 0)
{
source.PreviewPosition = source.Text.Length;
return(source.CreateToken(this.OutputTerminal));
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
{
return(source.CreateToken(this.OutputTerminal));
}
source.PreviewPosition++; //move to the next char and try again
}
return(source.CreateToken(this.OutputTerminal));
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
foreach (var entry in Constants)
{
source.PreviewPosition = source.Position;
var constant = entry.Key;
if (source.PreviewPosition + constant.Length > source.Length)
{
continue;
}
if (source.MatchSymbol(constant))
{
source.PreviewPosition += constant.Length;
if (!this.Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
{
continue; //make sure it is delimiter
}
return(source.CreateToken(this.OutputTerminal, entry.Value));
}
}
return(null);
}
protected virtual void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
if (!_suffixesFirsts.Contains(source.PreviewChar))
{
return;
}
var comparisonType = CaseSensitivePrefixesSuffixes ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
foreach (var sfx in Suffixes)
{
//Suffixes are usually case insensitive, even if language is case-sensitive. So we cannot use source.MatchSymbol here,
// we need case-specific comparison
if (string.Compare(source.Text, source.PreviewPosition, sfx, 0, sfx.Length, comparisonType) != 0)
{
continue;
}
//We found suffix
details.Suffix = sfx;
source.PreviewPosition += sfx.Length;
//Set TypeCode from suffix
if (!string.IsNullOrEmpty(details.Suffix) && SuffixTypeCodes.TryGetValue(details.Suffix, out TypeCode[] codes))
private Token TryMatchContentExtended(ParsingContext context, ISourceStream source)
{
StringBuilder tokenText = new StringBuilder();
while (true)
{
//Find next position of one of stop chars
var nextPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
if (nextPos == -1)
{
if (IsSet(FreeTextOptions.AllowEof))
{
source.PreviewPosition = source.Text.Length;
return(source.CreateToken(this.OutputTerminal));
}
else
{
return(null);
}
}
var newText = source.Text.Substring(source.PreviewPosition, nextPos - source.PreviewPosition);
tokenText.Append(newText);
source.PreviewPosition = nextPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, tokenText))
{
continue;
}
//check terminators
if (CheckTerminators(source, tokenText))
{
break; //from while (true); we reached
}
//The current stop is not at escape or terminator; add this char to token text and move on
tokenText.Append(source.PreviewChar);
source.PreviewPosition++;
}//while
return(source.CreateToken(this.OutputTerminal, tokenText.ToString()));
}
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9
// so we try to do a quick parse for these, without starting the whole general process
protected override Token QuickParse(ParsingContext context, ISourceStream source)
{
if (IsSet(NumberOptions.DisableQuickParse))
{
return(null);
}
char current = source.PreviewChar;
//it must be a digit followed by a whitespace or delimiter
if (!char.IsDigit(current))
{
return(null);
}
if (!Grammar.IsWhitespaceOrDelimiter(source.NextPreviewChar))
{
return(null);
}
int iValue = current - '0';
object value = null;
switch (DefaultIntTypes[0])
{
case TypeCode.Int32: value = iValue; break;
case TypeCode.UInt32: value = (UInt32)iValue; break;
case TypeCode.Byte: value = (byte)iValue; break;
case TypeCode.SByte: value = (sbyte)iValue; break;
case TypeCode.Int16: value = (Int16)iValue; break;
case TypeCode.UInt16: value = (UInt16)iValue; break;
default: return(null);
}
source.PreviewPosition++;
return(source.CreateToken(this.OutputTerminal, value));
}
private bool ReadStartSymbol(ISourceStream source, CompoundTokenDetails details)
{
if (_startSymbolsFirsts.IndexOf(source.PreviewChar) < 0)
{
return(false);
}
foreach (StringSubType subType in _subtypes)
{
if (!source.MatchSymbol(subType.Start, !CaseSensitive))
{
continue;
}
//We found start symbol
details.StartSymbol = subType.Start;
details.EndSymbol = subType.End;
details.Flags |= (short)subType.Flags;
details.SubTypeIndex = subType.Index;
source.PreviewPosition += subType.Start.Length;
return(true);
} //foreach
return(false);
} //method
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(_startSymbol, false))
{
return(null);
}
source.Position += _startSymbol.Length;
while (!source.EOF())
{
int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
if (_isLineComment)
{
return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
}
else
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
}
}
source.Position = firstCharPos;
if (source.MatchSymbol(_endSymbol, false))
{
source.Position += _endSymbol.Length;
return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
}
source.Position++;
}
throw new NotSupportedException();
}
private Token CompleteMatch(CompilerContext context, ISourceStream source)
{
//Find end symbol
while (!source.EOF())
{
int firstCharPos;
if (EndSymbols.Count == 1)
{
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
}
else
{
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
}
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
return(null); //indicating error
}
//We found a character that might start an end symbol; let's see if it is true.
source.Position = firstCharPos;
foreach (string endSymbol in EndSymbols)
{
if (source.MatchSymbol(endSymbol, !OwnerGrammar.CaseSensitive))
{
//We found end symbol; eat end symbol only if it is not line comment.
// For line comment, leave LF symbol there, it might be important to have a separate LF token
if (!_isLineComment)
{
source.Position += endSymbol.Length;
}
return(new Token(this, source.TokenStart, source.GetLexeme(), null));
} //if
} //foreach endSymbol
source.Position++; //move to the next char and try again
} //while
return(null); //might happen if we found a start char of end symbol, but not the full endSymbol
} //method
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token token = null;
if (IsSet(TermOptions.EnableQuickParse))
{
token = QuickParse(context, source);
if (token != null)
{
return(token);
}
}
source.Position = source.TokenStart.Position;
ScanDetails details = new ScanDetails();
details.Flags = DefaultFlags;
details.TypeCodes = _defaultTypes;
ReadPrefix(source, details);
if (!ReadBody(source, details))
{
return(null);
}
if (details.HasError())
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, details.Error));
}
ReadSuffix(source, details);
if (!ConvertValue(details))
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value: " + details.Error));
}
token = CreateToken(context, source, details);
return(token);
}
private bool ReadStartSymbol(ISourceStream source, CompoundTokenDetails details)
{
if (!_startSymbolsFirsts.Contains(source.PreviewChar))
{
return(false);
}
foreach (var subType in _subtypes)
{
if (!source.MatchSymbol(subType.Start))
{
continue;
}
//We found start symbol
details.StartSymbol = subType.Start;
details.EndSymbol = subType.End;
details.Flags |= (short)subType.Flags;
details.SubTypeIndex = subType.Index;
source.PreviewPosition += subType.Start.Length;
return(true);
}
return(false);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, int numberOfEquals)
{
//Find end symbol
string endSymbol = "]";
while (numberOfEquals-- > 0)
{
endSymbol += "=";
}
endSymbol += "]";
int firstCharPos = source.Text.IndexOf(endSymbol, source.PreviewPosition);
if (firstCharPos < 0)
{
source.PreviewPosition = source.Text.Length;
return(null); //indicating error
}
source.PreviewPosition = firstCharPos + endSymbol.Length;
return(source.CreateToken(this.OutputTerminal));
}//method
private static string ReadQuotedBody(ParsingContext context, ISourceStream source)
{
const char dQuoute = '"';
StringBuilder sb = null;
var from = source.Location.Position + 1; //skip initial double quote
while (true)
{
var until = source.Text.IndexOf(dQuoute, from);
if (until < 0)
{
throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value."
}
source.PreviewPosition = until; //now points at double-quote
var piece = source.Text.Substring(from, until - from);
source.PreviewPosition++; //move after double quote
if (source.PreviewChar != dQuoute && sb == null)
{
return(piece); //quick path - if sb (string builder) was not created yet, we are looking at the very first segment;
}
// and if we found a standalone dquote, then we are done - the "piece" is the result.
if (sb == null)
{
sb = new StringBuilder(100);
}
sb.Append(piece);
if (source.PreviewChar != dQuoute)
{
return(sb.ToString());
}
//we have doubled double-quote; add a single double-quoute char to the result and move over both symbols
sb.Append(dQuoute);
from = source.PreviewPosition + 1;
}
}
private Token CompleteMatch(ISourceStream source)
{
if (source.EOF)
{
return(null);
}
do
{
// Match NewLine
var lookAhead = source.PreviewChar;
if (LineTerminators.Contains(lookAhead))
{
source.PreviewPosition++;
// Treat \r\n as single NewLine
if (!source.EOF && lookAhead == '\r' && source.PreviewChar == '\n')
{
source.PreviewPosition++;
}
break;
}
// Eat up whitespace
if (Grammar.IsWhitespaceOrDelimiter(lookAhead))
{
source.PreviewPosition++;
continue;
}
// Fail on anything else
return(null);
}while (!source.EOF);
// Create output token
return(source.CreateToken(OutputTerminal));
}
/// <summary>
/// Tries to match the current position in the <paramref name="source"/> with the rules for a S/M terminal;
/// returning null if there is no match, returning an error if the terminal is malformed and returning the
/// the appropriate MEMBERNAME or STRING terminal otherwise.
/// </summary>
/// <param name="context">
/// The context in which the match is occuring.
/// </param>
/// <param name="source">
/// The source to try to match.
/// </param>
/// <returns>
/// Null if there is no match, an error token if the terminal is malformed and returning the appropriate
/// MEMBERNAME or STRING terminal otherwise.
/// otherwise.
/// </returns>
public override Token TryMatch([NotNull] ParsingContext context, [NotNull] ISourceStream source)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
Debug.Assert(this.OtherTerminal != null, "Other terminal has not been set");
this.start = source.PreviewPosition;
var isQuoted = false;
Token token;
if (source.PreviewChar == '\'')
{
token = this.TryMatchQuoted(context, source);
isQuoted = true;
}
else
{
token = this.TryMatchUnquoted(context, source);
}
if (token != null && !token.IsError())
{
token.Details =
new MarkupExtensionTokenDetails(
source.Text.Substring(this.start, source.PreviewPosition - this.start), isQuoted);
}
return(token);
}
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9
// so we try to do a quick parse for these, without starting the whole general process
protected override Token QuickParse(CompilerContext context, ISourceStream source)
{
if (IsSet(NumberFlags.DisableQuickParse))
{
return(null);
}
char current = source.CurrentChar;
if (char.IsDigit(current) && QuickParseTerminators.IndexOf(source.NextChar) >= 0)
{
int iValue = current - '0';
object value = null;
switch (DefaultIntTypes[0])
{
case TypeCode.Int32: value = iValue; break;
case TypeCode.UInt32: value = (UInt32)iValue; break;
case TypeCode.Byte: value = (byte)iValue; break;
case TypeCode.SByte: value = (sbyte)iValue; break;
case TypeCode.Int16: value = (Int16)iValue; break;
case TypeCode.UInt16: value = (UInt16)iValue; break;
default: return(null);
}
Token token = new Token(this, source.TokenStart, current.ToString(), value);
source.Position++;
return(token);
}
else
{
return(null);
}
}
protected virtual void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
if (_suffixesFirsts.IndexOf(source.PreviewChar) < 0)
{
return;
}
foreach (string sfx in Suffixes)
{
if (!source.MatchSymbol(sfx, !CaseSensitive))
{
continue;
}
//We found suffix
details.Suffix = sfx;
source.PreviewPosition += sfx.Length;
//Set TypeCode from suffix
TypeCode[] codes;
if (!string.IsNullOrEmpty(details.Suffix) && SuffixTypeCodes.TryGetValue(details.Suffix, out codes))
{
details.TypeCodes = codes;
}
return;
} //foreach
} //method
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(OpenTag, true))
{
return(null);
}
source.PreviewPosition += OpenTag.Length;
var endPos = source.Text.IndexOf(CloseTag, source.PreviewPosition);
string content;
if (endPos > 0)
{
content = source.Text.Substring(source.PreviewPosition, endPos - source.PreviewPosition);
source.PreviewPosition = endPos + CloseTag.Length;
}
else
{
content = source.Text.Substring(source.PreviewPosition, source.Text.Length - source.PreviewPosition);
source.PreviewPosition = source.Text.Length;
}
var token = source.CreateToken(this.OutputTerminal, content);
return(token);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
bool isEscape = source.PreviewChar == EscapeChar && EscapeChar != NoEscape;
if (isEscape)
{
//return a token containing only escaped char
var value = source.NextPreviewChar.ToString();
source.PreviewPosition += 2;
return(source.CreateToken(this.OutputTerminal, value));
}
var stopIndex = source.IndexOfAny(_stopChars, source.Location.Position + 1);
if (stopIndex == source.Location.Position)
{
return(null);
}
if (stopIndex < 0)
{
stopIndex = source.Length;
}
source.PreviewPosition = stopIndex;
return(source.CreateToken(this.OutputTerminal));
} //method
private bool CheckTerminators(ISourceStream source, StringBuilder tokenText)
{
foreach (var term in Terminators)
{
if (!source.MatchSymbol(term))
{
continue;
}
if (IsSet(FreeTextOptions.IncludeTerminator))
{
tokenText.Append(term);
}
if (IsSet(FreeTextOptions.ConsumeTerminator | FreeTextOptions.IncludeTerminator))
{
source.PreviewPosition += term.Length;
}
return(true);
}
return(false);
}
protected virtual void ReadPrefix(ISourceStream source, CompoundTokenDetails details)
{
if (_prefixesFirsts.IndexOf(source.PreviewChar) < 0)
{
return;
}
foreach (string pfx in Prefixes)
{
if (!source.MatchSymbol(pfx, !CaseSensitive))
{
continue;
}
//We found prefix
details.Prefix = pfx;
source.PreviewPosition += pfx.Length;
//Set flag from prefix
short pfxFlags;
if (!string.IsNullOrEmpty(details.Prefix) && PrefixFlags.TryGetValue(details.Prefix, out pfxFlags))
{
details.Flags |= (short)pfxFlags;
}
return;
} //foreach
} //method
private Token CompleteMatch(ISourceStream source)
{
if (source.EOF())
{
return(null);
}
do
{
// Match NewLine
var lookAhead = source.PreviewChar;
if (LineTerminators.IndexOf(lookAhead) >= 0)
{
source.PreviewPosition++;
// Treat \r\n as single NewLine
if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
{
source.PreviewPosition++;
}
break;
}
// Eat up whitespace
if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
{
source.PreviewPosition++;
continue;
}
// Fail on anything else
return(null);
}while (!source.EOF());
// Create output token
return(source.CreateToken(this.OutputTerminal));
}
private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
{
bool escapeEnabled = !details.IsSet((short)StringOptions.NoEscapes);
int start = source.PreviewPosition;
string endQuoteSymbol = details.EndSymbol;
string endQuoteDoubled = endQuoteSymbol + endQuoteSymbol; //doubled quote symbol
bool lineBreakAllowed = details.IsSet((short)StringOptions.AllowsLineBreak);
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = lineBreakAllowed ? -1 : source.Text.IndexOf('\n', source.PreviewPosition);
//fix by ashmind for EOF right after opening symbol
while (true)
{
int endPos = source.Text.IndexOf(endQuoteSymbol, source.PreviewPosition);
//Check for partial token in line-scanning mode
if (endPos < 0 && details.PartialOk && lineBreakAllowed)
{
ProcessPartialBody(source, details);
return(true);
}
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed)
{
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0)
{
endPos = nlPos;
}
if (endPos > 0)
{
source.PreviewPosition = endPos + 1;
}
details.Error = Resources.ErrBadStrLiteral; // "Mal-formed string literal - cannot find termination symbol.";
return(true); //we did find start symbol, so it is definitely string, only malformed
}//if malformed
if (source.EOF())
{
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && IsEndQuoteEscaped(source.Text, endPos))
{
source.PreviewPosition = endPos + endQuoteSymbol.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.PreviewPosition = endPos;
if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && source.MatchSymbol(endQuoteDoubled, !CaseSensitive))
{
source.PreviewPosition = endPos + endQuoteDoubled.Length;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.Text.Substring(start, endPos - start);
source.PreviewPosition = endPos + endQuoteSymbol.Length;
return(true); //if we come here it means we're done - we found string end.
} //end of loop to find string end;
}
