public override Token TryMatch(ParsingContext context, ISourceStream source)
{
while (true)
{
//Find next position
var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition + 1);
//we either didn't find it
if (newPos == -1)
{
return(context.CreateErrorToken(Resources.ErrNoEndForRegex));// "No end symbol for regex literal."
}
source.PreviewPosition = newPos;
if (source.PreviewChar != EndSymbol)
{
//we hit CR or LF, this is an error
return(context.CreateErrorToken(Resources.ErrNoEndForRegex));
}
if (!CheckEscaped(source))
{
break;
}
}
source.PreviewPosition++; //move after end symbol
//save pattern length, we will need it
var patternLen = source.PreviewPosition - source.Location.Position - 2; //exclude start and end symbol
//read switches and turn them into options
RegexOptions options = RegexOptions.None;
var switches = string.Empty;
while (ReadSwitch(source, ref options))
{
if (IsSet(RegexTermOptions.UniqueSwitches) && switches.Contains(source.PreviewChar))
{
return(context.CreateErrorToken(Resources.ErrDupRegexSwitch, source.PreviewChar)); // "Duplicate switch '{0}' for regular expression"
}
switches += source.PreviewChar.ToString();
source.PreviewPosition++;
}
//check following symbol
if (!IsSet(RegexTermOptions.AllowLetterAfter))
{
var currChar = source.PreviewChar;
if (char.IsLetter(currChar) || currChar == '_')
{
return(context.CreateErrorToken(Resources.ErrInvRegexSwitch, currChar)); // "Invalid switch '{0}' for regular expression"
}
}
var token = source.CreateToken(this.OutputTerminal);
//we have token, now what's left is to set its Value field. It is either pattern itself, or Regex instance
string pattern = token.Text.Substring(1, patternLen); //exclude start and end symbol
object value = pattern;
if (IsSet(RegexTermOptions.CreateRegExObject))
{
value = new Regex(pattern, options);
}
token.Value = value;
token.Details = switches; //save switches in token.Details
return(token);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
// Quick check
var lookAhead = source.PreviewChar;
var startIndex = _startSymbolsFirsts.IndexOf(lookAhead);
if (startIndex < 0)
{
return(null);
}
// Match start symbols
if (!BeginMatch(source, startIndex, lookAhead))
{
return(null);
}
// Match NewLine
var result = CompleteMatch(source);
if (result != null)
{
return(result);
}
// Report an error
return(context.CreateErrorToken(Resources.ErrNewLineExpected));
}
private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
{
var startPos = source.PreviewPosition;
var termLen = _singleTerminator.Length;
#if NETSTANDARD
var stringComp = Grammar.CaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
#else
var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
#endif
int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
{
termPos = source.Text.Length;
}
if (termPos < 0)
{
return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
}
var textEnd = termPos;
if (IsSet(FreeTextOptions.IncludeTerminator))
{
textEnd += termLen;
}
var tokenText = source.Text.Substring(startPos, textEnd - startPos);
// The following line is a fix submitted by user rmcase
source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
return(source.CreateToken(this.OutputTerminal, tokenText));
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
Token result;
if (context.VsLineScanState.Value != 0)
{
byte commentLevel = context.VsLineScanState.TokenSubType;
result = CompleteMatch(context, source, commentLevel);
}
else
{
byte commentLevel = 0;
if (!BeginMatch(context, source, ref commentLevel))
return null;
result = CompleteMatch(context, source, commentLevel);
}
if (result != null)
return result;
if (context.Mode == ParseMode.VsLineScan)
return CreateIncompleteToken(context, source);
return context.CreateErrorToken("unclosed comment");
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
if (context.VsLineScanState.Value != 0)
{
// we are continuing in line mode - restore internal env (none in this case)
context.VsLineScanState.Value = 0;
}
else
{
//we are starting from scratch
if (!BeginMatch(context, source))
{
return(null);
}
}
var result = CompleteMatch(context, source);
if (result != null)
{
return(result);
}
//if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
if (_isLineComment)
{
return(source.CreateToken(OutputTerminal));
}
if (context.Mode == ParseMode.VsLineScan)
{
return(CreateIncompleteToken(context, source));
}
return(context.CreateErrorToken(Resources.ErrUnclosedComment));
}
private Token MatchQuoted(ParsingContext context, ISourceStream source)
{
char quoteChar = source.PreviewChar;
if ((quoteChar != '\'') && (quoteChar != '"'))
{
return null;
}
source.PreviewPosition++;
while (!source.EOF())
{
if (source.PreviewChar == quoteChar)
{
source.PreviewPosition++;
return source.CreateToken(this.OutputTerminal);
}
// Escaped?
if (source.PreviewChar == '\\')
{
// Consume next
++source.PreviewPosition;
}
source.PreviewPosition++;
}
return context.CreateErrorToken("Unbalanced quoted string");
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
while (!source.EOF())
{
switch (source.PreviewChar)
{
case '\n':
case '\r':
case ' ':
case '}':
if (source.PreviewPosition > source.Position)
return source.CreateToken(this.OutputTerminal);
return context.CreateErrorToken(Name + " was expected");
}
source.PreviewPosition++;
}
return context.CreateErrorToken("Unbalanced " + Name);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
try {
var textValue = ReadBody(context, source);
if (textValue == null) return null;
var value = ConvertValue(context, textValue);
return source.CreateToken(this.OutputTerminal, value);
} catch(Exception ex) {
//we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
return context.CreateErrorToken(ex.Message);
}
}//method
protected internal override void OnValidateToken(ParsingContext context)
{
if (!IsSet(NumberOptions.AllowLetterAfter))
{
var current = context.Source.PreviewChar;
if (char.IsLetter(current) || current == '_')
{
context.CurrentToken = context.CreateErrorToken(Resources.ErrNoLetterAfterNum); // "Number cannot be followed by a letter."
}
}
base.OnValidateToken(context);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
try {
var textValue = ReadBody(context, source);
if (textValue == null)
{
return(null);
}
var value = ConvertValue(context, textValue);
return(source.CreateToken(this.OutputTerminal, value));
} catch (Exception ex) {
//we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
return(context.CreateErrorToken(ex.Message));
}
}//method
public override Token TryMatch(ParsingContext context, ISourceStream source) {
Token result;
if (context.VsLineScanState.Value != 0) {
// we are continuing in line mode - restore internal env (none in this case)
context.VsLineScanState.Value = 0;
} else {
//we are starting from scratch
if (!BeginMatch(context, source)) return null;
}
result = CompleteMatch(context, source);
if (result != null) return result;
//if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
if (_isLineComment)
return source.CreateToken(this.OutputTerminal);
if (context.Mode == ParseMode.VsLineScan)
return CreateIncompleteToken(context, source);
return context.CreateErrorToken(Resources.ErrUnclosedComment);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
// Quick check
var lookAhead = source.PreviewChar;
var startIndex = _startSymbolsFirsts.IndexOf(lookAhead);
if (startIndex < 0)
return null;
// Match start symbols
if (!BeginMatch(source, startIndex, lookAhead))
return null;
// Match NewLine
var result = CompleteMatch(source);
if (result != null)
return result;
// Report an error
return context.CreateErrorToken(Resources.ErrNewLineExpected);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
Token token;
//Try quick parse first, but only if we're not continuing
if (context.VsLineScanState.Value == 0)
{
token = QuickParse(context, source);
if (token != null)
{
return(token);
}
source.PreviewPosition = source.Position; //revert the position
}
CompoundTokenDetails details = new CompoundTokenDetails();
InitDetails(context, details);
if (context.VsLineScanState.Value == 0)
{
ReadPrefix(source, details);
}
if (!ReadBody(source, details))
{
return(null);
}
if (details.Error != null)
{
return(context.CreateErrorToken(details.Error));
}
if (details.IsPartial)
{
details.Value = details.Body;
}
else
{
ReadSuffix(source, details);
if (!ConvertValue(details))
{
if (string.IsNullOrEmpty(details.Error))
{
details.Error = Resources.ErrInvNumber;
}
return(context.CreateErrorToken(details.Error)); // "Failed to convert the value: {0}"
}
}
token = CreateToken(context, source, details);
if (details.IsPartial)
{
//Save terminal state so we can continue
context.VsLineScanState.TokenSubType = (byte)details.SubTypeIndex;
context.VsLineScanState.TerminalFlags = (short)details.Flags;
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
}
else
{
context.VsLineScanState.Value = 0;
}
return(token);
}
private Token TryMatchContentSimple(ParsingContext context, ISourceStream source) {
var startPos = source.PreviewPosition;
var termLen = _singleTerminator.Length;
var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
termPos = source.Text.Length;
if (termPos < 0)
return context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
var textEnd = termPos;
if (IsSet(FreeTextOptions.IncludeTerminator))
textEnd += termLen;
var tokenText = source.Text.Substring(startPos, textEnd - startPos);
if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
return null;
// The following line is a fix submitted by user rmcase
source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
return source.CreateToken(this.OutputTerminal, tokenText);
}
private Token MatchUnquoted(ParsingContext context, ISourceStream source)
{
if (source.PreviewChar == '{')
{
// Member names can't start with {
if (this.IsMemberName)
{
return null;
}
// Check for special {} at start of token indicating that this is a STRING token.
if (source.NextPreviewChar != '}')
{
return null;
}
source.PreviewPosition += 2;
}
var runningBraceTotal = 0;
// This variable tracks the position of the last non whitespace (or significant whitespace).
var lastNonWhitespacePosition = source.PreviewPosition;
while (!source.EOF())
{
bool isWhiteSpace = false;
switch (source.PreviewChar)
{
case '{':
runningBraceTotal++;
break;
case '}':
if (--runningBraceTotal < 0)
{
return this.CreateToken(source, lastNonWhitespacePosition);
}
break;
case ',':
if (runningBraceTotal == 0)
{
return this.CreateToken(source, lastNonWhitespacePosition);
}
break;
case '=':
if (runningBraceTotal == 0)
{
// Equal sign. Only allowed after MemberNames.
return this.IsMemberName
? this.CreateToken(source, lastNonWhitespacePosition)
: null;
}
break;
case '\\':
source.PreviewPosition++;
break;
default:
isWhiteSpace = Char.IsWhiteSpace(source.PreviewChar);
break;
}
source.PreviewPosition++;
if (!isWhiteSpace)
{
lastNonWhitespacePosition = source.PreviewPosition;
}
}
return context.CreateErrorToken("Unterminated string terminal");
}
