public override Token TryMatch(ParsingContext context, ISourceStream source)
{
string tokenText = string.Empty;
while (true)
{
//Find next position
var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
if (newPos == -1)
{
if (IsSet(FreeTextOptions.AllowEof))
{
source.PreviewPosition = source.Text.Length;
return(source.CreateToken(this.OutputTerminal));
}
else
{
return(null);
}
}
tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
source.PreviewPosition = newPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, ref tokenText))
{
continue;
}
//check terminators
if (CheckTerminators(source, ref tokenText))
{
break; //from while (true)
}
}
return(source.CreateToken(this.OutputTerminal, tokenText));
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var isEscape = source.PreviewChar == EscapeChar && EscapeChar != NoEscape;
if (isEscape)
{
//return a token containing only escaped char
var value = source.NextPreviewChar.ToString();
source.PreviewPosition += 2;
return(source.CreateToken(OutputTerminal, value));
}
var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position + 1);
if (stopIndex == source.Location.Position)
{
return(null);
}
if (stopIndex < 0)
{
stopIndex = source.Text.Length;
}
source.PreviewPosition = stopIndex;
return(source.CreateToken(OutputTerminal));
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
string tokenText = string.Empty;
while (true) {
//Find next position
var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
if(newPos == -1) {
if(IsSet(FreeTextOptions.AllowEof)) {
source.PreviewPosition = source.Text.Length;
return source.CreateToken(this.OutputTerminal);
} else
return null;
}
if (newPos == source.PreviewPosition) // DC
{
context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {});
break; // DC
}
tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
source.PreviewPosition = newPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, ref tokenText))
continue;
//check terminators
if (CheckTerminators(source, ref tokenText))
break; //from while (true)
}
return source.CreateToken(this.OutputTerminal, tokenText);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
{
if (commentLevel == 0)
{
var line_breaks = new char[] { '\n', '\r', '\v' };
var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
if (firstCharPos > 0)
{
source.PreviewPosition = firstCharPos;
}
else
{
source.PreviewPosition = source.Text.Length;
}
return(source.CreateToken(this.OutputTerminal));
}
while (!source.EOF())
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach (Match match in matches)
{
if (match.Groups[1].Value.Length == (int)commentLevel - 1)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
//We are using line-mode and begin terminal was on previous line.
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return(new Token(this, tokenStart, lexeme, null));
}
else
{
return(source.CreateToken(this.OutputTerminal));
}
}
}
source.PreviewPosition++;
}
//The full match wasn't found, store the state for future parsing.
// context.VsLineScanState.TerminalIndex = this.MultilineIndex;
context.VsLineScanState.TokenSubType = commentLevel;
return(null);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
Match m = Expression.Match(source.Text, source.PreviewPosition);
if (!m.Success || m.Index != source.PreviewPosition)
return null;
source.PreviewPosition += m.Length;
if (ValueSelector != null)
return source.CreateToken(this, ValueSelector.Invoke(m));
return source.CreateToken(OutputTerminal);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
bool isEscape = source.PreviewChar == EscapeChar && EscapeChar != NoEscape;
if(isEscape) {
//return a token containing only escaped char
var value = source.NextPreviewChar.ToString();
source.PreviewPosition += 2;
return source.CreateToken(this.OutputTerminal, value);
}
var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position + 1);
if (stopIndex == source.Location.Position) return null;
if (stopIndex < 0) stopIndex = source.Text.Length;
source.PreviewPosition = stopIndex;
return source.CreateToken(this.OutputTerminal);
}//method
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
{
if (commentLevel == 0)
{
char[] anyOf = new char[]
{
'\n',
'\r',
'\v'
};
int num = source.Text.IndexOfAny(anyOf, source.PreviewPosition);
if (num > 0)
{
source.PreviewPosition = num;
}
else
{
source.PreviewPosition = source.Text.Length;
}
return(source.CreateToken(base.OutputTerminal));
}
while (!source.EOF())
{
foreach (Match match in Regex.Matches(source.Text.Substring(source.PreviewPosition), "\\](=*)\\]"))
{
if (match.Groups[1].Value.Length == (int)(commentLevel - 1))
{
source.PreviewPosition += match.Index + match.Length;
Token result;
if (context.VsLineScanState.Value != 0)
{
SourceLocation location = default(SourceLocation);
location.Position = 0;
string text = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
result = new Token(this, location, text, null);
return(result);
}
result = source.CreateToken(base.OutputTerminal);
return(result);
}
}
int previewPosition = source.PreviewPosition;
source.PreviewPosition = previewPosition + 1;
}
context.VsLineScanState.TokenSubType = commentLevel;
return(null);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var text = source.Text;
foreach (var entry in Constants)
{
source.PreviewPosition = source.Position;
var constant = entry.Key;
if (source.PreviewPosition + constant.Length > text.Length)
{
continue;
}
if (source.MatchSymbol(constant))
{
source.PreviewPosition += constant.Length;
if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
{
continue; //make sure it is delimiter
}
return(source.CreateToken(OutputTerminal, entry.Value));
}
}
return(null);
}
/// <summary>
/// Tries to match the current position in the <paramref name="source"/> with the rules for a TYPENAME terminal;
/// returning null if there is no match, returning an error token if the TYPENAME is malformed or there is no
/// match and returning a TYPENAME token
/// otherwise.
/// </summary>
/// <param name="context">
/// The context in which the match is occuring.
/// </param>
/// <param name="source">
/// The source to try to match.
/// </param>
/// <returns>
/// An error token if the TYPENAME is malformed or there is no match and returning a TYPENAME token
/// otherwise.
/// </returns>
public override Token TryMatch([NotNull] ParsingContext context, [NotNull] ISourceStream source)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
while (true)
{
if (source.EOF())
{
return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
}
if (source.PreviewChar == '}' || char.IsWhiteSpace(source.PreviewChar))
{
var token = source.CreateToken(this);
if (string.IsNullOrWhiteSpace(token.ValueString))
{
return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
}
return(token);
}
++source.PreviewPosition;
}
}
private Token MatchQuoted(ParsingContext context, ISourceStream source)
{
char quoteChar = source.PreviewChar;
if ((quoteChar != '\'') && (quoteChar != '"'))
{
return null;
}
source.PreviewPosition++;
while (!source.EOF())
{
if (source.PreviewChar == quoteChar)
{
source.PreviewPosition++;
return source.CreateToken(this.OutputTerminal);
}
// Escaped?
if (source.PreviewChar == '\\')
{
// Consume next
++source.PreviewPosition;
}
source.PreviewPosition++;
}
return context.CreateErrorToken("Unbalanced quoted string");
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
foreach (var prefix in prefixes)
{
if (!PrefixMatches(source, prefix))
{
continue;
}
if (DirectivePrefix.StartsWith(prefix) && PrefixMatches(source, DirectivePrefix))
{
continue;
}
var startIndex = source.Location.Position + prefix.Length;
const string suffix = "%>";
var endIndex = source.Text.IndexOf(suffix, startIndex, StringComparison.Ordinal);
if (endIndex < 0)
{
continue;
}
source.PreviewPosition = endIndex + suffix.Length;
var tokenValue = source.Text.Substring(startIndex, endIndex - startIndex);
return(source.CreateToken(OutputTerminal, tokenValue));
}
return(null);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
if (context.VsLineScanState.Value != 0)
{
// we are continuing in line mode - restore internal env (none in this case)
context.VsLineScanState.Value = 0;
}
else
{
//we are starting from scratch
if (!BeginMatch(context, source))
{
return(null);
}
}
var result = CompleteMatch(context, source);
if (result != null)
{
return(result);
}
//if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
if (_isLineComment)
{
return(source.CreateToken(OutputTerminal));
}
if (context.Mode == ParseMode.VsLineScan)
{
return(CreateIncompleteToken(context, source));
}
return(context.CreateErrorToken(Resources.ErrUnclosedComment));
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
Match m = _expression.Match(source.Text, source.PreviewPosition);
if (!m.Success || m.Index != source.PreviewPosition)
return null;
source.PreviewPosition += m.Length;
return source.CreateToken(this.OutputTerminal);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach (Match match in matches)
{
if (match.Groups[1].Value.Length == (int)level)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return(new Token(this, tokenStart, lexeme, null));
}
else
{
return(source.CreateToken(this.OutputTerminal));
}
}
}
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
context.VsLineScanState.TokenSubType = level;
return(null);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
while (true)
{
//Find next position
var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition + 1);
//we either didn't find it
if (newPos == -1)
{
return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrNoEndForRegex));// "No end symbol for regex literal."
}
source.PreviewPosition = newPos;
if (source.PreviewChar != EndSymbol)
{
//we hit CR or LF, this is an error
return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrNoEndForRegex));
}
if (!CheckEscaped(source))
{
break;
}
}
source.PreviewPosition++; //move after end symbol
//save pattern length, we will need it
var patternLen = source.PreviewPosition - source.Location.Position - 2; //exclude start and end symbol
//read switches and turn them into options
RegexOptions options = RegexOptions.None;
var switches = string.Empty;
while (ReadSwitch(source, ref options))
{
if (IsSet(RegexTermOptions.UniqueSwitches) && switches.Contains(source.PreviewChar))
{
return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrDupRegexSwitch, source.PreviewChar)); // "Duplicate switch '{0}' for regular expression"
}
switches += source.PreviewChar.ToString();
source.PreviewPosition++;
}
//check following symbol
if (!IsSet(RegexTermOptions.AllowLetterAfter))
{
var currChar = source.PreviewChar;
if (char.IsLetter(currChar) || currChar == '_')
{
return(source.CreateErrorToken(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrInvRegexSwitch, currChar)); // "Invalid switch '{0}' for regular expression"
}
}
var token = source.CreateToken(this.OutputTerminal);
//we have token, now what's left is to set its Value field. It is either pattern itself, or Regex instance
string pattern = token.Text.Substring(1, patternLen); //exclude start and end symbol
object value = pattern;
if (IsSet(RegexTermOptions.CreateRegExObject))
{
value = new Regex(pattern, options);
}
token.Value = value;
token.Details = switches; //save switches in token.Details
return(token);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
if (isHeadingOrList)
{
bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
if (!isAfterNewLine)
{
return(null);
}
}
if (!source.MatchSymbol(OpenTag, true))
{
return(null);
}
source.PreviewPosition += OpenTag.Length;
//For headings and lists require space after
if (TermType == WikiTermType.Heading || TermType == WikiTermType.List)
{
const string whitespaces = " \t\r\n\v";
if (!whitespaces.Contains(source.PreviewChar))
{
return(null);
}
}
var token = source.CreateToken(this.OutputTerminal);
return(token);
}
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9
// so we try to do a quick parse for these, without starting the whole general process
protected override Token QuickParse(ParsingContext context, ISourceStream source)
{
if (IsSet(NumberOptions.DisableQuickParse))
{
return(null);
}
char current = source.PreviewChar;
//it must be a digit followed by a terminator
if (!char.IsDigit(current) || GrammarData.WhitespaceAndDelimiters.IndexOf(source.NextPreviewChar) < 0)
{
return(null);
}
int iValue = current - '0';
object value = null;
switch (DefaultIntTypes[0])
{
case TypeCode.Int32: value = iValue; break;
case TypeCode.UInt32: value = (UInt32)iValue; break;
case TypeCode.Byte: value = (byte)iValue; break;
case TypeCode.SByte: value = (sbyte)iValue; break;
case TypeCode.Int16: value = (Int16)iValue; break;
case TypeCode.UInt16: value = (UInt16)iValue; break;
default: return(null);
}
source.PreviewPosition++;
return(source.CreateToken(this.OutputTerminal, value));
}
private Token MatchQuoted(ParsingContext context, ISourceStream source)
{
char quoteChar = source.PreviewChar;
if (quoteChar != '\'' && quoteChar != '"')
{
return(null);
}
source.PreviewPosition++;
while (!source.EOF())
{
if (source.PreviewChar == quoteChar)
{
source.PreviewPosition++;
return(source.CreateToken(this.OutputTerminal));
}
// Escaped?
if (source.PreviewChar == '\\')
{
// Consume next
++source.PreviewPosition;
}
source.PreviewPosition++;
}
return(context.CreateErrorToken("Unbalanced quoted string"));
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
try
{
var left = GetByte(source.PreviewChar);
var right = GetByte(source.NextPreviewChar);
if (left >= 0 && right >= 0)
{
var list = new List <byte>();
do
{
list.Add(Convert.ToByte(left * 16 + right));
source.PreviewPosition += 2;
left = GetByte(source.PreviewChar);
right = GetByte(source.NextPreviewChar);
} while (left >= 0 && right >= 0);
return(source.CreateToken(this.OutputTerminal, list.ToArray()));
}
return(null);
}
catch (Exception ex)
{
return(context.CreateErrorToken(ex.Message));
}
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
{
foreach (Match match in Regex.Matches(source.Text.Substring(source.PreviewPosition), "\\](=*)\\]"))
{
if (match.Groups[1].Value.Length == (int)level)
{
source.PreviewPosition += match.Index + match.Length;
Token result;
if (context.VsLineScanState.Value != 0)
{
SourceLocation location = default(SourceLocation);
location.Position = 0;
string text = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
result = new Token(this, location, text, null);
return(result);
}
result = source.CreateToken(base.OutputTerminal);
return(result);
}
}
context.VsLineScanState.TerminalIndex = MultilineIndex;
context.VsLineScanState.TokenSubType = level;
return(null);
}
private Token CreateIncompleteToken(ParsingContext context, ISourceStream source) {
source.PreviewPosition = source.Text.Length;
Token result = source.CreateToken(this.OutputTerminal);
result.Flags |= TokenFlags.IsIncomplete;
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
return result;
}
public override Token?TryMatch(ParsingContext context, ISourceStream source)
{
var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position);
if (stopIndex == source.Location.Position)
{
return(null);
}
if (stopIndex < 0)
{
stopIndex = source.Text.Length;
}
source.PreviewPosition = stopIndex;
// preserve leading whitespace, if present.
int start = source.Location.Position;
if (preserveLeadingWhitespace)
{
while (start > 0 && char.IsWhiteSpace(source.Text, start - 1))
{
start--;
}
}
var content = source.Text.Substring(start, stopIndex - start);
return(source.CreateToken(this.OutputTerminal, content));
}
private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
{
var startPos = source.PreviewPosition;
var termLen = _singleTerminator.Length;
#if NETSTANDARD
var stringComp = Grammar.CaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
#else
var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
#endif
int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
{
termPos = source.Text.Length;
}
if (termPos < 0)
{
return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
}
var textEnd = termPos;
if (IsSet(FreeTextOptions.IncludeTerminator))
{
textEnd += termLen;
}
var tokenText = source.Text.Substring(startPos, textEnd - startPos);
// The following line is a fix submitted by user rmcase
source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
return(source.CreateToken(this.OutputTerminal, tokenText));
}
protected override Token QuickParse(ParsingContext context, ISourceStream source)
{
if (AllFirstChars.IndexOf(source.PreviewChar) < 0)
{
return(null);
}
source.PreviewPosition++;
while (AllChars.IndexOf(source.PreviewChar) >= 0 && !source.EOF())
{
source.PreviewPosition++;
}
//if it is not a terminator then cancel; we need to go through full algorithm
if (GrammarData.WhitespaceAndDelimiters.IndexOf(source.PreviewChar) < 0)
{
return(null);
}
var token = source.CreateToken(this.OutputTerminal);
if (CaseRestriction != CaseRestriction.None && !CheckCaseRestriction(token.ValueString))
{
return(null);
}
//!!! Do not convert to common case (all-lower) for case-insensitive grammar. Let identifiers remain as is,
// it is responsibility of interpreter to provide case-insensitive read/write operations for identifiers
// if (!this.GrammarData.Grammar.CaseSensitive)
// token.Value = token.Text.ToLower(CultureInfo.InvariantCulture);
CheckReservedWord(token);
token.Symbol = SymbolTable.Symbols.TextToSymbol(token.ValueString);
return(token);
}
/// <summary>
/// Creates a MEMBERNAME token.
/// </summary>
/// <param name="source">
/// The source from which to create the token.
/// </param>
/// <param name="builder">
/// The builder that contains the MEMBERNAME value.
/// </param>
/// <returns>
/// The <see cref="Token"/>.
/// </returns>
private Token CreateMemberNameToken(ISourceStream source, StringBuilder builder)
{
var terminal = this.TerminalType == MemberNameOrStringTerminalType.String
? this.OtherTerminal.OutputTerminal
: this.OutputTerminal;
return(source.CreateToken(terminal, builder.ToString().Trim()));
}
public Token CreateErrorToken(string message, params object[] args)
{
if (args != null && args.Length > 0)
{
message = string.Format(message, args);
}
return(Source.CreateToken(Language.Grammar.SyntaxError, message));
}
private Token TryMatchContentExtended(ParsingContext context, ISourceStream source)
{
StringBuilder tokenText = new StringBuilder();
while (true)
{
//Find next position of one of stop chars
var nextPos = source.IndexOfAny(_stopChars, source.PreviewPosition);
if (nextPos == -1)
{
if (IsSet(FreeTextOptions.AllowEof))
{
source.PreviewPosition = source.Length;
return(source.CreateToken(this.OutputTerminal));
}
else
{
return(null);
}
}
var newText = source.GetText(source.PreviewPosition, nextPos - source.PreviewPosition);
tokenText.Append(newText);
source.PreviewPosition = nextPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, tokenText))
{
continue;
}
//check terminators
if (CheckTerminators(source, tokenText))
{
break; //from while (true); we reached
}
//The current stop is not at escape or terminator; add this char to token text and move on
tokenText.Append(source.PreviewChar);
source.PreviewPosition++;
}//while
var text = tokenText.ToString();
if (string.IsNullOrEmpty(text) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
{
return(null);
}
return(source.CreateToken(this.OutputTerminal, text));
}
private Token CreateIncompleteToken(ParsingContext context, ISourceStream source)
{
source.PreviewPosition = source.Text.Length;
Token expr_1D = source.CreateToken(base.OutputTerminal);
expr_1D.Flags |= TokenFlags.IsIncomplete;
context.VsLineScanState.TerminalIndex = MultilineIndex;
return(expr_1D);
}
private Token CreateIncompleteToken(ParsingContext context, ISourceStream source)
{
source.PreviewPosition = source.Text.Length;
Token result = source.CreateToken(this.OutputTerminal);
result.Flags |= TokenFlags.IsIncomplete;
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
return(result);
}
protected virtual Token CreateToken(ParsingContext context, ISourceStream source, CompoundTokenDetails details)
{
var token = source.CreateToken(this.OutputTerminal, details.Value);
token.Details = details;
if (details.IsPartial)
{
token.Flags |= TokenFlags.IsIncomplete;
}
return(token);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
foreach (var m in _matchers)
{
var backupOffset = source.PreviewPosition;
var start = m.TryGetInitiator(source.Text, source.PreviewPosition, !context.Language.Grammar.CaseSensitive);
if (start != null)
{
//Locate the end terminal
var end = m.TryGetTerminator(start, !context.Language.Grammar.CaseSensitive);
if (end == null)
{
continue;
}
source.PreviewPosition += start.Length;
var startPos = source.PreviewPosition;
var endPos = -2;
while (endPos == -2)
{
endPos = source.Text.IndexOf(end, source.PreviewPosition);
if ((m.Flags & StringOptions.NoEscapes) != 0)
{
continue;
}
if (IsEndQuoteEscaped(source.Text, endPos))
{
source.PreviewPosition = endPos + end.Length;
endPos = -2;
}
}
if (endPos == -1)
{
continue;
}
source.PreviewPosition = endPos + end.Length;
var tokenText = source.Text.Substring(startPos, endPos - startPos);
if ((m.Flags & StringOptions.NoEscapes) == 0)
{
tokenText = ConvertValue(context, tokenText, m);
}
return(source.CreateToken(this.OutputTerminal, tokenText));
}
}
return(null);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
try {
var textValue = ReadBody(context, source);
if (textValue == null) return null;
var value = ConvertValue(context, textValue);
return source.CreateToken(this.OutputTerminal, value);
} catch(Exception ex) {
//we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
return source.CreateErrorToken(ex.Message);
}
}//method
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
Match m = _expression.Match(source.Text, source.PreviewPosition);
if (!m.Success || m.Index != source.PreviewPosition)
{
return(null);
}
source.PreviewPosition += m.Length;
return(source.CreateToken(this.OutputTerminal));
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
char current = source.PreviewChar;
if (!LineTerminators.Contains(current)) return null;
//Treat \r\n as a single terminator
bool doExtraShift = (current == '\r' && source.NextPreviewChar == '\n');
source.PreviewPosition++; //main shift
if (doExtraShift)
source.PreviewPosition++;
Token result = source.CreateToken(this.OutputTerminal);
return result;
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var match = binaryRegex.Match(source.Text, source.PreviewPosition);
if (!match.Success || match.Index != source.PreviewPosition)
{
return(null);
}
source.Position += 2;
source.PreviewPosition += match.Length;
return(source.CreateToken(OutputTerminal));
}
/// <summary>
/// Creates a STRING token.
/// </summary>
/// <param name="context">
/// The context in which the parse is occuring.
/// </param>
/// <param name="source">
/// The source from which to create the token.
/// </param>
/// <param name="builder">
/// The builder that contains the STRING value.
/// </param>
/// <returns>
/// The STRING <see cref="Token"/>.
/// </returns>
private Token CreateStringToken(ParsingContext context, ISourceStream source, StringBuilder builder)
{
var terminal = this.TerminalType == MemberNameOrStringTerminalType.MemberName
? this.OtherTerminal.OutputTerminal
: this.OutputTerminal;
var text = builder.ToString();
var parseTree = new Parser(context.Language).Parse(text);
var value = parseTree.Status == ParseTreeStatus.Parsed ? (object)parseTree : text;
return(source.CreateToken(terminal, value));
}
private Token CompleteMatch(ParsingContext context, ISourceStream source)
{
//Find end symbol
while (!source.EOF())
{
var firstCharPos = source.Text.IndexOf('{', source.PreviewPosition);
if (firstCharPos < 0)
{
source.PreviewPosition = source.Text.Length;
return(source.CreateToken(this.OutputTerminal));
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
{
return(source.CreateToken(this.OutputTerminal));
}
source.PreviewPosition++; //move to the next char and try again
}
return(source.CreateToken(this.OutputTerminal));
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
if (!source.MatchSymbol(OpenTag, true)) return null;
source.PreviewPosition += OpenTag.Length;
var endPos = source.Text.IndexOf(CloseTag, source.PreviewPosition);
string content;
if(endPos > 0) {
content = source.Text.Substring(source.PreviewPosition, endPos - source.PreviewPosition);
source.PreviewPosition = endPos + CloseTag.Length;
} else {
content = source.Text.Substring(source.PreviewPosition, source.Text.Length - source.PreviewPosition);
source.PreviewPosition = source.Text.Length;
}
var token = source.CreateToken(this.OutputTerminal, content);
return token;
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
if(isHeadingOrList) {
bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
if(!isAfterNewLine) return null;
}
if(!source.MatchSymbol(OpenTag)) return null;
source.PreviewPosition += OpenTag.Length;
//For headings and lists require space after
if(TermType == WikiTermType.Heading || TermType == WikiTermType.List) {
const string whitespaces = " \t\r\n\v";
if (!whitespaces.Contains(source.PreviewChar)) return null;
}
var token = source.CreateToken(this.OutputTerminal);
return token;
}
protected Token CreateToken(ISourceStream source, int lastNonWhitespacePosition)
{
if (lastNonWhitespacePosition > source.Position)
{
// Remember position
var pos = source.PreviewPosition;
// Move position back to last non whitespace (or significant whitespace)
source.PreviewPosition = lastNonWhitespacePosition;
var token = source.CreateToken(this.OutputTerminal);
// Restore position
source.PreviewPosition = pos;
return token;
}
return null;
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
Token result;
if (context.VsLineScanState.Value != 0) {
// we are continuing in line mode - restore internal env (none in this case)
context.VsLineScanState.Value = 0;
} else {
//we are starting from scratch
if (!BeginMatch(context, source)) return null;
}
result = CompleteMatch(context, source);
if (result != null) return result;
//if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
if (_isLineComment)
return source.CreateToken(this.OutputTerminal);
if (context.Mode == ParseMode.VsLineScan)
return CreateIncompleteToken(context, source);
return context.CreateErrorToken(Resources.ErrUnclosedComment);
}
Token MatchCodeTerminal(Terminal terminal, ParsingContext context, ISourceStream source)
{
var remaining = source.Text.Substring(source.Location.Position);
var stack = 1;
var token = "";
while (stack > 0)
{
var index = remaining.IndexOf('}') + 1;
var cut = remaining.Substring(0, index);
token += cut;
remaining = remaining.Substring(index);
stack += cut.Count(x => x == '{');
stack--;
}
token = token.Substring(0, token.Length - 1);
source.PreviewPosition += token.Length;
return source.CreateToken(terminal.OutputTerminal);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
while (!source.EOF())
{
switch (source.PreviewChar)
{
case '\n':
case '\r':
case ' ':
case '}':
if (source.PreviewPosition > source.Position)
return source.CreateToken(this.OutputTerminal);
return context.CreateErrorToken(Name + " was expected");
}
source.PreviewPosition++;
}
return context.CreateErrorToken("Unbalanced " + Name);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach(Match match in matches)
{
if (match.Groups[1].Value.Length == (int)level)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return new Token(this, tokenStart, lexeme, null);
}
else
{
return source.CreateToken(this.OutputTerminal);
}
}
}
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
context.VsLineScanState.TokenSubType = level;
return null;
}
private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
{
var startPos = source.PreviewPosition;
int p = source.Text.IndexOf(_singleTerminator, startPos, Grammar.StringComparisonMode);
if (p < 0 && IsSet(FreeTextOptions.AllowEof))
p = source.Text.Length;
if (p < 0)
return source.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
var tokenText = source.Text.Substring(startPos, p - startPos);
return source.CreateToken(this.OutputTerminal, tokenText);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var Match = MatchRegex.Match(source.Text, source.PreviewPosition);
if (!Match.Success || Match.Index != source.PreviewPosition) return null;
source.PreviewPosition += Match.Length;
return source.CreateToken(this.OutputTerminal);
}
private Token CompleteMatch(ISourceStream source) {
if (source.EOF())
return null;
do {
// Match NewLine
var lookAhead = source.PreviewChar;
if (LineTerminators.IndexOf(lookAhead) >= 0)
{
source.PreviewPosition++;
// Treat \r\n as single NewLine
if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
source.PreviewPosition++;
break;
}
// Eat up whitespace
if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
{
source.PreviewPosition++;
continue;
}
// Fail on anything else
return null;
}
while (!source.EOF());
// Create output token
return source.CreateToken(this.OutputTerminal);
}
public override Token TryMatch(ParsingContext context, ISourceStream source) {
return source.CreateToken(this); //Create an empty token representing an implied symbol.
}
private Token CompleteMatch(ParsingContext context, ISourceStream source) {
//Find end symbol
while (!source.EOF()) {
int firstCharPos;
if (EndSymbols.Count == 1)
firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
else
firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
if (firstCharPos < 0) {
source.PreviewPosition = source.Text.Length;
return null; //indicating error
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
foreach (string endSymbol in EndSymbols) {
if (source.MatchSymbol(endSymbol)) {
//We found end symbol; eat end symbol only if it is not line comment.
// For line comment, leave LF symbol there, it might be important to have a separate LF token
if (!_isLineComment)
source.PreviewPosition += endSymbol.Length;
return source.CreateToken(this.OutputTerminal);
}//if
}//foreach endSymbol
source.PreviewPosition++; //move to the next char and try again
}//while
return null; //might happen if we found a start char of end symbol, but not the full endSymbol
}//method
private Token TryMatchContentExtended(ParsingContext context, ISourceStream source) {
StringBuilder tokenText = new StringBuilder();
while (true) {
//Find next position of one of stop chars
var nextPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
if(nextPos == -1) {
if(IsSet(FreeTextOptions.AllowEof)) {
source.PreviewPosition = source.Text.Length;
return source.CreateToken(this.OutputTerminal);
} else
return null;
}
var newText = source.Text.Substring(source.PreviewPosition, nextPos - source.PreviewPosition);
tokenText.Append(newText);
source.PreviewPosition = nextPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, tokenText))
continue;
//check terminators
if (CheckTerminators(source, tokenText))
break; //from while (true); we reached
//The current stop is not at escape or terminator; add this char to token text and move on
tokenText.Append(source.PreviewChar);
source.PreviewPosition++;
}//while
var text = tokenText.ToString();
if (string.IsNullOrEmpty(text) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
return null;
return source.CreateToken(this.OutputTerminal, text);
}
private Token TryMatchContentSimple(ParsingContext context, ISourceStream source) {
var startPos = source.PreviewPosition;
var termLen = _singleTerminator.Length;
var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
int termPos = source.Text.IndexOf(_singleTerminator, startPos, stringComp);
if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
termPos = source.Text.Length;
if (termPos < 0)
return context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
var textEnd = termPos;
if (IsSet(FreeTextOptions.IncludeTerminator))
textEnd += termLen;
var tokenText = source.Text.Substring(startPos, textEnd - startPos);
if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
return null;
// The following line is a fix submitted by user rmcase
source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
return source.CreateToken(this.OutputTerminal, tokenText);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
{
if (commentLevel == 0)
{
var line_breaks = new char[] { '\n', '\r', '\v' };
var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
if (firstCharPos > 0)
{
source.PreviewPosition = firstCharPos;
}
else
{
source.PreviewPosition = source.Text.Length;
}
return source.CreateToken(this.OutputTerminal);
}
while (!source.EOF())
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach (Match match in matches)
{
if (match.Groups[1].Value.Length == (int)commentLevel - 1)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return new Token(this, tokenStart, lexeme, null);
}
else
{
return source.CreateToken(this.OutputTerminal);
}
}
}
source.PreviewPosition++;
}
context.VsLineScanState.TokenSubType = commentLevel;
return null;
}