private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
{
var startPos = source.PreviewPosition;
var termLen = _singleTerminator.Length;
var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
int termPos = source.IndexOf(_singleTerminator, startPos, stringComp);
if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
{
termPos = source.Length;
}
if (termPos < 0)
{
return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
}
var textEnd = termPos;
if (IsSet(FreeTextOptions.IncludeTerminator))
{
textEnd += termLen;
}
var tokenText = source.GetText(startPos, textEnd - startPos);
if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
{
return(null);
}
// The following line is a fix submitted by user rmcase
source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
return(source.CreateToken(this.OutputTerminal, tokenText));
}
private string ReadQuotedBody(ParsingContext context, ISourceStream source)
{
const char dQuoute = '"';
StringBuilder sb = null;
var from = source.Location.Position + 1; //skip initial double quote
while (true)
{
var until = source.IndexOf(dQuoute, from);
if (until < 0)
{
throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value."
}
source.PreviewPosition = until; //now points at double-quote
var piece = source.GetText(from, until - from);
source.PreviewPosition++; //move after double quote
if (source.PreviewChar != dQuoute && sb == null)
{
return(piece); //quick path - if sb (string builder) was not created yet, we are looking at the very first segment;
}
// and if we found a standalone dquote, then we are done - the "piece" is the result.
if (sb == null)
{
sb = new StringBuilder(100);
}
sb.Append(piece);
if (source.PreviewChar != dQuoute)
{
return(sb.ToString());
}
//we have doubled double-quote; add a single double-quoute char to the result and move over both symbols
sb.Append(dQuoute);
from = source.PreviewPosition + 1;
}
}
protected override string ReadBody(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(StartSymbol))
{
return(null); //this will result in null returned from TryMatch, no token
}
var start = source.Location.Position + StartSymbol.Length;
var end = source.IndexOf(EndSymbol, start);
if (end < 0)
{
return(null);
}
var body = source.GetText(start, end - start);
source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
return(body);
}
private Token CompleteMatch(ParsingContext context, ISourceStream source)
{
//Find end symbol
while (!source.EOF())
{
int firstCharPos;
if (EndSymbols.Count == 1)
{
firstCharPos = source.IndexOf(EndSymbols[0], source.PreviewPosition);
}
else
{
firstCharPos = source.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
}
if (firstCharPos < 0)
{
source.PreviewPosition = source.Length;
return(null); //indicating error
}
//We found a character that might start an end symbol; let's see if it is true.
source.PreviewPosition = firstCharPos;
foreach (string endSymbol in EndSymbols)
{
if (source.MatchSymbol(endSymbol))
{
//We found end symbol; eat end symbol only if it is not line comment.
// For line comment, leave LF symbol there, it might be important to have a separate LF token
if (!_isLineComment)
{
source.PreviewPosition += endSymbol.Length;
}
return(source.CreateToken(this.OutputTerminal));
} //if
} //foreach endSymbol
source.PreviewPosition++; //move to the next char and try again
} //while
return(null); //might happen if we found a start char of end symbol, but not the full endSymbol
} //method
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(OpenTag))
{
return(null);
}
source.PreviewPosition += OpenTag.Length;
var endPos = source.IndexOf(CloseTag, source.PreviewPosition);
string content;
if (endPos > 0)
{
content = source.GetText(source.PreviewPosition, endPos - source.PreviewPosition);
source.PreviewPosition = endPos + CloseTag.Length;
}
else
{
content = source.GetText(source.PreviewPosition, source.Length - source.PreviewPosition);
source.PreviewPosition = source.Length;
}
var token = source.CreateToken(this.OutputTerminal, content);
return(token);
}
public static int IndexOf(this ISourceStream source, char chr, int index)
{
return(source.IndexOf(chr, index, source.Length - index));
}
public static int IndexOf(this ISourceStream source, string str, int index, StringComparison comparison = StringComparison.Ordinal)
{
return(source.IndexOf(str, index, source.Length - index, comparison));
}
private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
{
bool escapeEnabled = !details.IsSet((short)StringOptions.NoEscapes);
int start = source.PreviewPosition;
string endQuoteSymbol = details.EndSymbol;
string endQuoteDoubled = endQuoteSymbol + endQuoteSymbol; //doubled quote symbol
bool lineBreakAllowed = details.IsSet((short)StringOptions.AllowsLineBreak);
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = lineBreakAllowed ? -1 : source.IndexOf('\n', source.PreviewPosition);
//fix by ashmind for EOF right after opening symbol
while (true)
{
int endPos = source.IndexOf(endQuoteSymbol, source.PreviewPosition);
//Check for partial token in line-scanning mode
if (endPos < 0 && details.PartialOk && lineBreakAllowed)
{
ProcessPartialBody(source, details);
return(true);
}
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed)
{
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0)
{
endPos = nlPos;
}
if (endPos > 0)
{
source.PreviewPosition = endPos + 1;
}
details.Error = Resources.ErrBadStrLiteral; // "Mal-formed string literal - cannot find termination symbol.";
return(true); //we did find start symbol, so it is definitely string, only malformed
}//if malformed
if (source.EOF())
{
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && IsEndQuoteEscaped(source, endPos))
{
source.PreviewPosition = endPos + endQuoteSymbol.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.PreviewPosition = endPos;
if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && source.MatchSymbol(endQuoteDoubled))
{
source.PreviewPosition = endPos + endQuoteDoubled.Length;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.GetText(start, endPos - start);
source.PreviewPosition = endPos + endQuoteSymbol.Length;
return(true); //if we come here it means we're done - we found string end.
} //end of loop to find string end;
}