private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
{
var startPos = source.PreviewPosition;
var termLen = _singleTerminator.Length;
var stringComp = Grammar.CaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
int termPos = source.IndexOf(_singleTerminator, startPos, stringComp);
if (termPos < 0 && IsSet(FreeTextOptions.AllowEof))
{
termPos = source.Length;
}
if (termPos < 0)
{
return(context.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator));
}
var textEnd = termPos;
if (IsSet(FreeTextOptions.IncludeTerminator))
{
textEnd += termLen;
}
var tokenText = source.GetText(startPos, textEnd - startPos);
if (string.IsNullOrEmpty(tokenText) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
{
return(null);
}
// The following line is a fix submitted by user rmcase
source.PreviewPosition = IsSet(FreeTextOptions.ConsumeTerminator) ? termPos + termLen : termPos;
return(source.CreateToken(this.OutputTerminal, tokenText));
}
protected virtual void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
if (!_suffixesFirsts.Contains(source.PreviewChar))
{
return;
}
var comparisonType = CaseSensitivePrefixesSuffixes ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
foreach (string sfx in Suffixes)
{
//Suffixes are usually case insensitive, even if language is case-sensitive. So we cannot use source.MatchSymbol here,
// we need case-specific comparison
if (string.Compare(source.GetText(source.PreviewPosition, sfx.Length), 0, sfx, 0, sfx.Length, comparisonType) != 0)
{
continue;
}
//We found suffix
details.Suffix = sfx;
source.PreviewPosition += sfx.Length;
//Set TypeCode from suffix
TypeCode[] codes;
if (!string.IsNullOrEmpty(details.Suffix) && SuffixTypeCodes.TryGetValue(details.Suffix, out codes))
{
details.TypeCodes = codes;
}
return;
} //foreach
} //method
private string ReadQuotedBody(ParsingContext context, ISourceStream source)
{
const char dQuoute = '"';
StringBuilder sb = null;
var from = source.Location.Position + 1; //skip initial double quote
while (true)
{
var until = source.IndexOf(dQuoute, from);
if (until < 0)
{
throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value."
}
source.PreviewPosition = until; //now points at double-quote
var piece = source.GetText(from, until - from);
source.PreviewPosition++; //move after double quote
if (source.PreviewChar != dQuoute && sb == null)
{
return(piece); //quick path - if sb (string builder) was not created yet, we are looking at the very first segment;
}
// and if we found a standalone dquote, then we are done - the "piece" is the result.
if (sb == null)
{
sb = new StringBuilder(100);
}
sb.Append(piece);
if (source.PreviewChar != dQuoute)
{
return(sb.ToString());
}
//we have doubled double-quote; add a single double-quoute char to the result and move over both symbols
sb.Append(dQuoute);
from = source.PreviewPosition + 1;
}
}
protected override string ReadBody(ParsingContext context, ISourceStream source)
{
source.PreviewPosition = source.Location.Position + Length;
var body = source.GetText(source.Location.Position, Length);
return(body);
}
}//method
private char ReadUnicodeEscape(ISourceStream source, CompoundTokenDetails details)
{
//Position is currently at "\" symbol
source.PreviewPosition++; //move to U/u char
int len;
switch (source.PreviewChar)
{
case 'u': len = 4; break;
case 'U': len = 8; break;
default:
details.Error = Resources.ErrInvEscSymbol; // "Invalid escape symbol, expected 'u' or 'U' only."
return('\0');
}
if (source.PreviewPosition + len > source.Length)
{
details.Error = Resources.ErrInvEscSeq; // "Invalid escape sequence";
return('\0');
}
source.PreviewPosition++; //move to the first digit
string digits = source.GetText(source.PreviewPosition, len);
char result = (char)Convert.ToUInt32(digits, 16);
source.PreviewPosition += len;
details.Flags |= (int)IdFlagsInternal.HasEscapes;
return(result);
}
private void ProcessPartialBody(ISourceStream source, CompoundTokenDetails details)
{
int from = source.PreviewPosition;
source.PreviewPosition = source.Length;
details.Body = source.GetText(from, source.PreviewPosition - from);
details.IsPartial = true;
}
public static string GetRestOfLine(this ISourceStream source, int position)
{
var endIndex = source.IndexOfAny(eol, position);
if (endIndex < 0)
{
endIndex = source.Length;
}
return(source.GetText(position, endIndex - position));
}
private string ReadNotQuotedBody(ParsingContext context, ISourceStream source)
{
var startPos = source.Location.Position;
var sepPos = source.IndexOfAny(_terminators, startPos);
if (sepPos < 0)
{
sepPos = source.Length;
}
source.PreviewPosition = sepPos;
var valueText = source.GetText(startPos, sepPos - startPos);
return(valueText);
}
public override Token TryMatch(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(OpenTag))
{
return(null);
}
source.PreviewPosition += OpenTag.Length;
var endPos = source.IndexOf(CloseTag, source.PreviewPosition);
string content;
if (endPos > 0)
{
content = source.GetText(source.PreviewPosition, endPos - source.PreviewPosition);
source.PreviewPosition = endPos + CloseTag.Length;
}
else
{
content = source.GetText(source.PreviewPosition, source.Length - source.PreviewPosition);
source.PreviewPosition = source.Length;
}
var token = source.CreateToken(this.OutputTerminal, content);
return(token);
}
private Token TryMatchContentExtended(ParsingContext context, ISourceStream source)
{
StringBuilder tokenText = new StringBuilder();
while (true)
{
//Find next position of one of stop chars
var nextPos = source.IndexOfAny(_stopChars, source.PreviewPosition);
if (nextPos == -1)
{
if (IsSet(FreeTextOptions.AllowEof))
{
source.PreviewPosition = source.Length;
return(source.CreateToken(this.OutputTerminal));
}
else
{
return(null);
}
}
var newText = source.GetText(source.PreviewPosition, nextPos - source.PreviewPosition);
tokenText.Append(newText);
source.PreviewPosition = nextPos;
//if it is escape, add escaped text and continue search
if (CheckEscape(source, tokenText))
{
continue;
}
//check terminators
if (CheckTerminators(source, tokenText))
{
break; //from while (true); we reached
}
//The current stop is not at escape or terminator; add this char to token text and move on
tokenText.Append(source.PreviewChar);
source.PreviewPosition++;
}//while
var text = tokenText.ToString();
if (string.IsNullOrEmpty(text) && (this.FreeTextOptions & Parsing.FreeTextOptions.AllowEmpty) == 0)
{
return(null);
}
return(source.CreateToken(this.OutputTerminal, text));
}
protected override string ReadBody(ParsingContext context, ISourceStream source)
{
if (!source.MatchSymbol(StartSymbol))
{
return(null); //this will result in null returned from TryMatch, no token
}
var start = source.Location.Position + StartSymbol.Length;
var end = source.IndexOf(EndSymbol, start);
if (end < 0)
{
return(null);
}
var body = source.GetText(start, end - start);
source.PreviewPosition = end + EndSymbol.Length; //move beyond the end of EndSymbol
return(body);
}
}//method
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
{
//remember start - it may be different from source.TokenStart, we may have skipped prefix
int start = source.PreviewPosition;
char current = source.PreviewChar;
if (IsSet(NumberOptions.AllowSign) && (current == '-' || current == '+'))
{
details.Sign = current.ToString();
source.PreviewPosition++;
}
//Figure out digits set
string digits = GetDigits(details);
bool isDecimal = !details.IsSet((short)(NumberOptions.Binary | NumberOptions.Octal | NumberOptions.Hex));
bool allowFloat = !IsSet(NumberOptions.IntOnly);
bool foundDigits = false;
while (!source.EOF())
{
current = source.PreviewChar;
//1. If it is a digit, just continue going; the same for '_' if it is allowed
if (digits.IndexOf(current) >= 0 || IsSet(NumberOptions.AllowUnderscore) && current == '_')
{
source.PreviewPosition++;
foundDigits = true;
continue;
}
//2. Check if it is a dot in float number
bool isDot = current == DecimalSeparator;
if (allowFloat && isDot)
{
//If we had seen already a dot or exponent, don't accept this one;
bool hasDotOrExp = details.IsSet((short)(NumberFlagsInternal.HasDot | NumberFlagsInternal.HasExp));
if (hasDotOrExp)
{
break; //from while loop
}
//In python number literals (NumberAllowPointFloat) a point can be the first and last character,
//We accept dot only if it is followed by a digit
if (digits.IndexOf(source.NextPreviewChar) < 0 && !IsSet(NumberOptions.AllowStartEndDot))
{
break; //from while loop
}
details.Flags |= (int)NumberFlagsInternal.HasDot;
source.PreviewPosition++;
continue;
}
//3. Check if it is int number followed by dot or exp symbol
bool isExpSymbol = (details.ExponentSymbol == null) && _exponentsTable.ContainsKey(current);
if (!allowFloat && foundDigits && (isDot || isExpSymbol))
{
//If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
if (IsSet(NumberOptions.NoDotAfterInt))
{
return(false);
}
//otherwise break, it is integer and we're done reading digits
break;
}
//4. Only for decimals - check if it is (the first) exponent symbol
if (allowFloat && isDecimal && isExpSymbol)
{
char next = source.NextPreviewChar;
bool nextIsSign = next == '-' || next == '+';
bool nextIsDigit = digits.IndexOf(next) >= 0;
if (!nextIsSign && !nextIsDigit)
{
break; //Exponent should be followed by either sign or digit
}
//ok, we've got real exponent
details.ExponentSymbol = current.ToString(); //remember the exp char
details.Flags |= (int)NumberFlagsInternal.HasExp;
source.PreviewPosition++;
if (nextIsSign)
{
source.PreviewPosition++; //skip +/- explicitly so we don't have to deal with them on the next iteration
}
continue;
}
//4. It is something else (not digit, not dot or exponent) - we're done
break; //from while loop
}//while
int end = source.PreviewPosition;
if (!foundDigits)
{
return(false);
}
details.Body = source.GetText(start, end - start);
return(true);
}
private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
{
bool escapeEnabled = !details.IsSet((short)StringOptions.NoEscapes);
int start = source.PreviewPosition;
string endQuoteSymbol = details.EndSymbol;
string endQuoteDoubled = endQuoteSymbol + endQuoteSymbol; //doubled quote symbol
bool lineBreakAllowed = details.IsSet((short)StringOptions.AllowsLineBreak);
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = lineBreakAllowed ? -1 : source.IndexOf('\n', source.PreviewPosition);
//fix by ashmind for EOF right after opening symbol
while (true)
{
int endPos = source.IndexOf(endQuoteSymbol, source.PreviewPosition);
//Check for partial token in line-scanning mode
if (endPos < 0 && details.PartialOk && lineBreakAllowed)
{
ProcessPartialBody(source, details);
return(true);
}
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed)
{
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0)
{
endPos = nlPos;
}
if (endPos > 0)
{
source.PreviewPosition = endPos + 1;
}
details.Error = Resources.ErrBadStrLiteral; // "Mal-formed string literal - cannot find termination symbol.";
return(true); //we did find start symbol, so it is definitely string, only malformed
}//if malformed
if (source.EOF())
{
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && IsEndQuoteEscaped(source, endPos))
{
source.PreviewPosition = endPos + endQuoteSymbol.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.PreviewPosition = endPos;
if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && source.MatchSymbol(endQuoteDoubled))
{
source.PreviewPosition = endPos + endQuoteDoubled.Length;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.GetText(start, endPos - start);
source.PreviewPosition = endPos + endQuoteSymbol.Length;
return(true); //if we come here it means we're done - we found string end.
} //end of loop to find string end;
}