private IEnumerable StringMode(Itr itr)
{
if (currentItem.Length == 0)
{
currentItem.Append('"');
}
currentItem.Append(itr.Current());
if (SwitchMode(itr.Mode))
{
yield return(PopString(currentItem));
}
else
{
if (itr.IsFinalIndex())
{
currentItem.Append('"');
yield return(PopString(currentItem));
}
else if ('\\'.Equals(itr.Current()))
{
currentItem.Append(itr.Peek(1));
itr.MoveNext(); //don't check next character since already added
}
}
}
private IEnumerable <string> NormalMode(Itr itr)
{
if (!SwitchMode(itr.Mode))
{
if (' '.Equals(itr.Current()))
{
if (currentItem.Length > 0)
{
yield return(PopString(currentItem));
}
}
else if (SplitOnDot(itr))
{
if (currentItem.Length > 0)
{
yield return(PopString(currentItem));
}
yield return(itr.Current().ToString());
}
else if (MatchDoubleSymbolDelimiter(__multi_symbol_delimiters__, itr))
{
if (currentItem.Length > 0)
{
yield return(PopString(currentItem));
}
currentItem.Append(itr.Current());
currentItem.Append(itr.Peek(1));
itr.MoveNext();
}
else if (MatchDelimiter(__Updated_Delimiters__, itr))
{
if (currentItem.Length > 0)
{
yield return(PopString(currentItem));
}
yield return(itr.Current().ToString());
}
else
{
currentItem.Append(itr.Current());
}
if (itr.IsFinalIndex() && currentItem.Length > 0)
{
yield return(PopString(currentItem));
}
}
else if (itr.Current() != ' ' && currentItem.Length > 0)
{
yield return(PopString(currentItem));
}
}
public IEnumerable GenerateLexemes(string input)
{
yield return(GetWhiteSpace(input));
itr.Line = TrimWhiteSpace(input, itr);
while (itr.MoveNext())
{
switch (itr.Mode)
{
case LexerMode.Normal:
foreach (var result in NormalMode(itr))
{
yield return(result);
}
break;
case LexerMode.String:
foreach (var result in StringMode(itr))
{
yield return(result);
}
break;
case LexerMode.Comment:
case LexerMode.MultiComment:
CommentMode(itr);
break;
}
}
itr.Reset();
if (itr.Mode == LexerMode.String)
{
yield return("+");
}
yield return(@"\n");
}
