public void CanPeekAheadNCharacters()
{
using (var empty = new LinedTextTestFixture(""))
{
empty.Peek(0).ShouldBe("");
empty.Peek(1).ShouldBe("");
empty.ReadLine().ShouldBeFalse();
empty.Peek(0).ShouldBe("");
empty.Peek(1).ShouldBe("");
}
using (var t = new StringReader("abc"))
{
var abc = new LinedInputText(t);
abc.ReadLine();
abc.Peek(0).ShouldBe("");
abc.Peek(1).ShouldBe("a");
abc.Peek(2).ShouldBe("ab");
abc.Peek(3).ShouldBe("abc");
abc.Peek(4).ShouldBe("abc");
abc.Peek(100).ShouldBe("abc");
}
}
public bool ReadLine()
{
return(_text.ReadLine());
}
public override IEnumerable <Token> Tokenize(TextReader textReader)
{
var text = new LinedInputText(textReader);
var context = CreateTokenizationContext(text, FlowExtents);
while (text.ReadLine())
{
context.OnLineTokenizingBegin();
for (int i = 0, indentsCount = _indents.Count; i < indentsCount; ++i)
{
if (text.EndOfLine)
{
break;
}
var indent = _indents[i];
if (!indent.Token.TryMatch(text, out Token token))
{
continue;
}
if (context.OnProcessTheFlow(token.Kind) == Flow.State.IncorrectExtent)
{
yield return(ScopeInconsistent.CreateTokenAtCurrentPosition(text));
yield break;
}
context.OnProcessIndent(indent, token);
var skipCurrentToken = ShouldSkipToken(token, context);
if (!skipCurrentToken && (indent.ScopePerToken == ScopePerToken.None || indent.ScopePerToken == ScopePerToken.EmitAfter))
{
yield return(token);
}
if (indent.ScopePerToken != ScopePerToken.None)
{
foreach (var scope in context.BalanceScope())
{
switch (scope)
{
case ScopeState.Begin:
yield return(ScopeBegin.CreateTokenAtCurrentPosition(text));
break;
case ScopeState.End:
yield return(ScopeEnd.CreateTokenAtCurrentPosition(text));
break;
case ScopeState.Inconsistent:
yield return(ScopeInconsistent.CreateTokenAtCurrentPosition(text));
yield break;
}
}
}
if (!skipCurrentToken && indent.ScopePerToken == ScopePerToken.EmitBefore)
{
yield return(token);
}
text.Advance(token.Lexeme.Length);
if (context.StopIndentLexing())
{
break;
}
i = -1; // start from the beginning of the _indents
}
context.OnIndentLexingComplete();
while (!text.EndOfLine)
{
var current = GetToken(text, context);
context.OnProcessToken(current);
if (current == null)
{
yield return(CreateUnknownToken(text));
yield break;
}
if (context.OnProcessTheFlow(current.Kind) == Flow.State.IncorrectExtent)
{
yield return(ScopeInconsistent.CreateTokenAtCurrentPosition(text));
yield break;
}
foreach (var scope in context.BalanceScope())
{
switch (scope)
{
case ScopeState.Begin:
yield return(ScopeBegin.CreateTokenAtCurrentPosition(text));
break;
case ScopeState.End:
yield return(ScopeEnd.CreateTokenAtCurrentPosition(text));
break;
case ScopeState.Inconsistent:
yield return(ScopeInconsistent.CreateTokenAtCurrentPosition(text));
yield break;
}
}
text.Advance(current.Lexeme.Length);
if (ShouldSkipToken(current, context))
{
continue;
}
yield return(current);
}
}
var scopesToClose = context.ResetIndentation();
for (int i = 0; i < scopesToClose; ++i)
{
yield return(ScopeEnd.CreateTokenAtCurrentPosition(text));
}
}