public void ShouldConsumeAllAvailableWhiteSpaceFromTokenizer()
{
var stringTokenizer = new StringTokenizer(" \t\t \r\n ");
var mockMatchWhiteSpace = new MockMatchWhiteSpace();
var token = mockMatchWhiteSpace.CallImplementation(stringTokenizer);
Assert.IsNotNull(token);
Assert.IsTrue(stringTokenizer.IsAtEnd());
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
var location = tokenizer.GetCurrentLocation();
bool foundWord = false;
var word = new StringBuilder();
while (!tokenizer.IsAtEnd() && !string.IsNullOrWhiteSpace(tokenizer.Current) && !this.SpecialCharacters.With(s => s.Any(c => c.IsMatch(tokenizer))))
{
foundWord = true;
word.Append(tokenizer.Current);
tokenizer.Consume();
}
return(foundWord ? location.CreateToken(TokenType.Word, word.ToString()) : null);
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
var location = tokenizer.GetCurrentLocation();
bool foundWhiteSpace = false;
var whitesSpace = new StringBuilder();
while (!tokenizer.IsAtEnd() && string.IsNullOrWhiteSpace(tokenizer.Current))
{
foundWhiteSpace = true;
whitesSpace.Append(tokenizer.Current);
tokenizer.Consume();
}
if (foundWhiteSpace)
{
return(location.CreateToken(TokenType.WhiteSpace, whitesSpace.ToString()));
}
return(null);
}
/// <summary>
/// Gets the token.
/// </summary>
/// <param name="tokenizer">The tokenizer.</param>
/// <returns>
/// A token if the matcher matches successfully; otherwise, null.
/// </returns>
protected override StringToken GetTokenImpl(StringTokenizer tokenizer)
{
if (tokenizer.Current == this.StringDelimiter)
{
var location = tokenizer.GetCurrentLocation();
var str = new StringBuilder();
tokenizer.Consume();
while (!tokenizer.IsAtEnd() && tokenizer.Current != this.StringDelimiter)
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
if (tokenizer.Current == this.StringDelimiter)
{
tokenizer.Consume();
return(location.CreateToken(TokenType.QuotedString, str.ToString()));
}
}
return(null);
}