private int SkipWhitespaces(LexerStringReader reader)
{
var whitespaceRuntimeInfo = new LexerRuntimeInfo(reader);
var whitespaceResult = _whitespaceTokenizer.Tokenize(ref whitespaceRuntimeInfo);
return(whitespaceResult.Token?.Length ?? 0);
}
public void GivenString_Split_MustTokenizeCorrectly(string input, IEnumerable <string> expected)
{
var tokenizer = new WhitespaceTokenizer();
var output = tokenizer.Tokenize(input);
Assert.Equal(expected, output);
}
public void TokenizeShouldReturnExpectedValue(Scenario scenario)
{
var tokenizer = new WhitespaceTokenizer();
var result = tokenizer.Tokenize(scenario.Input);
Assert.Equal(scenario.ExpectedResult, result);
}
public void ShouldTokenize()
{
var input = " hello pretty \r\n world ";
var tokenizer = new WhitespaceTokenizer();
using (var streamReader = input.AsStreamReader())
{
var tokens = tokenizer.Tokenize(streamReader);
CollectionAssert.AreEquivalent(new[] { "hello", "pretty", "world" }, tokens);
}
}
public void ShouldTokenizeLongText()
{
var word1 = Utils.RandomWord(1000, 1000);
var word2 = Utils.RandomWord(1000, 1000);
var word3 = Utils.RandomWord(1000, 1000);
var input = $" \r\n {word1} {word2} \r\n {word3} ";
var tokenizer = new WhitespaceTokenizer();
using (var streamReader = input.AsStreamReader())
{
var tokens = tokenizer.Tokenize(streamReader);
CollectionAssert.AreEquivalent(new[] { word1, word2, word3 }, tokens);
}
}
public IList <string> TestWhitespaceTokenizerReturnDelimiterTrue(string text)
{
var tokenizer = new WhitespaceTokenizer(true);
return(tokenizer.Tokenize(text));
}