public void OnNext_SplittedText_TextCorrectlyParsed(int size0, int size1)
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
var buf = Encoding.UTF8.GetBytes("id \tblabla");
var pos = 0;
var blocks = new[] { size0, size1 };
foreach (var blockSize in blocks.Where(_ => _ != 0))
{
tokenizer.OnNext(new Span <byte>(buf, pos, blockSize));
pos += blockSize;
}
if (pos < buf.Length)
{
tokenizer.OnNext(new Span <byte>(buf, pos, buf.Length - pos));
}
tokenizer.OnCompleted();
Console.WriteLine(string.Join(Environment.NewLine, observer.Tokens));
Assert.AreEqual(3, observer.Tokens.Count);
Assert.AreEqual("id", observer.Tokens[0].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[0].Type);
Assert.AreEqual(" \t", observer.Tokens[1].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Whitespace, observer.Tokens[1].Type);
Assert.AreEqual("blabla", observer.Tokens[2].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[2].Type);
Assert.IsNull(observer.Error);
Assert.AreEqual(true, observer.IsCompleted);
}
public void OnCompleted_NoTokensNoErrorCompleted()
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
tokenizer.OnCompleted();
Assert.AreEqual(0, observer.Tokens.Count);
Assert.IsNull(observer.Error);
Assert.AreEqual(true, observer.IsCompleted);
}
public void OnNext_Id_IdCorrectlyParsed()
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
tokenizer.OnNext(Encoding.UTF8.GetBytes("Bla"));
tokenizer.OnCompleted();
Assert.AreEqual(1, observer.Tokens.Count);
Assert.AreEqual("Bla", observer.Tokens[0].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[0].Type);
Assert.IsNull(observer.Error);
Assert.AreEqual(true, observer.IsCompleted);
}
public void OnNext_CorrectlyParsed(string text, SimpleTokenizer.TokenType type)
{
Console.WriteLine(float.MinValue.ToString("f", CultureInfo.InvariantCulture));
Console.WriteLine(float.MaxValue.ToString("f", CultureInfo.InvariantCulture));
var buf = Encoding.UTF8.GetBytes(text);
for (var i = 0; i < text.Length; ++i)
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
tokenizer.OnNext(new Span <byte>(buf, 0, i));
tokenizer.OnNext(new Span <byte>(buf, i, buf.Length - i));
tokenizer.OnCompleted();
Assert.AreEqual(1, observer.Tokens.Count);
Assert.AreEqual(text, observer.Tokens[0].Value);
Assert.AreEqual(type, observer.Tokens[0].Type);
}
}
