public void OnNext_SplittedText_TextCorrectlyParsed(int size0, int size1) { var observer = new TokenObserver(); var tokenizer = new SimpleTokenizer(observer); var buf = Encoding.UTF8.GetBytes("id \tblabla"); var pos = 0; var blocks = new[] { size0, size1 }; foreach (var blockSize in blocks.Where(_ => _ != 0)) { tokenizer.OnNext(new Span <byte>(buf, pos, blockSize)); pos += blockSize; } if (pos < buf.Length) { tokenizer.OnNext(new Span <byte>(buf, pos, buf.Length - pos)); } tokenizer.OnCompleted(); Console.WriteLine(string.Join(Environment.NewLine, observer.Tokens)); Assert.AreEqual(3, observer.Tokens.Count); Assert.AreEqual("id", observer.Tokens[0].Value); Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[0].Type); Assert.AreEqual(" \t", observer.Tokens[1].Value); Assert.AreEqual(SimpleTokenizer.TokenType.Whitespace, observer.Tokens[1].Type); Assert.AreEqual("blabla", observer.Tokens[2].Value); Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[2].Type); Assert.IsNull(observer.Error); Assert.AreEqual(true, observer.IsCompleted); }
public void OnCompleted_NoTokensNoErrorCompleted() { var observer = new TokenObserver(); var tokenizer = new SimpleTokenizer(observer); tokenizer.OnCompleted(); Assert.AreEqual(0, observer.Tokens.Count); Assert.IsNull(observer.Error); Assert.AreEqual(true, observer.IsCompleted); }
public void OnNext_Id_IdCorrectlyParsed() { var observer = new TokenObserver(); var tokenizer = new SimpleTokenizer(observer); tokenizer.OnNext(Encoding.UTF8.GetBytes("Bla")); tokenizer.OnCompleted(); Assert.AreEqual(1, observer.Tokens.Count); Assert.AreEqual("Bla", observer.Tokens[0].Value); Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[0].Type); Assert.IsNull(observer.Error); Assert.AreEqual(true, observer.IsCompleted); }
public void OnNext_CorrectlyParsed(string text, SimpleTokenizer.TokenType type) { Console.WriteLine(float.MinValue.ToString("f", CultureInfo.InvariantCulture)); Console.WriteLine(float.MaxValue.ToString("f", CultureInfo.InvariantCulture)); var buf = Encoding.UTF8.GetBytes(text); for (var i = 0; i < text.Length; ++i) { var observer = new TokenObserver(); var tokenizer = new SimpleTokenizer(observer); tokenizer.OnNext(new Span <byte>(buf, 0, i)); tokenizer.OnNext(new Span <byte>(buf, i, buf.Length - i)); tokenizer.OnCompleted(); Assert.AreEqual(1, observer.Tokens.Count); Assert.AreEqual(text, observer.Tokens[0].Value); Assert.AreEqual(type, observer.Tokens[0].Type); } }