public void OnNext_SplittedText_TextCorrectlyParsed(int size0, int size1)
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
var buf = Encoding.UTF8.GetBytes("id \tblabla");
var pos = 0;
var blocks = new[] { size0, size1 };
foreach (var blockSize in blocks.Where(_ => _ != 0))
{
tokenizer.OnNext(new Span <byte>(buf, pos, blockSize));
pos += blockSize;
}
if (pos < buf.Length)
{
tokenizer.OnNext(new Span <byte>(buf, pos, buf.Length - pos));
}
tokenizer.OnCompleted();
Console.WriteLine(string.Join(Environment.NewLine, observer.Tokens));
Assert.AreEqual(3, observer.Tokens.Count);
Assert.AreEqual("id", observer.Tokens[0].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[0].Type);
Assert.AreEqual(" \t", observer.Tokens[1].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Whitespace, observer.Tokens[1].Type);
Assert.AreEqual("blabla", observer.Tokens[2].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[2].Type);
Assert.IsNull(observer.Error);
Assert.AreEqual(true, observer.IsCompleted);
}
public void OnNext_SplittedId_IdCorrectlyParsed()
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
tokenizer.OnNext(Encoding.UTF8.GetBytes("iii"));
tokenizer.OnNext(Encoding.UTF8.GetBytes("bbb"));
tokenizer.OnCompleted();
Assert.AreEqual(1, observer.Tokens.Count);
Assert.AreEqual("iiibbb", observer.Tokens[0].Value);
Assert.AreEqual(SimpleTokenizer.TokenType.Id, observer.Tokens[0].Type);
Assert.IsNull(observer.Error);
Assert.AreEqual(true, observer.IsCompleted);
}
private static void Main(string[] args)
{
if (args.Length > 0)
{
var buf = new byte[4049];
using (var fileStream = File.Open(args[0], FileMode.Open, FileAccess.Read, FileShare.Write))
{
var tokenizer = new SimpleTokenizer(new AbstractTokenObserver <SimpleTokenizer.TokenType>(),
new Utf8TokenEncoding());
for (; ;)
{
var len = fileStream.Read(buf, 0, buf.Length);
if (len <= 0)
{
break;
}
tokenizer.OnNext(new ReadOnlySpan <byte>(buf, 0, len));
}
}
}
else
{
#if DEBUG
new Utf8TokenEncodingBenchmark().AseFile();
new Utf8TokenEncodingBenchmark().TwoByteChars();
#else
var summary = BenchmarkRunner.Run <Utf8TokenEncodingBenchmark>();
Console.WriteLine(summary);
#endif
}
}
public void OnNext_CorrectlyParsed(string text, SimpleTokenizer.TokenType type)
{
Console.WriteLine(float.MinValue.ToString("f", CultureInfo.InvariantCulture));
Console.WriteLine(float.MaxValue.ToString("f", CultureInfo.InvariantCulture));
var buf = Encoding.UTF8.GetBytes(text);
for (var i = 0; i < text.Length; ++i)
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
tokenizer.OnNext(new Span <byte>(buf, 0, i));
tokenizer.OnNext(new Span <byte>(buf, i, buf.Length - i));
tokenizer.OnCompleted();
Assert.AreEqual(1, observer.Tokens.Count);
Assert.AreEqual(text, observer.Tokens[0].Value);
Assert.AreEqual(type, observer.Tokens[0].Type);
}
}
public void OnNext_EmptyString_NoTokensNoErrorNotCompleted()
{
var observer = new TokenObserver();
var tokenizer = new SimpleTokenizer(observer);
tokenizer.OnNext(ReadOnlySpan <byte> .Empty);
Assert.AreEqual(0, observer.Tokens.Count);
Assert.IsNull(observer.Error);
Assert.AreEqual(false, observer.IsCompleted);
}