public TokenStream GenerateTokens()
{
StateStorage scannerStates = new StateStorage();
TokenConstruction constr = new TokenConstruction();
IScannerState currentState = scannerStates.Base;
while (true)
{
source.MoveNext();
if (source.EndOfStream)
{
break;
}
char curr = source.Current;
if (curr == '\n')
{
constr.IncrementLines();
}
if (!currentState.Equals(scannerStates.StringLiteral) &&
!currentState.Equals(scannerStates.EscapeCharacter) &&
!currentState.Equals(scannerStates.StringLiteralStart))
{
curr = char.ToLowerInvariant(curr);
}
currentState = currentState.Read(constr, curr, scannerStates);
}
if (!currentState.Equals(scannerStates.Base))
{
throw new LexerException("Unexpected end of input", constr.Line);
}
constr.End(Symbol.EndOfInput);
return(constr.CreateStream());
}
public async Task CopyAsyncStream()
{
var bytes = Helper.CreateRandomByteArray(190234);
var source = new SourceStream(bytes);
source.Position = 0;
var uut = Container.GetService <FlowerPower.IStreamer>();
var partitioner = Container.GetService <FlowerPower.Partitioners.MultiFilePartitioner>();
partitioner.PartitionSize = 4048;
uut.UsePartitionGenerator(partitioner);
var copied = uut.FromAsync(source, false, CancellationToken.None);
var target = new MemoryStream(bytes.Length);
var reader = uut.CreateReader();
var read = reader.CopyToAsync(target);
while (true)
{
Console.WriteLine("Moving ...");
await source.MoveNext(7209);
if (source.Position < source.OriginalData.Length)
{
Console.WriteLine("Should not yet be done..");
Assert.IsFalse(copied.IsCompleted, "We still have data to send");
Assert.IsFalse(read.IsCompleted, "We are still writing data");
}
else
{
Console.WriteLine("Should now be done.");
await Task.WhenAny(copied, Task.Delay(5000));
Assert.IsTrue(copied.IsCompleted, "Copy operation not yet completed?");
await Task.WhenAny(read, Task.Delay(5000));
Assert.IsTrue(read.IsCompleted, "Reader not yet completed?");
Assert.AreEqual(source.OriginalData.Length, reader.Position);
break;
}
}
Assert.AreEqual(bytes.Length, target.Position);
reader.Dispose();
uut.Dispose();
}