public async Task TestTwoLevelWhere()
{
var input = new List <int>()
{
5, 213, 23, -21, 23, 99
};
var output = new List <string>()
{
"213", "23", "23", "99"
};
var source = new StreamMessageSenderComposite <int>(_provider, 2);
var query = await source.Where(x => x >= 20, _factory).Select(x => x.ToString());
var queryOutputStreams = await query.GetOutputStreams();
var resultConsumer = new TransactionalStreamListConsumer <string>(_provider);
await resultConsumer.SetInput(queryOutputStreams);
await source.SendMessage(new ItemMessage <int>(input));
await query.TearDown();
await resultConsumer.TearDown();
}
public async Task TestTearDownStreamBroken()
{
var source = new StreamMessageSenderComposite <int>(_provider, 2);
var factory = new DefaultStreamProcessorAggregateFactory(GrainFactory);
var query = await source.Select(x => x, factory);
var queryOutputStreams = await query.GetOutputStreams();
var resultConsumer = new TestTransactionalTransactionalStreamConsumerAggregate <int>(_provider);
await resultConsumer.SetInput(queryOutputStreams);
Assert.AreEqual(2, queryOutputStreams.Count);
Assert.AreEqual(0, resultConsumer.Items.Count);
Assert.IsFalse(await resultConsumer.AllConsumersTearDownCalled());
await query.TearDown();
await resultConsumer.TearDown();
Assert.IsTrue(await resultConsumer.AllConsumersTearDownCalled());
await source.SendMessage(new ItemMessage <int>(new List <int>()
{
2, 3
}));
Assert.AreEqual(0, resultConsumer.Items.Count);
Assert.IsTrue(await resultConsumer.AllConsumersTearDownCalled());
}
public async Task TestItemTransfer()
{
var processorNodeGuid = Guid.NewGuid();
var processor = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectNodeGrain <int, int> >(processorNodeGuid);
await processor.SetFunction(new SerializableFunc <int, int>(_ => _));
var itemsToSend = new List <int> {
-1, 5, 30
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var testProvider = new StreamMessageSenderComposite <int>(provider, 1);
await processor.SubscribeToStreams(await testProvider.GetOutputStreams());
var testConsumer = new TransactionalStreamListConsumer <int>(provider);
await SubscribeConsumer(processor, testConsumer);
var tid = TransactionGenerator.GenerateTransactionId();
await testProvider.StartTransaction(tid);
await testProvider.SendMessage(new ItemMessage <int>(itemsToSend));
await testProvider.EndTransaction(tid);
CollectionAssert.AreEquivalent(itemsToSend, testConsumer.Items);
await testProvider.TearDown();
}
public async Task TestAggregateCleanupSuccessful()
{
var aggregate = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectAggregate <int, int> >(Guid.NewGuid());
await aggregate.SetFunction(new SerializableFunc <int, int>(_ => _));
var itemsToSend = new List <int> {
1, 5, 32, -12
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var inputAggregate = new StreamMessageSenderComposite <int>(provider, 2);;
await aggregate.SetInput(await inputAggregate.GetOutputStreams());
var streamIdentitiesProcessor = await aggregate.GetOutputStreams();
Assert.AreEqual(2, (await aggregate.GetOutputStreams()).Count);
var consumerAggregate = new TestTransactionalTransactionalStreamConsumerAggregate <int>(provider);
await consumerAggregate.SetInput(await aggregate.GetOutputStreams());
var subscriptionHdl1 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[0]);
var subscriptionHdl2 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[1]);
Assert.AreEqual(1, subscriptionHdl1.Count);
Assert.AreEqual(1, subscriptionHdl2.Count);
await inputAggregate.TearDown();
var tId = Guid.NewGuid();
await inputAggregate.StartTransaction(tId);
await inputAggregate.EndTransaction(tId);
var taskCompleted = consumerAggregate.TransactionComplete(tId).Wait(TimeSpan.FromSeconds(5));
Assert.IsFalse(taskCompleted);
subscriptionHdl1 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[0]);
subscriptionHdl2 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[1]);
Assert.AreEqual(0, subscriptionHdl1.Count);
Assert.AreEqual(0, subscriptionHdl2.Count);
}
public async Task TestOneLevelSelectManyDataPass()
{
var inputChunks = new List <int>()
{
5, 213, 23, -21, 23, 99
}.BatchIEnumerable(2).ToList();
var outputChunks = inputChunks.SelectMany(i => i).Select(i => (i > 0) ? Enumerable.Range(0, i).ToList() : i.SingleValueToList()).ToList();
var source = new StreamMessageSenderComposite <int>(_provider, 2);
var factory = new DefaultStreamProcessorAggregateFactory(GrainFactory);
var query = await source.SimpleSelectMany(i => (i > 0)?Enumerable.Range(0, i) : i.SingleValueToList(), factory);
var queryOutputStreams = await query.GetOutputStreams();
var resultConsumer = new TestTransactionalTransactionalStreamConsumerAggregate <int>(_provider);
await resultConsumer.SetInput(queryOutputStreams);
Assert.AreEqual(2, queryOutputStreams.Count);
Assert.AreEqual(0, resultConsumer.Items.Count);
for (int i = 0; i < inputChunks.Count; i++)
{
var input = inputChunks[i];
var expectedOutput = new List <int>();
expectedOutput.AddRange(outputChunks[2 * i]);
expectedOutput.AddRange(outputChunks[2 * i + 1]);
var tid = TransactionGenerator.GenerateTransactionId();
await source.StartTransaction(tid);
await source.SendMessage(new ItemMessage <int>(input));
await source.EndTransaction(tid);
CollectionAssert.AreEquivalent(expectedOutput, resultConsumer.Items);
resultConsumer.Items.Clear();
}
await query.TearDown();
await resultConsumer.TearDown();
}
private async Task TestMultiLevelDataPass <TIn, TOut>(
Func <StreamMessageSenderComposite <TIn>, DefaultStreamProcessorAggregateFactory, Task <IStreamProcessorChain <TOut, DefaultStreamProcessorAggregateFactory> > >
createStreamProcessingChainFunc, List <List <TIn> > inputChunks, List <List <TOut> > outputChunks,
Action <List <TOut>, List <TOut> > resultAssertion)
{
if (inputChunks.Count != outputChunks.Count)
{
throw new ArgumentException();
}
var source = new StreamMessageSenderComposite <TIn>(_provider, 2);
var query = await createStreamProcessingChainFunc(source, _factory);
var queryOutputStreams = await query.GetOutputStreams();
var resultConsumer = new TestTransactionalTransactionalStreamConsumerAggregate <TOut>(_provider);
await resultConsumer.SetInput(queryOutputStreams);
Assert.AreEqual(2, queryOutputStreams.Count);
Assert.AreEqual(0, resultConsumer.Items.Count);
for (int i = 0; i < inputChunks.Count; i++)
{
var input = inputChunks[i];
var expectedOutput = outputChunks[i];
var tid = TransactionGenerator.GenerateTransactionId();
await source.StartTransaction(tid);
await source.SendMessage(new ItemMessage <TIn>(input));
await source.EndTransaction(tid);
resultAssertion(expectedOutput, resultConsumer.Items);
resultConsumer.Items.Clear();
}
await query.TearDown();
await resultConsumer.TearDown();
}
public async Task TestTransactionNoItems()
{
var processorNodeGuid = Guid.NewGuid();
var processor = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectNodeGrain <int, int> >(processorNodeGuid);
await processor.SetFunction(new SerializableFunc <int, int>(_ => _));
var provider = GrainClient.GetStreamProvider(StreamProvider);
var testProvider = new StreamMessageSenderComposite <int>(provider, 1);
await processor.SubscribeToStreams(await testProvider.GetOutputStreams());
var testConsumer = new TransactionalStreamListConsumer <int>(provider);
await SubscribeConsumer(processor, testConsumer);
await testProvider.SendMessage(new ItemMessage <int>(new List <int>()));
Assert.AreEqual(0, testConsumer.Items.Count);
await testProvider.TearDown();
}
private async Task ValidateTwoLevelAggregateSetupAndTearDown()
{
int numberOfStreamsPerLevel = 2;
var source = new StreamMessageSenderComposite <int>(_provider, numberOfStreamsPerLevel);
var factory = new DefaultStreamProcessorAggregateFactory(GrainFactory);
var aggregateOne = await factory.CreateSelect <int, int>(_ => _, new StreamProcessorAggregateConfiguration(await source.GetOutputStreams()));
var aggregateTwo = await factory.CreateSelect <int, int>(_ => _, new StreamProcessorAggregateConfiguration(await aggregateOne.GetOutputStreams()));
var firstElement = new StreamProcessorChainStart <int, int, DefaultStreamProcessorAggregateFactory>(aggregateOne, source, new DefaultStreamProcessorAggregateFactory(GrainFactory));
var query = new StreamProcessorChain <int, int, DefaultStreamProcessorAggregateFactory>(aggregateTwo, firstElement);
Assert.IsFalse(await aggregateOne.IsTearedDown());
Assert.IsFalse(await aggregateTwo.IsTearedDown());
await query.TearDown();
Assert.IsTrue(await aggregateOne.IsTearedDown());
Assert.IsTrue(await aggregateTwo.IsTearedDown());
}
public async Task TestItemAggregation()
{
var aggregate = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectAggregate <int, int> >(Guid.NewGuid());
await aggregate.SetFunction(new SerializableFunc <int, int>(_ => _));
var itemsToSend = new List <int> {
1, 5, 32, -12
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var inputAggregate = new StreamMessageSenderComposite <int>(provider, 2);
await aggregate.SetInput(await inputAggregate.GetOutputStreams());
Assert.AreEqual(2, (await aggregate.GetOutputStreams()).Count);
var consumerAggregate = new TestTransactionalTransactionalStreamConsumerAggregate <int>(provider);
await consumerAggregate.SetInput(await aggregate.GetOutputStreams());
var tid = TransactionGenerator.GenerateTransactionId();
await inputAggregate.StartTransaction(tid);
await inputAggregate.SendMessage(new ItemMessage <int>(itemsToSend));
await inputAggregate.EndTransaction(tid);
var resultItems = consumerAggregate.Items;
Assert.AreEqual(4, resultItems.Count);
CollectionAssert.AreEquivalent(itemsToSend, resultItems);
Assert.IsFalse(await consumerAggregate.AllConsumersTearDownCalled());
await inputAggregate.TearDown();
Assert.IsTrue(await consumerAggregate.AllConsumersTearDownCalled());
}
public async Task Execute()
{
var provider = GetStreamProvider("CollectionStreamProvider");
// Simple collection sample
var container = GrainClient.GrainFactory.GetGrain <IContainerGrain <DummyInt> >(Guid.NewGuid());
await container.SetNumberOfNodes(4);
await container.BatchAdd(Enumerable.Range(0, 10).Select(x => new DummyInt(x)).ToList());
await container.ExecuteSync((i) => { i.Value += 2; }); // Pass action to container
var consumer = new TransactionalStreamListConsumer <ContainerElement <DummyInt> >(provider);
await consumer.SetInput(await container.GetOutputStreams());
var transactionId = await container.EnumerateToSubscribers();
await consumer.TransactionComplete(transactionId);
Console.WriteLine(consumer.Items);
// Sample with observable collection and data query.
var collection = GrainClient.GrainFactory.GetGrain <IContainerGrain <DummyInt> >(Guid.NewGuid());
int numContainers = 4;
await collection.SetNumberOfNodes(numContainers);
var aggregateFactory = new DefaultStreamProcessorAggregateFactory(GrainClient.GrainFactory);
var query = await collection.Select(x => x.Item.Value, aggregateFactory).Where(x => x > 500);
var matchingItemConsumer = new TransactionalStreamListConsumer <int>(provider);
await matchingItemConsumer.SetInput(await query.GetOutputStreams());
var observedCollectionConsumer = new TransactionalStreamListConsumer <ContainerElement <DummyInt> >(provider);
await observedCollectionConsumer.SetInput(await collection.GetOutputStreams());
var inputList = Enumerable.Range(0, 1000).Select(x => new DummyInt(x)).ToList();
await collection.BatchAdd(inputList);
Console.WriteLine("#Items: {0}", observedCollectionConsumer.Items.Count);
Console.WriteLine("#Items resultQuery: {0}", matchingItemConsumer.Items.Count);
// Simple query using stream provider and consumer.
var simpleProvider = new StreamMessageSenderComposite <int>(provider, numberOfOutputStreams: 10);
var queryNumbersLessThan1000 = await simpleProvider.Where(x => x < 1000, aggregateFactory);
var simpleResultConsumer = new TransactionalStreamListConsumer <int>(provider);
await simpleResultConsumer.SetInput(await queryNumbersLessThan1000.GetOutputStreams());
var rand = new Random(123);
List <int> numbers = Enumerable.Repeat(2000, 10000).Select(x => rand.Next(x)).ToList();
await simpleProvider.SendMessage(new ItemMessage <int>(numbers));
var tid = Guid.NewGuid();
await simpleProvider.StartTransaction(tid);
await simpleProvider.EndTransaction(tid);
Console.WriteLine("#Items less than 1000: {0}", simpleResultConsumer.Items.Count);
}