public async Task TestItemAggregation()
{
var aggregate = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectAggregate <int, int> >(Guid.NewGuid());
await aggregate.SetFunction(_ => _);
var itemsToSend = new List <int> {
1, 5, 32, -12
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var inputAggregate = new MultiStreamProvider <int>(provider, 2);
await aggregate.SetInput(await inputAggregate.GetStreamIdentities());
Assert.AreEqual(2, (await aggregate.GetStreamIdentities()).Count);
var consumerAggregate = new TestTransactionalStreamConsumerAggregate <int>(provider);
await consumerAggregate.SetInput(await aggregate.GetStreamIdentities());
var tid = await inputAggregate.SendItems(itemsToSend);
var waitForTransaction = consumerAggregate.TransactionComplete(tid);
var resultItems = consumerAggregate.Items;
Assert.AreEqual(4, resultItems.Count);
CollectionAssert.AreEquivalent(itemsToSend, resultItems);
Assert.IsFalse(await consumerAggregate.AllConsumersTearDownCalled());
await inputAggregate.TearDown();
Assert.IsTrue(await consumerAggregate.AllConsumersTearDownCalled());
}
public async Task TestItemTransfer()
{
var processorNodeGuid = Guid.NewGuid();
var processor = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectNodeGrain <int, int> >(processorNodeGuid);
await processor.SetFunction(_ => _);
var itemsToSend = new List <int> {
-1, 5, 30
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var testProvider = new SingleStreamProvider <int>(provider);
await SubscribeConsumer(testProvider, processor);
var testConsumer = new MultiStreamListConsumer <int>(provider);
await SubscribeConsumer(processor, testConsumer);
var waitForTransaction = testConsumer.TransactionComplete(1);
await testProvider.SendItems(itemsToSend, true, 1);
await waitForTransaction;
CollectionAssert.AreEquivalent(itemsToSend, testConsumer.Items);
await testProvider.TearDown();
}
public OrniscientObserver()
{
var streamprovider = GrainClient.GetStreamProvider(StreamKeys.StreamProvider);
_stream = streamprovider.GetStream <DiffModel>(Guid.Empty, StreamKeys.OrniscientClient);
_stream.SubscribeAsync(this);
}
private async Task GenerateEvents(string streamNamespace, List <Guid> streamGuids, int eventsInStream, int payloadSize)
{
IStreamProvider streamProvider = GrainClient.GetStreamProvider(StreamProviderName);
IAsyncStream <GeneratedEvent>[] producers = streamGuids
.Select(streamGuid => streamProvider.GetStream <GeneratedEvent>(streamGuid, streamNamespace))
.ToArray();
for (int i = 0; i < eventsInStream - 1; i++)
{
// send event on each stream
for (int j = 0; j < streamGuids.Count; j++)
{
await producers[j].OnNextAsync(new GeneratedEvent {
EventType = GeneratedEvent.GeneratedEventType.Fill, Payload = new int[payloadSize]
});
}
}
// send end events
for (int j = 0; j < streamGuids.Count; j++)
{
await producers[j].OnNextAsync(new GeneratedEvent {
EventType = GeneratedEvent.GeneratedEventType.Report, Payload = new int[payloadSize]
});
}
}
static void WriteALot()
{
var providerName = "FPSProvider";
var streamId = new Guid("00000000-0000-0000-0000-000000000000");
var provider = GrainClient.GetStreamProvider(providerName);
var stream = provider.GetStream <int>(streamId, "GlobalNamespace");
var start = DateTime.UtcNow;
var t1 = start;
DateTime t2;
for (int i = 0; i < 1000000; i++)
{
stream.OnNextAsync(i);
if (i % 10000 == 0)
{
t2 = DateTime.UtcNow;
Console.WriteLine($"New mark set. Time since last mark: {(t2 - t1).TotalMilliseconds}");
t1 = t2;
}
}
var end = DateTime.UtcNow;
Console.WriteLine($"{(end - start).TotalMilliseconds} ms");
//stream.OnCompletedAsync().Wait();
}
public async Task TestItemTransfer()
{
var processorNodeGuid = Guid.NewGuid();
var processor = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectNodeGrain <int, int> >(processorNodeGuid);
await processor.SetFunction(new SerializableFunc <int, int>(_ => _));
var itemsToSend = new List <int> {
-1, 5, 30
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var testProvider = new StreamMessageSenderComposite <int>(provider, 1);
await processor.SubscribeToStreams(await testProvider.GetOutputStreams());
var testConsumer = new TransactionalStreamListConsumer <int>(provider);
await SubscribeConsumer(processor, testConsumer);
var tid = TransactionGenerator.GenerateTransactionId();
await testProvider.StartTransaction(tid);
await testProvider.SendMessage(new ItemMessage <int>(itemsToSend));
await testProvider.EndTransaction(tid);
CollectionAssert.AreEquivalent(itemsToSend, testConsumer.Items);
await testProvider.TearDown();
}
static async void StartListening(string locationName)
{
var stream = GrainClient.GetStreamProvider("sms")
.GetStream <RunnerMetrics>(Streams.Challenge, locationName);
Subscription = await stream.SubscribeAsync(new LocationObserver());
}
public async Task DeactivationTest_ClientConsumer(Guid streamGuid, string streamNamespace)
{
// get producer and consumer
var producer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
var count = new Counter();
// get stream and subscribe
IStreamProvider streamProvider = GrainClient.GetStreamProvider(streamProviderName);
var stream = streamProvider.GetStream <int>(streamGuid, streamNamespace);
StreamSubscriptionHandle <int> subscriptionHandle = await stream.SubscribeAsync((e, t) => count.Increment());
// produce one message (PubSubRendezvousGrain will have one consumer and one producer)
await producer.BecomeProducer(streamGuid, streamNamespace, streamProviderName);
await producer.Produce();
Assert.AreEqual(count.Value, 1, "Client consumer grain has not received stream message");
//TODO: trigger deactivation programmatically
await Task.Delay(TimeSpan.FromMilliseconds(130000)); // wait for the PubSubRendezvousGrain and the SampleStreaming_ProducerGrain to be deactivated
// deactivating PubSubRendezvousGrain and SampleStreaming_ProducerGrain during the same GC cycle causes a deadlock
// resume producing after the PubSubRendezvousGrain and the SampleStreaming_ProducerGrain grains have been deactivated:
await producer.BecomeProducer(streamGuid, streamNamespace, streamProviderName).WithTimeout(Timeout, "BecomeProducer is hung due to deactivation deadlock");
await producer.Produce().WithTimeout(Timeout, "Produce is hung due to deactivation deadlock");
Assert.AreEqual(count.Value, 2, "Client consumer grain did not receive stream messages after PubSubRendezvousGrain and SampleStreaming_ProducerGrain reactivation");
}
static void Main(string[] args)
{
Console.WriteLine($"Starting client {ClientName}");
var configFile = Path.Combine(Path.GetDirectoryName(Assembly.GetCallingAssembly().Location), "ClientConfiguration.xml");
GrainClient.Initialize(configFile);
var streamProvider = GrainClient.GetStreamProvider("DefaultProvider");
var messageStream = streamProvider.GetStream <MessageBatch>(ClientName, "AccountSteams");
var watcher = new MessageWatcher();
var subscriptionHandle = messageStream.SubscribeAsync(watcher).Result;
var cts = new CancellationTokenSource();
var publisherTask = Task.Factory.StartNew(PublishMessages, cts.Token);
Console.WriteLine("client started");
Console.WriteLine("Press ENTER to stop this publisher.");
Console.ReadLine();
Console.WriteLine("Stopping client");
cts.Cancel();
publisherTask = null;
subscriptionHandle.UnsubscribeAsync().Wait();
GrainClient.Uninitialize();
Console.WriteLine("Publisher client");
}
public async Task StreamingPubSubStoreTest()
{
var strmId = Guid.NewGuid();
var providers = this.HostedCluster.ClientConfiguration.GetAllProviderConfigurations();
var sms = from x in providers
where
x.Name.Equals(StreamProviderName)
select x;
Assert.True(sms.Count() == 1);
try
{
var streamProv = GrainClient.GetStreamProvider(StreamProviderName);
IAsyncStream <int> stream = streamProv.GetStream <int>(strmId, "test1");
StreamSubscriptionHandle <int> handle = await stream.SubscribeAsync(
(e, t) => { return(TaskDone.Done); },
e => { return(TaskDone.Done); });
}
catch (Exception ex)
{
this.output.WriteLine(ex.ToString());
throw;
}
}
public async Task SubscribeFromClientTest(Guid streamGuid, string streamNamespace)
{
// get producer and consumer
var producer = GrainClient.GrainFactory.GetGrain <ISampleStreaming_ProducerGrain>(Guid.NewGuid());
int eventCount = 0;
var provider = GrainClient.GetStreamProvider(streamProviderName);
var stream = provider.GetStream <int>(streamGuid, streamNamespace);
var handle = await stream.SubscribeAsync((e, t) =>
{
eventCount++;
return(TaskDone.Done);
});
// produce some messages
await producer.BecomeProducer(streamGuid, streamNamespace, streamProviderName);
await producer.StartPeriodicProducing();
await Task.Delay(TimeSpan.FromMilliseconds(1000));
await producer.StopPeriodicProducing();
// check
await TestingUtils.WaitUntilAsync(lastTry => CheckCounters(producer, () => eventCount, lastTry), Timeout);
// unsubscribe
await handle.UnsubscribeAsync();
}
private async Task GenerateEvents(string streamNamespace, int streamCount, int eventsInStream)
{
IStreamProvider streamProvider = GrainClient.GetStreamProvider(StreamProviderName);
IAsyncStream <GeneratedEvent>[] producers =
Enumerable.Range(0, streamCount)
.Select(i => streamProvider.GetStream <GeneratedEvent>(Guid.NewGuid(), streamNamespace))
.ToArray();
for (int i = 0; i < eventsInStream - 1; i++)
{
// send event on each stream
for (int j = 0; j < streamCount; j++)
{
await producers[j].OnNextAsync(new GeneratedEvent {
EventType = GeneratedEvent.GeneratedEventType.Fill
});
}
}
// send end events
for (int j = 0; j < streamCount; j++)
{
await producers[j].OnNextAsync(new GeneratedEvent {
EventType = GeneratedEvent.GeneratedEventType.Report
});
}
}
async Task IMessagesMonitor.StartTracking()
{
var provider = GrainClient.GetStreamProvider("SMSProvider");
var room = provider.GetStream <RoomMessage>(Streams.Id, "default");
handle = await room.SubscribeAsync(this);
}
public async Task MultipleImplicitSubscriptionTest()
{
logger.Info("************************ MultipleImplicitSubscriptionTest *********************************");
var streamId = Guid.NewGuid();
const int nRedEvents = 5, nBlueEvents = 3;
var provider = GrainClient.GetStreamProvider(StreamTestsConstants.SMS_STREAM_PROVIDER_NAME);
var redStream = provider.GetStream <int>(streamId, "red");
var blueStream = provider.GetStream <int>(streamId, "blue");
for (int i = 0; i < nRedEvents; i++)
{
await redStream.OnNextAsync(i);
}
for (int i = 0; i < nBlueEvents; i++)
{
await blueStream.OnNextAsync(i);
}
var grain = GrainClient.GrainFactory.GetGrain <IMultipleImplicitSubscriptionGrain>(streamId);
var counters = await grain.GetCounters();
Assert.Equal(nRedEvents, counters.Item1);
Assert.Equal(nBlueEvents, counters.Item2);
}
List <IClusterTestListener> listeners = new List <IClusterTestListener>(); // keep them from being GCed
// stream-based notification
public void SubscribeStream(int i, IAsyncObserver <int> listener)
{
IStreamProvider streamProvider = GrainClient.GetStreamProvider("SMSProvider");
Guid guid = new Guid(i, 0, 0, new byte[8]);
IAsyncStream <int> stream = streamProvider.GetStream <int>(guid, "notificationtest");
handle = stream.SubscribeAsync(listener).Result;
}
private async Task <IList <StreamSubscriptionHandle <T> > > GetStreamSubscriptionHandles <T>(Tuple <Guid, string> identifier)
{
var result = await GrainClient.GetStreamProvider(StreamProvider)
.GetStream <T>(identifier.Item1, identifier.Item2)
.GetAllSubscriptionHandles();
return(result);
}
private Task SubscribeToStream(string streamProviderName, Guid streamGuid, string streamNamespace,
Func <int, StreamSequenceToken, Task> onNextAsync)
{
IStreamProvider streamProvider = GrainClient.GetStreamProvider(streamProviderName);
IAsyncObservable <int> stream = streamProvider.GetStream <int>(streamGuid, streamNamespace);
return(stream.SubscribeAsync(onNextAsync));
}
public Task EnableStreamNotifications()
{
IStreamProvider streamProvider = GrainClient.GetStreamProvider("SMSProvider");
Guid guid = new Guid((int)this.GetPrimaryKeyLong(), 0, 0, new byte[8]);
stream = streamProvider.GetStream <int>(guid, "notificationtest");
return(TaskDone.Done);
}
private async Task <IList <StreamSubscriptionHandle <T> > > GetStreamSubscriptionHandles <T>(StreamIdentity streamIdentity)
{
var result = await GrainClient.GetStreamProvider(StreamProvider)
.GetStream <T>(streamIdentity.Guid, streamIdentity.Namespace)
.GetAllSubscriptionHandles();
return(result);
}
static async Task SubscribeToStream()
{
StreamProvider = GrainClient.GetStreamProvider("SimpleStreamProvider");
ClusterSnapshotStream = StreamProvider
.GetStream <MetricsSnapshot>(Guid.Empty, "ClusterMetricSnapshots");
await ClusterSnapshotStream.SubscribeAsync(OnNewMetricSnapshot);
}
public async Task Configure(MetricsConfiguration config)
{
try
{
Configuration = config;
// TODO: figure out how to get MetricsTrackerTelemetryConsumer to get notifications
// using grain observables
if (Subscribers != null)
{
Subscribers.Notify(o => o.Configure(config));
}
// using streams
if (!string.IsNullOrWhiteSpace(Configuration.StreamingProviderName))
{
try
{
StreamProvider = GrainClient.GetStreamProvider(Configuration.StreamingProviderName);
ClusterSnapshotStream = StreamProvider.GetStream <MetricsSnapshot>(Guid.Empty, "ClusterMetricSnapshots");
SiloSnapshotStream = StreamProvider.GetStream <MetricsSnapshot>(Guid.Empty, "SiloMetricSnapshots");
ActivateClusterMetricsTimer();
}
catch (Exception ex)
{
// probably here because stream provider wasn't found
// TODO: handle better
logger.TrackException(ex);
// don't rethrow the exception
}
}
else
{
if (ClusterSnapshotStream != null)
{
await ClusterSnapshotStream.OnCompletedAsync();
}
if (SiloSnapshotStream != null)
{
await SiloSnapshotStream.OnCompletedAsync();
}
ClusterSnapshotStream = null;
SiloSnapshotStream = null;
}
logger.IncrementMetric("ClusterMetricsConfigured");
}
catch (Exception ex)
{
logger.TrackException(ex);
throw;
}
}
public async Task Subscribe()
{
var providerName = "FPSProvider";
var streamId = new Guid("00000000-0000-0000-0000-000000000000");
var provider = GrainClient.GetStreamProvider(providerName);
var stream = provider.GetStream <int>(streamId, "GlobalNamespace");
await stream.SubscribeAsync(this, new PipeStreamProvider.SimpleSequenceToken(0));
}
public async Task SetupTelemetryStreamPump()
{
var metricsStreamProvider = GrainClient.GetStreamProvider(MetricsStreamProvider);
var clusterStream = metricsStreamProvider.GetStream <MetricsSnapshot>(Guid.Empty, "ClusterMetricSnapshots");
var siloStream = metricsStreamProvider.GetStream <MetricsSnapshot>(Guid.Empty, "SiloMetricSnapshots");
var metricsGrain = GrainClient.GrainFactory.GetGrain <IClusterMetricsGrain>(Guid.Empty);
//config.StreamingProviderName = "SystemMetricsStream";
await metricsGrain.Configure(new MetricsConfiguration
{
Enabled = true, // default
SamplingInterval = TimeSpan.FromSeconds(1), // default
ConfigurationInterval = TimeSpan.FromSeconds(10), // default
StaleSiloMetricsDuration = TimeSpan.FromSeconds(30), // default
TrackExceptionCounters = true,
TrackMethodGrainCalls = true,
StreamingProviderName = MetricsStreamProvider,
HistoryLength = 30 // default
});
_rootDataObject = new VizceralRootObject
{
renderer = "global",
name = "edge",
serverUpdateTime = DateTime.UtcNow.Ticks,
maxVolume = 100000,
nodes = new List <Node>
{
new Node
{
renderer = "region",
name = "INTERNET",
updated = DateTime.UtcNow.Ticks,
@class = "normal",
nodes = new List <Node>(),
},
},
connections = new List <Connection>()
};
_metricsHub = GlobalHost.ConnectionManager.GetHubContext <VizceralHub>();
await clusterStream.SubscribeAsync((data, token) =>
{
PushClusterMetricsToConsumers();
return(TaskDone.Done);
});
await siloStream.SubscribeAsync((data, token) =>
{
data.Source = System.Uri.EscapeDataString(data.Source);
AddReplaceSiloStatsToMetrics(data, _snapshotHistoryCache.ContainsKey(data.Source) ? _snapshotHistoryCache[data.Source] : null);
PushSiloMetricsToConsumers();
return(TaskDone.Done);
});
}
private async Task GenerateEvents(string streamProviderName, Guid streamGuid, string streamNamespace, int produceCount)
{
IStreamProvider streamProvider = GrainClient.GetStreamProvider(streamProviderName);
IAsyncObserver <int> observer = streamProvider.GetStream <int>(streamGuid, streamNamespace);
for (int i = 0; i < produceCount; i++)
{
await observer.OnNextAsync(i);
}
}
public async Task StreamingPubSubStoreTest()
{
var strmId = Guid.NewGuid();
var streamProv = GrainClient.GetStreamProvider("SMSProvider");
IAsyncStream <int> stream = streamProv.GetStream <int>(strmId, "test1");
StreamSubscriptionHandle <int> handle = await stream.SubscribeAsync(
(e, t) => { return(TaskDone.Done); },
e => { return(TaskDone.Done); });
}
public async Task ReceiveAsync(WebSocket socket, WebSocketReceiveResult result, byte[] buffer)
{
var id = this.socketSubscriptions[socket].StreamIdentity.Guid;
var streamProvider = GrainClient.GetStreamProvider("Default");
var stream = streamProvider.GetStream <IAction>(id, "ActionsFromClient");
var message = Encoding.UTF8.GetString(buffer, 0, result.Count);
var actionData = JsonConvert.DeserializeObject <dynamic>(message);
IAction action = ActionHelper.ConstructTypedAction(actionData);
if (action != null)
{
await stream.OnNextAsync(action);
}
}
static void WriteSome()
{
var providerName = "FPSProvider";
var streamId = new Guid("00000000-0000-0000-0000-000000000000");
var provider = GrainClient.GetStreamProvider(providerName);
var stream = provider.GetStream <int>(streamId, "GlobalNamespace");
for (int i = 0; i < 1000; i++)
{
Task.Delay(500).Wait();
stream.OnNextAsync(i);
Console.WriteLine($"Writing....: {i}\t\t{DateTime.UtcNow.Millisecond}");
}
}
public async Task PreInvocationCallbackWithStreamTest()
{
var streamProvider = GrainClient.GetStreamProvider("SMSProvider");
var id = Guid.NewGuid();
var stream = streamProvider.GetStream <int>(id, "InterceptedStream");
var grain = this.GrainFactory.GetGrain <IStreamInterceptionGrain>(id);
// The intercepted grain should double the value passed to the stream.
const int TestValue = 43;
await stream.OnNextAsync(TestValue);
var actual = await grain.GetLastStreamValue();
Assert.Equal(TestValue * 2, actual);
}
public async Task OnConnected(WebSocket socket, Guid id)
{
var streamProvider = GrainClient.GetStreamProvider("Default");
var actionsStream = streamProvider.GetStream <IAction>(id, "ActionsToClient");
var actionsSubscription = await actionsStream.SubscribeAsync(
// subscribe to the stream of actions coming from grains that need to go to the client
async (action, st) => {
// Cast it to Redux Javascript format. The ActionName method is mirrored in the Typewriter Redux template, so typescript knows the same string constants
dynamic message = new { type = action.GetType().Name, payload = action };
await SendMessageAsync(socket, JsonConvert.SerializeObject(message));
});
this.socketSubscriptions[socket] = actionsSubscription;
//if (!this.socketSubscriptions.TryAdd(socket, streamSubscription))
// throw new Exception("Can't add subscription!");
}
public async Task TestAggregateCleanupSuccessful()
{
var aggregate = GrainClient.GrainFactory.GetGrain <IStreamProcessorSelectAggregate <int, int> >(Guid.NewGuid());
await aggregate.SetFunction(new SerializableFunc <int, int>(_ => _));
var itemsToSend = new List <int> {
1, 5, 32, -12
};
var provider = GrainClient.GetStreamProvider(StreamProvider);
var inputAggregate = new StreamMessageSenderComposite <int>(provider, 2);;
await aggregate.SetInput(await inputAggregate.GetOutputStreams());
var streamIdentitiesProcessor = await aggregate.GetOutputStreams();
Assert.AreEqual(2, (await aggregate.GetOutputStreams()).Count);
var consumerAggregate = new TestTransactionalTransactionalStreamConsumerAggregate <int>(provider);
await consumerAggregate.SetInput(await aggregate.GetOutputStreams());
var subscriptionHdl1 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[0]);
var subscriptionHdl2 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[1]);
Assert.AreEqual(1, subscriptionHdl1.Count);
Assert.AreEqual(1, subscriptionHdl2.Count);
await inputAggregate.TearDown();
var tId = Guid.NewGuid();
await inputAggregate.StartTransaction(tId);
await inputAggregate.EndTransaction(tId);
var taskCompleted = consumerAggregate.TransactionComplete(tId).Wait(TimeSpan.FromSeconds(5));
Assert.IsFalse(taskCompleted);
subscriptionHdl1 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[0]);
subscriptionHdl2 = await GetStreamSubscriptionHandles <IStreamMessage>(streamIdentitiesProcessor[1]);
Assert.AreEqual(0, subscriptionHdl1.Count);
Assert.AreEqual(0, subscriptionHdl2.Count);
}
