private async Task ObserverBasedClientNotification()
        {
            var x    = Next();
            var gref = Clients[0][0].GetGrainRef(x);

            Clients[0][0].GetRuntimeId(x);
            WriteLog("{0} created grain", gref);

            var promises = new List <Task <int> >();

            // create an observer on each client
            Parallel.For(0, Clients.Length, paralleloptions, i =>
            {
                for (int jj = 0; jj < Clients[i].Length; jj++)
                {
                    int j        = jj;
                    var promise  = new TaskCompletionSource <int>();
                    var listener = new ClusterTestListener((num) =>
                    {
                        WriteLog("{3} observedcall {2} on Client[{0}][{1}]", i, j, num, gref);
                        promise.TrySetResult(num);
                    });
                    lock (promises)
                        promises.Add(promise.Task);
                    Clients[i][j].Subscribe(x, listener);
                    WriteLog("{2} subscribed to Client[{0}][{1}]", i, j, gref);
                }
            });

            // call the grain
            Clients[0][0].CallGrain(x);

            await Task.WhenAll(promises);

            var sortedresults = promises.Select(p => p.Result).OrderBy(num => num).ToArray();

            // each client should get its own notification
            for (int i = 0; i < sortedresults.Length; i++)
            {
                AssertEqual(sortedresults[i], i, gref);
            }
        }
        private async Task StreamBasedClientNotification()
        {
            var x    = Next();
            var gref = Clients[0][0].GetGrainRef(x);

            Clients[0][0].EnableStreamNotifications(x);
            WriteLog("{0} created grain", gref);

            var listeners = new List <ClusterTestListener>();
            var promises  = new List <Task <int> >();

            // create an observer on each client
            Parallel.For(0, Clients.Length, paralleloptions, i =>
            {
                for (int jj = 0; jj < Clients[i].Length; jj++)
                {
                    int j        = jj;
                    var promise  = new TaskCompletionSource <int>();
                    var listener = new ClusterTestListener((num) =>
                    {
                        WriteLog("{3} observedcall {2} on Client[{0}][{1}]", i, j, num, gref);
                        promise.TrySetResult(num);
                    });
                    promises.Add(promise.Task);
                    listeners.Add(listener);
                    Clients[i][j].SubscribeStream(x, listener);
                    WriteLog("{2} subscribed to Client[{0}][{1}]", i, j, gref);
                }
            });
            // call the grain
            Clients[0][0].CallGrain(x);

            await Task.WhenAll(promises);

            // each client should get same value
            foreach (var p in promises)
            {
                AssertEqual(1, p.Result, gref);
            }
        }