Exemplo n.º 1
0
        public async Task SimpleOverview()
        {
            string projectId      = _fixture.ProjectId;
            string topicId        = _fixture.CreateTopicId();
            string subscriptionId = _fixture.CreateSubscriptionId();

            // Sample: SimpleOverview
            // First create a topic.
            PublisherClient publisher = await PublisherClient.CreateAsync();

            TopicName topicName = new TopicName(projectId, topicId);

            publisher.CreateTopic(topicName);

            // Subscribe to the topic.
            SubscriberClient subscriber = await SubscriberClient.CreateAsync();

            SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId);

            subscriber.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 60);

            // Publish a message to the topic using SimplePublisher.
            SimplePublisher simplePublisher = await SimplePublisher.CreateAsync(topicName);

            // PublishAsync() has various overloads. Here we're using the string overload.
            string messageId = await simplePublisher.PublishAsync("Hello, Pubsub");

            // SimplePublisher should be shutdown after use.
            // The TimeSpan specifies for how long to attempt to publish locally queued messages.
            await simplePublisher.ShutdownAsync(TimeSpan.FromSeconds(15));

            // Pull messages from the subscription using SimpleSubscriber.
            SimpleSubscriber simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName);

            List <PubsubMessage> receivedMessages = new List <PubsubMessage>();
            // Start the subscriber listening for messages.
            await simpleSubscriber.StartAsync((msg, cancellationToken) =>
            {
                receivedMessages.Add(msg);
                Console.WriteLine($"Received message {msg.MessageId} published at {msg.PublishTime.ToDateTime()}");
                Console.WriteLine($"Text: '{msg.Data.ToStringUtf8()}'");
                // Stop this subscriber after one message is received.
                // This is non-blocking, and the returned Task may be awaited.
                simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
                // Return Reply.Ack to indicate this message has been handled.
                return(Task.FromResult(SimpleSubscriber.Reply.Ack));
            });

            // Tidy up by deleting the subscription and the topic.
            subscriber.DeleteSubscription(subscriptionName);
            publisher.DeleteTopic(topicName);
            // End sample

            Assert.Equal(1, receivedMessages.Count);
            Assert.Equal("Hello, Pubsub", receivedMessages[0].Data.ToStringUtf8());
        }
Exemplo n.º 2
0
        public static SimplePublisher GetSimplePublisher(string projectId,
                                                         string topicId)
        {
            // [START publish_message]
            PublisherClient publisherClient = PublisherClient.Create();
            SimplePublisher publisher       = SimplePublisher.Create(
                new TopicName(projectId, topicId), new[] { publisherClient });

            // [END publish_message]
            return(publisher);
        }
Exemplo n.º 3
0
        public void TriggerStaticEvent()
        {
            SimplePublisher publisher = (SimplePublisher)_container["SimplePublisher"];
            SimpleListener  listener  = (SimpleListener)_container["SimpleListener"];

            Assert.IsFalse(listener.Listened);
            Assert.IsNull(listener.Sender);

            publisher.Trigger();

            Assert.IsTrue(listener.Listened);
            Assert.AreSame(publisher, listener.Sender);
        }
Exemplo n.º 4
0
        private void PublishAny <TModel>(TModel model, string rountingKey)
        {
            try
            {
                SimplePublisher.GetInstance().SendMessage(model, rountingKey);
            }
            catch (Exception ex)
            {
                //TODO add to redis cache.
                logger.LogE(ex);
#if DEBUG
                throw;
#endif
            }
        }
        public async Task OversizedMessage()
        {
            var topicId = _fixture.CreateTopicId();
            // Create topic
            var topicName = new TopicName(_fixture.ProjectId, topicId);
            var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);

            await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);

            // Create SimplePublisher
            var simplePublisher = await SimplePublisher.CreateAsync(topicName).ConfigureAwait(false);

            // Create oversized message
            Random rnd = new Random(1234);

            byte[] msg = new byte[10_000_001];
Exemplo n.º 6
0
        static void Main(string[] args)
        {
            List <Task> tasks = new List <Task>();

            for (var i = 0; i < 1; i++)
            {
                for (var j = 0; j < 100; j++)
                {
                    tasks.Add(Task.Run(() => SimplePublisher.Publish("queue" + i, "Hello World")));
                }
            }

            Console.Write(tasks.Count);

            Task.WaitAll(tasks.ToArray());
            Console.WriteLine("ok");
        }
Exemplo n.º 7
0
        /// <summary>
        /// Create a SimplePublisher with custom batch thresholds.
        /// </summary>
        public static SimplePublisher GetCustomPublisher(string projectId,
                                                         string topicId)
        {
            // [START pubsub_publisher_batch_settings]
            PublisherClient publisherClient = PublisherClient.Create();
            SimplePublisher publisher       = SimplePublisher.Create(
                new TopicName(projectId, topicId), new[] { publisherClient },
                new SimplePublisher.Settings()
            {
                BatchingSettings = new Google.Api.Gax.BatchingSettings(
                    elementCountThreshold: 100,
                    byteCountThreshold: 10240,
                    delayThreshold: TimeSpan.FromSeconds(3))
            });

            // [END pubsub_publisher_batch_settings]
            return(publisher);
        }
        public void Test_TInputSerializeException_IsHandled()
        {
            const string QUEUE = nameof(Test_NoResponse_IsTimedOut);

            // Arrange
            var publisher = new SimplePublisher <string>(
                this.UsingConnection.CreateModel()
                )
            {
                RoutingKey      = QUEUE,
                InputSerializer = new NotImplementedSerializer <string>()
            };

            // Act
            publisher.Send(string.Empty);

            // Assert
            Assert.IsNotNull(publisher.LastException);
        }
Exemplo n.º 9
0
        private static SimplePublisher GetGoogleBusWriter(Microsoft.Azure.WebJobs.ExecutionContext context)
        {
            string topicId   = ConfigurationManager.AppSettings["GoogleTopicId"];
            string projectId = ConfigurationManager.AppSettings["GoogleProjectId"];

            //Google credential expected to be a local file called googleCredential.json, copied to output

            TopicName topicName      = new TopicName(projectId, topicId);
            string    credentialPath = Path.Combine(context.FunctionDirectory, "..\\googleCredential.json");

            GoogleCredential credential = ReadGoogleCredentialFile(credentialPath);

            credential = credential.CreateScoped(PublisherClient.DefaultScopes);
            ClientCreationSettings clientSettings = new ClientCreationSettings(credentials: credential.ToChannelCredentials());

            Channel         channel = new Channel(PublisherClient.DefaultEndpoint.Host, PublisherClient.DefaultEndpoint.Port, credential.ToChannelCredentials());
            PublisherClient client  = PublisherClient.Create(channel);

            return(SimplePublisher.Create(topicName, new[] { client }));
        }
Exemplo n.º 10
0
        private static async Task GetAndSendSnapshotsAsync(string namespaceId, List <string> streamIds, QiClientSlim qiClient,
                                                           Microsoft.Azure.WebJobs.ExecutionContext context, TraceWriter log = null)
        {
            SimplePublisher busWriter = GetGoogleBusWriter(context);

            List <Task <Task> > snapshotTasks = new List <Task <Task> >(streamIds.Count);

            foreach (string streamId in streamIds)
            {
                string      streamIdCopy = streamId;
                Task <Task> snapshotTask = GetAndSendAsync(qiClient, namespaceId, streamIdCopy, busWriter, log);
                snapshotTasks.Add(snapshotTask);
            }
            await Task.WhenAll(snapshotTasks);

            //Wait until messages are flushed from the bus
            await busWriter.ShutdownAsync(CancellationToken.None);

            var publishingTasks = snapshotTasks.Select(outerTask => outerTask.Result);
            await Task.WhenAll(publishingTasks);
        }
Exemplo n.º 11
0
        public static object PublishMessages(SimplePublisher publisher,
                                             IEnumerable <string> messageTexts)
        {
            // [START publish_message]
            // [START pubsub_publisher_batch_settings]
            var publishTasks = new List <Task <string> >();

            // SimplePublisher collects messages into appropriately sized
            // batches.
            foreach (string text in messageTexts)
            {
                publishTasks.Add(publisher.PublishAsync(text));
            }
            foreach (var task in publishTasks)
            {
                Console.WriteLine("Published message {0}", task.Result);
            }
            // [END pubsub_publisher_batch_settings]
            // [END publish_message]
            return(0);
        }
Exemplo n.º 12
0
        public void MultiEvents()
        {
            MultiListener     listener             = (MultiListener)_container["MultiListener"];
            MultiPublisher    publisher            = (MultiPublisher)_container["MultiPublisher"];
            PublisherListener publisherThatListens = (PublisherListener)_container["PublisherListener"];
            SimplePublisher   anotherPublisher     = (SimplePublisher)_container["SimplePublisher"];

            Assert.IsFalse(listener.Listened);
            Assert.IsNull(listener.Sender);

            publisher.Trigger1();

            Assert.IsTrue(listener.Listened);
            Assert.AreSame(publisher, listener.Sender);

            listener.Reset();

            publisher.Trigger2();

            Assert.IsTrue(listener.Listened);
            Assert.AreSame(publisher, listener.Sender);

            listener.Reset();

            publisherThatListens.Trigger1();

            Assert.IsTrue(listener.Listened);
            Assert.AreSame(publisherThatListens, listener.Sender);


            Assert.IsFalse(publisherThatListens.Listened);
            Assert.IsNull(publisherThatListens.Sender);

            anotherPublisher.Trigger();

            Assert.IsTrue(publisherThatListens.Listened);
            Assert.AreSame(anotherPublisher, publisherThatListens.Sender);
        }
        private async Task RunBulkMessaging(
            int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
            TimeSpan?timeouts = null, int?cancelAfterRecvCount = null)
        {
            // Force messages to be at least 4 bytes long, so an int ID can be used.
            minMessageSize = Math.Max(4, minMessageSize);
            var topicId        = _fixture.CreateTopicId();
            var subscriptionId = _fixture.CreateSubscriptionId();

            Console.WriteLine("BulkMessaging test");
            Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " +
                              $"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}");

            // Create topic
            var topicName = new TopicName(_fixture.ProjectId, topicId);
            var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);

            await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);

            // Subscribe to the topic
            var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false);

            var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId);
            await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false);

            // Create SimplePublisher and SimpleSubscriber
            var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null :
                                                                    new SimplePublisher.ClientCreationSettings(
                                                                        publisherSettings: new PublisherSettings
            {
                PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
                                                                                       retryBackoff: PublisherSettings.GetMessagingRetryBackoff(),
                                                                                       timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0),
                                                                                       totalExpiration: Expiration.FromTimeout(timeouts.Value),
                                                                                       retryFilter: PublisherSettings.NonIdempotentRetryFilter
                                                                                       )))
            }
                                                                        )).ConfigureAwait(false);

            var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName,
                                                                      settings : new SimpleSubscriber.Settings
            {
                StreamAckDeadline   = timeouts,
                FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
            }).ConfigureAwait(false);

            Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created");

            // Subscribe
            object recvLock  = new object();
            int    recvCount = 0;  // Count of received messages
            int    dupCount  = 0;  // Count of duplicate messages
            long   recvSum   = 0L; // Sum of bytes of received messages
            var    recvedIds = new ConcurrentDictionary <int, bool>();
            var    nackedIds = new HashSet <int>();
            Task   subTask   = simpleSubscriber.StartAsync((msg, ct) =>
            {
                int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
                lock (nackedIds)
                {
                    if (nackedIds.Count < initialNackCount)
                    {
                        if (nackedIds.Add(id))
                        {
                            // This ID not already nacked
                            Interlocked.Increment(ref recvCount);
                            return(Task.FromResult(SimpleSubscriber.Reply.Nack));
                        }
                    }
                }
                bool wasAdded = recvedIds.TryAdd(id, false);
                if (wasAdded)
                {
                    var localRecvCount = Interlocked.Increment(ref recvCount);
                    Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
                    if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
                    {
                        // Test finished, so stop subscriber
                        Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
                    }
                }
                else
                {
                    Interlocked.Add(ref dupCount, 1);
                }
                // ACK all messages
                return(Task.FromResult(SimpleSubscriber.Reply.Ack));
            });

            // Publish
            var  rnd        = new Random(1234);
            var  activePubs = new HashSet <Task>();
            int  sentCount  = 0;
            long sentSum    = 0L; // Sum of bytes of sent messages

            // Watchdog to report progress and fail test on deadlock
            CancellationTokenSource watchdogCts = new CancellationTokenSource();

            Task.Run(async() =>
            {
                int prevSentCount   = -1;
                int prevRecvCount   = -1;
                int noProgressCount = 0;
                while (!watchdogCts.IsCancellationRequested)
                {
                    await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false);
                    var localSentCount = Interlocked.Add(ref sentCount, 0);
                    var localRecvCount = Interlocked.Add(ref recvCount, 0);
                    var localDupCount  = Interlocked.Add(ref dupCount, 0);
                    if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
                    {
                        if (noProgressCount > 100)
                        {
                            // Deadlock, shutdown subscriber, and cancel
                            Console.WriteLine("Deadlock detected. Cancelling test");
                            simpleSubscriber.StopAsync(new CancellationToken(true));
                            watchdogCts.Cancel();
                            break;
                        }
                        noProgressCount += 1;
                    }
                    else
                    {
                        noProgressCount = 0;
                    }
                    prevSentCount = localSentCount;
                    prevRecvCount = localRecvCount;
                    Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
                }
            });

            for (int i = 0; i < messageCount; i++)
            {
                if (watchdogCts.IsCancellationRequested)
                {
                    Assert.True(false, "Test cancelled by watchdog");
                }
                if (subTask.IsCompleted)
                {
                    break;
                }
                var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
                var msg     = new byte[msgSize];
                rnd.NextBytes(msg);
                // Insert an int ID into message
                Array.Copy(BitConverter.GetBytes(i), msg, 4);
                sentSum += msg.Sum(x => (long)x);
                // Send message, and record Task
                var pubTask = simplePublisher.PublishAsync(msg);
                Interlocked.Increment(ref sentCount);
                activePubs.Locked(() => activePubs.Add(pubTask));
                // Remove Task from active when the message has been sent to server
                pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
                // If too many messages are currently in flight, wait a bit
                while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
                {
                    await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
                }
            }
            Console.WriteLine("Publishing complete");
            // Wait for all messages to be sent to server
            await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);

            Console.WriteLine("Publishing completed sending to server");

            // Wait for subscriber to finish shutdown
            await subTask.ConfigureAwait(false);

            watchdogCts.Cancel();
            Console.WriteLine("Subscriber finished shutdown");
            Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");

            if (cancelAfterRecvCount is int cancelAfter)
            {
                Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
            }
            else
            {
                // Check that all messages are correctly received.
                Assert.Equal(messageCount + initialNackCount, recvCount);
                // This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
                Assert.Equal(sentSum, recvSum);
            }
        }
Exemplo n.º 14
0
        private static async Task <Task> GetAndSendAsync(QiClientSlim qiClient, string namespaceId, string streamId, SimplePublisher busWriter, TraceWriter log = null)
        {
            string snapshotJson = await qiClient.GetLatestValueJsonAsync(namespaceId, streamId).ConfigureAwait(false);

            if (snapshotJson == null)
            {
                return(Task.CompletedTask);
            }
            string identifiedSnapshotJson = $"{{ streamId: \"{streamId}\", values: {snapshotJson} }}";

            return(busWriter.PublishAsync(identifiedSnapshotJson));
        }