public async Task SimpleOverview() { string projectId = _fixture.ProjectId; string topicId = _fixture.CreateTopicId(); string subscriptionId = _fixture.CreateSubscriptionId(); // Sample: SimpleOverview // First create a topic. PublisherClient publisher = await PublisherClient.CreateAsync(); TopicName topicName = new TopicName(projectId, topicId); publisher.CreateTopic(topicName); // Subscribe to the topic. SubscriberClient subscriber = await SubscriberClient.CreateAsync(); SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId); subscriber.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 60); // Publish a message to the topic using SimplePublisher. SimplePublisher simplePublisher = await SimplePublisher.CreateAsync(topicName); // PublishAsync() has various overloads. Here we're using the string overload. string messageId = await simplePublisher.PublishAsync("Hello, Pubsub"); // SimplePublisher should be shutdown after use. // The TimeSpan specifies for how long to attempt to publish locally queued messages. await simplePublisher.ShutdownAsync(TimeSpan.FromSeconds(15)); // Pull messages from the subscription using SimpleSubscriber. SimpleSubscriber simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName); List <PubsubMessage> receivedMessages = new List <PubsubMessage>(); // Start the subscriber listening for messages. await simpleSubscriber.StartAsync((msg, cancellationToken) => { receivedMessages.Add(msg); Console.WriteLine($"Received message {msg.MessageId} published at {msg.PublishTime.ToDateTime()}"); Console.WriteLine($"Text: '{msg.Data.ToStringUtf8()}'"); // Stop this subscriber after one message is received. // This is non-blocking, and the returned Task may be awaited. simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15)); // Return Reply.Ack to indicate this message has been handled. return(Task.FromResult(SimpleSubscriber.Reply.Ack)); }); // Tidy up by deleting the subscription and the topic. subscriber.DeleteSubscription(subscriptionName); publisher.DeleteTopic(topicName); // End sample Assert.Equal(1, receivedMessages.Count); Assert.Equal("Hello, Pubsub", receivedMessages[0].Data.ToStringUtf8()); }
public static SimplePublisher GetSimplePublisher(string projectId, string topicId) { // [START publish_message] PublisherClient publisherClient = PublisherClient.Create(); SimplePublisher publisher = SimplePublisher.Create( new TopicName(projectId, topicId), new[] { publisherClient }); // [END publish_message] return(publisher); }
public void TriggerStaticEvent() { SimplePublisher publisher = (SimplePublisher)_container["SimplePublisher"]; SimpleListener listener = (SimpleListener)_container["SimpleListener"]; Assert.IsFalse(listener.Listened); Assert.IsNull(listener.Sender); publisher.Trigger(); Assert.IsTrue(listener.Listened); Assert.AreSame(publisher, listener.Sender); }
private void PublishAny <TModel>(TModel model, string rountingKey) { try { SimplePublisher.GetInstance().SendMessage(model, rountingKey); } catch (Exception ex) { //TODO add to redis cache. logger.LogE(ex); #if DEBUG throw; #endif } }
public async Task OversizedMessage() { var topicId = _fixture.CreateTopicId(); // Create topic var topicName = new TopicName(_fixture.ProjectId, topicId); var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false); await publisher.CreateTopicAsync(topicName).ConfigureAwait(false); // Create SimplePublisher var simplePublisher = await SimplePublisher.CreateAsync(topicName).ConfigureAwait(false); // Create oversized message Random rnd = new Random(1234); byte[] msg = new byte[10_000_001];
static void Main(string[] args) { List <Task> tasks = new List <Task>(); for (var i = 0; i < 1; i++) { for (var j = 0; j < 100; j++) { tasks.Add(Task.Run(() => SimplePublisher.Publish("queue" + i, "Hello World"))); } } Console.Write(tasks.Count); Task.WaitAll(tasks.ToArray()); Console.WriteLine("ok"); }
/// <summary> /// Create a SimplePublisher with custom batch thresholds. /// </summary> public static SimplePublisher GetCustomPublisher(string projectId, string topicId) { // [START pubsub_publisher_batch_settings] PublisherClient publisherClient = PublisherClient.Create(); SimplePublisher publisher = SimplePublisher.Create( new TopicName(projectId, topicId), new[] { publisherClient }, new SimplePublisher.Settings() { BatchingSettings = new Google.Api.Gax.BatchingSettings( elementCountThreshold: 100, byteCountThreshold: 10240, delayThreshold: TimeSpan.FromSeconds(3)) }); // [END pubsub_publisher_batch_settings] return(publisher); }
public void Test_TInputSerializeException_IsHandled() { const string QUEUE = nameof(Test_NoResponse_IsTimedOut); // Arrange var publisher = new SimplePublisher <string>( this.UsingConnection.CreateModel() ) { RoutingKey = QUEUE, InputSerializer = new NotImplementedSerializer <string>() }; // Act publisher.Send(string.Empty); // Assert Assert.IsNotNull(publisher.LastException); }
private static SimplePublisher GetGoogleBusWriter(Microsoft.Azure.WebJobs.ExecutionContext context) { string topicId = ConfigurationManager.AppSettings["GoogleTopicId"]; string projectId = ConfigurationManager.AppSettings["GoogleProjectId"]; //Google credential expected to be a local file called googleCredential.json, copied to output TopicName topicName = new TopicName(projectId, topicId); string credentialPath = Path.Combine(context.FunctionDirectory, "..\\googleCredential.json"); GoogleCredential credential = ReadGoogleCredentialFile(credentialPath); credential = credential.CreateScoped(PublisherClient.DefaultScopes); ClientCreationSettings clientSettings = new ClientCreationSettings(credentials: credential.ToChannelCredentials()); Channel channel = new Channel(PublisherClient.DefaultEndpoint.Host, PublisherClient.DefaultEndpoint.Port, credential.ToChannelCredentials()); PublisherClient client = PublisherClient.Create(channel); return(SimplePublisher.Create(topicName, new[] { client })); }
private static async Task GetAndSendSnapshotsAsync(string namespaceId, List <string> streamIds, QiClientSlim qiClient, Microsoft.Azure.WebJobs.ExecutionContext context, TraceWriter log = null) { SimplePublisher busWriter = GetGoogleBusWriter(context); List <Task <Task> > snapshotTasks = new List <Task <Task> >(streamIds.Count); foreach (string streamId in streamIds) { string streamIdCopy = streamId; Task <Task> snapshotTask = GetAndSendAsync(qiClient, namespaceId, streamIdCopy, busWriter, log); snapshotTasks.Add(snapshotTask); } await Task.WhenAll(snapshotTasks); //Wait until messages are flushed from the bus await busWriter.ShutdownAsync(CancellationToken.None); var publishingTasks = snapshotTasks.Select(outerTask => outerTask.Result); await Task.WhenAll(publishingTasks); }
public static object PublishMessages(SimplePublisher publisher, IEnumerable <string> messageTexts) { // [START publish_message] // [START pubsub_publisher_batch_settings] var publishTasks = new List <Task <string> >(); // SimplePublisher collects messages into appropriately sized // batches. foreach (string text in messageTexts) { publishTasks.Add(publisher.PublishAsync(text)); } foreach (var task in publishTasks) { Console.WriteLine("Published message {0}", task.Result); } // [END pubsub_publisher_batch_settings] // [END publish_message] return(0); }
public void MultiEvents() { MultiListener listener = (MultiListener)_container["MultiListener"]; MultiPublisher publisher = (MultiPublisher)_container["MultiPublisher"]; PublisherListener publisherThatListens = (PublisherListener)_container["PublisherListener"]; SimplePublisher anotherPublisher = (SimplePublisher)_container["SimplePublisher"]; Assert.IsFalse(listener.Listened); Assert.IsNull(listener.Sender); publisher.Trigger1(); Assert.IsTrue(listener.Listened); Assert.AreSame(publisher, listener.Sender); listener.Reset(); publisher.Trigger2(); Assert.IsTrue(listener.Listened); Assert.AreSame(publisher, listener.Sender); listener.Reset(); publisherThatListens.Trigger1(); Assert.IsTrue(listener.Listened); Assert.AreSame(publisherThatListens, listener.Sender); Assert.IsFalse(publisherThatListens.Listened); Assert.IsNull(publisherThatListens.Sender); anotherPublisher.Trigger(); Assert.IsTrue(publisherThatListens.Listened); Assert.AreSame(anotherPublisher, publisherThatListens.Sender); }
private async Task RunBulkMessaging( int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount, TimeSpan?timeouts = null, int?cancelAfterRecvCount = null) { // Force messages to be at least 4 bytes long, so an int ID can be used. minMessageSize = Math.Max(4, minMessageSize); var topicId = _fixture.CreateTopicId(); var subscriptionId = _fixture.CreateSubscriptionId(); Console.WriteLine("BulkMessaging test"); Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " + $"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}"); // Create topic var topicName = new TopicName(_fixture.ProjectId, topicId); var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false); await publisher.CreateTopicAsync(topicName).ConfigureAwait(false); // Subscribe to the topic var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false); var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId); await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false); // Create SimplePublisher and SimpleSubscriber var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null : new SimplePublisher.ClientCreationSettings( publisherSettings: new PublisherSettings { PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings( retryBackoff: PublisherSettings.GetMessagingRetryBackoff(), timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0), totalExpiration: Expiration.FromTimeout(timeouts.Value), retryFilter: PublisherSettings.NonIdempotentRetryFilter ))) } )).ConfigureAwait(false); var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName, settings : new SimpleSubscriber.Settings { StreamAckDeadline = timeouts, FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null) }).ConfigureAwait(false); Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created"); // Subscribe object recvLock = new object(); int recvCount = 0; // Count of received messages int dupCount = 0; // Count of duplicate messages long recvSum = 0L; // Sum of bytes of received messages var recvedIds = new ConcurrentDictionary <int, bool>(); var nackedIds = new HashSet <int>(); Task subTask = simpleSubscriber.StartAsync((msg, ct) => { int id = BitConverter.ToInt32(msg.Data.ToArray(), 0); lock (nackedIds) { if (nackedIds.Count < initialNackCount) { if (nackedIds.Add(id)) { // This ID not already nacked Interlocked.Increment(ref recvCount); return(Task.FromResult(SimpleSubscriber.Reply.Nack)); } } } bool wasAdded = recvedIds.TryAdd(id, false); if (wasAdded) { var localRecvCount = Interlocked.Increment(ref recvCount); Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x)); if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount) { // Test finished, so stop subscriber Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15)); } } else { Interlocked.Add(ref dupCount, 1); } // ACK all messages return(Task.FromResult(SimpleSubscriber.Reply.Ack)); }); // Publish var rnd = new Random(1234); var activePubs = new HashSet <Task>(); int sentCount = 0; long sentSum = 0L; // Sum of bytes of sent messages // Watchdog to report progress and fail test on deadlock CancellationTokenSource watchdogCts = new CancellationTokenSource(); Task.Run(async() => { int prevSentCount = -1; int prevRecvCount = -1; int noProgressCount = 0; while (!watchdogCts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false); var localSentCount = Interlocked.Add(ref sentCount, 0); var localRecvCount = Interlocked.Add(ref recvCount, 0); var localDupCount = Interlocked.Add(ref dupCount, 0); if (prevSentCount == localSentCount && prevRecvCount == localRecvCount) { if (noProgressCount > 100) { // Deadlock, shutdown subscriber, and cancel Console.WriteLine("Deadlock detected. Cancelling test"); simpleSubscriber.StopAsync(new CancellationToken(true)); watchdogCts.Cancel(); break; } noProgressCount += 1; } else { noProgressCount = 0; } prevSentCount = localSentCount; prevRecvCount = localRecvCount; Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})"); } }); for (int i = 0; i < messageCount; i++) { if (watchdogCts.IsCancellationRequested) { Assert.True(false, "Test cancelled by watchdog"); } if (subTask.IsCompleted) { break; } var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1); var msg = new byte[msgSize]; rnd.NextBytes(msg); // Insert an int ID into message Array.Copy(BitConverter.GetBytes(i), msg, 4); sentSum += msg.Sum(x => (long)x); // Send message, and record Task var pubTask = simplePublisher.PublishAsync(msg); Interlocked.Increment(ref sentCount); activePubs.Locked(() => activePubs.Add(pubTask)); // Remove Task from active when the message has been sent to server pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask))); // If too many messages are currently in flight, wait a bit while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight) { await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false); } } Console.WriteLine("Publishing complete"); // Wait for all messages to be sent to server await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false); Console.WriteLine("Publishing completed sending to server"); // Wait for subscriber to finish shutdown await subTask.ConfigureAwait(false); watchdogCts.Cancel(); Console.WriteLine("Subscriber finished shutdown"); Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}"); if (cancelAfterRecvCount is int cancelAfter) { Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}"); } else { // Check that all messages are correctly received. Assert.Equal(messageCount + initialNackCount, recvCount); // This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator. Assert.Equal(sentSum, recvSum); } }
private static async Task <Task> GetAndSendAsync(QiClientSlim qiClient, string namespaceId, string streamId, SimplePublisher busWriter, TraceWriter log = null) { string snapshotJson = await qiClient.GetLatestValueJsonAsync(namespaceId, streamId).ConfigureAwait(false); if (snapshotJson == null) { return(Task.CompletedTask); } string identifiedSnapshotJson = $"{{ streamId: \"{streamId}\", values: {snapshotJson} }}"; return(busWriter.PublishAsync(identifiedSnapshotJson)); }