public async Task SimpleOverview()
{
string projectId = _fixture.ProjectId;
string topicId = _fixture.CreateTopicId();
string subscriptionId = _fixture.CreateSubscriptionId();
// Sample: SimpleOverview
// First create a topic.
PublisherClient publisher = await PublisherClient.CreateAsync();
TopicName topicName = new TopicName(projectId, topicId);
publisher.CreateTopic(topicName);
// Subscribe to the topic.
SubscriberClient subscriber = await SubscriberClient.CreateAsync();
SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId);
subscriber.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 60);
// Publish a message to the topic using SimplePublisher.
SimplePublisher simplePublisher = await SimplePublisher.CreateAsync(topicName);
// PublishAsync() has various overloads. Here we're using the string overload.
string messageId = await simplePublisher.PublishAsync("Hello, Pubsub");
// SimplePublisher should be shutdown after use.
// The TimeSpan specifies for how long to attempt to publish locally queued messages.
await simplePublisher.ShutdownAsync(TimeSpan.FromSeconds(15));
// Pull messages from the subscription using SimpleSubscriber.
SimpleSubscriber simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName);
List <PubsubMessage> receivedMessages = new List <PubsubMessage>();
// Start the subscriber listening for messages.
await simpleSubscriber.StartAsync((msg, cancellationToken) =>
{
receivedMessages.Add(msg);
Console.WriteLine($"Received message {msg.MessageId} published at {msg.PublishTime.ToDateTime()}");
Console.WriteLine($"Text: '{msg.Data.ToStringUtf8()}'");
// Stop this subscriber after one message is received.
// This is non-blocking, and the returned Task may be awaited.
simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
// Return Reply.Ack to indicate this message has been handled.
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
// Tidy up by deleting the subscription and the topic.
subscriber.DeleteSubscription(subscriptionName);
publisher.DeleteTopic(topicName);
// End sample
Assert.Equal(1, receivedMessages.Count);
Assert.Equal("Hello, Pubsub", receivedMessages[0].Data.ToStringUtf8());
}
public static SimplePublisher GetSimplePublisher(string projectId,
string topicId)
{
// [START publish_message]
PublisherClient publisherClient = PublisherClient.Create();
SimplePublisher publisher = SimplePublisher.Create(
new TopicName(projectId, topicId), new[] { publisherClient });
// [END publish_message]
return(publisher);
}
public void TriggerStaticEvent()
{
SimplePublisher publisher = (SimplePublisher)_container["SimplePublisher"];
SimpleListener listener = (SimpleListener)_container["SimpleListener"];
Assert.IsFalse(listener.Listened);
Assert.IsNull(listener.Sender);
publisher.Trigger();
Assert.IsTrue(listener.Listened);
Assert.AreSame(publisher, listener.Sender);
}
private void PublishAny <TModel>(TModel model, string rountingKey)
{
try
{
SimplePublisher.GetInstance().SendMessage(model, rountingKey);
}
catch (Exception ex)
{
//TODO add to redis cache.
logger.LogE(ex);
#if DEBUG
throw;
#endif
}
}
public async Task OversizedMessage()
{
var topicId = _fixture.CreateTopicId();
// Create topic
var topicName = new TopicName(_fixture.ProjectId, topicId);
var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);
await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);
// Create SimplePublisher
var simplePublisher = await SimplePublisher.CreateAsync(topicName).ConfigureAwait(false);
// Create oversized message
Random rnd = new Random(1234);
byte[] msg = new byte[10_000_001];
static void Main(string[] args)
{
List <Task> tasks = new List <Task>();
for (var i = 0; i < 1; i++)
{
for (var j = 0; j < 100; j++)
{
tasks.Add(Task.Run(() => SimplePublisher.Publish("queue" + i, "Hello World")));
}
}
Console.Write(tasks.Count);
Task.WaitAll(tasks.ToArray());
Console.WriteLine("ok");
}
/// <summary>
/// Create a SimplePublisher with custom batch thresholds.
/// </summary>
public static SimplePublisher GetCustomPublisher(string projectId,
string topicId)
{
// [START pubsub_publisher_batch_settings]
PublisherClient publisherClient = PublisherClient.Create();
SimplePublisher publisher = SimplePublisher.Create(
new TopicName(projectId, topicId), new[] { publisherClient },
new SimplePublisher.Settings()
{
BatchingSettings = new Google.Api.Gax.BatchingSettings(
elementCountThreshold: 100,
byteCountThreshold: 10240,
delayThreshold: TimeSpan.FromSeconds(3))
});
// [END pubsub_publisher_batch_settings]
return(publisher);
}
public void Test_TInputSerializeException_IsHandled()
{
const string QUEUE = nameof(Test_NoResponse_IsTimedOut);
// Arrange
var publisher = new SimplePublisher <string>(
this.UsingConnection.CreateModel()
)
{
RoutingKey = QUEUE,
InputSerializer = new NotImplementedSerializer <string>()
};
// Act
publisher.Send(string.Empty);
// Assert
Assert.IsNotNull(publisher.LastException);
}
private static SimplePublisher GetGoogleBusWriter(Microsoft.Azure.WebJobs.ExecutionContext context)
{
string topicId = ConfigurationManager.AppSettings["GoogleTopicId"];
string projectId = ConfigurationManager.AppSettings["GoogleProjectId"];
//Google credential expected to be a local file called googleCredential.json, copied to output
TopicName topicName = new TopicName(projectId, topicId);
string credentialPath = Path.Combine(context.FunctionDirectory, "..\\googleCredential.json");
GoogleCredential credential = ReadGoogleCredentialFile(credentialPath);
credential = credential.CreateScoped(PublisherClient.DefaultScopes);
ClientCreationSettings clientSettings = new ClientCreationSettings(credentials: credential.ToChannelCredentials());
Channel channel = new Channel(PublisherClient.DefaultEndpoint.Host, PublisherClient.DefaultEndpoint.Port, credential.ToChannelCredentials());
PublisherClient client = PublisherClient.Create(channel);
return(SimplePublisher.Create(topicName, new[] { client }));
}
private static async Task GetAndSendSnapshotsAsync(string namespaceId, List <string> streamIds, QiClientSlim qiClient,
Microsoft.Azure.WebJobs.ExecutionContext context, TraceWriter log = null)
{
SimplePublisher busWriter = GetGoogleBusWriter(context);
List <Task <Task> > snapshotTasks = new List <Task <Task> >(streamIds.Count);
foreach (string streamId in streamIds)
{
string streamIdCopy = streamId;
Task <Task> snapshotTask = GetAndSendAsync(qiClient, namespaceId, streamIdCopy, busWriter, log);
snapshotTasks.Add(snapshotTask);
}
await Task.WhenAll(snapshotTasks);
//Wait until messages are flushed from the bus
await busWriter.ShutdownAsync(CancellationToken.None);
var publishingTasks = snapshotTasks.Select(outerTask => outerTask.Result);
await Task.WhenAll(publishingTasks);
}
public static object PublishMessages(SimplePublisher publisher,
IEnumerable <string> messageTexts)
{
// [START publish_message]
// [START pubsub_publisher_batch_settings]
var publishTasks = new List <Task <string> >();
// SimplePublisher collects messages into appropriately sized
// batches.
foreach (string text in messageTexts)
{
publishTasks.Add(publisher.PublishAsync(text));
}
foreach (var task in publishTasks)
{
Console.WriteLine("Published message {0}", task.Result);
}
// [END pubsub_publisher_batch_settings]
// [END publish_message]
return(0);
}
public void MultiEvents()
{
MultiListener listener = (MultiListener)_container["MultiListener"];
MultiPublisher publisher = (MultiPublisher)_container["MultiPublisher"];
PublisherListener publisherThatListens = (PublisherListener)_container["PublisherListener"];
SimplePublisher anotherPublisher = (SimplePublisher)_container["SimplePublisher"];
Assert.IsFalse(listener.Listened);
Assert.IsNull(listener.Sender);
publisher.Trigger1();
Assert.IsTrue(listener.Listened);
Assert.AreSame(publisher, listener.Sender);
listener.Reset();
publisher.Trigger2();
Assert.IsTrue(listener.Listened);
Assert.AreSame(publisher, listener.Sender);
listener.Reset();
publisherThatListens.Trigger1();
Assert.IsTrue(listener.Listened);
Assert.AreSame(publisherThatListens, listener.Sender);
Assert.IsFalse(publisherThatListens.Listened);
Assert.IsNull(publisherThatListens.Sender);
anotherPublisher.Trigger();
Assert.IsTrue(publisherThatListens.Listened);
Assert.AreSame(anotherPublisher, publisherThatListens.Sender);
}
private async Task RunBulkMessaging(
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
var topicId = _fixture.CreateTopicId();
var subscriptionId = _fixture.CreateSubscriptionId();
Console.WriteLine("BulkMessaging test");
Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " +
$"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}");
// Create topic
var topicName = new TopicName(_fixture.ProjectId, topicId);
var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);
await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);
// Subscribe to the topic
var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false);
var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId);
await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false);
// Create SimplePublisher and SimpleSubscriber
var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null :
new SimplePublisher.ClientCreationSettings(
publisherSettings: new PublisherSettings
{
PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
retryBackoff: PublisherSettings.GetMessagingRetryBackoff(),
timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0),
totalExpiration: Expiration.FromTimeout(timeouts.Value),
retryFilter: PublisherSettings.NonIdempotentRetryFilter
)))
}
)).ConfigureAwait(false);
var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName,
settings : new SimpleSubscriber.Settings
{
StreamAckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created");
// Subscribe
object recvLock = new object();
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = simpleSubscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SimpleSubscriber.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (!watchdogCts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 100)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
simpleSubscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = simplePublisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
}
private static async Task <Task> GetAndSendAsync(QiClientSlim qiClient, string namespaceId, string streamId, SimplePublisher busWriter, TraceWriter log = null)
{
string snapshotJson = await qiClient.GetLatestValueJsonAsync(namespaceId, streamId).ConfigureAwait(false);
if (snapshotJson == null)
{
return(Task.CompletedTask);
}
string identifiedSnapshotJson = $"{{ streamId: \"{streamId}\", values: {snapshotJson} }}";
return(busWriter.PublishAsync(identifiedSnapshotJson));
}