public override Task <StartResponse> Start(StartRequest request,
ServerCallContext context)
{
mutex = new Mutex();
latencies = new List <long>();
received_messages = new List <MessageIdentifier>();
try {
client = SimpleSubscriber.CreateAsync(
new SubscriptionName(request.Project,
request.PubsubOptions.Subscription)
).Result;
Console.WriteLine("Client created successfully.");
} catch (Exception e) {
Console.WriteLine("Error starting client: " + e.ToString());
}
client.StartAsync((msg, ct) =>
{
long now = CurrentTimeMillis();
var identifier = new MessageIdentifier();
mutex.WaitOne();
latencies.Add(now - long.Parse(msg.Attributes["sendTime"]));
identifier.PublisherClientId =
long.Parse(msg.Attributes["clientId"]);
identifier.SequenceNumber =
int.Parse(msg.Attributes["sequenceNumber"]);
received_messages.Add(identifier);
mutex.ReleaseMutex();
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
return(Task.FromResult(new StartResponse()));
}
public async Task SimpleOverview()
{
string projectId = _fixture.ProjectId;
string topicId = _fixture.CreateTopicId();
string subscriptionId = _fixture.CreateSubscriptionId();
// Sample: SimpleOverview
// First create a topic.
PublisherClient publisher = await PublisherClient.CreateAsync();
TopicName topicName = new TopicName(projectId, topicId);
publisher.CreateTopic(topicName);
// Subscribe to the topic.
SubscriberClient subscriber = await SubscriberClient.CreateAsync();
SubscriptionName subscriptionName = new SubscriptionName(projectId, subscriptionId);
subscriber.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 60);
// Publish a message to the topic using SimplePublisher.
SimplePublisher simplePublisher = await SimplePublisher.CreateAsync(topicName);
// PublishAsync() has various overloads. Here we're using the string overload.
string messageId = await simplePublisher.PublishAsync("Hello, Pubsub");
// SimplePublisher should be shutdown after use.
// The TimeSpan specifies for how long to attempt to publish locally queued messages.
await simplePublisher.ShutdownAsync(TimeSpan.FromSeconds(15));
// Pull messages from the subscription using SimpleSubscriber.
SimpleSubscriber simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName);
List <PubsubMessage> receivedMessages = new List <PubsubMessage>();
// Start the subscriber listening for messages.
await simpleSubscriber.StartAsync((msg, cancellationToken) =>
{
receivedMessages.Add(msg);
Console.WriteLine($"Received message {msg.MessageId} published at {msg.PublishTime.ToDateTime()}");
Console.WriteLine($"Text: '{msg.Data.ToStringUtf8()}'");
// Stop this subscriber after one message is received.
// This is non-blocking, and the returned Task may be awaited.
simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
// Return Reply.Ack to indicate this message has been handled.
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
// Tidy up by deleting the subscription and the topic.
subscriber.DeleteSubscription(subscriptionName);
publisher.DeleteTopic(topicName);
// End sample
Assert.Equal(1, receivedMessages.Count);
Assert.Equal("Hello, Pubsub", receivedMessages[0].Data.ToStringUtf8());
}
public void Start(PushRuntimeSettings limitations)
{
SimpleSubscriber.CreateAsync(subscriptionName)
.ContinueWith(async t =>
{
var s = await t.ConfigureAwait(false);
await s.StartAsync(Consume)
.ConfigureAwait(false);
subscriber = s;
});
}
private async Task RunBulkMessaging(
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
var topicId = _fixture.CreateTopicId();
var subscriptionId = _fixture.CreateSubscriptionId();
Console.WriteLine("BulkMessaging test");
Console.WriteLine($"{messageCount} messages; of size [{minMessageSize}, {maxMessageSize}]; " +
$"max in-flight: {maxMessagesInFlight}, initialNacks: {initialNackCount}, cancelAfterRecvCount: {cancelAfterRecvCount}");
// Create topic
var topicName = new TopicName(_fixture.ProjectId, topicId);
var publisher = await PublisherClient.CreateAsync().ConfigureAwait(false);
await publisher.CreateTopicAsync(topicName).ConfigureAwait(false);
// Subscribe to the topic
var subscriber = await SubscriberClient.CreateAsync().ConfigureAwait(false);
var subscriptionName = new SubscriptionName(_fixture.ProjectId, subscriptionId);
await subscriber.CreateSubscriptionAsync(subscriptionName, topicName, null, 60).ConfigureAwait(false);
// Create SimplePublisher and SimpleSubscriber
var simplePublisher = await SimplePublisher.CreateAsync(topicName, clientCreationSettings : timeouts == null?null :
new SimplePublisher.ClientCreationSettings(
publisherSettings: new PublisherSettings
{
PublishSettings = CallSettings.FromCallTiming(CallTiming.FromRetry(new RetrySettings(
retryBackoff: PublisherSettings.GetMessagingRetryBackoff(),
timeoutBackoff: new BackoffSettings(timeouts.Value, timeouts.Value, 1.0),
totalExpiration: Expiration.FromTimeout(timeouts.Value),
retryFilter: PublisherSettings.NonIdempotentRetryFilter
)))
}
)).ConfigureAwait(false);
var simpleSubscriber = await SimpleSubscriber.CreateAsync(subscriptionName,
settings : new SimpleSubscriber.Settings
{
StreamAckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, SimplePublisher and SimpleSubscriber all created");
// Subscribe
object recvLock = new object();
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = simpleSubscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SimpleSubscriber.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Task unused = simpleSubscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SimpleSubscriber.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (!watchdogCts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1), watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 100)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
simpleSubscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = simplePublisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
}