public async Task MultipleRetryableErrors_ForSameResumeToken_CausesTimeout(System.Type type)
{
var results = CreateResultSets("token1", "token2", "token3");
var filter = CreateExceptionFilter("token3", RetriableStatusCode, 2);
var client = new FakeSpannerClient(results, filter);
var clock = new FakeClock();
client.Settings.Clock = clock;
client.Settings.Scheduler = new AdvanceFakeClockScheduler(clock);
// The call will have a timeout of 15 seconds and a backoff of 10 seconds.
// token3 will return two consecutive retryable errors. After the second backoff
// the timeout of the call has been exceeded and a DeadlineExceeded error is thrown.
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(15)));
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(10),
maxBackoff: TimeSpan.FromSeconds(10),
backoffMultiplier: 1.0,
retryFilter: ignored => false,
RetrySettings.NoJitter);
var stream = CreateResultStream(type, client, maxBufferSize: 10, callSettings, retrySettings);
await AssertResultsThenExceptionAsync(stream, results, 2);
// 2 calls == 1 initial call + 1 retry of token3.
// A second retry of token3 is never executed as the deadline has already been exceeded.
Assert.Equal(2, client.Calls);
}
public async Task RetryableError_ExceedsTimeout(System.Type type)
{
var results = CreateResultSets("token1", "token2", "token3");
var filter = CreateExceptionFilter("token2", RetriableStatusCode);
var client = new FakeSpannerClient(results, filter);
var clock = new FakeClock();
client.Settings.Clock = clock;
client.Settings.Scheduler = new AdvanceFakeClockScheduler(clock);
// Create a call that will timeout if it has to backoff and retry once.
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(10)));
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(20),
maxBackoff: TimeSpan.FromSeconds(30),
backoffMultiplier: 2.0,
retryFilter: ignored => false,
RetrySettings.NoJitter);
var stream = CreateResultStream(type, client, maxBufferSize: 10, callSettings, retrySettings);
await AssertResultsThenExceptionAsync(stream, results, 1);
Assert.Equal(1, client.Calls);
}
public async Task PublishMessageWithRetrySettingsAsync(string projectId, string topicId, string messageText)
{
TopicName topicName = TopicName.FromProjectTopic(projectId, topicId);
// Retry settings control how the publisher handles retry-able failures
var maxAttempts = 3;
var initialBackoff = TimeSpan.FromMilliseconds(110); // default: 100 ms
var maxBackoff = TimeSpan.FromSeconds(70); // default : 60 seconds
var backoffMultiplier = 1.3; // default: 1.0
var totalTimeout = TimeSpan.FromSeconds(100); // default: 600 seconds
var publisher = await PublisherClient.CreateAsync(topicName,
clientCreationSettings : new PublisherClient.ClientCreationSettings(
publisherServiceApiSettings: new PublisherServiceApiSettings
{
PublishSettings = CallSettings.FromRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: maxAttempts,
initialBackoff: initialBackoff,
maxBackoff: maxBackoff,
backoffMultiplier: backoffMultiplier,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
.WithTimeout(totalTimeout)
}
)).ConfigureAwait(false);
string message = await publisher.PublishAsync(messageText);
Console.WriteLine($"Published message {message}");
}
public async Task RecordErrorAndWait_RetrySettingsObeyed()
{
RetrySettings retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue, // Ignored in SqlResultStream
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
ignored => false, // Ignored in SqlResultStream
RetrySettings.NoJitter);
var mock = new Mock <IScheduler>(MockBehavior.Strict);
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(2), default)).Returns(Task.FromResult(0));
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(4), default)).Returns(Task.FromResult(0));
// Retry maxes out at 5 seconds
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(5), default)).Returns(Task.FromResult(0));
// After reset
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
var state = new RetryState(new FakeClock(), mock.Object, retrySettings, s_callSettings);
await state.WaitAsync(exception, default);
await state.WaitAsync(exception, default);
await state.WaitAsync(exception, default);
await state.WaitAsync(exception, default);
state.Reset();
await state.WaitAsync(exception, default);
}
public async Task MultipleRetryableErrors_ForDifferentResumeTokens_ResetsTimeout(System.Type type)
{
var results = CreateResultSets("token1", "token2", "token3");
var filter1 = CreateExceptionFilter("token2", RetriableStatusCode);
var filter2 = CreateExceptionFilter("token3", RetriableStatusCode);
var client = new FakeSpannerClient(results, prs => filter2(filter1(prs)));
var clock = new FakeClock();
client.Settings.Clock = clock;
client.Settings.Scheduler = new AdvanceFakeClockScheduler(clock);
// The call will have a timeout of 15 seconds and a backoff of 10 seconds.
// One retry will therefore not cause it to fail. Two retries for different
// calls will also not cause it to fail, as the timeout is reset after each
// successful RPC.
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(15)));
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(10),
maxBackoff: TimeSpan.FromSeconds(10),
backoffMultiplier: 1.0,
retryFilter: ignored => false,
RetrySettings.NoJitter);
var stream = CreateResultStream(type, client, maxBufferSize: 10, callSettings, retrySettings);
await AssertResultsAsync(stream, results);
Assert.Equal(3, client.Calls);
}
private async Task IncrementByOneAsync(SpannerConnection connection, bool orphanTransaction = false)
{
var retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromMilliseconds(250),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 1.5,
retryFilter: ignored => false,
RetrySettings.RandomJitter);
TimeSpan nextDelay = TimeSpan.Zero;
SpannerException spannerException;
DateTime deadline = DateTime.UtcNow.AddSeconds(30);
while (true)
{
spannerException = null;
try
{
// We use manually created transactions here so the tests run on .NET Core.
using (var transaction = await connection.BeginTransactionAsync())
{
long current;
using (var cmd = connection.CreateSelectCommand($"SELECT Int64Value FROM {_fixture.TableName} WHERE K=@k"))
{
cmd.Parameters.Add("k", SpannerDbType.String, _key);
cmd.Transaction = transaction;
var fetched = await cmd.ExecuteScalarAsync().ConfigureAwait(false);
current = fetched is DBNull ? 0L : (long)fetched;
}
using (var cmd = connection.CreateUpdateCommand(_fixture.TableName))
{
cmd.Parameters.Add("k", SpannerDbType.String, _key);
cmd.Parameters.Add("Int64Value", SpannerDbType.Int64, current + 1);
cmd.Transaction = transaction;
await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
if (!orphanTransaction)
{
await transaction.CommitAsync().ConfigureAwait(false);
}
}
}
return;
}
// Keep trying for up to 30 seconds
catch (SpannerException ex) when(ex.IsRetryable && DateTime.UtcNow < deadline)
{
nextDelay = retrySettings.NextBackoff(nextDelay);
await Task.Delay(retrySettings.BackoffJitter.GetDelay(nextDelay));
spannerException = ex;
}
}
}
public Task DeadlineRespected()
{
var settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 10, initialBackoff: OneSecond, maxBackoff: FiveSeconds,
backoffMultiplier: 2, retryFilter: ex => true, backoffJitter: RetrySettings.NoJitter);
var scheduler = new FakeScheduler();
var deadline = scheduler.Clock.GetCurrentDateTimeUtc() + FiveSeconds;
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler, deadline, scheduler.Clock);
// Should attempt at T=0, T=1, T=3, then stop because the next attempt would be after the deadline.
return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 3));
}
public Task PredicateRespected()
{
int count = 0;
Func <Exception> exceptionProvider = () => ++ count == 3 ? new Exception() : new RpcException(Status.DefaultCancelled);
var settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 4, initialBackoff: OneSecond, maxBackoff: FiveSeconds,
backoffMultiplier: 1, retryFilter: ex => ex is RpcException, backoffJitter: RetrySettings.NoJitter);
var scheduler = new FakeScheduler();
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler);
return(AssertAttemptsAsync(sequence, scheduler, exceptionProvider, 0, 1, 2));
}
public void RetrySettingsTiming()
{
// Sample: RetrySettingsTiming
// Each delay is double the previous one, with a maximum of 5s.
// The first delay is 1s, then 2s, then 4s, then 5s, then 5s, etc.
// Only aborted RPCs are retried in this example.
RetrySettings settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 10,
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Aborted));
// End sample
}
public Task JitterRespected()
{
var settings = RetrySettings.FromExponentialBackoff(
maxAttempts: 6, initialBackoff: TimeSpan.FromSeconds(2), maxBackoff: TimeSpan.FromSeconds(10),
backoffMultiplier: 2, retryFilter: ex => true, backoffJitter: new HalvingJitter());
var scheduler = new FakeScheduler();
var sequence = RetryAttempt.CreateRetrySequence(settings, scheduler);
// Sequence of theoretical backoffs is 2, 4, 8, 10, 10, 10
// Sequence of jittered backoffs is 1, 2, 4, 5, 5.
return(AssertAttemptsAsync(sequence, scheduler, () => new Exception(), 0, 1, 3, 7, 12, 17));
}
public void FromExponentialBackoff_SpecificJitter()
{
var initialBackoff = TimeSpan.FromSeconds(3);
var maxBackoff = TimeSpan.FromSeconds(5);
var filter = RetrySettings.FilterForStatusCodes(StatusCode.Aborted);
var settings = RetrySettings.FromExponentialBackoff(10, initialBackoff, maxBackoff, 1.5, filter, RetrySettings.NoJitter);
Assert.Equal(10, settings.MaxAttempts);
Assert.Equal(initialBackoff, settings.InitialBackoff);
Assert.Equal(maxBackoff, settings.MaxBackoff);
Assert.Equal(1.5, settings.BackoffMultiplier);
Assert.Same(filter, settings.RetryFilter);
Assert.Same(RetrySettings.NoJitter, settings.BackoffJitter);
}
internal WatchStream(IScheduler scheduler, IWatchState state, Target target, FirestoreDb db, CancellationToken cancellationToken)
{
_scheduler = scheduler;
_state = state;
_target = target;
_db = db;
_callbackCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
_networkCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_callbackCancellationTokenSource.Token);
_listenCallSettings = CallSettings.FromHeader(FirestoreClientImpl.ResourcePrefixHeader, db.RootPath);
// TODO: Make these configurable?
_backoffSettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(30),
backoffMultiplier: 2.0,
retryFilter: _ => false, // Ignored
backoffJitter: RetrySettings.RandomJitter);
}
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext)
{
ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
Client = GaxPreconditions.CheckNotNull(client, nameof(client));
// TODO: Investigate using DatabaseName and DocumentPathName.
RootPath = $"projects/{ProjectId}/databases/{DatabaseId}";
DocumentsPath = $"{RootPath}/documents";
WarningLogger = warningLogger;
// TODO: Validate these settings, and potentially make them configurable
var batchGetRetry = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromMilliseconds(500),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable));
_batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10));
SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext));
}
public async Task <long> CustomTimeoutsAndRetriesAsync(string projectId, string instanceId, string databaseId)
{
// Create a SessionPool.
SpannerClient client = SpannerClient.Create();
SessionPool sessionPool = new SessionPool(client, new SessionPoolOptions());
// Acquire a session with a read-write transaction to run a query.
DatabaseName databaseName =
DatabaseName.FromProjectInstanceDatabase(projectId, instanceId, databaseId);
TransactionOptions transactionOptions = new TransactionOptions
{
ReadWrite = new ReadWrite()
};
using PooledSession session = await sessionPool.AcquireSessionAsync(
databaseName, transactionOptions, CancellationToken.None);
ExecuteSqlRequest request = new ExecuteSqlRequest
{
Sql = "INSERT Singers (SingerId, FirstName, LastName) VALUES (20, 'George', 'Washington')"
};
// Prepare the call settings with custom timeout and retry settings.
CallSettings settings = CallSettings
.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(60)))
.WithRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: 12,
initialBackoff: TimeSpan.FromMilliseconds(500),
maxBackoff: TimeSpan.FromMilliseconds(6400),
backoffMultiplier: 1.5,
retryFilter: RetrySettings.FilterForStatusCodes(
new StatusCode[] { StatusCode.Unavailable, StatusCode.DeadlineExceeded })));
ResultSet result = await session.ExecuteSqlAsync(request, settings);
await session.CommitAsync(new CommitRequest(), null);
return(result.Stats.RowCountExact);
}
private FirestoreDb(string projectId, string databaseId, FirestoreClient client, Action <string> warningLogger, SerializationContext serializationContext)
{
ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
DatabaseId = GaxPreconditions.CheckNotNull(databaseId, nameof(databaseId));
Client = GaxPreconditions.CheckNotNull(client, nameof(client));
// TODO: Investigate using DatabaseName and DocumentPathName.
RootPath = $"projects/{ProjectId}/databases/{DatabaseId}";
DocumentsPath = $"{RootPath}/documents";
WarningLogger = warningLogger;
// TODO: Potentially make these configurable.
// The retry settings are taken from firestore_grpc_service_config.json.
var batchGetRetry = RetrySettings.FromExponentialBackoff(
maxAttempts: 5,
initialBackoff: TimeSpan.FromMilliseconds(100),
maxBackoff: TimeSpan.FromSeconds(60),
backoffMultiplier: 1.3,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable, StatusCode.Internal, StatusCode.DeadlineExceeded));
_batchGetCallSettings = CallSettings.FromRetry(batchGetRetry).WithTimeout(TimeSpan.FromMinutes(10));
SerializationContext = GaxPreconditions.CheckNotNull(serializationContext, nameof(serializationContext));
}
/// <inheritdoc/>
public override void ActivateOptions()
{
// Validate configuration
GaxPreconditions.CheckState(string.IsNullOrWhiteSpace(CredentialFile) || string.IsNullOrWhiteSpace(CredentialJson),
$"{nameof(CredentialFile)} and {nameof(CredentialJson)} must not both be set.");
GaxPreconditions.CheckState(LogId != null, $"{nameof(LogId)} must be set.");
GaxPreconditions.CheckState(MaxUploadBatchSize > 0, $"{nameof(MaxUploadBatchSize)} must be > 0");
GaxPreconditions.CheckEnumValue <LocalQueueType>(LocalQueueType, nameof(LocalQueueType));
base.ActivateOptions();
// Initialise services if not already initialised for testing
_client = _client ?? BuildLoggingServiceClient();
_scheduler = _scheduler ?? SystemScheduler.Instance;
_clock = _clock ?? SystemClock.Instance;
_platform = _platform ?? Platform.Instance();
// Normalize string configuration
ResourceType = string.IsNullOrWhiteSpace(ResourceType) ? null : ResourceType;
ProjectId = string.IsNullOrWhiteSpace(ProjectId) ? null : ProjectId;
LogId = string.IsNullOrWhiteSpace(LogId) ? null : LogId;
switch (LocalQueueType)
{
case LocalQueueType.Memory:
GaxPreconditions.CheckState(MaxMemoryCount > 0 || MaxMemorySize > 0,
$"Either {nameof(MaxMemoryCount)} or {nameof(MaxMemorySize)} must be configured to be > 0");
break;
default:
throw new InvalidOperationException($"Invalid {nameof(Log4Net.LocalQueueType)}: '{LocalQueueType}'");
}
GaxPreconditions.CheckState(ServerErrorBackoffDelaySeconds >= 1,
$"{nameof(ServerErrorBackoffDelaySeconds)} must be >= 1 second.");
GaxPreconditions.CheckState(ServerErrorBackoffMultiplier > 1.1999999,
$"{nameof(ServerErrorBackoffMultiplier)} must be >= 1.2");
GaxPreconditions.CheckState(ServerErrorBackoffMaxDelaySeconds >= 20,
$"{nameof(ServerErrorBackoffMaxDelaySeconds)} must be >= 20 seconds.");
ActivateLogIdAndResource();
switch (LocalQueueType)
{
case LocalQueueType.Memory:
_logQ = new MemoryLogQueue(MaxMemorySize, MaxMemoryCount);
break;
default:
throw new InvalidOperationException($"Invalid {nameof(Log4Net.LocalQueueType)}: '{LocalQueueType}'");
}
_initIdTask = Task.Run(_logQ.GetPreviousExecutionIdAsync);
var logsLostWarningEntry = new LogEntry
{
TextPayload = s_logsLostWarningMessage,
Severity = LogSeverity.Warning,
LogName = _logName,
Resource = _resource,
// Patterns included in custom labels will not be used in this "logs lost" entry.
// The pattern itself will be logged, regardless of the "UsePatternWithinCustomLabels" setting.
// This is acceptable as most patterns will be irrelevant in this context.
Labels = { _customLabels.ToDictionary(x => x.Key, x => x.Value) },
};
var serverErrorRetrySettings = RetrySettings.FromExponentialBackoff(maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromSeconds(ServerErrorBackoffDelaySeconds),
maxBackoff: TimeSpan.FromSeconds(ServerErrorBackoffMaxDelaySeconds),
backoffMultiplier: ServerErrorBackoffMultiplier,
retryFilter: _ => true); // Ignored
_logUploader = new LogUploader(
_client, _scheduler, _clock,
_logQ, logsLostWarningEntry, MaxUploadBatchSize,
serverErrorRetrySettings);
if (_usePatternWithinCustomLabels)
{
// Initialize a pattern layout for each custom label.
_customLabelsPatterns = _customLabels.Select(x => new CustomLabelPattern(x.Key, new PatternLayout(x.Value))).ToArray();
}
_isActivated = true;
}
private async Task RunBulkMessagingImpl(
TopicName topicName, SubscriptionName subscriptionName,
int messageCount, int minMessageSize, int maxMessageSize, int maxMessagesInFlight, int initialNackCount,
TimeSpan?timeouts = null, int?cancelAfterRecvCount = null, TimeSpan?interPublishDelay = null,
TimeSpan?debugOutputPeriod = null, int?publisherChannelCount = null, int?clientCount = null)
{
// Force messages to be at least 4 bytes long, so an int ID can be used.
minMessageSize = Math.Max(4, minMessageSize);
// Create PublisherClient and SubscriberClient
var publisher = await PublisherClient.CreateAsync(topicName,
clientCreationSettings : new PublisherClient.ClientCreationSettings(
clientCount: publisherChannelCount,
publisherServiceApiSettings: timeouts == null ? null : new PublisherServiceApiSettings
{
PublishSettings = CallSettings
.FromRetry(RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue,
initialBackoff: TimeSpan.FromMilliseconds(100),
maxBackoff: TimeSpan.FromSeconds(6),
backoffMultiplier: 1.3,
retryFilter: RetrySettings.FilterForStatusCodes(StatusCode.Unavailable)))
.WithTimeout(timeouts.Value)
}
)).ConfigureAwait(false);
var subscriber = await SubscriberClient.CreateAsync(subscriptionName,
clientCreationSettings : new SubscriberClient.ClientCreationSettings(clientCount: clientCount),
settings : new SubscriberClient.Settings
{
AckDeadline = timeouts,
FlowControlSettings = new FlowControlSettings(maxMessagesInFlight, null)
}).ConfigureAwait(false);
Console.WriteLine("Topic, Subscription, Publisher and Subscriber all created");
// Subscribe
int recvCount = 0; // Count of received messages
int dupCount = 0; // Count of duplicate messages
long recvSum = 0L; // Sum of bytes of received messages
var recvedIds = new ConcurrentDictionary <int, bool>();
var nackedIds = new HashSet <int>();
Task subTask = subscriber.StartAsync((msg, ct) =>
{
int id = BitConverter.ToInt32(msg.Data.ToArray(), 0);
lock (nackedIds)
{
if (nackedIds.Count < initialNackCount)
{
if (nackedIds.Add(id))
{
// This ID not already nacked
Interlocked.Increment(ref recvCount);
return(Task.FromResult(SubscriberClient.Reply.Nack));
}
}
}
bool wasAdded = recvedIds.TryAdd(id, false);
if (wasAdded)
{
var localRecvCount = Interlocked.Increment(ref recvCount);
Interlocked.Add(ref recvSum, msg.Data.Sum(x => (long)x));
if (localRecvCount == cancelAfterRecvCount || localRecvCount >= messageCount + initialNackCount)
{
// Test finished, so stop subscriber
Console.WriteLine("All msgs received, stopping subscriber.");
Task unused = subscriber.StopAsync(TimeSpan.FromSeconds(15));
}
}
else
{
Interlocked.Add(ref dupCount, 1);
}
// ACK all messages
return(Task.FromResult(SubscriberClient.Reply.Ack));
});
// Publish
var rnd = new Random(1234);
var activePubs = new HashSet <Task>();
int sentCount = 0;
long sentSum = 0L; // Sum of bytes of sent messages
// Watchdog to report progress and fail test on deadlock
CancellationTokenSource watchdogCts = new CancellationTokenSource();
Task.Run(async() =>
{
var debugOutputPeriod1 = debugOutputPeriod ?? TimeSpan.FromSeconds(1);
int prevSentCount = -1;
int prevRecvCount = -1;
int noProgressCount = 0;
while (true)
{
await Task.Delay(debugOutputPeriod1, watchdogCts.Token).ConfigureAwait(false);
var localSentCount = Interlocked.Add(ref sentCount, 0);
var localRecvCount = Interlocked.Add(ref recvCount, 0);
var localDupCount = Interlocked.Add(ref dupCount, 0);
if (prevSentCount == localSentCount && prevRecvCount == localRecvCount)
{
if (noProgressCount > 60)
{
// Deadlock, shutdown subscriber, and cancel
Console.WriteLine("Deadlock detected. Cancelling test");
subscriber.StopAsync(new CancellationToken(true));
watchdogCts.Cancel();
break;
}
noProgressCount += 1;
}
else
{
noProgressCount = 0;
}
prevSentCount = localSentCount;
prevRecvCount = localRecvCount;
Console.WriteLine($"[{DateTime.Now}] Sent: {localSentCount} (in-flight: {activePubs.Locked(() => activePubs.Count)}); Recv: {localRecvCount} (dups: {localDupCount})");
}
});
for (int i = 0; i < messageCount; i++)
{
if (watchdogCts.IsCancellationRequested)
{
Assert.True(false, "Test cancelled by watchdog");
}
if (subTask.IsCompleted)
{
break;
}
if (i > 0 && interPublishDelay is TimeSpan delay)
{
await Task.Delay(delay, watchdogCts.Token).ConfigureAwait(false);
}
var msgSize = rnd.Next(minMessageSize, maxMessageSize + 1);
var msg = new byte[msgSize];
rnd.NextBytes(msg);
// Insert an int ID into message
Array.Copy(BitConverter.GetBytes(i), msg, 4);
sentSum += msg.Sum(x => (long)x);
// Send message, and record Task
var pubTask = publisher.PublishAsync(msg);
Interlocked.Increment(ref sentCount);
activePubs.Locked(() => activePubs.Add(pubTask));
// Remove Task from active when the message has been sent to server
pubTask.ContinueWith(t => activePubs.Locked(() => activePubs.Remove(pubTask)));
// If too many messages are currently in flight, wait a bit
while (activePubs.Locked(() => activePubs.Count) >= maxMessagesInFlight)
{
await Task.Delay(TimeSpan.FromMilliseconds(1)).ConfigureAwait(false);
}
}
Console.WriteLine("Publishing complete");
// Wait for all messages to be sent to server
await Task.WhenAll(activePubs.Locked(() => activePubs.ToArray())).ConfigureAwait(false);
Console.WriteLine("Publishing completed sending to server");
// Wait for subscriber to finish shutdown
await subTask.ConfigureAwait(false);
watchdogCts.Cancel();
Console.WriteLine("Subscriber finished shutdown");
Console.WriteLine($"Sent: {sentCount}; Recv: {recvCount}");
if (cancelAfterRecvCount is int cancelAfter)
{
Assert.True(recvCount >= cancelAfter && recvCount <= cancelAfter + maxMessagesInFlight, $"Incorrect recvCount: {recvCount}");
}
else
{
// Check that all messages are correctly received.
Assert.Equal(messageCount + initialNackCount, recvCount);
// This isn't foolproof (we can get to the right sum with wrong values) but it's a pretty strong indicator.
Assert.Equal(sentSum, recvSum);
}
Console.WriteLine("Test complete.");
}