private async Task HandleExceptions(DynamicSemaphoreSlim semaphore, SendMetrics sendMetrics, AggregateException ex)
{
bool wait = false;
ex.Handle((x) =>
{
if (x is ServiceBusCommunicationException)
{
if (((ServiceBusCommunicationException)x).InnerException is SocketException &&
((SocketException)((ServiceBusCommunicationException)x).InnerException).SocketErrorCode == SocketError.HostNotFound)
{
return(false);
}
}
if (x is ServerBusyException)
{
sendMetrics.BusyErrors = 1;
if (!this.CancellationToken.IsCancellationRequested)
{
wait = true;
}
}
else
{
sendMetrics.Errors = 1;
}
return(true);
});
if (wait)
{
await Task.Delay(3000, this.CancellationToken).ConfigureAwait(false);
}
semaphore.Release();
Metrics.PushSendMetrics(sendMetrics);
}
async Task SendTask()
{
var sender = new MessageSender(this.Settings.ConnectionString, this.Settings.SendPath, NoRetry.Default);
var payload = new byte[this.Settings.MessageSizeInBytes];
var semaphore = new DynamicSemaphoreSlim(this.Settings.MaxInflightSends.Value);
var done = new SemaphoreSlim(1);
done.Wait();
long totalSends = 0;
this.Settings.MaxInflightSends.Changing += (a, e) => AdjustSemaphore(e, semaphore);
var sw = Stopwatch.StartNew();
// first send will fail out if the cxn string is bad
await sender.SendAsync(new Message(payload) { TimeToLive = TimeSpan.FromMinutes(5) });
for (int j = 0; j < Settings.MessageCount && !this.CancellationToken.IsCancellationRequested; j++)
{
var sendMetrics = new SendMetrics()
{
Tick = sw.ElapsedTicks
};
var nsec = sw.ElapsedTicks;
semaphore.Wait();
//await semaphore.WaitAsync().ConfigureAwait(false);
sendMetrics.InflightSends = this.Settings.MaxInflightSends.Value - semaphore.CurrentCount;
sendMetrics.GateLockDuration100ns = sw.ElapsedTicks - nsec;
if (Settings.SendDelay > 0)
{
await Task.Delay(Settings.SendDelay);
}
if (Settings.SendBatchCount <= 1)
{
sender.SendAsync(new Message(payload)
{
TimeToLive = TimeSpan.FromMinutes(5)
})
.ContinueWith(async(t) =>
{
if (t.IsFaulted || t.IsCanceled)
{
await HandleExceptions(semaphore, sendMetrics, t.Exception);
}
else
{
sendMetrics.SendDuration100ns = sw.ElapsedTicks - nsec;
sendMetrics.Sends = 1;
sendMetrics.Messages = 1;
semaphore.Release();
Metrics.PushSendMetrics(sendMetrics);
}
if (Interlocked.Increment(ref totalSends) >= Settings.MessageCount)
{
done.Release();
}
}).Fork();
}
else
{
List <Message> batch = new List <Message>();
for (int i = 0; i < Settings.SendBatchCount && j < Settings.MessageCount && !this.CancellationToken.IsCancellationRequested; i++, j++)
{
batch.Add(new Message(payload)
{
TimeToLive = TimeSpan.FromMinutes(5)
});
}
sender.SendAsync(batch)
.ContinueWith(async(t) =>
{
if (t.IsFaulted || t.IsCanceled)
{
await HandleExceptions(semaphore, sendMetrics, t.Exception);
}
else
{
sendMetrics.SendDuration100ns = sw.ElapsedTicks - nsec;
sendMetrics.Sends = 1;
sendMetrics.Messages = Settings.SendBatchCount;
semaphore.Release();
Metrics.PushSendMetrics(sendMetrics);
}
if (Interlocked.Increment(ref totalSends) >= Settings.MessageCount)
{
done.Release();
}
}).Fork();
}
}
await done.WaitAsync();
}
async Task ReceiveTask(string path)
{
var receiver = new MessageReceiver(this.Settings.ConnectionString, path, this.Settings.ReceiveMode);
receiver.PrefetchCount = Settings.PrefetchCount;
var semaphore = new DynamicSemaphoreSlim(this.Settings.MaxInflightReceives.Value + 1);
var done = new SemaphoreSlim(1); done.Wait();
var sw = Stopwatch.StartNew();
long totalReceives = 0;
await Task.Delay(TimeSpan.FromMilliseconds(Settings.WorkDuration));
this.Settings.MaxInflightReceives.Changing += (a, e) => AdjustSemaphore(e, semaphore);
for (int j = 0; j < Settings.MessageCount && !this.CancellationToken.IsCancellationRequested; j++)
{
var receiveMetrics = new ReceiveMetrics()
{
Tick = sw.ElapsedTicks
};
var nsec = sw.ElapsedTicks;
await semaphore.WaitAsync().ConfigureAwait(false);
receiveMetrics.GateLockDuration100ns = sw.ElapsedTicks - nsec;
if (Settings.ReceiveBatchCount <= 1)
{
nsec = sw.ElapsedTicks;
// we're going to unblock the receives after 10 seconds if there's no pending message
receiver.ReceiveAsync(TimeSpan.FromSeconds(10)).ContinueWith(async(t) =>
{
receiveMetrics.ReceiveDuration100ns = sw.ElapsedTicks - nsec;
if (t.IsFaulted || t.IsCanceled || t.Result == null)
{
if (t.Exception?.GetType() == typeof(ServerBusyException))
{
receiveMetrics.BusyErrors = 1;
if (!this.CancellationToken.IsCancellationRequested)
{
await Task.Delay(3000, this.CancellationToken).ConfigureAwait(false);
}
}
else
{
receiveMetrics.Errors = 1;
}
Metrics.PushReceiveMetrics(receiveMetrics);
semaphore.Release();
if (Interlocked.Increment(ref totalReceives) >= Settings.MessageCount)
{
done.Release();
}
}
else
{
receiveMetrics.Receives = receiveMetrics.Messages = 1;
nsec = sw.ElapsedTicks;
// simulate work by introducing a delay, if needed
if (Settings.WorkDuration > 0)
{
await Task.Delay(TimeSpan.FromMilliseconds(Settings.WorkDuration));
}
receiver.CompleteAsync(t.Result.SystemProperties.LockToken).ContinueWith(async(t1) =>
{
receiveMetrics.CompleteDuration100ns = sw.ElapsedTicks - nsec;
if (t1.IsFaulted)
{
if (t1.Exception?.GetType() == typeof(ServerBusyException))
{
receiveMetrics.BusyErrors = 1;
if (!this.CancellationToken.IsCancellationRequested)
{
await Task.Delay(3000, this.CancellationToken).ConfigureAwait(false);
}
}
else
{
receiveMetrics.Errors = 1;
}
}
else
{
receiveMetrics.Completions = receiveMetrics.CompleteCalls = 1;
}
Metrics.PushReceiveMetrics(receiveMetrics);
semaphore.Release();
if (Interlocked.Increment(ref totalReceives) >= Settings.MessageCount)
{
done.Release();
}
}).Fork();
};
}).Fork();
}
else
{
nsec = sw.ElapsedTicks;
// we're going to unblock the receives after 10 seconds if there's no pending message
receiver.ReceiveAsync(Settings.ReceiveBatchCount, TimeSpan.FromSeconds(10)).ContinueWith(async(t) =>
{
receiveMetrics.ReceiveDuration100ns = sw.ElapsedTicks - nsec;
if (t.IsFaulted || t.IsCanceled || t.Result == null)
{
if (t.Exception?.GetType() == typeof(ServerBusyException))
{
receiveMetrics.BusyErrors = 1;
if (!this.CancellationToken.IsCancellationRequested)
{
await Task.Delay(3000, this.CancellationToken).ConfigureAwait(false);
}
}
else
{
receiveMetrics.Errors = 1;
}
Metrics.PushReceiveMetrics(receiveMetrics);
semaphore.Release();
if (Interlocked.Increment(ref totalReceives) >= Settings.MessageCount)
{
done.Release();
}
}
else
{
receiveMetrics.Messages = t.Result.Count;
receiveMetrics.Receives = 1;
nsec = sw.ElapsedTicks;
if (Settings.ReceiveMode == ReceiveMode.PeekLock)
{
if (Settings.WorkDuration > 0)
{
// handle completes singly
for (int i = 0; i < t.Result.Count; i++)
{
await Task.Delay(TimeSpan.FromMilliseconds(Settings.WorkDuration));
await receiver.CompleteAsync(t.Result[i].SystemProperties.LockToken).ContinueWith(async(t1) =>
{
receiveMetrics.CompleteDuration100ns = sw.ElapsedTicks - nsec;
if (t1.IsFaulted)
{
if (t1.Exception?.GetType() == typeof(ServerBusyException))
{
receiveMetrics.BusyErrors = 1;
if (!this.CancellationToken.IsCancellationRequested)
{
await Task.Delay(3000, this.CancellationToken).ConfigureAwait(false);
}
}
else
{
receiveMetrics.Errors = 1;
}
}
else
{
receiveMetrics.CompleteCalls = 1;
receiveMetrics.Completions = 1;
}
Metrics.PushReceiveMetrics(receiveMetrics);
semaphore.Release();
if (Interlocked.Increment(ref totalReceives) >= Settings.MessageCount)
{
done.Release();
}
});
}
}
else
{
// batch complete
await receiver.CompleteAsync(t.Result.Select((m) => { return(m.SystemProperties.LockToken); })).ContinueWith(async(t1) =>
{
receiveMetrics.CompleteDuration100ns = sw.ElapsedTicks - nsec;
if (t1.IsFaulted)
{
if (t1.Exception?.GetType() == typeof(ServerBusyException))
{
receiveMetrics.BusyErrors = 1;
if (!this.CancellationToken.IsCancellationRequested)
{
await Task.Delay(3000, this.CancellationToken).ConfigureAwait(false);
}
}
else
{
receiveMetrics.Errors = 1;
}
}
else
{
receiveMetrics.CompleteCalls = 1;
receiveMetrics.Completions = t.Result.Count;
}
Metrics.PushReceiveMetrics(receiveMetrics);
semaphore.Release();
// count all the messages
for (int k = 0; k < t.Result.Count; k++)
{
if (Interlocked.Increment(ref totalReceives) >= Settings.MessageCount)
{
done.Release();
}
}
});
}
}
else
{
if (Settings.WorkDuration > 0)
{
await Task.Delay(TimeSpan.FromMilliseconds(Settings.WorkDuration));
}
Metrics.PushReceiveMetrics(receiveMetrics);
semaphore.Release();
if (Interlocked.Increment(ref totalReceives) >= Settings.MessageCount)
{
done.Release();
}
}
};
}).Fork();
}
}
await done.WaitAsync();
}