/// <summary>
/// Requests the tracker to update its data set.
/// </summary>
/// <remarks>
/// May be called multiple times concurrently.
///
/// The method returns to signal that the trackerss of all containers
/// when the method was called have attempted an update to their data.
/// It may be that some updates failed - all we can say is that we tried.
///
/// Method does not throw exceptions on transient failures, merely logs and ignores them.
/// </remarks>
public async Task TryUpdateAsync()
{
using var cts = new CancellationTokenSource(Constants.MaxTotalUpdateDuration);
// If we get this lock, we will actually perform the update.
using var writeLock = await SemaphoreLock.TryTakeAsync(_updateLock, TimeSpan.Zero);
if (writeLock == null)
{
// Otherwise, we just no-op once the earlier probe request has updated the data.
await WaitForPredecessorUpdateAsync(cts.Token);
return;
}
using var probeDurationTimer = DockerTrackerMetrics.ProbeDuration.NewTimer();
IList <ContainerListResponse> allContainers;
try
{
using var listDurationTimer = DockerTrackerMetrics.ListContainersDuration.NewTimer();
allContainers = await _client.Containers.ListContainersAsync(new ContainersListParameters
{
All = true
}, cts.Token);
}
catch (Exception ex)
{
DockerTrackerMetrics.ListContainersErrorCount.Inc();
_log.Error(Helpers.Debug.GetAllExceptionMessages(ex));
_log.Debug(ex.ToString()); // Only to verbose output.
// Errors are ignored - if we fail to get data, we just skip an update and log the failure.
// The next update will hopefully get past the error.
// We will not remove the trackers yet but we will unpublish so we don't keep stale data published.
foreach (var tracker in _containerTrackers.Values)
{
tracker.Unpublish();
}
return;
}
DockerTrackerMetrics.ContainerCount.Set(allContainers.Count);
SynchronizeTrackerSet(allContainers);
// Update each tracker. We do them in parallel to minimize the total time span spent on probing.
var updateTasks = new List <Task>();
foreach (var tracker in _containerTrackers.Values)
{
updateTasks.Add(tracker.TryUpdateAsync(_client, cts.Token));
}
// Only exceptions from the update calls should be terminal exceptions,
// so it is fine not to catch anything that may be thrown here.
await Task.WhenAll(updateTasks);
DockerTrackerMetrics.SuccessfulProbeTime.SetToCurrentTimeUtc();
}
public void LockingLogic_WithTimeout_SeemsToWork()
{
// Start one long-running operation and two short-running ones, where one of the shorts times out on the lock.
using (var semaphore = new SemaphoreSlim(1))
{
int?longCompletedAt = null;
int?short1CompletedAt = null;
int?short2CompletedAt = null;
int?short1TimedOutAt = null;
var longTask = Task.Run(async() =>
{
Debug.WriteLine("Long entered.");
using (await SemaphoreLock.TakeAsync(semaphore))
{
Debug.WriteLine("Long acquired lock.");
await Task.Delay(5000);
Debug.WriteLine("Long completed.");
longCompletedAt = Environment.TickCount;
}
});
var shortTask1 = Task.Run(async() =>
{
Debug.WriteLine("Short1 entered.");
await Task.Delay(500);
Debug.WriteLine("Short1 starting work.");
var lockInstance = await SemaphoreLock.TryTakeAsync(semaphore, TimeSpan.FromSeconds(1));
if (lockInstance == null)
{
Debug.WriteLine("Short1 timed out.");
short1TimedOutAt = Environment.TickCount;
}
else
{
using (lockInstance)
{
Debug.WriteLine("Short1 acquired lock.");
await Task.Delay(50);
Debug.WriteLine("Short1 completed.");
short1CompletedAt = Environment.TickCount;
}
}
});
var shortTask2 = Task.Run(async() =>
{
Debug.WriteLine("Short2 entered.");
await Task.Delay(500);
Debug.WriteLine("Short2 starting work.");
using (await SemaphoreLock.TakeAsync(semaphore))
{
Debug.WriteLine("Short2 acquired lock.");
await Task.Delay(50);
Debug.WriteLine("Short2 completed.");
short2CompletedAt = Environment.TickCount;
}
});
Task.WaitAll(longTask, shortTask1, shortTask2);
Assert.IsFalse(short1CompletedAt.HasValue);
Assert.IsTrue(short1TimedOutAt.HasValue);
Assert.IsTrue(short2CompletedAt.HasValue);
Assert.IsTrue(longCompletedAt.HasValue);
Assert.IsTrue(short1TimedOutAt.Value < longCompletedAt.Value);
Assert.IsTrue(short1TimedOutAt.Value < short2CompletedAt.Value);
Assert.IsTrue(longCompletedAt.Value < short2CompletedAt.Value);
}
}