private async Task WithContentLocationEventStore(Func <Context, IClock, IAbsFileSystem, ContentLocationEventStore, Task> action, ContentLocationEventStoreConfiguration configuration, IContentLocationEventHandler eventHandler, string localMachineName = "Worker")
{
string centralStateKeyBase = "ThisIsUnused";
var clock = new MemoryClock();
using var fileSystem = new PassThroughFileSystem(TestGlobal.Logger);
var tracingContext = new Context(TestGlobal.Logger);
{
using var localDiskCentralStoreWorkingDirectory = new DisposableDirectory(fileSystem);
var localDiskCentralStoreConfiguration = new LocalDiskCentralStoreConfiguration(localDiskCentralStoreWorkingDirectory.Path, centralStateKeyBase);
var centralStorage = new LocalDiskCentralStorage(localDiskCentralStoreConfiguration);
{
using var eventHubWorkingDirectory = new DisposableDirectory(fileSystem);
var eventStore = CreateEventStore(configuration, eventHandler, localMachineName, centralStorage, eventHubWorkingDirectory, clock);
(await eventStore.StartupAsync(tracingContext)).ShouldBeSuccess();
await action(tracingContext, clock, fileSystem, eventStore);
if (!eventStore.ShutdownStarted)
{
(await eventStore.ShutdownAsync(tracingContext)).ShouldBeSuccess();
}
}
}
}
private Task <byte[]> CorruptStoreWithReplicasAsync(
Context context,
MemoryClock clock,
DisposableDirectory tempDirectory,
Func <ContentHash, Task> corruptFunc)
{
return(CorruptStoreAsync(context, tempDirectory, async contentHash =>
{
// ReSharper disable once UnusedVariable
foreach (var x in Enumerable.Range(0, 1500))
{
AbsolutePath tempPath = tempDirectory.CreateRandomFileName();
clock.Increment();
var result = await PlaceFileAsync(
context,
contentHash,
tempPath,
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink,
null);
result.Code.Should().Be(PlaceFileResult.ResultCode.PlacedWithHardLink);
}
await corruptFunc(contentHash);
}));
}
public void OperationCountLimitTest()
{
var clock = new MemoryClock();
clock.UtcNow = DateTime.Today;
var operationLimit = 2;
var span = TimeSpan.FromSeconds(1);
var halfSpan = new TimeSpan(span.Ticks / 2);
var limit = new OperationThrottle(span, operationLimit, clock);
for (var iteration = 0; iteration < 3; iteration++)
{
// Use all available operations
for (var i = 0; i < operationLimit; i++)
{
limit.CheckAndRegisterOperation().ShouldBeSuccess();
}
// Doing more operations should fail.
limit.CheckAndRegisterOperation().ShouldBeError();
// ... even if time passes, as long as we don't cross the span limit.
clock.Increment(halfSpan);
limit.CheckAndRegisterOperation().ShouldBeError();
// Once the span has been completed, operations should succeed again.
clock.Increment(halfSpan);
}
}
public Task ReleasedInstancesArentRemovedIfTheyHaveAliveReferences()
{
var clock = new MemoryClock();
var configuration = new ResourcePoolConfiguration()
{
MaximumAge = TimeSpan.FromSeconds(1)
};
return(RunTest <Key, Resource>(async(context, pool) =>
{
var key = new Key(0);
await pool.UseAsync(context, key, async wrapper =>
{
// Invalidate resource. This will force GC to release it. Notice time doesn't change, so this isn't
// removed due to TTL
await pool.UseAsync(context, key, wrapper =>
{
wrapper.Invalidate(context);
return BoolResult.SuccessTask;
}).ShouldBeSuccess();
await pool.GarbageCollectAsync(context).IgnoreFailure();
wrapper.ShutdownToken.IsCancellationRequested.Should().BeTrue();
pool.Counter[ResourcePoolV2Counters.ReleasedResources].Value.Should().Be(1);
pool.Counter[ResourcePoolV2Counters.ShutdownAttempts].Value.Should().Be(0);
return BoolResult.Success;
}).ShouldBeSuccess();
},
configuration,
clock));
}
public async Task ValidateInvalidation()
{
var maxCapacity = 10;
var clock = new MemoryClock();
var maxAgeMinutes = maxCapacity + 10; // No resources should expire.
var pool = new ResourcePool <Key, Resource>(new Context(TestGlobal.Logger), new ResourcePoolConfiguration()
{
MaximumAge = TimeSpan.FromMinutes(maxAgeMinutes),
MaximumResourceCount = maxCapacity,
EnableInstanceInvalidation = true,
}, resourceFactory: _ => new Resource(), clock);
using (var wrapper = await pool.CreateAsync(new Key(1)))
{
wrapper.Invalidate();
}
// We actually do the cleanup during the next operation
using (var wrapper = await pool.CreateAsync(new Key(2)))
{
}
Assert.Equal(1, pool.Counter[ResourcePoolCounters.Cleaned].Value);
Assert.Equal(2, pool.Counter[ResourcePoolCounters.Created].Value);
}
public Task ExpiredInstancesGetGarbageCollected()
{
var clock = new MemoryClock();
var configuration = new ResourcePoolConfiguration()
{
MaximumAge = TimeSpan.FromSeconds(1)
};
return(RunTest <Key, Resource>(async(context, pool) =>
{
var key = new Key(0);
await pool.UseAsync(context, key, wrapper =>
{
return BoolResult.SuccessTask;
}).ShouldBeSuccess();
clock.Increment(TimeSpan.FromMinutes(1));
await pool.GarbageCollectAsync(context).IgnoreFailure();
pool.Counter[ResourcePoolV2Counters.CreatedResources].Value.Should().Be(1);
pool.Counter[ResourcePoolV2Counters.ReleasedResources].Value.Should().Be(1);
pool.Counter[ResourcePoolV2Counters.ShutdownSuccesses].Value.Should().Be(1);
pool.Counter[ResourcePoolV2Counters.GarbageCollectionSuccesses].Value.Should().Be(1);
},
configuration,
clock));
}
public void NonRareContentShouldBeImportantInSomeCases(int machineCount)
{
// Using non random hash to make the test deterministic
var hash = VsoHashInfo.Instance.EmptyHash;
var machineIds = Enumerable.Range(1, machineCount).Select(v => (ushort)v).ToArray();
var clock = new MemoryClock();
// Creating an entry with 'machineCount' locations.
// But instead of using instance as is, we call Copy to serialize/deserialize the instance.
// In this case, we'll create different types at runtime instead of using just a specific type like ArrayMachineIdSet.
var machineIdSet = Copy(new ArrayMachineIdSet(machineIds));
var entry = ContentLocationEntry.Create(
locations: machineIdSet,
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
// Then checking all the "machines" (by creating MachineId for each id in machineIds)
// to figure out if the replica is important.
var importantMachineCount = machineIds.Select(
machineId => EffectiveLastAccessTimeProvider.IsImportantReplica(
hash,
entry,
new MachineId(machineId),
Configuration.DesiredReplicaRetention)).Count(important => important);
// We should get roughly 'configuration.DesiredReplicaRetention' important replicas.
// The number is not exact, because when we're computing the importance we're computing a hash of the first bytes of the hash
// plus the machine id. So it is possible that we can be slightly off here.
var desiredReplicaCount = Configuration.DesiredReplicaRetention;
importantMachineCount.Should().BeInRange(desiredReplicaCount - 2, desiredReplicaCount + 3);
}
protected override ITestRedisDatabase GetRedisDatabase(
MemoryClock clock,
IDictionary <RedisKey, RedisValue> initialData = null,
IDictionary <RedisKey, DateTime> expiryData = null,
IDictionary <RedisKey, RedisValue[]> setData = null)
{
return(LocalRedisProcessDatabase.CreateAndStart(Redis, TestGlobal.Logger, clock, initialData, expiryData, setData));
}
public void TestAverage()
{
var clock = new MemoryClock();
var activityTracker = new ActivityTracker <MyCounters>(clock, TimeSpan.FromSeconds(10));
var collection = new CounterCollection <MyCounters>();
// Initial rates are (total: 0, ratePerSecond: 0)
Dictionary <MyCounters, (long total, double ratePerSecond)> rates = activityTracker.GetRates();
rates[MyCounters.Value1].Should().Be((0, 0));
rates[MyCounters.Value2].Should().Be((0, 0));
clock.AddSeconds(1);
collection.AddToCounter(MyCounters.Value1, 1);
collection.AddToCounter(MyCounters.Value2, 2);
activityTracker.ProcessSnapshot(collection);
rates = activityTracker.GetRates();
// Totals should be changed, but the rates are still 0. We don't have enough data.
rates[MyCounters.Value1].Should().Be((1, 0));
rates[MyCounters.Value2].Should().Be((2, 0));
clock.AddSeconds(1);
collection.AddToCounter(MyCounters.Value1, 1);
collection.AddToCounter(MyCounters.Value2, 2);
activityTracker.ProcessSnapshot(collection);
rates = activityTracker.GetRates();
rates[MyCounters.Value1].Should().Be((2, 1));
rates[MyCounters.Value2].Should().Be((4, 2));
// Moving the time to the end of the window
clock.AddSeconds(9);
collection.AddToCounter(MyCounters.Value1, 2);
collection.AddToCounter(MyCounters.Value2, 4);
activityTracker.ProcessSnapshot(collection);
rates = activityTracker.GetRates();
rates[MyCounters.Value1].Should().Be((4, 0.3));
rates[MyCounters.Value2].Should().Be((8, 0.6));
// Move even further, should have only one record in the window.
clock.AddSeconds(3);
rates = activityTracker.GetRates();
rates[MyCounters.Value1].Should().Be((4, 0));
rates[MyCounters.Value2].Should().Be((8, 0));
clock.AddSeconds(2); // 5 seconds since the last snapshot
collection.AddToCounter(MyCounters.Value1, 5);
collection.AddToCounter(MyCounters.Value2, 10);
activityTracker.ProcessSnapshot(collection);
rates = activityTracker.GetRates();
rates[MyCounters.Value1].Should().Be((9, 1));
rates[MyCounters.Value2].Should().Be((18, 2));
}
public Task <byte[]> CorruptWithMissingReplicaAsync(Context context, MemoryClock clock, DisposableDirectory tempDirectory)
{
return(CorruptStoreWithReplicasAsync(context, clock, tempDirectory, hash =>
{
var pathForSecondReplica = GetReplicaPathForTest(hash, 1);
FileSystem.DeleteFile(pathForSecondReplica);
return Task.FromResult(true);
}));
}
public void OnlyOneImportantReplicaIsUnprotected()
{
Configuration.ThrottledEvictionInterval = TimeSpan.FromMinutes(20);
var hash = ContentHash.Random();
var clock = new MemoryClock();
var machineIds = new ushort[] { 1, 2, 3 };
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(machineIds),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
int rangesWithUnprotected = 0;
for (int i = 0; i < Configuration.DesiredReplicaRetention; i++)
{
clock.UtcNow += Configuration.ThrottledEvictionInterval;
int protectedCount = 0;
int importantCount = 0;
foreach (var machineId in machineIds)
{
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(machineId), Configuration, clock.UtcNow);
switch (rank)
{
case ReplicaRank.Important:
importantCount++;
break;
case ReplicaRank.Protected:
protectedCount++;
break;
default:
XAssert.Fail($"Rank is '{rank}' but should be Important or Protected since content is rare");
break;
}
}
var importantOrProtectedCount = protectedCount + importantCount;
importantCount.Should().BeLessOrEqualTo(1, "At most 1 out of DesiredReplicaCount of the time content should just be important allowing eviction");
importantOrProtectedCount.Should().Be(Configuration.DesiredReplicaRetention, "All replicas should be important or protected");
if (importantCount == 1)
{
rangesWithUnprotected++;
protectedCount.Should().Be(Configuration.DesiredReplicaRetention - 1, "(DesiredReplicaCount - 1) out of DesiredReplicaCount of the time content should be protected");
}
}
rangesWithUnprotected.Should().Be(1, "There should be one time range out of DesiredReplicaCount ranges where one of replicas is unprotected");
}
public void TestContentEvictionWithDesignatedLocation()
{
var clock = new MemoryClock();
var entries = new List <ContentLocationEntry>();
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
var hashes = new[] { ContentHash.Random(), ContentHash.Random(), ContentHash.Random() };
var mock = new EffectiveLastAccessTimeProviderMock(
localMachineId: new MachineId(1024),
isDesignatedLocation: hash => hash == hashes[0]); // The first hash will be designated, and thus important
mock.Map = new Dictionary <ContentHash, ContentLocationEntry>()
{
[hashes[0]] = entries[0],
[hashes[1]] = entries[1],
[hashes[2]] = entries[2],
};
var provider = new EffectiveLastAccessTimeProvider(Configuration, clock, mock);
var context = new OperationContext(new Context(Logger));
// A given machine id index is higher then the max number of locations used in this test.
// This will prevent the provider to consider non-important locations randomly important
var input = hashes.Select(hash => new ContentHashWithLastAccessTime(hash, mock.Map[hash].LastAccessTimeUtc.ToDateTime())).ToList();
var result = provider.GetEffectiveLastAccessTimes(context, input).ShouldBeSuccess();
var output = result.Value.ToList();
output.Sort(ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance);
// We know that the first hash should be the last one, because this is only important hash in the list.
output[output.Count - 1].ContentHash.Should().Be(hashes[0]);
}
public async Task EnsureContentIsNotPinned(Context context, MemoryClock clock, ContentHash contentHash)
{
Assert.True(await ContainsAsync(context, contentHash, null));
clock.Increment();
await ClearStoreOfUnpinnedContent(context, clock);
Assert.False(await ContainsAsync(context, contentHash, null));
clock.Increment();
}
public Task <byte[]> CorruptWithExtraReplicaAsync(Context context, MemoryClock clock, DisposableDirectory tempDirectory)
{
return(CorruptStoreWithReplicasAsync(context, clock, tempDirectory, async hash =>
await ContentDirectory.UpdateAsync(hash, true, Clock, fileInfo =>
{
fileInfo.ReplicaCount.Should().Be(2);
fileInfo.ReplicaCount--;
return Task.FromResult(fileInfo);
})));
}
private static LifetimeTrackerHelper CreateLifetimeTracker(MemoryClock memoryClock)
{
var processStartTimeUtc = memoryClock.UtcNow;
memoryClock.UtcNow += TimeSpan.FromSeconds(1);
var instance = LifetimeTrackerHelper.Starting(memoryClock.UtcNow, processStartTimeUtc, new Result <DateTime>(processStartTimeUtc.Subtract(TimeSpan.FromMinutes(10))));
return(instance);
}
public void TestDistributedCacheServiceTracker()
{
var context = new OperationContext(new Context(Logger));
using var testDirectory = new DisposableDirectory(FileSystem);
var logFilePath = testDirectory.Path;
// Intentionally making the interval longer to test the logic manually instead of relying on the tracker's
// logic to write to the file when the timer fires.
var logIntervalSeconds = TimeSpan.FromSeconds(10_000);
var expectedOfflineTime = 10;
var testClock = new MemoryClock();
using (var tracker = ServiceOfflineDurationTracker.Create(context, testClock, FileSystem, logIntervalSeconds, logFilePath).ThrowIfFailure())
{
// Offline time is not available, because the file is missing.
tracker.GetLastServiceHeartbeatTime(context).ShouldBeError();
tracker.LogCurrentTimeStampToFile(context);
// Moving the clock forward
testClock.UtcNow += TimeSpan.FromSeconds(expectedOfflineTime);
// GetLastServiceHeartbeatTime is memoized, so we can't change the state and get different results!
tracker.GetLastServiceHeartbeatTime(context).ShouldBeError();
// Need to create another tracker to test the behavior
using var nestedTracker = ServiceOfflineDurationTracker.Create(context, testClock, FileSystem, logIntervalSeconds, logFilePath).ThrowIfFailure();
var lastHeartbeat = nestedTracker.GetLastServiceHeartbeatTime(context).ShouldBeSuccess();
// From the previous simulated interval, we created a new file and wrote a timestamp to it.
// Now we should be able to determine previous offlineTIme
// Intentionally converting the offline time to int to simplify the comparison.
int offlineDuration = (int)testClock.UtcNow.Subtract(lastHeartbeat.Value.lastServiceHeartbeatTime).TotalSeconds;
offlineDuration.Should().Be(expectedOfflineTime);
lastHeartbeat.Value.shutdownCorrectly.Should().BeFalse();
}
// Moving the clock forward
testClock.UtcNow += TimeSpan.FromSeconds(expectedOfflineTime);
using (var tracker = ServiceOfflineDurationTracker.Create(context, testClock, FileSystem, logIntervalSeconds, logFilePath).ThrowIfFailure())
{
// The previous tracker should have written the files.
var result = tracker.GetLastServiceHeartbeatTime(context).ShouldBeSuccess();
result.Value.shutdownCorrectly.Should().BeTrue();
// From the previous simulated interval, we created a new file and wrote a timestamp to it.
// Now we should be able to determine previous offlineTIme
// Intentionally converting the offline time to int to simplify the comparison.
int offlineDuration = (int)testClock.UtcNow.Subtract(result.Value.lastServiceHeartbeatTime).TotalSeconds;
offlineDuration.Should().Be(expectedOfflineTime);
}
}
public async Task IcmClientTestAsync()
{
Debugger.Launch();
var appKey = GetApplicationKey().ShouldBeSuccess();
var config = new App.Monitor.Configuration
{
AzureAppKey = appKey.Value !
};
var clock = new MemoryClock();
clock.Increment();
var keyVault = new KeyVaultClient(
config.KeyVaultUrl,
config.AzureTenantId,
config.AzureAppId,
config.AzureAppKey,
clock,
cacheTimeToLive: TimeSpan.FromSeconds(1));
keyVault.IcmCallsCounter.Value.Should().Be(0);
// Simulate that the certificate has been acquired before.
_ = await keyVault.GetCertificateAsync(config.IcmCertificateName);
keyVault.IcmCallsCounter.Value.Should().Be(1);
var icmClient = new IcmClient(keyVault, config.IcmUrl, config.IcmConnectorId, config.IcmCertificateName, clock);
var incident = new IcmIncident(
stamp: "Test",
environment: "PROD",
machines: new [] { "MachineA", "MachineB" },
correlationIds: new[] { "GuidA", "GuidB" },
severity: 4,
description: "This incident was created for testing the cache monitor",
title: "Cache Monitor Test Incident",
incidentTime: DateTime.Now,
cacheTimeToLive: null);
await icmClient.EmitIncidentAsync(incident);
// Should have used cached cert.
keyVault.IcmCallsCounter.Value.Should().Be(1);
// Simulate that the certificate will be acquired in the future.
clock.AddSeconds(2);
_ = await keyVault.GetCertificateAsync(config.IcmCertificateName);
keyVault.IcmCallsCounter.Value.Should().Be(2);
}
}
public void ToStringShouldNotHaveSuccessWordInIt()
{
var memoryClock = new MemoryClock();
var instance = CreateLifetimeTracker(memoryClock);
instance.Ready();
var stringRep = instance.ToString();
stringRep.Should().NotContain("Success");
}
public void TestVolatileSetAddExpired()
{
var clock = new MemoryClock();
var set = new VolatileSet<int>(clock);
set.Add(0, TimeSpan.FromSeconds(100));
clock.UtcNow += TimeSpan.FromSeconds(101);
set.Add(0, TimeSpan.FromSeconds(10)).Should().BeTrue("Adding for an expired entry should return true (as if the entry was missing)");
}
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
bool isImportant = EffectiveLastAccessTimeProvider.IsImportantReplica(hash, entry, new MachineId(1), Configuration.DesiredReplicaRetention);
isImportant.Should().BeTrue();
}
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(1), Configuration, clock.UtcNow);
rank.Should().Be(ReplicaRank.Important);
}
public void PropertiesAvailableAfterReadyIsCalled()
{
var memoryClock = new MemoryClock();
var instance = CreateLifetimeTracker(memoryClock);
instance.FullInitializationDuration.ShouldBeError();
instance.OfflineTime.ShouldBeError();
instance.Ready();
instance.FullInitializationDuration.ShouldBeSuccess();
instance.OfflineTime.ShouldBeSuccess();
}
public void TestVolatileWithShortHashSet()
{
var clock = new MemoryClock();
var set = new VolatileSet <ShortHash>(clock);
for (int i = 0; i < 500_000; i++)
{
var hash = ContentHash.Random(HashType.Vso0);
var shortHash = new ShortHash(hash);
set.Add(shortHash, TimeSpan.FromMinutes(30));
set.Contains(shortHash).Should().BeTrue();
}
}
public override int GetHashCode()
{
unchecked
{
var hashCode = 1;
hashCode = (hashCode * 397) ^ (Voltage != null ? Voltage.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (TempLimit != null ? TempLimit.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (PowerLimit != null ? PowerLimit.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (CoreClock != null ? CoreClock.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (MemoryClock != null ? MemoryClock.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (FanSpeed != null ? FanSpeed.GetHashCode() : 0);
return(hashCode);
}
}
private static LifetimeTracker CreateLifetimeTracker(MemoryClock memoryClock)
{
var processStartTimeUtc = memoryClock.UtcNow;
var startupDuration = TimeSpan.FromSeconds(1);
memoryClock.UtcNow += TimeSpan.FromSeconds(1);
Result <DateTime> shutdownTimeUtc = memoryClock.UtcNow - TimeSpan.FromMinutes(5);
var instance = LifetimeTracker.Started(memoryClock, startupDuration, shutdownTimeUtc, processStartTimeUtc);
return(instance);
}
public void RareContentShouldBeImportantOrProtectedWithThrottling(ushort[] machineIds)
{
Configuration.ThrottledEvictionInterval = TimeSpan.FromMinutes(20);
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(machineIds),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
int totalImportantFound = 0;
foreach (var machineId in machineIds)
{
int protectedCount = 0;
int importantCount = 0;
for (int i = 0; i < Configuration.DesiredReplicaRetention; i++)
{
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(machineId), Configuration, clock.UtcNow);
clock.UtcNow += Configuration.ThrottledEvictionInterval;
switch (rank)
{
case ReplicaRank.Important:
importantCount++;
totalImportantFound++;
break;
case ReplicaRank.Protected:
protectedCount++;
break;
default:
XAssert.Fail($"Rank is '{rank}' but should be Important or Protected since content is rare");
break;
}
}
protectedCount.Should().BeInRange(Configuration.DesiredReplicaRetention - 1, Configuration.DesiredReplicaRetention, "At least (DesiredReplicaCount - 1) out of DesiredReplicaCount of the time content should be protected");
importantCount.Should().BeInRange(0, 1, "At most 1 out of DesiredReplicaCount of the time content should just be important allowing eviction");
}
totalImportantFound.Should().Be(1, "Within the time interval (DesiredReplicaCount * ThrottledEvictionInterval), " +
"during exactly one ThrottledEvictionInterval exactly one machine should have content unproteced");
}
public Task CheckpointMaxAgeIsRespected()
{
var clock = new MemoryClock();
return(RunTest(async(context, service, iteration) =>
{
if (iteration == 0)
{
// First heartbeat lets the service know its master. This also restores the latest checkpoint and clears old ones if needed
await service.OnRoleUpdatedAsync(context, Role.Master);
// Create a checkpoint and make sure it shows up in the registry
await service.CreateCheckpointAsync(context).ShouldBeSuccess();
var r = await service.CheckpointManager.CheckpointRegistry.GetCheckpointStateAsync(context);
r.Succeeded.Should().BeTrue();
r.Value !.CheckpointAvailable.Should().BeTrue();
}
else if (iteration == 1)
{
clock.Increment(TimeSpan.FromHours(0.5));
// First heartbeat lets the service know its master. This also restores the latest checkpoint and clears old ones if needed
await service.OnRoleUpdatedAsync(context, Role.Master);
var r = await service.CheckpointManager.CheckpointRegistry.GetCheckpointStateAsync(context);
r.Succeeded.Should().BeTrue();
r.Value !.CheckpointAvailable.Should().BeTrue();
}
else
{
clock.Increment(TimeSpan.FromHours(1));
// First heartbeat lets the service know its master. This also restores the latest checkpoint and clears old ones if needed
await service.OnRoleUpdatedAsync(context, Role.Master);
var r = await service.CheckpointManager.CheckpointRegistry.GetCheckpointStateAsync(context);
r.Succeeded.Should().BeTrue();
r.Value !.CheckpointAvailable.Should().BeFalse();
}
},
clock: clock,
iterations: 3,
modifyConfig: configuration =>
{
configuration.CheckpointMaxAge = TimeSpan.FromHours(1);
}));
}
public void PropertiesAvailableAfterStartedAndReadyCalled()
{
var memoryClock = new MemoryClock();
var instance = CreateLifetimeTracker(memoryClock);
instance.Started(memoryClock.UtcNow);
instance.IsFullyInitialized().Should().BeFalse();
var readyTime = memoryClock.AddSeconds(1);
instance.Ready(readyTime);
instance.IsFullyInitialized().Should().BeTrue();
instance.ServiceReadyTimeUtc.Should().Be(readyTime);
}
public async Task IssueSameClientManyTimes()
{
var clock = new MemoryClock();
var pool = new ResourcePool <Key, Resource>(new Context(TestGlobal.Logger), maxResourceCount: 1, maxAgeMinutes: 1, resourceFactory: _ => new Resource(), clock);
var key = new Key(1);
using var originalWrapper = await pool.CreateAsync(key);
var originalResource = originalWrapper.Value;
for (var i = 0; i < 1000; i++)
{
using var newWrapper = await pool.CreateAsync(key);
Assert.Same(newWrapper.Value, originalResource);
}
}
public void TestVolatileSet()
{
var clock = new MemoryClock();
var set = new VolatileSet<int>(clock);
set.Add(0, TimeSpan.FromSeconds(100));
set.Add(1, TimeSpan.FromSeconds(10));
set.Contains(1).Should().BeTrue();
// Increment time to invalidate item
clock.UtcNow += TimeSpan.FromSeconds(11);
// Item 1 should be invalidated, but 0 should remain
set.Contains(1).Should().BeFalse();
set.Contains(0).Should().BeTrue();
// Explicitly invalidating item should remove it
set.Invalidate(0);
set.Contains(0).Should().BeFalse();
set = new VolatileSet<int>(clock);
set.Add(1, TimeSpan.FromSeconds(10));
set.Add(2, TimeSpan.FromSeconds(20));
set.Add(3, TimeSpan.FromSeconds(30));
set.Add(4, TimeSpan.FromSeconds(40));
set.Add(5, TimeSpan.FromSeconds(50));
clock.UtcNow += TimeSpan.FromSeconds(25);
// Should only be able to remove items 1 and 2 since they have expired
set.CleanStaleItems(20).Should().Be(2);
set.Contains(1).Should().BeFalse();
set.Contains(2).Should().BeFalse();
set.Contains(3).Should().BeTrue();
set.Contains(4).Should().BeTrue();
set.Contains(5).Should().BeTrue();
// Move time past original timeout for 3 but add with 20 seconds from current time
// 3 should be retained
set.Add(3, TimeSpan.FromSeconds(20));
clock.UtcNow += TimeSpan.FromSeconds(10);
set.Contains(3).Should().BeTrue();
}
