public void NonRareContentShouldBeImportantInSomeCases(int machineCount)
{
// Using non random hash to make the test deterministic
var hash = VsoHashInfo.Instance.EmptyHash;
var machineIds = Enumerable.Range(1, machineCount).Select(v => (ushort)v).ToArray();
var clock = new MemoryClock();
// Creating an entry with 'machineCount' locations.
// But instead of using instance as is, we call Copy to serialize/deserialize the instance.
// In this case, we'll create different types at runtime instead of using just a specific type like ArrayMachineIdSet.
var machineIdSet = Copy(new ArrayMachineIdSet(machineIds));
var entry = ContentLocationEntry.Create(
locations: machineIdSet,
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
// Then checking all the "machines" (by creating MachineId for each id in machineIds)
// to figure out if the replica is important.
var importantMachineCount = machineIds.Select(
machineId => EffectiveLastAccessTimeProvider.IsImportantReplica(
hash,
entry,
new MachineId(machineId),
Configuration.DesiredReplicaRetention)).Count(important => important);
// We should get roughly 'configuration.DesiredReplicaRetention' important replicas.
// The number is not exact, because when we're computing the importance we're computing a hash of the first bytes of the hash
// plus the machine id. So it is possible that we can be slightly off here.
var desiredReplicaCount = Configuration.DesiredReplicaRetention;
importantMachineCount.Should().BeInRange(desiredReplicaCount - 2, desiredReplicaCount + 3);
}
private void CheckSerializationRoundtrip(RuntimeTypeModel model, ContentLocationEntry obj)
{
var deserialized = Roundtrip(model, obj);
Assert.Equal(obj.ContentSize, deserialized.ContentSize);
XAssert.SetEqual(obj.Locations, deserialized.Locations);
}
private ContentLocationEntry GetContentLocationEntry(ShortHash hash)
{
if (_map.TryGetValue(hash, out var entry))
{
return(ContentLocationEntry.Create(entry.Locations, entry.ContentSize, lastAccessTimeUtc: Clock.UtcNow, creationTimeUtc: null));
}
return(ContentLocationEntry.Missing);
}
public void OnlyOneImportantReplicaIsUnprotected()
{
Configuration.ThrottledEvictionInterval = TimeSpan.FromMinutes(20);
var hash = ContentHash.Random();
var clock = new MemoryClock();
var machineIds = new ushort[] { 1, 2, 3 };
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(machineIds),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
int rangesWithUnprotected = 0;
for (int i = 0; i < Configuration.DesiredReplicaRetention; i++)
{
clock.UtcNow += Configuration.ThrottledEvictionInterval;
int protectedCount = 0;
int importantCount = 0;
foreach (var machineId in machineIds)
{
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(machineId), Configuration, clock.UtcNow);
switch (rank)
{
case ReplicaRank.Important:
importantCount++;
break;
case ReplicaRank.Protected:
protectedCount++;
break;
default:
XAssert.Fail($"Rank is '{rank}' but should be Important or Protected since content is rare");
break;
}
}
var importantOrProtectedCount = protectedCount + importantCount;
importantCount.Should().BeLessOrEqualTo(1, "At most 1 out of DesiredReplicaCount of the time content should just be important allowing eviction");
importantOrProtectedCount.Should().Be(Configuration.DesiredReplicaRetention, "All replicas should be important or protected");
if (importantCount == 1)
{
rangesWithUnprotected++;
protectedCount.Should().Be(Configuration.DesiredReplicaRetention - 1, "(DesiredReplicaCount - 1) out of DesiredReplicaCount of the time content should be protected");
}
}
rangesWithUnprotected.Should().Be(1, "There should be one time range out of DesiredReplicaCount ranges where one of replicas is unprotected");
}
public void TestContentEvictionWithDesignatedLocation()
{
var clock = new MemoryClock();
var entries = new List <ContentLocationEntry>();
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
var hashes = new[] { ContentHash.Random(), ContentHash.Random(), ContentHash.Random() };
var mock = new EffectiveLastAccessTimeProviderMock(
localMachineId: new MachineId(1024),
isDesignatedLocation: hash => hash == hashes[0]); // The first hash will be designated, and thus important
mock.Map = new Dictionary <ContentHash, ContentLocationEntry>()
{
[hashes[0]] = entries[0],
[hashes[1]] = entries[1],
[hashes[2]] = entries[2],
};
var provider = new EffectiveLastAccessTimeProvider(Configuration, clock, mock);
var context = new OperationContext(new Context(Logger));
// A given machine id index is higher then the max number of locations used in this test.
// This will prevent the provider to consider non-important locations randomly important
var input = hashes.Select(hash => new ContentHashWithLastAccessTime(hash, mock.Map[hash].LastAccessTimeUtc.ToDateTime())).ToList();
var result = provider.GetEffectiveLastAccessTimes(context, input).ShouldBeSuccess();
var output = result.Value.ToList();
output.Sort(ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance);
// We know that the first hash should be the last one, because this is only important hash in the list.
output[output.Count - 1].ContentHash.Should().Be(hashes[0]);
}
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
bool isImportant = EffectiveLastAccessTimeProvider.IsImportantReplica(hash, entry, new MachineId(1), Configuration.DesiredReplicaRetention);
isImportant.Should().BeTrue();
}
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(1), Configuration, clock.UtcNow);
rank.Should().Be(ReplicaRank.Important);
}
public void TestRoundtripRedisValue()
{
Random r = new Random();
for (int machineIdIndex = 0; machineIdIndex < 2048; machineIdIndex++)
{
long randomSize = (long)Math.Pow(2, 63 * r.NextDouble());
byte[] entryBytes = ContentLocationEntry.ConvertSizeAndMachineIdToRedisValue(randomSize, new MachineId(machineIdIndex));
var deserializedEntry = ContentLocationEntry.FromRedisValue(entryBytes, DateTime.UtcNow, missingSizeHandling: true);
deserializedEntry.ContentSize.Should().Be(randomSize);
deserializedEntry.Locations[machineIdIndex].Should().BeTrue();
}
}
public void RareContentShouldBeImportantOrProtectedWithThrottling(ushort[] machineIds)
{
Configuration.ThrottledEvictionInterval = TimeSpan.FromMinutes(20);
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(machineIds),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
int totalImportantFound = 0;
foreach (var machineId in machineIds)
{
int protectedCount = 0;
int importantCount = 0;
for (int i = 0; i < Configuration.DesiredReplicaRetention; i++)
{
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(machineId), Configuration, clock.UtcNow);
clock.UtcNow += Configuration.ThrottledEvictionInterval;
switch (rank)
{
case ReplicaRank.Important:
importantCount++;
totalImportantFound++;
break;
case ReplicaRank.Protected:
protectedCount++;
break;
default:
XAssert.Fail($"Rank is '{rank}' but should be Important or Protected since content is rare");
break;
}
}
protectedCount.Should().BeInRange(Configuration.DesiredReplicaRetention - 1, Configuration.DesiredReplicaRetention, "At least (DesiredReplicaCount - 1) out of DesiredReplicaCount of the time content should be protected");
importantCount.Should().BeInRange(0, 1, "At most 1 out of DesiredReplicaCount of the time content should just be important allowing eviction");
}
totalImportantFound.Should().Be(1, "Within the time interval (DesiredReplicaCount * ThrottledEvictionInterval), " +
"during exactly one ThrottledEvictionInterval exactly one machine should have content unproteced");
}
public void ContentLocationEntryRoundtrip()
{
var model = MetadataServiceSerializer.TypeModel;
CheckSerializationRoundtrip(model, ContentLocationEntry.Create(
ArrayMachineIdSet.Create(new[] { new MachineId(12), new MachineId(23) }),
12345,
DateTime.UtcNow,
DateTime.UtcNow - TimeSpan.FromDays(1)));
CheckSerializationRoundtrip(model, ContentLocationEntry.Create(
MachineIdSet.Empty,
46456,
new UnixTime(1)));
}
public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ShortHash> contentHashes)
{
var entries = new ContentLocationEntry[contentHashes.Count];
for (var i = 0; i < contentHashes.Count; i++)
{
var hash = contentHashes[i];
if (!Database.TryGetEntry(context, hash, out var entry))
{
entry = ContentLocationEntry.Missing;
}
entries[i] = entry;
}
return(Task.FromResult(Result.Success <IReadOnlyList <ContentLocationEntry> >(entries)));
}
public void ZeroLocationsShouldNotCauseRuntimeError()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[0]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
var configuration = new LocalLocationStoreConfiguration()
{
ThrottledEvictionInterval = TimeSpan.FromSeconds(1)
};
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(1), configuration, clock.UtcNow);
rank.Should().Be(ReplicaRank.None);
}
/// <nodoc />
public void StoreResult(OperationContext context, string path, List <string> machines)
{
foreach (var machine in machines)
{
_knownMachines.GetOrAdd(machine, _ =>
{
var machineId = new MachineId(Interlocked.Increment(ref _currentId));
_clusterState.AddMachine(machineId, new MachineLocation(machine));
return(machineId);
});
}
var pathHash = ComputePathHash(path);
var entry = ContentLocationEntry.Create(MachineIdSet.Empty.SetExistence(machines.SelectList(machine => _knownMachines[machine]), true), 0, DateTime.UtcNow);
_database.Store(context, pathHash, entry);
}
/// <inheritdoc />
protected override void Store(OperationContext context, ShortHash hash, ContentLocationEntry entry)
{
// consider merging the values. Right now we always reconstruct the entry.
_map.AddOrUpdate(hash, key => entry, (key, old) => entry);
}
/// <inheritdoc />
protected override bool TryGetEntryCore(OperationContext context, ShortHash hash, out ContentLocationEntry entry)
{
entry = GetContentLocationEntry(hash);
return(!entry.IsMissing);
}
/// <nodoc />
protected ContentLocationEntry ToContentLocationEntry(RedisValue contentHashInfo)
{
return(ContentLocationEntry.FromRedisValue(contentHashInfo, _clock.UtcNow));
}
public static ContentHashWithSizeAndLocations Merge(ContentHashWithSizeAndLocations left, ContentHashWithSizeAndLocations right)
{
Contract.Requires(left.ContentHash == right.ContentHash);
Contract.Requires(left.Size == -1 || right.Size == -1 || right.Size == left.Size);
var finalList = (left.Locations ?? Enumerable.Empty <MachineLocation>()).Union(right.Locations ?? Enumerable.Empty <MachineLocation>());
return(new ContentHashWithSizeAndLocations(left.ContentHash, Math.Max(left.Size, right.Size), finalList.ToList(), ContentLocationEntry.MergeEntries(left.Entry, right.Entry)));
}
