/// <summary>
/// Tries to resolve <see cref="MachineId"/> by <paramref name="machineLocation"/>.
/// </summary>
public bool TryResolveMachineId(MachineLocation machineLocation, out MachineId machineId)
{
return(_idByLocationMap.TryGetValue(machineLocation, out machineId));
}
public static ReplicaRank GetReplicaRank(
ContentHash hash,
ContentLocationEntry entry,
MachineId localMachineId,
LocalLocationStoreConfiguration configuration,
DateTime now)
{
var locationsCount = entry.Locations.Count;
var desiredReplicaCount = configuration.DesiredReplicaRetention;
if (desiredReplicaCount == 0 ||
locationsCount == 0) // It is possible for the entry to have 0 locations.
// For instance, the database has 1 location but the machine is in the bad state
{
return(ReplicaRank.None);
}
if (locationsCount <= desiredReplicaCount
// If using throttled eviction, we might need to upgrade the rank to Protected
// so don't return here
&& configuration.ThrottledEvictionInterval == TimeSpan.Zero)
{
return(ReplicaRank.Important);
}
// Making sure that probabilistically, some locations are considered important for the current machine.
long contentHashCode = unchecked ((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, hash[1]));
var importantRangeStart = contentHashCode % locationsCount;
// Getting an index of a current location in the location list
int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId);
if (currentMachineLocationIndex == -1)
{
// This is used for testing only. The machine Id should be part of the machines.
// But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important.
return(ReplicaRank.None);
}
// In case of important range wrapping around end of location list to start of location list
// we need to compute a positive offset from the range start to see if the replica exists in the range
// i.e. range start = 5, location count = 7, and desired location count = 3
// important range contains [5, 6 and 0] since it overflows the end of the list
var offset = currentMachineLocationIndex - importantRangeStart;
if (offset < 0)
{
offset += locationsCount;
}
var lastImportantReplicaOffset = Math.Min(desiredReplicaCount, locationsCount) - 1;
if (offset >= desiredReplicaCount)
{
return(ReplicaRank.None);
}
if (offset != lastImportantReplicaOffset)
{
// All but last important replica are always Protected
return(ReplicaRank.Protected);
}
if (configuration.ThrottledEvictionInterval == TimeSpan.Zero)
{
// Throttled eviction is disabled. Just mark the replica as important
// since its in the important range
return(ReplicaRank.Important);
}
// How throttled eviction works:
// 1. Compute which machines consider the content important
// This is done by computing a hash code from the content hash modulo location count to
// generate a start index into the list replicas.
// For instance,
// given locations: [4, 11, 22, 35, 73, 89]
// locationCount = 6,
// if contentHashCode % locationCount = 2 and DesiredReplicaCount = 3
// then the machines considering content important are [22, 35, 73]
// 2. All but last important replica must be consider Protected (i.e. 22, 35 have rank Protected)
// 3. Compute if last replica is protected.
// This is based of to time ranges or buckets of duration ThrottledEvictionInterval
// For instance,
// if ThrottleInterval = 20 minutes
// 10:00AM-10:20AM -> (timeBucketIndex = 23045230) % DesiredReplicaCount = 2 = evictableOffset
// 10:20AM-10:40AM -> (timeBucketIndex = 23045231) % DesiredReplicaCount = 0 = evictableOffset
// 10:40AM-11:00AM -> (timeBucketIndex = 23045232) % DesiredReplicaCount = 1 = evictableOffset
// 11:00AM-11:20AM -> (timeBucketIndex = 23045233) % DesiredReplicaCount = 2 = evictableOffset
// So for times 10:00AM-10:20AM and 11:00AM-11:20AM the last important replica is evictable
var timeBucketIndex = now.Ticks / configuration.ThrottledEvictionInterval.Ticks;
// NOTE: We add contentHashCode to timeBucketIndex so that not all Protected content is considered evictable
// at the same time
var evictableOffset = (contentHashCode + timeBucketIndex) % desiredReplicaCount;
if (evictableOffset == offset)
{
return(ReplicaRank.Important);
}
else
{
// The replica is not currently evictable. Mark it as protected which will give it the minimum effective age
// so that it is only evicted as a last resort
return(ReplicaRank.Protected);
}
}
public ClusterState(MachineId primaryMachineId, IReadOnlyList <MachineMapping> localMachineMappings)
: this(new ClusterStateInternal(primaryMachineId, localMachineMappings))
{
}
public bool IsMachineMarkedClosed(MachineId machineId)
{
return(_closedMachinesSet[machineId]);
}
internal ClusterStateInternal(MachineId primaryMachineId, IReadOnlyList <MachineMapping> localMachineMappings)
{
PrimaryMachineId = primaryMachineId;
LocalMachineMappings = localMachineMappings;
}
public bool IsMachineMarkedInactive(MachineId machineId)
{
return(_inactiveMachinesSet[machineId]);
}
private async Task <Unit> SetLocationBitAndExpireAsync(OperationContext context, IBatch batch, RedisKey key, ContentHashWithSize hash, MachineId machineId)
{
var tasks = new List <Task>();
// NOTE: The order here matters. KeyExpire must be after creation of the entry. SetBit creates the entry if needed.
tasks.Add(batch.StringSetBitAsync(key, machineId.GetContentLocationEntryBitOffset(), true));
tasks.Add(batch.KeyExpireAsync(key, Configuration.LocationEntryExpiry));
// NOTE: We don't set the size when using optimistic location registration because the entry should have already been created at this point (the prior set
// if not exists call failed indicating the entry already exists).
// There is a small race condition if the entry was near-expiry and this call ends up recreating the entry without the size being set. We accept
// this possibility since we have to handle unknown size and either case and the occurrence of the race should be rare. Also, we can mitigate by
// applying the size from the local database which should be known if the entry is that old.
if (!Configuration.UseOptimisticRegisterLocalLocation && hash.Size >= 0)
{
tasks.Add(batch.StringSetRangeAsync(key, 0, ContentLocationEntry.ConvertSizeToRedisRangeBytes(hash.Size)));
}
await Task.WhenAll(tasks);
return(Unit.Void);
}
private Task <BoolResult> RegisterLocationAsync(
OperationContext context,
IReadOnlyList <ContentHashWithSize> contentHashes,
MachineId machineId,
[CallerMemberName] string caller = null)
{
const int operationsPerHash = 3;
var hashBatchSize = Math.Max(1, Configuration.RedisBatchPageSize / operationsPerHash);
return(context.PerformOperationAsync(
Tracer,
async() =>
{
foreach (var page in contentHashes.GetPages(hashBatchSize))
{
var batchResult = await RaidedRedis.ExecuteRedisAsync(context, async(redisDb, token) =>
{
Counters[GlobalStoreCounters.RegisterLocalLocationHashCount].Add(page.Count);
int requiresSetBitCount;
ConcurrentBitArray requiresSetBit;
if (Configuration.UseOptimisticRegisterLocalLocation)
{
requiresSetBitCount = 0;
requiresSetBit = new ConcurrentBitArray(page.Count);
var redisBatch = redisDb.CreateBatch(RedisOperation.RegisterLocalSetNonExistentHashEntries);
// Perform initial pass to set redis entries in single operation. Fallback to more elaborate
// flow where we use SetBit + KeyExpire
foreach (var indexedHash in page.WithIndices())
{
var hash = indexedHash.value;
var key = GetRedisKey(hash.Hash);
redisBatch.AddOperationAndTraceIfFailure(context, key, async batch =>
{
bool set = await batch.StringSetAsync(key, ContentLocationEntry.ConvertSizeAndMachineIdToRedisValue(hash.Size, machineId), Configuration.LocationEntryExpiry, When.NotExists);
if (!set)
{
requiresSetBit[indexedHash.index] = true;
Interlocked.Increment(ref requiresSetBitCount);
}
return set;
}, operationName: "ConvertSizeAndMachineIdToRedisValue");
}
var result = await redisDb.ExecuteBatchOperationAsync(context, redisBatch, token);
if (!result || requiresSetBitCount == 0)
{
return result;
}
}
else
{
requiresSetBitCount = page.Count;
requiresSetBit = null;
}
// Some keys already exist and require that we set the bit and update the expiry on the existing entry
using (Counters[GlobalStoreCounters.RegisterLocalLocationUpdate].Start())
{
Counters[GlobalStoreCounters.RegisterLocalLocationUpdateHashCount].Add(requiresSetBitCount);
var updateRedisBatch = redisDb.CreateBatch(RedisOperation.RegisterLocalSetHashEntries);
foreach (var hash in page.Where((h, index) => requiresSetBit?[index] ?? true))
{
var key = GetRedisKey(hash.Hash);
updateRedisBatch.AddOperationAndTraceIfFailure(
context,
key,
batch => SetLocationBitAndExpireAsync(context, batch, key, hash, machineId),
operationName: "SetLocationBitAndExpireAsync");
}
return await redisDb.ExecuteBatchOperationAsync(context, updateRedisBatch, token);
}
}, Configuration.RetryWindow);
if (!batchResult)
{
return batchResult;
}
}
return BoolResult.Success;
},
Counters[GlobalStoreCounters.RegisterLocalLocation],
caller: caller,
traceErrorsOnly: true));
}
/// <inheritdoc />
public Task <BoolResult> RegisterLocationAsync(OperationContext context, MachineId machineId, IReadOnlyList <ContentHashWithSize> contentHashes)
{
return(RegisterLocationAsync(context, contentHashes, machineId));
}
public MachineWithBinAssigments(MachineId machineId) => MachineId = machineId;
