/// <nodoc />
public void Store(OperationContext context, ShortHash hash, ContentLocationEntry entry)
{
using (_exchangeLock.AcquireReadLock())
{
_cache[hash] = entry;
}
}
private ContentLocationEntry MergeEntries(ContentLocationEntry entry, ContentLocationEntry originalEntry)
{
if (entry.IsMissing)
{
return(originalEntry);
}
if (originalEntry.IsMissing)
{
return(entry);
}
return(entry.SetMachineExistence(originalEntry.Locations, exists: true));;
}
public static bool IsImportantReplica(ContentHash hash, ContentLocationEntry entry, MachineId localMachineId, long desiredReplicaCount)
{
var locationsCount = entry.Locations.Count;
if (locationsCount <= desiredReplicaCount)
{
return(true);
}
if (desiredReplicaCount == 0)
{
return(false);
}
// Making sure that probabilistically, some locations are considered important for the current machine.
long replicaHash = unchecked ((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, localMachineId.Index + 1));
var offset = replicaHash % locationsCount;
var importantRangeStart = offset;
var importantRangeStop = (offset + desiredReplicaCount) % locationsCount;
// Getting an index of a current location in the location list
int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId);
if (currentMachineLocationIndex == -1)
{
// This is used for testing only. The machine Id should be part of the machines.
// But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important.
return(false);
}
// For instance, for locations [1, 20]
// the important range can be [5, 7]
// or [19, 1]
if (importantRangeStart < importantRangeStop)
{
// This is the first case: the start index is less then the stop,
// so the desired location should be within the range
return(currentMachineLocationIndex >= importantRangeStart && currentMachineLocationIndex <= importantRangeStop);
}
// Important range is broken because the start is greater then the end.
// like [19, 1], so the location is important if it's index is >= start or <= the stop.
// For instance, for 10 locations the start is 9 and the end is 2
return(currentMachineLocationIndex >= importantRangeStart || currentMachineLocationIndex <= importantRangeStop);
}
public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ShortHash> contentHashes)
{
var entries = new ContentLocationEntry[contentHashes.Count];
for (int i = 0; i < contentHashes.Count; i++)
{
var hash = contentHashes[i];
if (!_database.TryGetEntry(context, hash, out var entry))
{
entry = ContentLocationEntry.Missing;
}
entries[i] = entry;
}
return(Task.FromResult(Result.Success <IReadOnlyList <ContentLocationEntry> >(entries)));
}
/// <nodoc />
public static ContentLocationEntry MergeEntries(ContentLocationEntry entry1, ContentLocationEntry entry2)
{
if (entry1 == null || entry1.IsMissing)
{
return(entry2);
}
if (entry2 == null || entry2.IsMissing)
{
return(entry1);
}
return(new ContentLocationEntry(
entry1.Locations.SetExistence(entry2.Locations, true),
entry1.ContentSize,
UnixTime.Min(entry1.LastAccessTimeUtc, entry2.LastAccessTimeUtc),
UnixTime.Min(entry1.CreationTimeUtc, entry2.CreationTimeUtc)));
}
private async Task<Unit> SetLocationBitAndExpireAsync(OperationContext context, IBatch batch, RedisKey key, ShortHashWithSize hash, MachineId machineId)
{
var tasks = new List<Task>();
// NOTE: The order here matters. KeyExpire must be after creation of the entry. SetBit creates the entry if needed.
tasks.Add(batch.StringSetBitAsync(key, machineId.GetContentLocationEntryBitOffset(), true));
tasks.Add(batch.KeyExpireAsync(key, Configuration.LocationEntryExpiry));
// NOTE: We don't set the size when using optimistic location registration because the entry should have already been created at this point (the prior set
// if not exists call failed indicating the entry already exists).
// There is a small race condition if the entry was near-expiry and this call ends up recreating the entry without the size being set. We accept
// this possibility since we have to handle unknown size and either case and the occurrence of the race should be rare. Also, we can mitigate by
// applying the size from the local database which should be known if the entry is that old.
if (!Configuration.UseOptimisticRegisterLocalLocation && hash.Size >= 0)
{
tasks.Add(batch.StringSetRangeAsync(key, 0, ContentLocationEntry.ConvertSizeToRedisRangeBytes(hash.Size)));
}
await Task.WhenAll(tasks);
return Unit.Void;
}
/// <nodoc />
public bool TryGetEntry(ShortHash hash, out ContentLocationEntry entry)
{
using (_exchangeLock.AcquireReadLock())
{
// The entry could be a tombstone, so we need to make sure the user knows content has actually been
// deleted, which is why we check for null.
if (_cache.TryGetValue(hash, out entry))
{
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheHit].Increment();
return(entry != null);
}
else if (_flushingCache.TryGetValue(hash, out entry))
{
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheHit].Increment();
return(entry != null);
}
}
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheMiss].Increment();
return(false);
}
private Task <BoolResult> RegisterLocationAsync(
OperationContext context,
IReadOnlyList <ShortHashWithSize> contentHashes,
MachineId machineId,
[CallerMemberName] string caller = null)
{
const int operationsPerHash = 3;
var hashBatchSize = Math.Max(1, Configuration.RedisBatchPageSize / operationsPerHash);
return(context.PerformOperationAsync(
Tracer,
async() =>
{
foreach (var page in contentHashes.GetPages(hashBatchSize))
{
var batchResult = await RaidedRedis.ExecuteRedisAsync(context, async(redisDb, token) =>
{
Counters[GlobalStoreCounters.RegisterLocalLocationHashCount].Add(page.Count);
int requiresSetBitCount;
ConcurrentBitArray requiresSetBit;
if (Configuration.UseOptimisticRegisterLocalLocation)
{
requiresSetBitCount = 0;
requiresSetBit = new ConcurrentBitArray(page.Count);
var redisBatch = redisDb.CreateBatch(RedisOperation.RegisterLocalSetNonExistentHashEntries);
// Perform initial pass to set redis entries in single operation. Fallback to more elaborate
// flow where we use SetBit + KeyExpire
foreach (var indexedHash in page.WithIndices())
{
var hash = indexedHash.value;
var key = GetRedisKey(hash.Hash);
redisBatch.AddOperationAndTraceIfFailure(context, key, async batch =>
{
bool set = await batch.StringSetAsync(key, ContentLocationEntry.ConvertSizeAndMachineIdToRedisValue(hash.Size, machineId), Configuration.LocationEntryExpiry, When.NotExists);
if (!set)
{
requiresSetBit[indexedHash.index] = true;
Interlocked.Increment(ref requiresSetBitCount);
}
return set;
}, operationName: "ConvertSizeAndMachineIdToRedisValue");
}
var result = await redisDb.ExecuteBatchOperationAsync(context, redisBatch, token);
if (!result || requiresSetBitCount == 0)
{
return result;
}
}
else
{
requiresSetBitCount = page.Count;
requiresSetBit = null;
}
// Some keys already exist and require that we set the bit and update the expiry on the existing entry
using (Counters[GlobalStoreCounters.RegisterLocalLocationUpdate].Start())
{
Counters[GlobalStoreCounters.RegisterLocalLocationUpdateHashCount].Add(requiresSetBitCount);
var updateRedisBatch = redisDb.CreateBatch(RedisOperation.RegisterLocalSetHashEntries);
foreach (var hash in page.Where((h, index) => requiresSetBit?[index] ?? true))
{
var key = GetRedisKey(hash.Hash);
updateRedisBatch.AddOperationAndTraceIfFailure(
context,
key,
batch => SetLocationBitAndExpireAsync(context, batch, key, hash, machineId),
operationName: "SetLocationBitAndExpireAsync");
}
return await redisDb.ExecuteBatchOperationAsync(context, updateRedisBatch, token);
}
}, Configuration.RetryWindow);
if (!batchResult)
{
return batchResult;
}
}
return BoolResult.Success;
},
Counters[GlobalStoreCounters.RegisterLocalLocation],
caller: caller,
traceErrorsOnly: true));
}
/// <inheritdoc />
public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ShortHash> contentHashes)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var results = new ContentLocationEntry[contentHashes.Count];
UnixTime now = _clock.UtcNow;
int dualResultCount = 0;
foreach (var page in contentHashes.AsIndexed().GetPages(Configuration.RedisBatchPageSize))
{
var batchResult = await RaidedRedis.ExecuteRedisAsync(context, async(redisDb, token) =>
{
var redisBatch = redisDb.CreateBatch(RedisOperation.GetBulkGlobal);
foreach (var indexedHash in page)
{
var key = GetRedisKey(indexedHash.Item);
redisBatch.AddOperationAndTraceIfFailure(context, key, async batch =>
{
var redisEntry = await batch.StringGetAsync(key);
ContentLocationEntry entry;
if (redisEntry.IsNullOrEmpty)
{
entry = ContentLocationEntry.Missing;
}
else
{
entry = ContentLocationEntry.FromRedisValue(redisEntry, now, missingSizeHandling: true);
}
var originalEntry = Interlocked.CompareExchange(ref results[indexedHash.Index], entry, null);
if (originalEntry != null)
{
// Existing entry was there. Merge the entries.
entry = ContentLocationEntry.MergeEntries(entry, originalEntry);
Interlocked.Exchange(ref results[indexedHash.Index], entry);
Interlocked.Increment(ref dualResultCount);
}
return Unit.Void;
});
}
// TODO ST: now this operation may fail with TaskCancelledException. But this should be traced differently!
return await redisDb.ExecuteBatchOperationAsync(context, redisBatch, token);
}, Configuration.RetryWindow);
if (!batchResult)
{
return new Result <IReadOnlyList <ContentLocationEntry> >(batchResult);
}
}
if (RaidedRedis.HasSecondary)
{
Counters[GlobalStoreCounters.GetBulkEntrySingleResult].Add(contentHashes.Count - dualResultCount);
}
return Result.Success <IReadOnlyList <ContentLocationEntry> >(results);
},
Counters[GlobalStoreCounters.GetBulk],
traceErrorsOnly: true));
}
/// <inheritdoc />
public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ContentHash> contentHashes)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var results = new ContentLocationEntry[contentHashes.Count];
UnixTime now = _clock.UtcNow;
int dualResultCount = 0;
foreach (var page in contentHashes.AsIndexed().GetPages(_configuration.RedisBatchPageSize))
{
var batchResult = await ExecuteRedisAsync(context, async redisDb =>
{
var redisBatch = redisDb.CreateBatch(RedisOperation.GetBulkGlobal);
foreach (var indexedHash in page)
{
var key = GetRedisKey(indexedHash.Item);
redisBatch.AddOperation(key, async batch =>
{
var redisEntry = await batch.StringGetAsync(key);
ContentLocationEntry entry;
if (redisEntry.IsNullOrEmpty)
{
entry = ContentLocationEntry.Missing;
}
else
{
entry = ContentLocationEntry.FromRedisValue(redisEntry, now, missingSizeHandling: true);
}
var originalEntry = Interlocked.CompareExchange(ref results[indexedHash.Index], entry, null);
if (originalEntry != null)
{
// Existing entry was there. Merge the entries.
entry = MergeEntries(entry, originalEntry);
results[indexedHash.Index] = entry;
Interlocked.Increment(ref dualResultCount);
}
return Unit.Void;
}).FireAndForget(context);
}
return await redisDb.ExecuteBatchOperationAsync(context, redisBatch, context.Token);
});
if (!batchResult)
{
return new Result <IReadOnlyList <ContentLocationEntry> >(batchResult);
}
}
if (HasSecondary)
{
Counters[GlobalStoreCounters.GetBulkEntrySingleResult].Add(contentHashes.Count - dualResultCount);
}
return Result.Success <IReadOnlyList <ContentLocationEntry> >(results);
},
Counters[GlobalStoreCounters.GetBulk]));
}
public static ReplicaRank GetReplicaRank(
ContentHash hash,
ContentLocationEntry entry,
MachineId localMachineId,
LocalLocationStoreConfiguration configuration,
DateTime now)
{
var locationsCount = entry.Locations.Count;
var desiredReplicaCount = configuration.DesiredReplicaRetention;
if (desiredReplicaCount == 0 ||
locationsCount == 0) // It is possible for the entry to have 0 locations.
// For instance, the database has 1 location but the machine is in the bad state
{
return(ReplicaRank.None);
}
if (locationsCount <= desiredReplicaCount
// If using throttled eviction, we might need to upgrade the rank to Protected
// so don't return here
&& configuration.ThrottledEvictionInterval == TimeSpan.Zero)
{
return(ReplicaRank.Important);
}
// Making sure that probabilistically, some locations are considered important for the current machine.
long contentHashCode = unchecked ((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, hash[1]));
var importantRangeStart = contentHashCode % locationsCount;
// Getting an index of a current location in the location list
int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId);
if (currentMachineLocationIndex == -1)
{
// This is used for testing only. The machine Id should be part of the machines.
// But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important.
return(ReplicaRank.None);
}
// In case of important range wrapping around end of location list to start of location list
// we need to compute a positive offset from the range start to see if the replica exists in the range
// i.e. range start = 5, location count = 7, and desired location count = 3
// important range contains [5, 6 and 0] since it overflows the end of the list
var offset = currentMachineLocationIndex - importantRangeStart;
if (offset < 0)
{
offset += locationsCount;
}
var lastImportantReplicaOffset = Math.Min(desiredReplicaCount, locationsCount) - 1;
if (offset >= desiredReplicaCount)
{
return(ReplicaRank.None);
}
if (offset != lastImportantReplicaOffset)
{
// All but last important replica are always Protected
return(ReplicaRank.Protected);
}
if (configuration.ThrottledEvictionInterval == TimeSpan.Zero)
{
// Throttled eviction is disabled. Just mark the replica as important
// since its in the important range
return(ReplicaRank.Important);
}
// How throttled eviction works:
// 1. Compute which machines consider the content important
// This is done by computing a hash code from the content hash modulo location count to
// generate a start index into the list replicas.
// For instance,
// given locations: [4, 11, 22, 35, 73, 89]
// locationCount = 6,
// if contentHashCode % locationCount = 2 and DesiredReplicaCount = 3
// then the machines considering content important are [22, 35, 73]
// 2. All but last important replica must be consider Protected (i.e. 22, 35 have rank Protected)
// 3. Compute if last replica is protected.
// This is based of to time ranges or buckets of duration ThrottledEvictionInterval
// For instance,
// if ThrottleInterval = 20 minutes
// 10:00AM-10:20AM -> (timeBucketIndex = 23045230) % DesiredReplicaCount = 2 = evictableOffset
// 10:20AM-10:40AM -> (timeBucketIndex = 23045231) % DesiredReplicaCount = 0 = evictableOffset
// 10:40AM-11:00AM -> (timeBucketIndex = 23045232) % DesiredReplicaCount = 1 = evictableOffset
// 11:00AM-11:20AM -> (timeBucketIndex = 23045233) % DesiredReplicaCount = 2 = evictableOffset
// So for times 10:00AM-10:20AM and 11:00AM-11:20AM the last important replica is evictable
var timeBucketIndex = now.Ticks / configuration.ThrottledEvictionInterval.Ticks;
// NOTE: We add contentHashCode to timeBucketIndex so that not all Protected content is considered evictable
// at the same time
var evictableOffset = (contentHashCode + timeBucketIndex) % desiredReplicaCount;
if (evictableOffset == offset)
{
return(ReplicaRank.Important);
}
else
{
// The replica is not currently evictable. Mark it as protected which will give it the minimum effective age
// so that it is only evicted as a last resort
return(ReplicaRank.Protected);
}
}
