/// <nodoc /> public void Store(OperationContext context, ShortHash hash, ContentLocationEntry entry) { using (_exchangeLock.AcquireReadLock()) { _cache[hash] = entry; } }
private ContentLocationEntry MergeEntries(ContentLocationEntry entry, ContentLocationEntry originalEntry) { if (entry.IsMissing) { return(originalEntry); } if (originalEntry.IsMissing) { return(entry); } return(entry.SetMachineExistence(originalEntry.Locations, exists: true));; }
public static bool IsImportantReplica(ContentHash hash, ContentLocationEntry entry, MachineId localMachineId, long desiredReplicaCount) { var locationsCount = entry.Locations.Count; if (locationsCount <= desiredReplicaCount) { return(true); } if (desiredReplicaCount == 0) { return(false); } // Making sure that probabilistically, some locations are considered important for the current machine. long replicaHash = unchecked ((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, localMachineId.Index + 1)); var offset = replicaHash % locationsCount; var importantRangeStart = offset; var importantRangeStop = (offset + desiredReplicaCount) % locationsCount; // Getting an index of a current location in the location list int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId); if (currentMachineLocationIndex == -1) { // This is used for testing only. The machine Id should be part of the machines. // But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important. return(false); } // For instance, for locations [1, 20] // the important range can be [5, 7] // or [19, 1] if (importantRangeStart < importantRangeStop) { // This is the first case: the start index is less then the stop, // so the desired location should be within the range return(currentMachineLocationIndex >= importantRangeStart && currentMachineLocationIndex <= importantRangeStop); } // Important range is broken because the start is greater then the end. // like [19, 1], so the location is important if it's index is >= start or <= the stop. // For instance, for 10 locations the start is 9 and the end is 2 return(currentMachineLocationIndex >= importantRangeStart || currentMachineLocationIndex <= importantRangeStop); }
public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ShortHash> contentHashes) { var entries = new ContentLocationEntry[contentHashes.Count]; for (int i = 0; i < contentHashes.Count; i++) { var hash = contentHashes[i]; if (!_database.TryGetEntry(context, hash, out var entry)) { entry = ContentLocationEntry.Missing; } entries[i] = entry; } return(Task.FromResult(Result.Success <IReadOnlyList <ContentLocationEntry> >(entries))); }
/// <nodoc /> public static ContentLocationEntry MergeEntries(ContentLocationEntry entry1, ContentLocationEntry entry2) { if (entry1 == null || entry1.IsMissing) { return(entry2); } if (entry2 == null || entry2.IsMissing) { return(entry1); } return(new ContentLocationEntry( entry1.Locations.SetExistence(entry2.Locations, true), entry1.ContentSize, UnixTime.Min(entry1.LastAccessTimeUtc, entry2.LastAccessTimeUtc), UnixTime.Min(entry1.CreationTimeUtc, entry2.CreationTimeUtc))); }
private async Task<Unit> SetLocationBitAndExpireAsync(OperationContext context, IBatch batch, RedisKey key, ShortHashWithSize hash, MachineId machineId) { var tasks = new List<Task>(); // NOTE: The order here matters. KeyExpire must be after creation of the entry. SetBit creates the entry if needed. tasks.Add(batch.StringSetBitAsync(key, machineId.GetContentLocationEntryBitOffset(), true)); tasks.Add(batch.KeyExpireAsync(key, Configuration.LocationEntryExpiry)); // NOTE: We don't set the size when using optimistic location registration because the entry should have already been created at this point (the prior set // if not exists call failed indicating the entry already exists). // There is a small race condition if the entry was near-expiry and this call ends up recreating the entry without the size being set. We accept // this possibility since we have to handle unknown size and either case and the occurrence of the race should be rare. Also, we can mitigate by // applying the size from the local database which should be known if the entry is that old. if (!Configuration.UseOptimisticRegisterLocalLocation && hash.Size >= 0) { tasks.Add(batch.StringSetRangeAsync(key, 0, ContentLocationEntry.ConvertSizeToRedisRangeBytes(hash.Size))); } await Task.WhenAll(tasks); return Unit.Void; }
/// <nodoc /> public bool TryGetEntry(ShortHash hash, out ContentLocationEntry entry) { using (_exchangeLock.AcquireReadLock()) { // The entry could be a tombstone, so we need to make sure the user knows content has actually been // deleted, which is why we check for null. if (_cache.TryGetValue(hash, out entry)) { _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheHit].Increment(); return(entry != null); } else if (_flushingCache.TryGetValue(hash, out entry)) { _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheHit].Increment(); return(entry != null); } } _database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheMiss].Increment(); return(false); }
private Task <BoolResult> RegisterLocationAsync( OperationContext context, IReadOnlyList <ShortHashWithSize> contentHashes, MachineId machineId, [CallerMemberName] string caller = null) { const int operationsPerHash = 3; var hashBatchSize = Math.Max(1, Configuration.RedisBatchPageSize / operationsPerHash); return(context.PerformOperationAsync( Tracer, async() => { foreach (var page in contentHashes.GetPages(hashBatchSize)) { var batchResult = await RaidedRedis.ExecuteRedisAsync(context, async(redisDb, token) => { Counters[GlobalStoreCounters.RegisterLocalLocationHashCount].Add(page.Count); int requiresSetBitCount; ConcurrentBitArray requiresSetBit; if (Configuration.UseOptimisticRegisterLocalLocation) { requiresSetBitCount = 0; requiresSetBit = new ConcurrentBitArray(page.Count); var redisBatch = redisDb.CreateBatch(RedisOperation.RegisterLocalSetNonExistentHashEntries); // Perform initial pass to set redis entries in single operation. Fallback to more elaborate // flow where we use SetBit + KeyExpire foreach (var indexedHash in page.WithIndices()) { var hash = indexedHash.value; var key = GetRedisKey(hash.Hash); redisBatch.AddOperationAndTraceIfFailure(context, key, async batch => { bool set = await batch.StringSetAsync(key, ContentLocationEntry.ConvertSizeAndMachineIdToRedisValue(hash.Size, machineId), Configuration.LocationEntryExpiry, When.NotExists); if (!set) { requiresSetBit[indexedHash.index] = true; Interlocked.Increment(ref requiresSetBitCount); } return set; }, operationName: "ConvertSizeAndMachineIdToRedisValue"); } var result = await redisDb.ExecuteBatchOperationAsync(context, redisBatch, token); if (!result || requiresSetBitCount == 0) { return result; } } else { requiresSetBitCount = page.Count; requiresSetBit = null; } // Some keys already exist and require that we set the bit and update the expiry on the existing entry using (Counters[GlobalStoreCounters.RegisterLocalLocationUpdate].Start()) { Counters[GlobalStoreCounters.RegisterLocalLocationUpdateHashCount].Add(requiresSetBitCount); var updateRedisBatch = redisDb.CreateBatch(RedisOperation.RegisterLocalSetHashEntries); foreach (var hash in page.Where((h, index) => requiresSetBit?[index] ?? true)) { var key = GetRedisKey(hash.Hash); updateRedisBatch.AddOperationAndTraceIfFailure( context, key, batch => SetLocationBitAndExpireAsync(context, batch, key, hash, machineId), operationName: "SetLocationBitAndExpireAsync"); } return await redisDb.ExecuteBatchOperationAsync(context, updateRedisBatch, token); } }, Configuration.RetryWindow); if (!batchResult) { return batchResult; } } return BoolResult.Success; }, Counters[GlobalStoreCounters.RegisterLocalLocation], caller: caller, traceErrorsOnly: true)); }
/// <inheritdoc /> public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ShortHash> contentHashes) { return(context.PerformOperationAsync( Tracer, async() => { var results = new ContentLocationEntry[contentHashes.Count]; UnixTime now = _clock.UtcNow; int dualResultCount = 0; foreach (var page in contentHashes.AsIndexed().GetPages(Configuration.RedisBatchPageSize)) { var batchResult = await RaidedRedis.ExecuteRedisAsync(context, async(redisDb, token) => { var redisBatch = redisDb.CreateBatch(RedisOperation.GetBulkGlobal); foreach (var indexedHash in page) { var key = GetRedisKey(indexedHash.Item); redisBatch.AddOperationAndTraceIfFailure(context, key, async batch => { var redisEntry = await batch.StringGetAsync(key); ContentLocationEntry entry; if (redisEntry.IsNullOrEmpty) { entry = ContentLocationEntry.Missing; } else { entry = ContentLocationEntry.FromRedisValue(redisEntry, now, missingSizeHandling: true); } var originalEntry = Interlocked.CompareExchange(ref results[indexedHash.Index], entry, null); if (originalEntry != null) { // Existing entry was there. Merge the entries. entry = ContentLocationEntry.MergeEntries(entry, originalEntry); Interlocked.Exchange(ref results[indexedHash.Index], entry); Interlocked.Increment(ref dualResultCount); } return Unit.Void; }); } // TODO ST: now this operation may fail with TaskCancelledException. But this should be traced differently! return await redisDb.ExecuteBatchOperationAsync(context, redisBatch, token); }, Configuration.RetryWindow); if (!batchResult) { return new Result <IReadOnlyList <ContentLocationEntry> >(batchResult); } } if (RaidedRedis.HasSecondary) { Counters[GlobalStoreCounters.GetBulkEntrySingleResult].Add(contentHashes.Count - dualResultCount); } return Result.Success <IReadOnlyList <ContentLocationEntry> >(results); }, Counters[GlobalStoreCounters.GetBulk], traceErrorsOnly: true)); }
/// <inheritdoc /> public Task <Result <IReadOnlyList <ContentLocationEntry> > > GetBulkAsync(OperationContext context, IReadOnlyList <ContentHash> contentHashes) { return(context.PerformOperationAsync( Tracer, async() => { var results = new ContentLocationEntry[contentHashes.Count]; UnixTime now = _clock.UtcNow; int dualResultCount = 0; foreach (var page in contentHashes.AsIndexed().GetPages(_configuration.RedisBatchPageSize)) { var batchResult = await ExecuteRedisAsync(context, async redisDb => { var redisBatch = redisDb.CreateBatch(RedisOperation.GetBulkGlobal); foreach (var indexedHash in page) { var key = GetRedisKey(indexedHash.Item); redisBatch.AddOperation(key, async batch => { var redisEntry = await batch.StringGetAsync(key); ContentLocationEntry entry; if (redisEntry.IsNullOrEmpty) { entry = ContentLocationEntry.Missing; } else { entry = ContentLocationEntry.FromRedisValue(redisEntry, now, missingSizeHandling: true); } var originalEntry = Interlocked.CompareExchange(ref results[indexedHash.Index], entry, null); if (originalEntry != null) { // Existing entry was there. Merge the entries. entry = MergeEntries(entry, originalEntry); results[indexedHash.Index] = entry; Interlocked.Increment(ref dualResultCount); } return Unit.Void; }).FireAndForget(context); } return await redisDb.ExecuteBatchOperationAsync(context, redisBatch, context.Token); }); if (!batchResult) { return new Result <IReadOnlyList <ContentLocationEntry> >(batchResult); } } if (HasSecondary) { Counters[GlobalStoreCounters.GetBulkEntrySingleResult].Add(contentHashes.Count - dualResultCount); } return Result.Success <IReadOnlyList <ContentLocationEntry> >(results); }, Counters[GlobalStoreCounters.GetBulk])); }
public static ReplicaRank GetReplicaRank( ContentHash hash, ContentLocationEntry entry, MachineId localMachineId, LocalLocationStoreConfiguration configuration, DateTime now) { var locationsCount = entry.Locations.Count; var desiredReplicaCount = configuration.DesiredReplicaRetention; if (desiredReplicaCount == 0 || locationsCount == 0) // It is possible for the entry to have 0 locations. // For instance, the database has 1 location but the machine is in the bad state { return(ReplicaRank.None); } if (locationsCount <= desiredReplicaCount // If using throttled eviction, we might need to upgrade the rank to Protected // so don't return here && configuration.ThrottledEvictionInterval == TimeSpan.Zero) { return(ReplicaRank.Important); } // Making sure that probabilistically, some locations are considered important for the current machine. long contentHashCode = unchecked ((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, hash[1])); var importantRangeStart = contentHashCode % locationsCount; // Getting an index of a current location in the location list int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId); if (currentMachineLocationIndex == -1) { // This is used for testing only. The machine Id should be part of the machines. // But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important. return(ReplicaRank.None); } // In case of important range wrapping around end of location list to start of location list // we need to compute a positive offset from the range start to see if the replica exists in the range // i.e. range start = 5, location count = 7, and desired location count = 3 // important range contains [5, 6 and 0] since it overflows the end of the list var offset = currentMachineLocationIndex - importantRangeStart; if (offset < 0) { offset += locationsCount; } var lastImportantReplicaOffset = Math.Min(desiredReplicaCount, locationsCount) - 1; if (offset >= desiredReplicaCount) { return(ReplicaRank.None); } if (offset != lastImportantReplicaOffset) { // All but last important replica are always Protected return(ReplicaRank.Protected); } if (configuration.ThrottledEvictionInterval == TimeSpan.Zero) { // Throttled eviction is disabled. Just mark the replica as important // since its in the important range return(ReplicaRank.Important); } // How throttled eviction works: // 1. Compute which machines consider the content important // This is done by computing a hash code from the content hash modulo location count to // generate a start index into the list replicas. // For instance, // given locations: [4, 11, 22, 35, 73, 89] // locationCount = 6, // if contentHashCode % locationCount = 2 and DesiredReplicaCount = 3 // then the machines considering content important are [22, 35, 73] // 2. All but last important replica must be consider Protected (i.e. 22, 35 have rank Protected) // 3. Compute if last replica is protected. // This is based of to time ranges or buckets of duration ThrottledEvictionInterval // For instance, // if ThrottleInterval = 20 minutes // 10:00AM-10:20AM -> (timeBucketIndex = 23045230) % DesiredReplicaCount = 2 = evictableOffset // 10:20AM-10:40AM -> (timeBucketIndex = 23045231) % DesiredReplicaCount = 0 = evictableOffset // 10:40AM-11:00AM -> (timeBucketIndex = 23045232) % DesiredReplicaCount = 1 = evictableOffset // 11:00AM-11:20AM -> (timeBucketIndex = 23045233) % DesiredReplicaCount = 2 = evictableOffset // So for times 10:00AM-10:20AM and 11:00AM-11:20AM the last important replica is evictable var timeBucketIndex = now.Ticks / configuration.ThrottledEvictionInterval.Ticks; // NOTE: We add contentHashCode to timeBucketIndex so that not all Protected content is considered evictable // at the same time var evictableOffset = (contentHashCode + timeBucketIndex) % desiredReplicaCount; if (evictableOffset == offset) { return(ReplicaRank.Important); } else { // The replica is not currently evictable. Mark it as protected which will give it the minimum effective age // so that it is only evicted as a last resort return(ReplicaRank.Protected); } }