private Task <BoolResult> ProactiveReplicationAsync(
OperationContext context,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var proactiveCopySession = await ProactiveCopySession.Value.ThrowIfFailureAsync();
await contentLocationStore.LocalLocationStore.EnsureInitializedAsync().ThrowIfFailure();
while (!context.Token.IsCancellationRequested)
{
// Create task before starting operation to ensure uniform intervals assuming operation takes less than the delay.
var delayTask = Task.Delay(_settings.ProactiveReplicationInterval, context.Token);
await ProactiveReplicationIterationAsync(context, proactiveCopySession, localContentStore, contentLocationStore).ThrowIfFailure();
if (_settings.InlineOperationsForTests)
{
// Inlining is used only for testing purposes. In those cases,
// we only perform one proactive replication.
break;
}
await delayTask;
}
return BoolResult.Success;
})
.FireAndForgetOrInlineAsync(context, _settings.InlineOperationsForTests));
}
/// <inheritdoc />
public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation)
{
IContentLocationStore contentLocationStore = null;
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent);
var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations);
contentLocationStore = new RedisContentLocationStore(
redisDatabaseAdapter,
machineLocationRedisDatabaseAdapter,
Clock,
_contentHashBumpTime,
localMachineLocation.Data,
Configuration);
}
if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore))
{
Contract.Assert(RedisDatabaseFactoryForRedisGlobalStore != null);
var redisDatabaseForGlobalStore = CreateDatabase(RedisDatabaseFactoryForRedisGlobalStore);
var secondaryRedisDatabaseForGlobalStore = CreateDatabase(RedisDatabaseFactoryForRedisGlobalStoreSecondary, optional: true);
IGlobalLocationStore globalStore = new RedisGlobalStore(Clock, Configuration, localMachineLocation, redisDatabaseForGlobalStore, secondaryRedisDatabaseForGlobalStore);
var localLocationStore = new LocalLocationStore(Clock, globalStore, Configuration);
contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, Clock);
}
return(Task.FromResult(contentLocationStore));
}
/// <inheritdoc />
public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation, ILocalContentStore localContentStore)
{
IContentLocationStore contentLocationStore = null;
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent, "RedisDatabaseFactoryForContent");
var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations, "RedisDatabaseFactoryForMachineLocations");
contentLocationStore = new RedisContentLocationStore(
redisDatabaseAdapter,
machineLocationRedisDatabaseAdapter,
Clock,
_contentHashBumpTime,
localMachineLocation.Data,
Configuration);
}
if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore))
{
var localLocationStore = _lazyLocalLocationStore.Value;
contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, localMachineLocation, localContentStore);
}
return(Task.FromResult(contentLocationStore));
}
public static Task <GetBulkLocationsResult> GetBulkLocalAsync(
this TransitioningContentLocationStore store,
Context context,
params ContentHash[] contentHashes)
{
return(store.GetBulkAsync(context, contentHashes, CancellationToken.None, UrgencyHint.Nominal, GetBulkOrigin.Local));
}
public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation, ILocalContentStore?localContentStore)
{
IContentLocationStore contentLocationStore = new TransitioningContentLocationStore(
Configuration,
Services.LocalLocationStore.Instance,
localMachineLocation,
localContentStore);
return(Task.FromResult(contentLocationStore));
}
private Task <ProactiveReplicationResult> ProactiveReplicationIterationAsync(
OperationContext context,
ReadOnlyDistributedContentSession <T> proactiveCopySession,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
// Important to yield as GetContentInfoAsync has a synchronous implementation.
await Task.Yield();
var localContent = (await localContentStore.GetContentInfoAsync(context.Token))
.OrderByDescending(info => info.LastAccessTimeUtc) // GetHashesInEvictionOrder expects entries to already be ordered by last access time.
.Select(info => new ContentHashWithLastAccessTimeAndReplicaCount(info.ContentHash, info.LastAccessTimeUtc))
.ToArray();
var contents = contentLocationStore.GetHashesInEvictionOrder(context, localContent, reverse: true);
var succeeded = 0;
var failed = 0;
var skipped = 0;
var scanned = 0;
var rejected = 0;
var delayTask = Task.CompletedTask;
var wasPreviousCopyNeeded = true;
ContentEvictionInfo?lastVisited = default;
IEnumerable <ContentEvictionInfo> getReplicableHashes()
{
foreach (var content in contents)
{
scanned++;
if (content.ReplicaCount < _settings.ProactiveCopyLocationsThreshold)
{
yield return content;
}
else
{
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
}
}
}
foreach (var page in getReplicableHashes().GetPages(_settings.ProactiveCopyGetBulkBatchSize))
{
var contentInfos = await proactiveCopySession.GetLocationsForProactiveCopyAsync(context, page.SelectList(c => c.ContentHash));
for (int i = 0; i < contentInfos.Count; i++)
{
context.Token.ThrowIfCancellationRequested();
var contentInfo = contentInfos[i];
lastVisited = page[i];
if (wasPreviousCopyNeeded)
{
await delayTask;
delayTask = Task.Delay(_settings.DelayForProactiveReplication, context.Token);
}
var result = await proactiveCopySession.ProactiveCopyIfNeededAsync(
context,
contentInfo,
tryBuildRing: false,
reason: CopyReason.Replication);
wasPreviousCopyNeeded = true;
switch (result.Status)
{
case ProactiveCopyStatus.Success:
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
succeeded++;
break;
case ProactiveCopyStatus.Skipped:
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
wasPreviousCopyNeeded = false;
break;
case ProactiveCopyStatus.Rejected:
rejected++;
CounterCollection[Counters.ProactiveReplication_Rejected].Increment();
break;
case ProactiveCopyStatus.Error:
CounterCollection[Counters.ProactiveReplication_Failed].Increment();
failed++;
break;
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
return new ProactiveReplicationResult(succeeded, failed, skipped, rejected, localContent.Length, scanned, lastVisited);
},
private Task <ProactiveReplicationResult> ProactiveReplicationIterationAsync(
OperationContext context,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
// Important to yield as GetContentInfoAsync has a synchronous implementation.
await Task.Yield();
var localContent = (await localContentStore.GetContentInfoAsync(context.Token))
.OrderByDescending(info => info.LastAccessTimeUtc) // GetHashesInEvictionOrder expects entries to already be ordered by last access time.
.Select(info => new ContentHashWithLastAccessTimeAndReplicaCount(info.ContentHash, info.LastAccessTimeUtc))
.ToArray();
var contents = contentLocationStore.GetHashesInEvictionOrder(context, localContent, reverse: true);
var succeeded = 0;
var failed = 0;
var skipped = 0;
var scanned = 0;
var rejected = 0;
var delayTask = Task.CompletedTask;
var wasPreviousCopyNeeded = true;
ContentEvictionInfo?lastVisited = default;
foreach (var content in contents)
{
context.Token.ThrowIfCancellationRequested();
lastVisited = content;
scanned++;
if (content.ReplicaCount < _settings.ProactiveCopyLocationsThreshold)
{
if (wasPreviousCopyNeeded)
{
await delayTask;
delayTask = Task.Delay(_settings.DelayForProactiveReplication, context.Token);
}
var result = await ProactiveCopyIfNeededAsync(context, content.ContentHash);
wasPreviousCopyNeeded = true;
switch (result.Status)
{
case ProactiveCopyStatus.Success:
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
succeeded++;
break;
case ProactiveCopyStatus.Skipped:
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
wasPreviousCopyNeeded = false;
break;
case ProactiveCopyStatus.Rejected:
rejected++;
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
break;
case ProactiveCopyStatus.Error:
CounterCollection[Counters.ProactiveReplication_Failed].Increment();
failed++;
break;
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
}
return new ProactiveReplicationResult(succeeded, failed, skipped, rejected, localContent.Length, scanned, lastVisited);
},
