private Task <BoolResult> ProactiveReplicationAsync(
OperationContext context,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var proactiveCopySession = await ProactiveCopySession.Value.ThrowIfFailureAsync();
await contentLocationStore.LocalLocationStore.EnsureInitializedAsync().ThrowIfFailure();
while (!context.Token.IsCancellationRequested)
{
// Create task before starting operation to ensure uniform intervals assuming operation takes less than the delay.
var delayTask = Task.Delay(_settings.ProactiveReplicationInterval, context.Token);
await ProactiveReplicationIterationAsync(context, proactiveCopySession, localContentStore, contentLocationStore).ThrowIfFailure();
if (_settings.InlineOperationsForTests)
{
// Inlining is used only for testing purposes. In those cases,
// we only perform one proactive replication.
break;
}
await delayTask;
}
return BoolResult.Success;
})
.FireAndForgetOrInlineAsync(context, _settings.InlineOperationsForTests));
}
/// <inheritdoc />
public Task <BoolResult> InvalidateLocalMachineAsync(Context context, ILocalContentStore localStore, CancellationToken cts)
{
var operationContext = new OperationContext(context, cts);
return(MultiExecuteAsync(
executeLegacyStore: () => _redisContentLocationStore.InvalidateLocalMachineAsync(context, localStore, cts),
executeLocalLocationStore: () => _localLocationStore.InvalidateLocalMachineAsync(operationContext)));
}
/// <nodoc />
public TransitioningContentLocationStore(
RedisContentLocationStoreConfiguration configuration,
LocalLocationStore localLocationStore,
MachineLocation localMachineLocation,
ILocalContentStore localContentStore)
{
Contract.Requires(configuration != null);
Contract.Requires(localMachineLocation.IsValid);
LocalContentStore = localContentStore;
_configuration = configuration;
LocalLocationStore = localLocationStore;
LocalMachineLocation = localMachineLocation;
}
/// <nodoc />
public TransitioningContentLocationStore(
RedisContentLocationStoreConfiguration configuration,
RedisContentLocationStore redisContentLocationStore,
LocalLocationStore localLocationStore,
MachineLocation localMachineLocation,
ILocalContentStore localContentStore)
{
Contract.Requires(configuration != null);
Contract.Requires(localMachineLocation.IsValid);
LocalContentStore = localContentStore;
_configuration = configuration;
_localLocationStore = localLocationStore;
LocalMachineLocation = localMachineLocation;
_redisContentLocationStore = redisContentLocationStore;
Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.Redis) || _redisContentLocationStore != null);
Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore) || _localLocationStore != null);
}
/// <inheritdoc /> public Task <BoolResult> InvalidateLocalMachineAsync(Context context, ILocalContentStore localStore, CancellationToken cts) => null;
private Task <ProactiveReplicationResult> ProactiveReplicationIterationAsync(
OperationContext context,
ReadOnlyDistributedContentSession <T> proactiveCopySession,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
// Important to yield as GetContentInfoAsync has a synchronous implementation.
await Task.Yield();
var localContent = (await localContentStore.GetContentInfoAsync(context.Token))
.OrderByDescending(info => info.LastAccessTimeUtc) // GetHashesInEvictionOrder expects entries to already be ordered by last access time.
.Select(info => new ContentHashWithLastAccessTimeAndReplicaCount(info.ContentHash, info.LastAccessTimeUtc))
.ToArray();
var contents = contentLocationStore.GetHashesInEvictionOrder(context, localContent, reverse: true);
var succeeded = 0;
var failed = 0;
var skipped = 0;
var scanned = 0;
var rejected = 0;
var delayTask = Task.CompletedTask;
var wasPreviousCopyNeeded = true;
ContentEvictionInfo?lastVisited = default;
IEnumerable <ContentEvictionInfo> getReplicableHashes()
{
foreach (var content in contents)
{
scanned++;
if (content.ReplicaCount < _settings.ProactiveCopyLocationsThreshold)
{
yield return content;
}
else
{
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
}
}
}
foreach (var page in getReplicableHashes().GetPages(_settings.ProactiveCopyGetBulkBatchSize))
{
var contentInfos = await proactiveCopySession.GetLocationsForProactiveCopyAsync(context, page.SelectList(c => c.ContentHash));
for (int i = 0; i < contentInfos.Count; i++)
{
context.Token.ThrowIfCancellationRequested();
var contentInfo = contentInfos[i];
lastVisited = page[i];
if (wasPreviousCopyNeeded)
{
await delayTask;
delayTask = Task.Delay(_settings.DelayForProactiveReplication, context.Token);
}
var result = await proactiveCopySession.ProactiveCopyIfNeededAsync(
context,
contentInfo,
tryBuildRing: false,
reason: CopyReason.Replication);
wasPreviousCopyNeeded = true;
switch (result.Status)
{
case ProactiveCopyStatus.Success:
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
succeeded++;
break;
case ProactiveCopyStatus.Skipped:
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
wasPreviousCopyNeeded = false;
break;
case ProactiveCopyStatus.Rejected:
rejected++;
CounterCollection[Counters.ProactiveReplication_Rejected].Increment();
break;
case ProactiveCopyStatus.Error:
CounterCollection[Counters.ProactiveReplication_Failed].Increment();
failed++;
break;
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
return new ProactiveReplicationResult(succeeded, failed, skipped, rejected, localContent.Length, scanned, lastVisited);
},
private Task <ProactiveReplicationResult> ProactiveReplicationIterationAsync(
OperationContext context,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
// Important to yield as GetContentInfoAsync has a synchronous implementation.
await Task.Yield();
var localContent = (await localContentStore.GetContentInfoAsync(context.Token))
.OrderByDescending(info => info.LastAccessTimeUtc) // GetHashesInEvictionOrder expects entries to already be ordered by last access time.
.Select(info => new ContentHashWithLastAccessTimeAndReplicaCount(info.ContentHash, info.LastAccessTimeUtc))
.ToArray();
var contents = contentLocationStore.GetHashesInEvictionOrder(context, localContent, reverse: true);
var succeeded = 0;
var failed = 0;
var skipped = 0;
var scanned = 0;
var rejected = 0;
var delayTask = Task.CompletedTask;
var wasPreviousCopyNeeded = true;
ContentEvictionInfo?lastVisited = default;
foreach (var content in contents)
{
context.Token.ThrowIfCancellationRequested();
lastVisited = content;
scanned++;
if (content.ReplicaCount < _settings.ProactiveCopyLocationsThreshold)
{
if (wasPreviousCopyNeeded)
{
await delayTask;
delayTask = Task.Delay(_settings.DelayForProactiveReplication, context.Token);
}
var result = await ProactiveCopyIfNeededAsync(context, content.ContentHash);
wasPreviousCopyNeeded = true;
switch (result.Status)
{
case ProactiveCopyStatus.Success:
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
succeeded++;
break;
case ProactiveCopyStatus.Skipped:
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
wasPreviousCopyNeeded = false;
break;
case ProactiveCopyStatus.Rejected:
rejected++;
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
break;
case ProactiveCopyStatus.Error:
CounterCollection[Counters.ProactiveReplication_Failed].Increment();
failed++;
break;
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
}
return new ProactiveReplicationResult(succeeded, failed, skipped, rejected, localContent.Length, scanned, lastVisited);
},
/// <inheritdoc />
public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation, ILocalContentStore localContentStore)
{
IContentLocationStore contentLocationStore = null;
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent, "RedisDatabaseFactoryForContent");
var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations, "RedisDatabaseFactoryForMachineLocations");
contentLocationStore = new RedisContentLocationStore(
redisDatabaseAdapter,
machineLocationRedisDatabaseAdapter,
Clock,
_contentHashBumpTime,
localMachineLocation.Data,
Configuration);
}
if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore))
{
var localLocationStore = _lazyLocalLocationStore.Value;
contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, localMachineLocation, localContentStore);
}
return(Task.FromResult(contentLocationStore));
}
public async Task <IContentLocationStore> CreateAsync(MachineLocation machineLocation, ILocalContentStore localContentStore)
{
var connection = MockRedisDatabaseFactory.CreateConnection(RedisDatabase);
RedisDatabaseAdapter = RedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), connection), RedisContentLocationStoreFactory.DefaultKeySpace);
var machineLocationConnection = MockRedisDatabaseFactory.CreateConnection(MachineLocationRedisDatabase);
MachineRedisDatabaseAdapter = MachineRedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), machineLocationConnection), RedisContentLocationStoreFactory.DefaultKeySpace);
IContentLocationStore store = new RedisContentLocationStore(
RedisDatabaseAdapter,
MachineRedisDatabaseAdapter,
_mockClock,
BumpTime,
machineLocation.Data,
_configuration);
var redisStore = (RedisContentLocationStore)store;
redisStore.DisableReplica = true;
return(store);
}