/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
IReadOnlyList <TimeSpan> retryIntervalForCopies = null,
PinConfiguration pinConfiguration = null,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
bool useTrustedHash = false,
int trustedHashFileSizeBoundary = -1,
long parallelHashingFileSizeBoundary = -1,
int maxConcurrentCopyOperations = 512,
ContentStoreSettings contentStoreSettings = null,
bool enableProactiveCopy = false)
: this(
localMachineLocation,
innerContentStoreFunc,
contentLocationStoreFactory,
fileExistenceChecker,
fileCopier,
pathTransform,
copyRequester,
contentAvailabilityGuarantee,
tempFolderForCopies,
fileSystem,
locationStoreBatchSize,
new DistributedContentStoreSettings()
{
UseTrustedHash = useTrustedHash,
TrustedHashFileSizeBoundary = trustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = parallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = maxConcurrentCopyOperations,
RetryIntervalForCopies = retryIntervalForCopies,
PinConfiguration = pinConfiguration,
EnableProactiveCopy = enableProactiveCopy
},
replicaCreditInMinutes,
clock,
enableRepairHandling,
contentHashBumpTime,
contentStoreSettings)
{
// This constructor is used from tests,
// so we need to complete _postInitializationCompletion when startup is done.
_setPostInitializationCompletionAfterStartup = true;
}
public async Task PinWithRedundantRecordAvailability()
{
ContentAvailabilityGuarantee = ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee.RedundantFileRecordsOrCheckFileExistence;
await RunTestAsync(
new Context(Logger),
3,
async context =>
{
var sessions = context.Sessions;
// Insert random file in session 0
var randomBytes1 = ThreadSafeRandom.GetBytes(0x40);
var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token);
var localContentPath1 = PathUtilities.GetContentPath(context.Directories[0].Path / "Root", putResult1.ContentHash);
Assert.True(putResult1.Succeeded);
// Case 1: Underreplicated file exists
context.FileCopier.SetNextFileExistenceResult(localContentPath1, FileExistenceResult.ResultCode.FileExists);
var pinResult = await sessions[2].PinAsync(context, putResult1.ContentHash, Token);
Assert.Equal(PinResult.ResultCode.Success, pinResult.Code);
// Case 2: Underreplicated file does not exist
context.FileCopier.SetNextFileExistenceResult(localContentPath1, FileExistenceResult.ResultCode.FileNotFound);
pinResult = await sessions[2].PinAsync(context, putResult1.ContentHash, Token);
Assert.Equal(PinResult.ResultCode.ContentNotFound, pinResult.Code);
// Now insert the content into session 1 to ensure it is sufficiently replicated
var putResult2 = await sessions[1].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token);
var localContentPath2 = PathUtilities.GetContentPath(context.Directories[1].Path / "Root", putResult2.ContentHash);
Assert.True(putResult2.Succeeded);
// Ensure both files don't exist from file copier's point of view to verify that existence isn't checked
var priorExistenceCheckCount1 = context.FileCopier.GetExistenceCheckCount(localContentPath1);
// Should have checked existence already for content in session0 for Case1 and Case 2
Assert.Equal(2, priorExistenceCheckCount1);
var priorExistenceCheckCount2 = context.FileCopier.GetExistenceCheckCount(localContentPath2);
Assert.Equal(0, priorExistenceCheckCount2);
pinResult = await sessions[2].PinAsync(context, putResult1.ContentHash, Token);
Assert.Equal(PinResult.ResultCode.Success, pinResult.Code);
});
}
/// <inheritdoc />
public CreateSessionResult <IReadOnlyContentSession> CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateReadOnlySessionCall.Run(_tracer, OperationContext(context), name, () =>
{
CreateSessionResult <IContentSession> innerSessionResult = InnerContentStore.CreateSession(context, name, implicitPin);
if (innerSessionResult.Succeeded)
{
var session = new ReadOnlyDistributedContentSession <T>(
name,
innerSessionResult.Session,
_contentLocationStore,
_distributedCopier,
LocalMachineLocation,
pinCache: _pinCache,
contentTrackerUpdater: _contentTrackerUpdater,
settings: _settings);
return new CreateSessionResult <IReadOnlyContentSession>(session);
}
return new CreateSessionResult <IReadOnlyContentSession>(innerSessionResult, "Could not initialize inner content session with error");
}));
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
DistributedContentStoreSettings settings,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = new MachineLocation(localMachineLocation);
_enableRepairHandling = enableRepairHandling;
_contentLocationStoreFactory = contentLocationStoreFactory;
_contentAvailabilityGuarantee = contentAvailabilityGuarantee;
_locationStoreBatchSize = locationStoreBatchSize;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_locationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
copyRequester,
pathTransform,
contentLocationStore));
};
_enableDistributedEviction = replicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;
var enableTouch = contentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.UsePinCache == true)
{
_pinCache = new PinCache(clock: clock);
}
}
private Task <ProactiveReplicationResult> ProactiveReplicationIterationAsync(
OperationContext context,
ReadOnlyDistributedContentSession <T> proactiveCopySession,
ILocalContentStore localContentStore,
TransitioningContentLocationStore contentLocationStore)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
// Important to yield as GetContentInfoAsync has a synchronous implementation.
await Task.Yield();
var localContent = (await localContentStore.GetContentInfoAsync(context.Token))
.OrderByDescending(info => info.LastAccessTimeUtc) // GetHashesInEvictionOrder expects entries to already be ordered by last access time.
.Select(info => new ContentHashWithLastAccessTimeAndReplicaCount(info.ContentHash, info.LastAccessTimeUtc))
.ToArray();
var contents = contentLocationStore.GetHashesInEvictionOrder(context, localContent, reverse: true);
var succeeded = 0;
var failed = 0;
var skipped = 0;
var scanned = 0;
var rejected = 0;
var delayTask = Task.CompletedTask;
var wasPreviousCopyNeeded = true;
ContentEvictionInfo?lastVisited = default;
IEnumerable <ContentEvictionInfo> getReplicableHashes()
{
foreach (var content in contents)
{
scanned++;
if (content.ReplicaCount < _settings.ProactiveCopyLocationsThreshold)
{
yield return content;
}
else
{
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
}
}
}
foreach (var page in getReplicableHashes().GetPages(_settings.ProactiveCopyGetBulkBatchSize))
{
var contentInfos = await proactiveCopySession.GetLocationsForProactiveCopyAsync(context, page.SelectList(c => c.ContentHash));
for (int i = 0; i < contentInfos.Count; i++)
{
context.Token.ThrowIfCancellationRequested();
var contentInfo = contentInfos[i];
lastVisited = page[i];
if (wasPreviousCopyNeeded)
{
await delayTask;
delayTask = Task.Delay(_settings.DelayForProactiveReplication, context.Token);
}
var result = await proactiveCopySession.ProactiveCopyIfNeededAsync(
context,
contentInfo,
tryBuildRing: false,
reason: CopyReason.Replication);
wasPreviousCopyNeeded = true;
switch (result.Status)
{
case ProactiveCopyStatus.Success:
CounterCollection[Counters.ProactiveReplication_Succeeded].Increment();
succeeded++;
break;
case ProactiveCopyStatus.Skipped:
CounterCollection[Counters.ProactiveReplication_Skipped].Increment();
skipped++;
wasPreviousCopyNeeded = false;
break;
case ProactiveCopyStatus.Rejected:
rejected++;
CounterCollection[Counters.ProactiveReplication_Rejected].Increment();
break;
case ProactiveCopyStatus.Error:
CounterCollection[Counters.ProactiveReplication_Failed].Increment();
failed++;
break;
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
if ((succeeded + failed) >= _settings.ProactiveReplicationCopyLimit)
{
break;
}
}
return new ProactiveReplicationResult(succeeded, failed, skipped, rejected, localContent.Length, scanned, lastVisited);
},