public ColdStorage(IAbsFileSystem fileSystem, ColdStorageSettings coldStorageSettings, DistributedContentCopier distributedContentCopier)
{
_fileSystem = fileSystem;
_copier = distributedContentCopier;
_rootPath = coldStorageSettings.GetAbsoulutePath();
ConfigurationModel configurationModel
= new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(coldStorageSettings.CacheSizeQuotaString !)));
ContentStoreSettings contentStoreSettings = FromColdStorageSettings(coldStorageSettings);
_store = new FileSystemContentStore(fileSystem, SystemClock.Instance, _rootPath, configurationModel, null, contentStoreSettings, null);
HashType hashType;
if (!Enum.TryParse <HashType>(coldStorageSettings.ConsistentHashingHashType, true, out hashType))
{
hashType = HashType.SHA256;
}
_contentHasher = HashInfoLookup.GetContentHasher(hashType);
_copiesQuantity = coldStorageSettings.ConsistentHashingCopiesQuantity;
_maxParallelPlaces = coldStorageSettings.MaxParallelPlaces;
// Starts empty and is created during the first update
_ring = new RingNode[0];
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// NOTE: We create and start the content location store before the inner content store just in case the
// inner content store starts background eviction after startup. We need the content store to be initialized
// so that it can be queried and used to unregister content.
await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure();
_contentLocationStore = await _contentLocationStoreFactory.CreateAsync();
_distributedCopier = _distributedCopierFactory(_contentLocationStore);
await _distributedCopier.StartupAsync(context).ThrowIfFailure();
if (_contentLocationStore is TransitioningContentLocationStore tcs)
{
tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier);
}
// Initializing inner store before initializing LocalLocationStore because
// LocalLocationStore may use inner store for reconciliation purposes
await InnerContentStore.StartupAsync(context).ThrowIfFailure();
await _contentLocationStore.StartupAsync(context).ThrowIfFailure();
Func <ContentHash[], Task> evictionHandler;
var localContext = new Context(context);
if (_enableDistributedEviction)
{
evictionHandler = hashes => EvictContentAsync(localContext, hashes);
}
else
{
evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes);
}
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup. So we start the queue here.
_evictionNagleQueue.Start(evictionHandler);
var touchContext = new Context(context);
_touchNagleQueue = NagleQueue <ContentHashWithSize> .Create(
hashes => TouchBulkAsync(touchContext, hashes),
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
batchSize : _locationStoreBatchSize);
return(BoolResult.Success);
}
/// <nodoc />
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier <T> distributedCopier,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_settings.LocationStoreBatchSize);
_enableDistributedEviction = _settings.ReplicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null;
var enableTouch = _settings.ContentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.IsPinCachingEnabled == true)
{
_pinCache = new PinCache(clock: _clock);
}
}
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <IDistributedLocationStore, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier distributedCopier,
IClock?clock = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock ?? SystemClock.Instance;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
_settings = settings;
InnerContentStore = innerContentStoreFunc(this);
}
/// <nodoc />
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <ContentStoreSettings, IDistributedLocationStore, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier <T> distributedCopier,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
contentStoreSettings ??= ContentStoreSettings.DefaultSettings;
_settings = settings;
InnerContentStore = innerContentStoreFunc(contentStoreSettings, this);
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// NOTE: We create and start the content location store before the inner content store just in case the
// inner content store starts background eviction after startup. We need the content store to be initialized
// so that it can be queried and used to unregister content.
await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure();
_contentLocationStore = await _contentLocationStoreFactory.CreateAsync(LocalMachineLocation);
_distributedCopier = _distributedCopierFactory(_contentLocationStore);
await _distributedCopier.StartupAsync(context).ThrowIfFailure();
if (_contentLocationStore is TransitioningContentLocationStore tcs && tcs.IsLocalLocationStoreEnabled)
{
// Define proactive copy logic.
async Task <ResultBase> proactiveCopyTaskFactory(OperationContext operationContext, ContentHash hash)
{
var sessionResult = await _proactiveCopySession.Value;
if (sessionResult)
{
return(await sessionResult.Value.ProactiveCopyIfNeededAsync(operationContext, hash, tryBuildRing : false));
}
return(new BoolResult("Failed to retrieve session for proactive copies."));
}
tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier, proactiveCopyTaskFactory);
}
// Initializing inner store before initializing LocalLocationStore because
// LocalLocationStore may use inner store for reconciliation purposes
await InnerContentStore.StartupAsync(context).ThrowIfFailure();
await _contentLocationStore.StartupAsync(context).ThrowIfFailure();
Func <ContentHash[], Task> evictionHandler;
var localContext = context.CreateNested();
if (_enableDistributedEviction)
{
evictionHandler = hashes => EvictContentAsync(localContext, hashes);
}
else
{
evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes);
}
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup. So we start the queue here.
_evictionNagleQueue.Start(evictionHandler);
var touchContext = context.CreateNested();
_touchNagleQueue = NagleQueue <ContentHashWithSize> .Create(
hashes => TouchBulkAsync(touchContext, hashes),
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
batchSize : _locationStoreBatchSize);
return(BoolResult.Success);
}
