public DistributedCentralStorage(
DistributedCentralStoreConfiguration configuration,
ILocationStore locationStore,
DistributedContentCopier copier,
CentralStorage fallbackStorage)
{
_configuration = configuration;
_copier = copier;
_fallbackStorage = fallbackStorage;
_locationStore = locationStore;
var maxRetentionMb = (int)Math.Ceiling(configuration.MaxRetentionGb * 1024);
var softRetentionMb = (int)(maxRetentionMb * 0.8);
var cacheFolder = configuration.CacheRoot / CacheSubFolderName;
_copierWorkingDirectory = new DisposableDirectory(copier.FileSystem, cacheFolder / "Temp");
// Create a private CAS for storing checkpoint data
// Avoid introducing churn into primary CAS
_privateCas = new FileSystemContentStoreInternal(
copier.FileSystem,
SystemClock.Instance,
cacheFolder,
new ConfigurationModel(
new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")),
ConfigurationSelection.RequireAndUseInProcessConfiguration),
settings: new ContentStoreSettings()
{
TraceFileSystemContentStoreDiagnosticMessages = _configuration.TraceFileSystemContentStoreDiagnosticMessages,
SelfCheckSettings = _configuration.SelfCheckSettings,
});
}
public DistributedContentStoreFactory(DistributedCacheServiceArguments arguments)
{
_logger = arguments.Logger;
_arguments = arguments;
_distributedSettings = arguments.Configuration.DistributedContentSettings;
_keySpace = string.IsNullOrWhiteSpace(_arguments.Keyspace) ? ContentLocationStoreFactory.DefaultKeySpace : _arguments.Keyspace;
_fileSystem = new PassThroughFileSystem(_logger);
_secretRetriever = new DistributedCacheSecretRetriever(arguments);
var bandwidthCheckedCopier = new BandwidthCheckedCopier(_arguments.Copier, BandwidthChecker.Configuration.FromDistributedContentSettings(_distributedSettings));
_orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments);
Contract.Assert(_orderedResolvedCacheSettings.Count != 0);
RedisContentLocationStoreConfiguration = CreateRedisConfiguration();
_distributedContentStoreSettings = CreateDistributedStoreSettings(_arguments, RedisContentLocationStoreConfiguration);
_copier = new DistributedContentCopier <AbsolutePath>(
_distributedContentStoreSettings,
_fileSystem,
fileCopier: bandwidthCheckedCopier,
fileExistenceChecker: _arguments.Copier,
_arguments.CopyRequester,
_arguments.PathTransformer,
_arguments.Overrides.Clock
);
_redisMemoizationStoreFactory = new Lazy <RedisMemoizationStoreFactory>(() => CreateRedisCacheFactory());
}
/// <nodoc />
public RedisMemoizationStoreFactory(
IClock clock,
RedisMemoizationStoreConfiguration configuration,
DistributedContentCopier copier)
: base(clock, configuration, copier)
{
}
public DistributedContentStoreFactory(DistributedCacheServiceArguments arguments)
{
_logger = arguments.Logger;
_arguments = arguments;
_distributedSettings = arguments.Configuration.DistributedContentSettings;
_keySpace = string.IsNullOrWhiteSpace(_arguments.Keyspace) ? ContentLocationStoreFactory.DefaultKeySpace : _arguments.Keyspace;
_fileSystem = arguments.FileSystem;
_secretRetriever = new DistributedCacheSecretRetriever(arguments);
_orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments);
Contract.Assert(_orderedResolvedCacheSettings.Count != 0);
RedisContentLocationStoreConfiguration = CreateRedisConfiguration();
_distributedContentStoreSettings = CreateDistributedStoreSettings(_arguments, RedisContentLocationStoreConfiguration);
_copier = new DistributedContentCopier(
_distributedContentStoreSettings,
_fileSystem,
fileCopier: _arguments.Copier,
copyRequester: _arguments.CopyRequester,
_arguments.Overrides.Clock,
_logger
);
_redisMemoizationStoreFactory = new Lazy <RedisMemoizationStoreFactory>(() => CreateRedisCacheFactory());
}
public ContentLocationStoreFactory(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
DistributedContentCopier copier)
: this(
new ContentLocationStoreFactoryArguments()
{
Clock = clock,
Copier = copier
},
configuration)
{
}
public ContentLocationStoreFactory(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
DistributedContentCopier copier)
{
Contract.Requires(configuration != null);
Contract.Requires(!string.IsNullOrEmpty(configuration.RedisGlobalStoreConnectionString));
Contract.Requires(!string.IsNullOrWhiteSpace(configuration.Keyspace));
Contract.Requires(copier != null);
Clock = clock;
Copier = copier;
_lazyLocalLocationStore = new Lazy <LocalLocationStore>(() => CreateLocalLocationStore());
Configuration = configuration;
}
public DistributedCentralStorage(
DistributedCentralStoreConfiguration configuration,
ILocationStore locationStore,
DistributedContentCopier copier,
CentralStorage fallbackStorage,
IClock clock)
: base(configuration, fallbackStorage, copier.FileSystem)
{
_copier = copier;
_locationStore = locationStore;
_checkpointCopies = new VolatileMap <ShortHash, CopyOperation>(clock);
_checkpointStore = configuration.IsCheckpointAware ? _locationStore as ICheckpointStore : null;
_copierWorkingDirectory = new DisposableDirectory(copier.FileSystem, PrivateCas !.RootPath / "Temp");
}
/// <nodoc />
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier contentCopier,
IDistributedContentCopierHost copierHost,
MachineLocation localMachineLocation,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
copierHost,
localMachineLocation,
settings)
{
}
private async Task <(DistributedContentCopier <AbsolutePath>, MockFileCopier)> CreateAsync(Context context, AbsolutePath rootDirectory, int retries = 1)
{
var mockFileCopier = new MockFileCopier();
var existenceChecker = new TestFileCopier();
var contentCopier = new DistributedContentCopier <AbsolutePath>(
rootDirectory,
// Need to use exactly one retry.
new DistributedContentStoreSettings()
{
RetryIntervalForCopies = Enumerable.Range(0, retries).Select(r => TimeSpan.Zero).ToArray()
},
FileSystem,
mockFileCopier,
existenceChecker,
new NoOpPathTransformer(rootDirectory),
new MockContentLocationStore());
await contentCopier.StartupAsync(context).ThrowIfFailure();
return(contentCopier, mockFileCopier);
}
/// <nodoc />
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier contentCopier,
DistributedContentStore distributedStore,
MachineLocation localMachineLocation,
ColdStorage coldStorage,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
distributedStore,
localMachineLocation,
coldStorage,
settings)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier <T> contentCopier,
MachineLocation localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
_putFileGate = new SemaphoreSlim(settings.MaximumConcurrentPutFileOperations);
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
var settings = CreateSettings();
var distributedCopier = new DistributedContentCopier <AbsolutePath>(
settings,
FileSystem,
fileCopier,
fileCopier,
copyRequester: null,
storeFactory.PathTransformer,
SystemClock.Instance);
return(new DistributedContentStore <AbsolutePath>(
localMachineLocation,
rootPath,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
settings: settings,
distributedCopier: distributedCopier));
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
ContentAvailabilityGuarantee contentAvailabilityGuarantee,
DistributedContentCopier <T> contentCopier,
byte[] localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentAvailabilityGuarantee,
contentCopier,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier <T> contentCopier,
IDistributedContentCopierHost copierHost,
MachineLocation localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
copierHost,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
}
