/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
DistributedContentStoreSettings settings,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
_localMachineLocation = localMachineLocation;
_enableRepairHandling = enableRepairHandling;
_contentLocationStoreFactory = contentLocationStoreFactory;
_contentAvailabilityGuarantee = contentAvailabilityGuarantee;
_locationStoreBatchSize = locationStoreBatchSize;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_locationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
pathTransform,
contentLocationStore));
};
_enableDistributedEviction = replicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;
var enableTouch = contentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.UsePinCache == true)
{
_pinCache = new PinCache(clock: clock);
}
}
private TestFileSystemContentStoreInternal CreateStore(DisposableDirectory testDirectory, ContentStoreSettings settings)
{
return(new TestFileSystemContentStoreInternal(FileSystem, Clock, testDirectory.Path, Config, settings: settings));
}
private TestFileSystemContentStoreInternal CreateStore(DisposableDirectory testDirectory, ContentStoreSettings settings, IDistributedLocationStore distributedStore)
{
return(new TestFileSystemContentStoreInternal(FileSystem, Clock, testDirectory.Path, Config, distributedStore: distributedStore, settings: settings));
}
private TestFileSystemContentStoreInternal CreateStore(DisposableDirectory testDirectory, ContentStoreSettings settings, DistributedEvictionSettings distributedEvictionSettings)
{
return(new TestFileSystemContentStoreInternal(FileSystem, Clock, testDirectory.Path, Config, distributedEvictionSettings: distributedEvictionSettings, nagleQueue: EmptyNagleQueue(), settings: settings));
}
public Task PutSameContentManyTimesTest(bool useRedundantPutFileShortcut)
{
var context = new Context(Logger);
ContentStoreSettings = new ContentStoreSettings()
{
UseRedundantPutFileShortcut = useRedundantPutFileShortcut
};
return TestStore(context, Clock, async store =>
{
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
ContentHash hashFromPut;
using (var pinContext = store.CreatePinContext())
{
var concurrency = 24;
var iterations = 100;
var items = Enumerable.Range(0, concurrency).Select(i =>
{
AbsolutePath pathToContent = tempDirectory.Path / $"tempContent{i}.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
return (pathToContent, iterations);
}).ToArray();
int nonDuplicatedPuts = 0;
await ParallelAlgorithms.WhenDoneAsync(24, CancellationToken.None, async (scheduleItem, item) =>
{
// Put the content into the store w/ hard link
var r = await store.PutFileAsync(
context, item.pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext));
hashFromPut = r.ContentHash;
if (!r.ContentAlreadyExistsInCache)
{
Interlocked.Increment(ref nonDuplicatedPuts);
}
Clock.Increment();
Assert.True(pinContext.Contains(hashFromPut));
if (item.iterations != 0)
{
scheduleItem((item.pathToContent, item.iterations - 1));
}
},
items);
Assert.Equal(1, nonDuplicatedPuts);
}
}
});
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
IContentCommunicationManager copyRequester,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
DistributedContentStoreSettings settings,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = new MachineLocation(localMachineLocation);
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_settings.LocationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
copyRequester,
pathTransform,
_clock,
contentLocationStore));
};
_enableDistributedEviction = _settings.ReplicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null;
var enableTouch = _settings.ContentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.IsPinCachingEnabled == true)
{
_pinCache = new PinCache(clock: _clock);
}
}
