/// <nodoc /> public DistributedContentStore( byte[] localMachineLocation, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, IFileExistenceChecker <T> fileExistenceChecker, IFileCopier <T> fileCopier, IPathTransformer <T> pathTransform, ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee, AbsolutePath tempFolderForCopies, IAbsFileSystem fileSystem, int locationStoreBatchSize, DistributedContentStoreSettings settings, int?replicaCreditInMinutes = null, IClock clock = null, bool enableRepairHandling = false, TimeSpan?contentHashBumpTime = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); _localMachineLocation = localMachineLocation; _enableRepairHandling = enableRepairHandling; _contentLocationStoreFactory = contentLocationStoreFactory; _contentAvailabilityGuarantee = contentAvailabilityGuarantee; _locationStoreBatchSize = locationStoreBatchSize; contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _locationStoreBatchSize); _distributedCopierFactory = (contentLocationStore) => { return(new DistributedContentCopier <T>( tempFolderForCopies, _settings, fileSystem, fileCopier, fileExistenceChecker, pathTransform, contentLocationStore)); }; _enableDistributedEviction = replicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null; var enableTouch = contentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.UsePinCache == true) { _pinCache = new PinCache(clock: clock); } }
private TestFileSystemContentStoreInternal CreateStore(DisposableDirectory testDirectory, ContentStoreSettings settings) { return(new TestFileSystemContentStoreInternal(FileSystem, Clock, testDirectory.Path, Config, settings: settings)); }
private TestFileSystemContentStoreInternal CreateStore(DisposableDirectory testDirectory, ContentStoreSettings settings, IDistributedLocationStore distributedStore) { return(new TestFileSystemContentStoreInternal(FileSystem, Clock, testDirectory.Path, Config, distributedStore: distributedStore, settings: settings)); }
private TestFileSystemContentStoreInternal CreateStore(DisposableDirectory testDirectory, ContentStoreSettings settings, DistributedEvictionSettings distributedEvictionSettings) { return(new TestFileSystemContentStoreInternal(FileSystem, Clock, testDirectory.Path, Config, distributedEvictionSettings: distributedEvictionSettings, nagleQueue: EmptyNagleQueue(), settings: settings)); }
public Task PutSameContentManyTimesTest(bool useRedundantPutFileShortcut) { var context = new Context(Logger); ContentStoreSettings = new ContentStoreSettings() { UseRedundantPutFileShortcut = useRedundantPutFileShortcut }; return TestStore(context, Clock, async store => { byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize); ContentHash contentHash = bytes.CalculateHash(ContentHashType); // Verify content doesn't exist yet in store Assert.False(await store.ContainsAsync(context, contentHash, null)); using (var tempDirectory = new DisposableDirectory(FileSystem)) { ContentHash hashFromPut; using (var pinContext = store.CreatePinContext()) { var concurrency = 24; var iterations = 100; var items = Enumerable.Range(0, concurrency).Select(i => { AbsolutePath pathToContent = tempDirectory.Path / $"tempContent{i}.txt"; FileSystem.WriteAllBytes(pathToContent, bytes); return (pathToContent, iterations); }).ToArray(); int nonDuplicatedPuts = 0; await ParallelAlgorithms.WhenDoneAsync(24, CancellationToken.None, async (scheduleItem, item) => { // Put the content into the store w/ hard link var r = await store.PutFileAsync( context, item.pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext)); hashFromPut = r.ContentHash; if (!r.ContentAlreadyExistsInCache) { Interlocked.Increment(ref nonDuplicatedPuts); } Clock.Increment(); Assert.True(pinContext.Contains(hashFromPut)); if (item.iterations != 0) { scheduleItem((item.pathToContent, item.iterations - 1)); } }, items); Assert.Equal(1, nonDuplicatedPuts); } } }); }
/// <nodoc /> public DistributedContentStore( byte[] localMachineLocation, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, IFileExistenceChecker <T> fileExistenceChecker, IFileCopier <T> fileCopier, IPathTransformer <T> pathTransform, IContentCommunicationManager copyRequester, AbsolutePath tempFolderForCopies, IAbsFileSystem fileSystem, DistributedContentStoreSettings settings, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = new MachineLocation(localMachineLocation); _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _settings.LocationStoreBatchSize); _distributedCopierFactory = (contentLocationStore) => { return(new DistributedContentCopier <T>( tempFolderForCopies, _settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransform, _clock, contentLocationStore)); }; _enableDistributedEviction = _settings.ReplicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null; var enableTouch = _settings.ContentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.IsPinCachingEnabled == true) { _pinCache = new PinCache(clock: _clock); } }