public AzureBlobStorageLog(AzureBlobStorageLogConfiguration configuration, OperationContext context, IClock clock, IAbsFileSystem fileSystem, ITelemetryFieldsProvider telemetryFieldsProvider, AzureBlobStorageCredentials credentials)
{
_configuration = configuration;
_context = context;
_clock = clock;
_fileSystem = fileSystem;
_telemetryFieldsProvider = telemetryFieldsProvider;
var cloudBlobClient = credentials.CreateCloudBlobClient();
_container = cloudBlobClient.GetContainerReference(configuration.ContainerName);
_writeQueue = NagleQueue <string> .CreateUnstarted(
configuration.WriteMaxDegreeOfParallelism,
configuration.WriteMaxInterval,
configuration.WriteMaxBatchSize);
_uploadQueue = NagleQueue <LogFile> .CreateUnstarted(
configuration.UploadMaxDegreeOfParallelism,
configuration.UploadMaxInterval,
1);
// TODO: this component doesn't have a quota, which could potentially be useful. If Azure Blob Storage
// becomes unavailable for an extended period of time, we might cause disk space issues.
}
public AzureBlobStorageLog(
AzureBlobStorageLogConfiguration configuration,
OperationContext context,
IClock clock,
IAbsFileSystem fileSystem,
ITelemetryFieldsProvider telemetryFieldsProvider,
CloudBlobContainer container,
IReadOnlyDictionary <string, string> additionalBlobMetadata)
{
_configuration = configuration;
_context = context;
_clock = clock;
_fileSystem = fileSystem;
_telemetryFieldsProvider = telemetryFieldsProvider;
_container = container;
_additionalBlobMetadata = additionalBlobMetadata;
_writeQueue = NagleQueue <string> .CreateUnstarted(
configuration.WriteMaxDegreeOfParallelism,
configuration.WriteMaxInterval,
configuration.WriteMaxBatchSize);
_uploadQueue = NagleQueue <LogFile> .CreateUnstarted(
configuration.UploadMaxDegreeOfParallelism,
configuration.UploadMaxInterval,
1);
// TODO: this component doesn't have a quota, which could potentially be useful. If Azure Blob Storage
// becomes unavailable for an extended period of time, we might cause disk space issues.
}
public void TestExceptionHandling()
{
int callbackCount = 0;
var queue = NagleQueue <int> .CreateUnstarted(
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMilliseconds(10),
batchSize : 2);
queue.Start(
processBatch: async data =>
{
callbackCount++;
await Task.Yield();
var e = new InvalidOperationException(string.Join(", ", data.Select(n => n.ToString())));
throw e;
});
queue.Enqueue(1);
queue.Enqueue(2);
queue.Enqueue(3);
// And if callback fails, the queue itself moves to a faulted state.
// This will manifest itself in an error during Dispose invocation.
// This is actually quite problematic, because Dispose method can be called
// from the finally block (explicitly, or implicitly via using block)
// and in this case the original exception that caused the finally block invocation
// will be masked by the exception from Dispose method.
// Work item: 1741215
// Dispose method propagates the error thrown in the callback.
Assert.Throws <InvalidOperationException>(() => queue.Dispose());
// Once callback fails, it won't be called any more
callbackCount.Should().Be(1);
}
/// <nodoc />
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier <T> distributedCopier,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_settings.LocationStoreBatchSize);
_enableDistributedEviction = _settings.ReplicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null;
var enableTouch = _settings.ContentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.IsPinCachingEnabled == true)
{
_pinCache = new PinCache(clock: _clock);
}
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
DistributedContentStoreSettings settings,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = new MachineLocation(localMachineLocation);
_enableRepairHandling = enableRepairHandling;
_contentLocationStoreFactory = contentLocationStoreFactory;
_contentAvailabilityGuarantee = contentAvailabilityGuarantee;
_locationStoreBatchSize = locationStoreBatchSize;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_locationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
copyRequester,
pathTransform,
contentLocationStore));
};
_enableDistributedEviction = replicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;
var enableTouch = contentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.UsePinCache == true)
{
_pinCache = new PinCache(clock: clock);
}
}
private NagleQueue <ContentHash> EmptyNagleQueue()
{
return(NagleQueue <ContentHash> .CreateUnstarted(1, TimeSpan.MaxValue, 42));
}