public async Task EvictionAnnouncesHash()
{
bool batchProcessWasCalled = false;
var nagleQueue = NagleQueue <ContentHash> .Create(
hashes => { batchProcessWasCalled = true; return(Task.FromResult(42)); },
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : 1);
await TestStore(
_context,
_clock,
async store =>
{
var cas = store as IContentStoreInternal;
var blobSize = BlobSizeToStartSoftPurging(2);
using (var stream1 = new MemoryStream(ThreadSafeRandom.GetBytes(blobSize)))
using (var stream2 = new MemoryStream(ThreadSafeRandom.GetBytes(blobSize)))
{
await cas.PutStreamAsync(_context, stream1, ContentHashType).ShouldBeSuccess();
_clock.Increment();
await cas.PutStreamAsync(_context, stream2, ContentHashType).ShouldBeSuccess();
_clock.Increment();
await store.SyncAsync(_context);
}
},
nagleQueue);
batchProcessWasCalled.Should().BeTrue();
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
public void TestExceptionHandling()
{
int callbackCount = 0;
var queue = NagleQueue <int> .CreateUnstarted(
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMilliseconds(10),
batchSize : 2);
queue.Start(
processBatch: async data =>
{
callbackCount++;
await Task.Yield();
var e = new InvalidOperationException(string.Join(", ", data.Select(n => n.ToString())));
throw e;
});
queue.Enqueue(1);
queue.Enqueue(2);
queue.Enqueue(3);
// And if callback fails, the queue itself moves to a faulted state.
// This will manifest itself in an error during Dispose invocation.
// This is actually quite problematic, because Dispose method can be called
// from the finally block (explicitly, or implicitly via using block)
// and in this case the original exception that caused the finally block invocation
// will be masked by the exception from Dispose method.
// Work item: 1741215
// Dispose method propagates the error thrown in the callback.
Assert.Throws <InvalidOperationException>(() => queue.Dispose());
// Once callback fails, it won't be called any more
callbackCount.Should().Be(1);
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
IConfiguration configuration,
string downLoadedPreviousFingerprintStoreSavedPath,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_batchLoggingQueue = configuration.Logging.CacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(5),
batchSize : 100) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
m_configuration = configuration;
m_downLoadedPreviousFingerprintStoreSavedPath = downLoadedPreviousFingerprintStoreSavedPath;
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
CacheMissDiffFormat cacheMissDiffFormat,
bool cacheMissBatch,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_cacheMissDiffFormat = cacheMissDiffFormat;
m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value;
m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
}
public AzureBlobStorageLog(AzureBlobStorageLogConfiguration configuration, OperationContext context, IClock clock, IAbsFileSystem fileSystem, ITelemetryFieldsProvider telemetryFieldsProvider, AzureBlobStorageCredentials credentials)
{
_configuration = configuration;
_context = context;
_clock = clock;
_fileSystem = fileSystem;
_telemetryFieldsProvider = telemetryFieldsProvider;
var cloudBlobClient = credentials.CreateCloudBlobClient();
_container = cloudBlobClient.GetContainerReference(configuration.ContainerName);
_writeQueue = NagleQueue <string> .CreateUnstarted(
configuration.WriteMaxDegreeOfParallelism,
configuration.WriteMaxInterval,
configuration.WriteMaxBatchSize);
_uploadQueue = NagleQueue <LogFile> .CreateUnstarted(
configuration.UploadMaxDegreeOfParallelism,
configuration.UploadMaxInterval,
1);
// TODO: this component doesn't have a quota, which could potentially be useful. If Azure Blob Storage
// becomes unavailable for an extended period of time, we might cause disk space issues.
}
public TestFileSystemContentStoreInternal(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ContentStoreConfiguration configuration,
Action <ContentHashWithSize> onContentAdded = null,
Action <ContentHashWithSize> onContentEvicted = null,
NagleQueue <ContentHash> nagleQueue = null,
ContentStoreSettings settings = null,
DistributedEvictionSettings distributedEvictionSettings = null)
: base(fileSystem, clock, rootPath, new ConfigurationModel(configuration), nagleQueue: nagleQueue, settings: settings, distributedEvictionSettings: distributedEvictionSettings)
{
Contract.Requires(fileSystem != null);
Contract.Requires(clock != null);
Contract.Requires(rootPath != null);
Contract.Requires(configuration != null);
_onContentAdded = onContentAdded;
_onContentEvicted = onContentEvicted;
if (_onContentAdded != null || _onContentEvicted != null)
{
Announcer = this;
}
}
public void ResumeShouldTriggerBatchOnTime()
{
bool processBatchIsCalled = false;
var queue = NagleQueue <int> .Create(
processBatch : data =>
{
processBatchIsCalled = true;
return(Task.FromResult(42));
},
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMilliseconds(1),
batchSize : 10);
var suspender = queue.Suspend();
Thread.Sleep(1000);
queue.Enqueue(42);
suspender.Dispose(); // This should resume the queue and restart the timer
Thread.Sleep(1000); // Definitely longer than the configured interval provided to NagleQueue
// It means that the queue should call the callback and we can rely on that.
Assert.True(processBatchIsCalled);
}
public void TwoItemsAreProcessedInParallel()
{
var threads = new List <int>();
using (var queue = NagleQueue <int> .Create(
processBatch: async data =>
{
lock (threads)
{
threads.Add(Thread.CurrentThread.ManagedThreadId);
}
await Task.Delay(1);
},
maxDegreeOfParallelism: 2,
interval: TimeSpan.FromSeconds(1),
batchSize: 2))
{
queue.Enqueue(1);
queue.Enqueue(2);
queue.Enqueue(3);
queue.Enqueue(4);
}
Assert.Equal(2, threads.Count);
}
public void ItemsInEagerBlocksAreProcessedEagerly()
{
int dataLength = 0;
var processBatchWasCalled = false;
var processBatchEvent = new ManualResetEvent(false);
using (var queue = NagleQueue <int> .Create(
processBatch: data =>
{
dataLength = data.Length;
processBatchWasCalled = true;
processBatchEvent.Set();
return(Task.FromResult(42));
},
maxDegreeOfParallelism: 1,
interval: TimeSpan.FromMinutes(1),
// batchSize is one, so the processing is eager.
batchSize: 1))
{
queue.Enqueue(42);
processBatchEvent.WaitOne(5000);
Assert.True(processBatchWasCalled, "processBatch should be called eagerly.");
Assert.Equal(1, dataLength);
}
}
/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
m_nagleQueue = NagleQueue <AddFileItem> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessAddFilesAsync);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
public AzureBlobStorageLog(
AzureBlobStorageLogConfiguration configuration,
OperationContext context,
IClock clock,
IAbsFileSystem fileSystem,
ITelemetryFieldsProvider telemetryFieldsProvider,
CloudBlobContainer container,
IReadOnlyDictionary <string, string> additionalBlobMetadata)
{
_configuration = configuration;
_context = context;
_clock = clock;
_fileSystem = fileSystem;
_telemetryFieldsProvider = telemetryFieldsProvider;
_container = container;
_additionalBlobMetadata = additionalBlobMetadata;
_writeQueue = NagleQueue <string> .CreateUnstarted(
configuration.WriteMaxDegreeOfParallelism,
configuration.WriteMaxInterval,
configuration.WriteMaxBatchSize);
_uploadQueue = NagleQueue <LogFile> .CreateUnstarted(
configuration.UploadMaxDegreeOfParallelism,
configuration.UploadMaxInterval,
1);
// TODO: this component doesn't have a quota, which could potentially be useful. If Azure Blob Storage
// becomes unavailable for an extended period of time, we might cause disk space issues.
}
/// <summary>
/// Creates a <see cref="FileSystemContentStore"/> at <see cref="rootPath"/>
/// </summary>
public static IContentStore CreateContentStore(
IAbsFileSystem fileSystem,
AbsolutePath rootPath,
NagleQueue <ContentHash> evictionAnnouncer,
DistributedEvictionSettings distributedEvictionSettings,
ContentStoreSettings contentStoreSettings,
TrimBulkAsync trimBulkAsync,
ConfigurationModel configurationModel = null)
=> new FileSystemContentStore(
fileSystem, SystemClock.Instance, rootPath, configurationModel, evictionAnnouncer, distributedEvictionSettings, trimBulkAsync, contentStoreSettings);
/// <nodoc />
protected override Task <BoolResult> StartupCoreAsync(OperationContext context)
{
TouchNagleQueue = NagleQueue <ContentHashWithSize> .Create(
hashes => BackgroundTouchBulkAsync(context, hashes),
RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
RedisContentLocationStoreConstants.BatchInterval,
RedisContentLocationStoreConstants.DefaultBatchSize);
return(BoolResult.SuccessTask);
}
protected Task TestStore
(
Context context,
ITestClock clock,
Func <TestFileSystemContentStoreInternal, Task> func,
NagleQueue <ContentHash> nagleBlock
)
{
return(TestStoreImpl(context, clock, func, nagleBlock: nagleBlock));
}
/// <summary>
/// Backward-compat constructor.
/// </summary>
public FileSystemContentStore(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ConfigurationModel?configurationModel = null,
NagleQueue <ContentHash>?nagleQueue = null,
DistributedEvictionSettings?distributedEvictionSettings = null,
TrimBulkAsync?trimBulkAsync = null,
ContentStoreSettings?settings = null)
: this(fileSystem, clock, rootPath, configurationModel, distributedEvictionSettings?.DistributedStore, settings)
{
}
public void PostAfterDispose()
{
var queue = NagleQueue <int> .Create(
processBatch : data =>
{
return(Task.FromResult(42));
},
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : 10);
queue.Dispose();
Assert.Throws <ObjectDisposedException>(() => queue.Enqueue(42));
}
protected virtual TestFileSystemContentStoreInternal CreateElastic(
AbsolutePath rootPath,
ITestClock clock,
NagleQueue <ContentHash> nagleBlock = null,
MaxSizeQuota initialQuota = null,
int?windowSize = default(int?))
{
var maxSizeQuota = initialQuota ?? new MaxSizeQuota(MaxSizeHard, MaxSizeSoft);
// Some tests rely on maxSizeQuota being set in the configuration although it is ignored if elasticity is enabled.
var config = new ContentStoreConfiguration(maxSizeQuota: maxSizeQuota, enableElasticity: true, initialElasticSize: maxSizeQuota, historyWindowSize: windowSize);
return(new TestFileSystemContentStoreInternal(FileSystem, clock, rootPath, config, nagleQueue: nagleBlock, settings: ContentStoreSettings));
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// NOTE: We create and start the content location store before the inner content store just in case the
// inner content store starts background eviction after startup. We need the content store to be initialized
// so that it can be queried and used to unregister content.
await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure();
_contentLocationStore = await _contentLocationStoreFactory.CreateAsync();
_distributedCopier = _distributedCopierFactory(_contentLocationStore);
await _distributedCopier.StartupAsync(context).ThrowIfFailure();
if (_contentLocationStore is TransitioningContentLocationStore tcs)
{
tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier);
}
// Initializing inner store before initializing LocalLocationStore because
// LocalLocationStore may use inner store for reconciliation purposes
await InnerContentStore.StartupAsync(context).ThrowIfFailure();
await _contentLocationStore.StartupAsync(context).ThrowIfFailure();
Func <ContentHash[], Task> evictionHandler;
var localContext = new Context(context);
if (_enableDistributedEviction)
{
evictionHandler = hashes => EvictContentAsync(localContext, hashes);
}
else
{
evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes);
}
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup. So we start the queue here.
_evictionNagleQueue.Start(evictionHandler);
var touchContext = new Context(context);
_touchNagleQueue = NagleQueue <ContentHashWithSize> .Create(
hashes => TouchBulkAsync(touchContext, hashes),
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
batchSize : _locationStoreBatchSize);
return(BoolResult.Success);
}
/// <inheritdoc />
public void Start(OperationContext context)
{
// Using nagle queues to "batch" messages together and to avoid writing them to the logs one by one.
var outputMessagesNagleQueue = NagleQueue <string> .Create(
messages =>
{
_tracer.Debug(context, $"Service Output: {string.Join(Environment.NewLine, messages)}");
return(Task.CompletedTask);
},
maxDegreeOfParallelism : 1, interval : TimeSpan.FromSeconds(1), batchSize : 1024);
var errorMessagesNagleQueue = NagleQueue <string> .Create(
messages =>
{
_tracer.Error(context, $"Service Error: {string.Join(Environment.NewLine, messages)}");
return(Task.CompletedTask);
},
maxDegreeOfParallelism : 1, interval : TimeSpan.FromSeconds(1), batchSize : 1024);
_process.OutputDataReceived += (s, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
{
outputMessagesNagleQueue.Enqueue(e.Data);
}
};
_process.ErrorDataReceived += (s, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
{
errorMessagesNagleQueue.Enqueue(e.Data);
}
};
_process.Exited += (sender, args) =>
{
// Dispose will drain all the existing items from the message queues.
outputMessagesNagleQueue.Dispose();
errorMessagesNagleQueue.Dispose();
};
_process.Start();
_process.BeginOutputReadLine();
_process.BeginErrorReadLine();
_started = true;
}
/// <nodoc />
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier <T> distributedCopier,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_settings.LocationStoreBatchSize);
_enableDistributedEviction = _settings.ReplicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null;
var enableTouch = _settings.ContentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.IsPinCachingEnabled == true)
{
_pinCache = new PinCache(clock: _clock);
}
}
/// <summary>
/// Finish setting up distributed eviction.
/// </summary>
public void InitializeDistributedEviction(
UpdateContentWithLastAccessTimeAsync updateMetadataFunc,
Tracer tracer,
PinnedSizeChecker pinnedSizeChecker,
NagleQueue <ContentHash> reregisterHashQueue)
{
Contract.Assert(updateMetadataFunc != null);
Contract.Assert(pinnedSizeChecker != null);
Contract.Assert(tracer != null);
Contract.Assert(reregisterHashQueue != null);
UpdateContentWithLastAccessTimeAsync = updateMetadataFunc;
Tracer = tracer;
PinnedSizeChecker = pinnedSizeChecker;
ReregisterHashQueue = reregisterHashQueue;
IsInitialized = true;
}
public void PendingItemsAreProcessedOnDispose()
{
bool processBatchWasCalled = false;
var queue = NagleQueue <int> .Create(
processBatch : data =>
{
processBatchWasCalled = true;
return(Task.FromResult(42));
},
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : 10);
queue.Enqueue(42);
Assert.False(processBatchWasCalled, "processBatch should not be called yet.");
queue.Dispose();
Assert.True(processBatchWasCalled, "processBatch should be called during disposal");
}
public KustoNotifier(Configuration configuration, ILogger logger, IKustoIngestClient kustoIngestClient)
{
_configuration = configuration;
_logger = logger;
_kustoIngestClient = kustoIngestClient;
_kustoIngestionProperties = new KustoIngestionProperties(_configuration.KustoDatabaseName, _configuration.KustoTableName)
{
Format = DataSourceFormat.json,
};
Contract.RequiresNotNullOrEmpty(_configuration.KustoTableIngestionMappingName,
"Kusto ingestion will fail to authenticate without a proper ingestion mapping.");
_kustoIngestionProperties.JSONMappingReference = _configuration.KustoTableIngestionMappingName;
_queue = NagleQueue <T> .Create(FlushAsync,
_configuration.MaxDegreeOfParallelism,
_configuration.FlushInterval,
_configuration.BatchSize);
}
public void ItemsAreProcessedInBatches()
{
int batchSize = 0;
using (var queue = NagleQueue <int> .Create(
processBatch: data =>
{
batchSize = data.Length;
return(Task.FromResult(42));
},
maxDegreeOfParallelism: 1,
interval: TimeSpan.FromMilliseconds(10),
batchSize: 2))
{
queue.Enqueue(1);
queue.Enqueue(2);
}
Assert.Equal(2, batchSize);
}
/// <summary>
/// Backward-compat constructor.
/// </summary>
public FileSystemContentStore(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ConfigurationModel configurationModel,
NagleQueue <ContentHash> nagleQueue,
RefCountdown sensitiveSessionCount,
DistributedEvictionSettings distributedEvictionSettings,
bool checkFiles,
TrimBulkAsync trimBulkAsync)
: this(
fileSystem,
clock,
rootPath,
configurationModel,
nagleQueue,
distributedEvictionSettings,
trimBulkAsync,
settings : new ContentStoreSettings() { CheckFiles = checkFiles })
{
}
private async Task TestStoreImpl
(
Context context,
ITestClock clock,
Func <TestFileSystemContentStoreInternal, Task> func,
DisposableDirectory testDirectory = null,
IContentChangeAnnouncer announcer = null,
NagleQueue <ContentHash> nagleBlock = null,
Action <TestFileSystemContentStoreInternal> preStartupAction = null
)
{
using (var tempTestDirectory = new DisposableDirectory(FileSystem))
{
DisposableDirectory disposableDirectory = testDirectory ?? tempTestDirectory;
using (var store = Create(disposableDirectory.Path, clock, nagleBlock))
{
if (announcer != null)
{
store.Announcer = announcer;
}
store.Should().NotBeNull();
try
{
preStartupAction?.Invoke(store);
var r = await store.StartupAsync(context);
r.ShouldBeSuccess();
await func(store);
}
finally
{
if (!store.ShutdownStarted)
{
await store.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
}
}
public void ItemsAreProcessedBasedOnInterval()
{
var processBatchWasCalled = false;
var processBatchEvent = new ManualResetEvent(false);
using (var queue = NagleQueue <int> .Create(
processBatch: data =>
{
processBatchWasCalled = true;
processBatchEvent.Set();
return(Task.FromResult(42));
},
maxDegreeOfParallelism: 1,
interval: TimeSpan.FromMilliseconds(10),
batchSize: 2))
{
queue.Enqueue(1);
Assert.False(processBatchWasCalled);
processBatchEvent.WaitOne(5000);
Assert.True(processBatchWasCalled);
}
}
public void EnqueingItemsInfrequentlyShouldAlwaysTriggerCallbackOnTime()
{
int processBatchIsCalled = 0;
var queue = NagleQueue <int> .Create(
processBatch : data =>
{
processBatchIsCalled++;
return(Task.FromResult(42));
},
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMilliseconds(10),
batchSize : 10);
queue.Enqueue(42);
Thread.Sleep(100);
Assert.Equal(1, processBatchIsCalled);
queue.Enqueue(42);
Thread.Sleep(100);
Assert.Equal(2, processBatchIsCalled);
}
static async Task testCore(int attempt)
{
var log = TestGlobal.Logger;
TestGlobal.Logger.Debug($"Running {attempt} attempt.");
Task task = null;
using (var queue = NagleQueue <int> .Create(
processBatch: data => { return(Task.FromResult(42)); },
maxDegreeOfParallelism: 1,
interval: TimeSpan.FromMilliseconds(1),
batchSize: 100))
{
var itemsEnqueuedSource = TaskSourceSlim.Create <object>();
task = Task.Run(
() =>
{
using (queue.Suspend())
{
for (int i = 0; i < 1_000_000; i++)
{
queue.Enqueue(i);
}
itemsEnqueuedSource.SetResult(null);
}
});
// The items are added to the queue and the suspender is about to push all the items to the queue
await itemsEnqueuedSource.Task;
// Meanwhile, the queue itself will be disposed at the end of the block.
// So we're introducing a race condition between the suspender and the dispose method of the queue
}
await task; // the task should not fail!
}