public ChunkedPhase(
IStageIdentity stageIdentity,
IGenerationPhaseIdentity phaseIdentity,
IReadOnlyLargeCollection <TKey> chunkKeys,
IAsyncFactory <TKey, IDisposableValue <TChunk> > chunkFactory,
IChunkProcessor <TChunk> chunkProcessor,
GenerationOptions options = null)
: base(options?.CancellationToken ?? DefaultCancellationToken)
{
Contracts.Requires.That(stageIdentity != null);
Contracts.Requires.That(phaseIdentity != null);
Contracts.Requires.That(chunkKeys != null);
Contracts.Requires.That(chunkFactory != null);
Contracts.Requires.That(chunkProcessor != null);
this.StageIdentity = stageIdentity;
this.PhaseIdentity = phaseIdentity;
this.chunkKeys = chunkKeys;
this.chunkFactory = chunkFactory;
this.chunkProcessor = chunkProcessor;
this.Progress = this.progress.AsObservable();
var dataflowOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = options?.MaxDegreeOfParallelism ?? DefaultMaxDegreeOfParallelism,
CancellationToken = this.CancellationToken,
};
this.processChunks = new ActionBlock <TKey>(this.ProcessChunk, dataflowOptions);
this.chunksCompleted = new AsyncLongCountdownEvent(this.ProgressTotalCount);
}
public ChunkedBatchingPhase(
IStageIdentity stageIdentity,
IGenerationPhaseIdentity phaseIdentity,
IReadOnlyLargeCollection <TKey> chunkKeys,
IAsyncFactory <TKey, TPersistable> chunkFactory,
IChunkStore <TKey, TPersistable> chunkStore,
int maxBatchedChunks,
BatchGenerationOptions options = null)
: base(options?.CancellationToken ?? DefaultCancellationToken)
{
Contracts.Requires.That(stageIdentity != null);
Contracts.Requires.That(phaseIdentity != null);
Contracts.Requires.That(chunkKeys != null);
Contracts.Requires.That(chunkFactory != null);
Contracts.Requires.That(chunkStore != null);
Contracts.Requires.That(maxBatchedChunks > 0);
this.StageIdentity = stageIdentity;
this.PhaseIdentity = phaseIdentity;
this.chunkKeys = chunkKeys;
this.chunkFactory = chunkFactory;
this.chunkStore = chunkStore;
this.Progress = this.progress.AsObservable();
var dataflowOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = options?.MaxDegreeOfParallelism ?? DefaultMaxDegreeOfParallelism,
BoundedCapacity = maxBatchedChunks,
CancellationToken = this.CancellationToken,
};
this.generateChunks = new TransformBlock <TKey, TPersistable>(
chunkIndex => this.chunkFactory.CreateAsync(chunkIndex), dataflowOptions);
this.saveChunks = BatchActionBlock.Create <TPersistable>(
this.SaveChunks,
maxBatchedChunks,
options?.Batching ?? DefaultBatching,
dataflowOptions);
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = true
};
this.generateChunks.LinkTo(this.saveChunks, linkOptions);
this.chunksCompleted = new AsyncLongCountdownEvent(this.ProgressTotalCount);
}
public static bool TrySignalAllRemaining(this AsyncLongCountdownEvent countdown)
{
Contracts.Requires.That(countdown != null);
return(countdown.TrySignal(countdown.CurrentCount));
}