protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase)
{
return new[] {firstPhase};
}
protected EventSubscriptionBasedProjectionProcessingPhase(
IPublisher publisher, ICoreProjectionForProcessingPhase coreProjection, Guid projectionCorrelationId,
ICoreProjectionCheckpointManager checkpointManager, ProjectionConfig projectionConfig, string projectionName,
ILogger logger, CheckpointTag zeroCheckpointTag, PartitionStateCache partitionStateCache,
IResultWriter resultWriter, Action updateStatistics, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, bool useCheckpoints, bool stopOnEof)
{
_publisher = publisher;
_coreProjection = coreProjection;
_projectionCorrelationId = projectionCorrelationId;
_checkpointManager = checkpointManager;
_projectionConfig = projectionConfig;
_projectionName = projectionName;
_logger = logger;
_zeroCheckpointTag = zeroCheckpointTag;
_partitionStateCache = partitionStateCache;
_resultWriter = resultWriter;
_processingQueue = new CoreProjectionQueue(
projectionCorrelationId, publisher, projectionConfig.PendingEventsThreshold, updateStatistics);
_processingQueue.EnsureTickPending += EnsureTickPending;
_subscriptionDispatcher = subscriptionDispatcher;
_readerStrategy = readerStrategy;
_useCheckpoints = useCheckpoints;
_stopOnEof = stopOnEof;
}
protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase)
{
return(new [] { firstPhase });
}
public override sealed IProjectionProcessingPhase[] CreateProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ProjectionNamesBuilder namingBuilder,
ITimeProvider timeProvider, IODispatcher ioDispatcher,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
{
var definesFold = _sourceDefinition.DefinesFold;
var readerStrategy = CreateReaderStrategy(timeProvider);
var zeroCheckpointTag = readerStrategy.PositionTagger.MakeZeroCheckpointTag();
var checkpointManager = CreateCheckpointManager(
projectionCorrelationId, publisher, ioDispatcher, namingBuilder, coreProjectionCheckpointWriter,
definesFold, readerStrategy);
var resultWriter = CreateFirstPhaseResultWriter(
checkpointManager as IEmittedEventWriter, zeroCheckpointTag, namingBuilder);
var firstPhase = CreateFirstProcessingPhase(
publisher, projectionCorrelationId, partitionStateCache, updateStatistics, coreProjection,
_subscriptionDispatcher, zeroCheckpointTag, checkpointManager, readerStrategy, resultWriter);
return(CreateProjectionProcessingPhases(
publisher, projectionCorrelationId, namingBuilder, partitionStateCache, coreProjection, ioDispatcher,
firstPhase));
}
public WriteQueryResultProjectionProcessingPhaseBase(
IPublisher publisher,
int phase,
string resultStream,
ICoreProjectionForProcessingPhase coreProjection,
PartitionStateCache stateCache,
ICoreProjectionCheckpointManager checkpointManager,
IEmittedEventWriter emittedEventWriter,
IEmittedStreamsTracker emittedStreamsTracker)
{
if (resultStream == null) throw new ArgumentNullException("resultStream");
if (coreProjection == null) throw new ArgumentNullException("coreProjection");
if (stateCache == null) throw new ArgumentNullException("stateCache");
if (checkpointManager == null) throw new ArgumentNullException("checkpointManager");
if (emittedEventWriter == null) throw new ArgumentNullException("emittedEventWriter");
if (emittedStreamsTracker == null) throw new ArgumentNullException("emittedStreamsTracker");
if (string.IsNullOrEmpty(resultStream)) throw new ArgumentException("resultStream");
_publisher = publisher;
_phase = phase;
_resultStream = resultStream;
_coreProjection = coreProjection;
_stateCache = stateCache;
_checkpointManager = checkpointManager;
_emittedEventWriter = emittedEventWriter;
_emittedStreamsTracker = emittedStreamsTracker;
}
public void given()
{
//given
_cache = new PartitionStateCache(CheckpointTag.FromPosition(0, -1));
_cachedAtCheckpointTag = CheckpointTag.FromPosition(1000, 900);
_cache.CacheAndLockPartitionState("partition", new PartitionStateCache.State("data", _cachedAtCheckpointTag), _cachedAtCheckpointTag);
}
public WriteQueryResultProjectionProcessingPhase(
int phase, string resultStream, ICoreProjectionForProcessingPhase coreProjection,
PartitionStateCache stateCache, ICoreProjectionCheckpointManager checkpointManager,
IEmittedEventWriter emittedEventWriter)
: base(phase, resultStream, coreProjection, stateCache, checkpointManager, emittedEventWriter)
{
}
public override sealed IProjectionProcessingPhase[] CreateProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ProjectionNamesBuilder namingBuilder,
ITimeProvider timeProvider, IODispatcher ioDispatcher,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
{
var definesFold = _sourceDefinition.DefinesFold;
var readerStrategy = CreateReaderStrategy(timeProvider);
var zeroCheckpointTag = readerStrategy.PositionTagger.MakeZeroCheckpointTag();
var checkpointManager = CreateCheckpointManager(
projectionCorrelationId, publisher, ioDispatcher, namingBuilder, coreProjectionCheckpointWriter,
definesFold, readerStrategy);
var resultWriter = CreateFirstPhaseResultWriter(
checkpointManager as IEmittedEventWriter, zeroCheckpointTag, namingBuilder);
var firstPhase = CreateFirstProcessingPhase(
publisher, projectionCorrelationId, partitionStateCache, updateStatistics, coreProjection,
_subscriptionDispatcher, zeroCheckpointTag, checkpointManager, readerStrategy, resultWriter);
return CreateProjectionProcessingPhases(
publisher, projectionCorrelationId, namingBuilder, partitionStateCache, coreProjection, ioDispatcher,
firstPhase);
}
protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase)
{
var coreProjectionCheckpointWriter =
new CoreProjectionCheckpointWriter(
namingBuilder.MakeCheckpointStreamName(), ioDispatcher, _projectionVersion, _name);
var checkpointManager2 = new DefaultCheckpointManager(
publisher, projectionCorrelationId, _projectionVersion, _projectionConfig.RunAs, ioDispatcher,
_projectionConfig, _name, new PhasePositionTagger(1), namingBuilder, GetUseCheckpoints(), false,
_sourceDefinition.DefinesFold, coreProjectionCheckpointWriter);
IProjectionProcessingPhase writeResultsPhase;
if (GetProducesRunningResults() ||
!string.IsNullOrEmpty(_sourceDefinition.CatalogStream) && _sourceDefinition.ByStreams)
{
writeResultsPhase = new WriteQueryEofProjectionProcessingPhase(
1, namingBuilder.GetResultStreamName(), coreProjection, partitionStateCache, checkpointManager2,
checkpointManager2);
}
else
{
writeResultsPhase = new WriteQueryResultProjectionProcessingPhase(
1, namingBuilder.GetResultStreamName(), coreProjection, partitionStateCache, checkpointManager2,
checkpointManager2);
}
return(new[] { firstPhase, writeResultsPhase });
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
IEmittedStreamsTracker emittedStreamsTracker)
{
return(new ParallelQueryMasterProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_spoolProcessingResponseDispatcher,
emittedStreamsTracker));
}
public WriteQueryEofProjectionProcessingPhase(
int phase, string resultStream, ICoreProjectionForProcessingPhase coreProjection,
PartitionStateCache stateCache, ICoreProjectionCheckpointManager checkpointManager,
IEmittedEventWriter emittedEventWriter)
: base(phase, resultStream, coreProjection, stateCache, checkpointManager, emittedEventWriter)
{
}
public void when()
{
_cache = new PartitionStateCache(CheckpointTag.FromPosition(0, -1));
_cachedAtCheckpointTag = CheckpointTag.FromPosition(1000, 900);
_cache.CachePartitionState(
"partition", new PartitionState("data", null, _cachedAtCheckpointTag));
}
protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase)
{
var coreProjectionCheckpointWriter =
new CoreProjectionCheckpointWriter(
namingBuilder.MakeCheckpointStreamName(), ioDispatcher, _projectionVersion, _name);
var checkpointManager2 = new DefaultCheckpointManager(
publisher, projectionCorrelationId, _projectionVersion, _projectionConfig.RunAs, ioDispatcher,
_projectionConfig, _name, new PhasePositionTagger(1), namingBuilder, GetUseCheckpoints(), false,
_sourceDefinition.DefinesFold, coreProjectionCheckpointWriter);
IProjectionProcessingPhase writeResultsPhase;
if (GetProducesRunningResults()
|| !string.IsNullOrEmpty(_sourceDefinition.CatalogStream) && _sourceDefinition.ByStreams)
writeResultsPhase = new WriteQueryEofProjectionProcessingPhase(
1, namingBuilder.GetResultStreamName(), coreProjection, partitionStateCache, checkpointManager2,
checkpointManager2);
else
writeResultsPhase = new WriteQueryResultProjectionProcessingPhase(
1, namingBuilder.GetResultStreamName(), coreProjection, partitionStateCache, checkpointManager2,
checkpointManager2);
return new[] {firstPhase, writeResultsPhase};
}
public override IProjectionProcessingPhase[] CreateProcessingPhases(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ProjectionNamesBuilder namingBuilder,
ITimeProvider timeProvider, IODispatcher ioDispatcher,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
{
return new IProjectionProcessingPhase[] {_phase1, _phase2};
}
public void given()
{
//given
_cache = new PartitionStateCache();
_cachedAtCheckpointTag = CheckpointTag.FromPosition(1000, 900);
_cache.CacheAndLockPartitionState("partition", new PartitionStateCache.State("data", _cachedAtCheckpointTag), _cachedAtCheckpointTag);
_relockedData = _cache.TryGetAndLockPartitionState("partition", CheckpointTag.FromPosition(2000, 1900));
}
protected abstract IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache,
CoreProjection coreProjection,
IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase);
public CoreProjection Create(
Guid projectionCorrelationId,
IPublisher inputQueue,
Guid workerId,
IPrincipal runAs,
IPublisher publisher,
IODispatcher ioDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher,
ITimeProvider timeProvider)
{
if (inputQueue == null)
{
throw new ArgumentNullException("inputQueue");
}
//if (runAs == null) throw new ArgumentNullException("runAs");
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (ioDispatcher == null)
{
throw new ArgumentNullException("ioDispatcher");
}
if (timeProvider == null)
{
throw new ArgumentNullException("timeProvider");
}
var namingBuilder = new ProjectionNamesBuilder(_name, GetSourceDefinition());
var coreProjectionCheckpointWriter =
new CoreProjectionCheckpointWriter(
namingBuilder.MakeCheckpointStreamName(),
ioDispatcher,
_projectionVersion,
namingBuilder.EffectiveProjectionName);
var partitionStateCache = new PartitionStateCache();
return(new CoreProjection(
this,
_projectionVersion,
projectionCorrelationId,
inputQueue,
workerId,
runAs,
publisher,
ioDispatcher,
subscriptionDispatcher,
_logger,
namingBuilder,
coreProjectionCheckpointWriter,
partitionStateCache,
namingBuilder.EffectiveProjectionName,
timeProvider));
}
public void it_can_be_created()
{
var coreProjection = new FakeCoreProjection();
var stateCache = new PartitionStateCache();
var bus = new InMemoryBus("test");
var fakeCheckpointManager = new specification_with_multi_phase_core_projection.FakeCheckpointManager(bus, Guid.NewGuid());
var it = new WriteQueryResultProjectionProcessingPhase(
1, "result-stream", coreProjection, stateCache,
fakeCheckpointManager, fakeCheckpointManager);
}
public abstract IProjectionProcessingPhase[] CreateProcessingPhases(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ProjectionNamesBuilder namingBuilder,
ITimeProvider timeProvider,
IODispatcher ioDispatcher,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter);
public abstract IProjectionProcessingPhase[] CreateProcessingPhases(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ProjectionNamesBuilder namingBuilder,
ITimeProvider timeProvider,
IODispatcher ioDispatcher,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter);
public void when()
{
try
{
_cache = new PartitionStateCache(CheckpointTag.FromPosition(0, -1));
}
catch (Exception ex)
{
_exception = ex;
}
}
public void when()
{
try
{
_cache = new PartitionStateCache();
}
catch (Exception ex)
{
_exception = ex;
}
}
protected virtual PartitionStateCache GivenStateCache()
{
var stateCache = new PartitionStateCache();
stateCache.CachePartitionState(
"a", new PartitionState("{}", null, CheckpointTag.FromPhase(0, completed: false)));
stateCache.CachePartitionState(
"b", new PartitionState("{}", null, CheckpointTag.FromPhase(0, completed: false)));
stateCache.CachePartitionState(
"c", new PartitionState("{}", null, CheckpointTag.FromPhase(0, completed: false)));
return stateCache;
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag, ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy, IResultWriter resultWriter)
{
return new ParallelQueryMasterProjectionProcessingPhase(
coreProjection, projectionCorrelationId, publisher, _projectionConfig, updateStatistics,
partitionStateCache, _name, _logger, zeroCheckpointTag, checkpointManager, subscriptionDispatcher,
readerStrategy, resultWriter, _projectionConfig.CheckpointsEnabled, this.GetStopOnEof(),
_spoolProcessingResponseDispatcher);
}
public EventProcessingProjectionProcessingPhase(
CoreProjection coreProjection,
Guid projectionCorrelationId,
IPublisher publisher,
IPublisher inputQueue,
ProjectionConfig projectionConfig,
Action updateStatistics,
IProjectionStateHandler projectionStateHandler,
PartitionStateCache partitionStateCache,
bool definesStateTransform,
string projectionName,
ILogger logger,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager coreProjectionCheckpointManager,
StatePartitionSelector statePartitionSelector,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
bool useCheckpoints,
bool stopOnEof,
bool isBiState,
bool orderedPartitionProcessing,
IEmittedStreamsTracker emittedStreamsTracker,
bool enableContentTypeValidation)
: base(
publisher,
inputQueue,
coreProjection,
projectionCorrelationId,
coreProjectionCheckpointManager,
projectionConfig,
projectionName,
logger,
zeroCheckpointTag,
partitionStateCache,
resultWriter,
updateStatistics,
subscriptionDispatcher,
readerStrategy,
useCheckpoints,
stopOnEof,
orderedPartitionProcessing,
isBiState,
emittedStreamsTracker,
enableContentTypeValidation)
{
_projectionStateHandler = projectionStateHandler;
_definesStateTransform = definesStateTransform;
_statePartitionSelector = statePartitionSelector;
_isBiState = isBiState;
_stopwatch = new Stopwatch();
}
protected abstract IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter);
public void SetUp()
{
_stateCache = GivenStateCache();
_publisher = new InMemoryBus("test");
_coreProjection = new FakeCoreProjection();
_checkpointManager = new specification_with_multi_phase_core_projection.FakeCheckpointManager(
_publisher, Guid.NewGuid());
_resultStreamName = "result-stream";
_phase = new WriteQueryResultProjectionProcessingPhase(
1, _resultStreamName, _coreProjection, _stateCache, _checkpointManager, _checkpointManager);
When();
}
public CoreProjection(
string name, Guid projectionCorrelationId, IPublisher publisher,
IProjectionStateHandler projectionStateHandler, ProjectionConfig projectionConfig,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher <ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ILogger logger = null)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
if (name == "")
{
throw new ArgumentException("name");
}
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (projectionStateHandler == null)
{
throw new ArgumentNullException("projectionStateHandler");
}
_projectionStateHandler = projectionStateHandler;
_projectionCorrelationId = projectionCorrelationId;
var namingBuilder = new ProjectionNamesBuilder();
_projectionStateHandler.ConfigureSourceProcessingStrategy(namingBuilder);
_name = namingBuilder.ForceProjectionName ?? name;
//TODO: move into name builder
_stateStreamNamePattern = namingBuilder.StateStreamName ?? ProjectionsStreamPrefix + _name + "-{0}" + ProjectionsStateStreamSuffix;
_stateStreamName = namingBuilder.StateStreamName ?? ProjectionsStreamPrefix + _name + ProjectionsStateStreamSuffix;
_projectionConfig = projectionConfig;
_logger = logger;
_publisher = publisher;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
var builder = new CheckpointStrategy.Builder();
_projectionStateHandler.ConfigureSourceProcessingStrategy(builder);
_checkpointStrategy = builder.Build(_projectionConfig.Mode);
_partitionStateCache = new PartitionStateCache();
_processingQueue = new CoreProjectionQueue(
projectionCorrelationId, publisher, projectionConfig.PendingEventsThreshold, UpdateStatistics);
_checkpointManager = this._checkpointStrategy.CreateCheckpointManager(
this, projectionCorrelationId, this._publisher, this._readDispatcher,
this._writeDispatcher, this._projectionConfig, this._name, _stateStreamName);
GoToState(State.Initial);
}
public void setup()
{
//given
_cache = new PartitionStateCache();
_cachedAtCheckpointTag1 = CheckpointTag.FromPosition(1000, 900);
_cachedAtCheckpointTag2 = CheckpointTag.FromPosition(1200, 1100);
_cachedAtCheckpointTag3 = CheckpointTag.FromPosition(1400, 1300);
_cache.CacheAndLockPartitionState("partition1", "data1", _cachedAtCheckpointTag1);
_cache.CacheAndLockPartitionState("partition2", "data2", _cachedAtCheckpointTag2);
_cache.CacheAndLockPartitionState("partition3", "data3", _cachedAtCheckpointTag3);
// when
_cache.Unlock(_cachedAtCheckpointTag2);
}
public GetStateWorkItem(
IEnvelope envelope, Guid correlationId, Guid projectionId, CoreProjection projection,
PartitionStateCache partitionStateCache, string partition)
: base(projection, string.Empty)
{
if (envelope == null) throw new ArgumentNullException("envelope");
if (partitionStateCache == null) throw new ArgumentNullException("partitionStateCache");
if (partition == null) throw new ArgumentNullException("partition");
_partition = partition;
_envelope = envelope;
_correlationId = correlationId;
_projectionId = projectionId;
_partitionStateCache = partitionStateCache;
}
public ParallelQueryMasterProjectionProcessingPhase(
CoreProjection coreProjection, Guid projectionCorrelationId, IPublisher publisher,
ProjectionConfig projectionConfig, Action updateStatistics, PartitionStateCache partitionStateCache,
string name, ILogger logger, CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, IResultWriter resultWriter, bool checkpointsEnabled, bool stopOnEof,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher)
: base(
publisher, coreProjection, projectionCorrelationId, checkpointManager, projectionConfig, name, logger,
zeroCheckpointTag, partitionStateCache, resultWriter, updateStatistics, subscriptionDispatcher,
readerStrategy, checkpointsEnabled, stopOnEof)
{
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag, ICoreProjectionCheckpointManager checkpointManager, IReaderStrategy readerStrategy, IResultWriter resultWriter)
{
var statePartitionSelector = CreateStatePartitionSelector(
_stateHandler, _sourceDefinition.ByCustomPartitions, _sourceDefinition.ByStreams);
return(new EventProcessingProjectionProcessingPhase(
coreProjection, projectionCorrelationId, publisher, _projectionConfig, updateStatistics,
_stateHandler, partitionStateCache, _sourceDefinition.DefinesStateTransform, _name, _logger,
zeroCheckpointTag, checkpointManager, statePartitionSelector, subscriptionDispatcher, readerStrategy,
resultWriter, _projectionConfig.CheckpointsEnabled, this.GetStopOnEof()));
}
public WriteQueryResultProjectionProcessingPhaseBase(
IPublisher publisher,
int phase,
string resultStream,
ICoreProjectionForProcessingPhase coreProjection,
PartitionStateCache stateCache,
ICoreProjectionCheckpointManager checkpointManager,
IEmittedEventWriter emittedEventWriter,
IEmittedStreamsTracker emittedStreamsTracker)
{
if (resultStream == null)
{
throw new ArgumentNullException("resultStream");
}
if (coreProjection == null)
{
throw new ArgumentNullException("coreProjection");
}
if (stateCache == null)
{
throw new ArgumentNullException("stateCache");
}
if (checkpointManager == null)
{
throw new ArgumentNullException("checkpointManager");
}
if (emittedEventWriter == null)
{
throw new ArgumentNullException("emittedEventWriter");
}
if (emittedStreamsTracker == null)
{
throw new ArgumentNullException("emittedStreamsTracker");
}
if (string.IsNullOrEmpty(resultStream))
{
throw new ArgumentException("resultStream");
}
_publisher = publisher;
_phase = phase;
_resultStream = resultStream;
_coreProjection = coreProjection;
_stateCache = stateCache;
_checkpointManager = checkpointManager;
_emittedEventWriter = emittedEventWriter;
_emittedStreamsTracker = emittedStreamsTracker;
}
protected EventSubscriptionBasedProjectionProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
ICoreProjectionForProcessingPhase coreProjection,
Guid projectionCorrelationId,
ICoreProjectionCheckpointManager checkpointManager,
ProjectionConfig projectionConfig,
string projectionName,
ILogger logger,
CheckpointTag zeroCheckpointTag,
PartitionStateCache partitionStateCache,
IResultWriter resultWriter,
Action updateStatistics,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
bool useCheckpoints,
bool stopOnEof,
bool orderedPartitionProcessing,
bool isBiState,
IEmittedStreamsTracker emittedStreamsTracker,
bool enableContentTypeValidation)
{
_publisher = publisher;
_inputQueue = inputQueue;
_coreProjection = coreProjection;
_projectionCorrelationId = projectionCorrelationId;
_checkpointManager = checkpointManager;
_projectionConfig = projectionConfig;
_projectionName = projectionName;
_logger = logger;
_zeroCheckpointTag = zeroCheckpointTag;
_partitionStateCache = partitionStateCache;
_resultWriter = resultWriter;
_updateStatistics = updateStatistics;
_processingQueue = new CoreProjectionQueue(publisher,
projectionConfig.PendingEventsThreshold,
orderedPartitionProcessing);
_processingQueue.EnsureTickPending += EnsureTickPending;
_subscriptionDispatcher = subscriptionDispatcher;
_readerStrategy = readerStrategy;
_useCheckpoints = useCheckpoints;
_stopOnEof = stopOnEof;
_isBiState = isBiState;
_progressResultWriter = new ProgressResultWriter(this, _resultWriter);
_inutQueueEnvelope = new PublishEnvelope(_inputQueue);
_emittedStreamsTracker = emittedStreamsTracker;
_enableContentTypeValidation = enableContentTypeValidation;
}
public void setup()
{
//given
_cache = new PartitionStateCache();
_cachedAtCheckpointTag1 = CheckpointTag.FromPosition(0, 1000, 900);
_cachedAtCheckpointTag2 = CheckpointTag.FromPosition(0, 1200, 1100);
_cachedAtCheckpointTag3 = CheckpointTag.FromPosition(0, 1400, 1300);
_cache.CacheAndLockPartitionState(
"partition1", new PartitionState("data1", null, _cachedAtCheckpointTag1), _cachedAtCheckpointTag1);
_cache.CacheAndLockPartitionState(
"partition2", new PartitionState("data2", null, _cachedAtCheckpointTag2), _cachedAtCheckpointTag2);
_cache.CacheAndLockPartitionState(
"partition3", new PartitionState("data3", null, _cachedAtCheckpointTag3), _cachedAtCheckpointTag3);
// when
_cache.Unlock(_cachedAtCheckpointTag2, forgetUnlocked: true);
}
//NOTE: this is only for slave projections (TBD)
public CoreProjection(
ProjectionProcessingStrategy projectionProcessingStrategy, ProjectionVersion version,
Guid projectionCorrelationId, IPublisher inputQueue, IPrincipal runAs, IPublisher publisher, IODispatcher ioDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher, ILogger logger, ProjectionNamesBuilder namingBuilder, CoreProjectionCheckpointWriter coreProjectionCheckpointWriter,
PartitionStateCache partitionStateCache, string effectiveProjectionName, ITimeProvider timeProvider, bool isSlaveProjection)
{
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (ioDispatcher == null)
{
throw new ArgumentNullException("ioDispatcher");
}
if (subscriptionDispatcher == null)
{
throw new ArgumentNullException("subscriptionDispatcher");
}
_projectionProcessingStrategy = projectionProcessingStrategy;
_projectionCorrelationId = projectionCorrelationId;
_inputQueue = inputQueue;
_runAs = runAs;
_name = effectiveProjectionName;
_version = version;
_stopOnEof = projectionProcessingStrategy.GetStopOnEof();
_logger = logger;
_publisher = publisher;
_partitionStateCache = partitionStateCache;
_requiresRootPartition = projectionProcessingStrategy.GetRequiresRootPartition();
_isSlaveProjection = isSlaveProjection;
var useCheckpoints = projectionProcessingStrategy.GetUseCheckpoints();
_coreProjectionCheckpointWriter = coreProjectionCheckpointWriter;
_projectionProcessingPhases = projectionProcessingStrategy.CreateProcessingPhases(
publisher, projectionCorrelationId, partitionStateCache, UpdateStatistics, this, namingBuilder,
timeProvider, ioDispatcher, coreProjectionCheckpointWriter);
//NOTE: currently assuming the first checkpoint manager to be able to load any state
_checkpointReader = new CoreProjectionCheckpointReader(
publisher, _projectionCorrelationId, ioDispatcher, namingBuilder.MakeCheckpointStreamName(), _version,
useCheckpoints);
_enrichStatistics = projectionProcessingStrategy.EnrichStatistics;
GoToState(State.Initial);
}
public void setup()
{
//given
_cache = new PartitionStateCache(10);
_cachedAtCheckpointTag1 = CheckpointTag.FromPosition(1000, 900);
for (var i = 0; i < 15; i++)
_cache.CacheAndLockPartitionState(
"partition1", "data1", CheckpointTag.FromPosition(1000 + (i*100), 1000 + (i*100) - 50));
_cachedAtCheckpointTag2 = CheckpointTag.FromPosition(20100, 20050);
_cachedAtCheckpointTag3 = CheckpointTag.FromPosition(20200, 20150);
_cache.CacheAndLockPartitionState("partition1", "data1", _cachedAtCheckpointTag1);
_cache.CacheAndLockPartitionState("partition2", "data2", _cachedAtCheckpointTag2);
_cache.CacheAndLockPartitionState("partition3", "data3", _cachedAtCheckpointTag3);
// when
_cache.Unlock(_cachedAtCheckpointTag2);
}
public ParallelQueryMasterProjectionProcessingPhase(
CoreProjection coreProjection,
Guid projectionCorrelationId,
IPublisher publisher,
IPublisher inputQueue,
ProjectionConfig projectionConfig,
Action updateStatistics,
IProjectionStateHandler stateHandler,
PartitionStateCache partitionStateCache,
string name,
ILogger logger,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
bool checkpointsEnabled,
bool stopOnEof,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
IEmittedStreamsTracker emittedStreamsTracker)
: base(
publisher,
inputQueue,
coreProjection,
projectionCorrelationId,
checkpointManager,
projectionConfig,
name,
logger,
zeroCheckpointTag,
partitionStateCache,
resultWriter,
updateStatistics,
subscriptionDispatcher,
readerStrategy,
checkpointsEnabled,
stopOnEof,
orderedPartitionProcessing: true,
isBiState: false,
emittedStreamsTracker: emittedStreamsTracker)
{
_stateHandler = stateHandler;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
_spoolProcessingWorkItems = new Dictionary <Guid, SpoolStreamProcessingWorkItem>();
}
public EventProcessingProjectionProcessingPhase(
CoreProjection coreProjection, Guid projectionCorrelationId, IPublisher publisher,
ProjectionConfig projectionConfig, Action updateStatistics, IProjectionStateHandler projectionStateHandler,
PartitionStateCache partitionStateCache, bool definesStateTransform, string projectionName, ILogger logger,
CheckpointTag zeroCheckpointTag, ICoreProjectionCheckpointManager coreProjectionCheckpointManager,
StatePartitionSelector statePartitionSelector, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, IResultWriter resultWriter, bool useCheckpoints, bool stopOnEof)
: base(
publisher, coreProjection, projectionCorrelationId, coreProjectionCheckpointManager, projectionConfig,
projectionName, logger, zeroCheckpointTag, partitionStateCache, resultWriter, updateStatistics,
subscriptionDispatcher, readerStrategy, useCheckpoints, stopOnEof)
{
_projectionStateHandler = projectionStateHandler;
_definesStateTransform = definesStateTransform;
_statePartitionSelector = statePartitionSelector;
_stopwatch = new Stopwatch();
}
public CoreProjection Create(
Guid projectionCorrelationId,
IPublisher inputQueue,
Guid workerId,
IPrincipal runAs,
IPublisher publisher,
IODispatcher ioDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher,
ITimeProvider timeProvider)
{
if (inputQueue == null) throw new ArgumentNullException("inputQueue");
//if (runAs == null) throw new ArgumentNullException("runAs");
if (publisher == null) throw new ArgumentNullException("publisher");
if (ioDispatcher == null) throw new ArgumentNullException("ioDispatcher");
if (timeProvider == null) throw new ArgumentNullException("timeProvider");
var namingBuilder = new ProjectionNamesBuilder(_name, GetSourceDefinition());
var coreProjectionCheckpointWriter =
new CoreProjectionCheckpointWriter(
namingBuilder.MakeCheckpointStreamName(),
ioDispatcher,
_projectionVersion,
namingBuilder.EffectiveProjectionName);
var partitionStateCache = new PartitionStateCache();
return new CoreProjection(
this,
_projectionVersion,
projectionCorrelationId,
inputQueue,
workerId,
runAs,
publisher,
ioDispatcher,
subscriptionDispatcher,
_logger,
namingBuilder,
coreProjectionCheckpointWriter,
partitionStateCache,
namingBuilder.EffectiveProjectionName,
timeProvider);
}
protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache,
CoreProjection coreProjection,
IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase)
{
var coreProjectionCheckpointWriter =
new CoreProjectionCheckpointWriter(
namingBuilder.MakeCheckpointStreamName(),
ioDispatcher,
_projectionVersion,
_name);
var checkpointManager2 = new DefaultCheckpointManager(
publisher,
projectionCorrelationId,
_projectionVersion,
_projectionConfig.RunAs,
ioDispatcher,
_projectionConfig,
_name,
new PhasePositionTagger(1),
namingBuilder,
GetUseCheckpoints(),
false,
_sourceDefinition.DefinesFold,
coreProjectionCheckpointWriter);
var writeResultsPhase = new WriteQueryEofProjectionProcessingPhase(
publisher,
1,
namingBuilder.GetResultStreamName(),
coreProjection,
partitionStateCache,
checkpointManager2,
checkpointManager2,
firstPhase.EmittedStreamsTracker);
return new[] {firstPhase, writeResultsPhase};
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
IEmittedStreamsTracker emittedStreamsTracker)
{
var statePartitionSelector = CreateStatePartitionSelector();
var orderedPartitionProcessing = _sourceDefinition.ByStreams && _sourceDefinition.IsBiState;
return(new EventProcessingProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_sourceDefinition.DefinesStateTransform,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
statePartitionSelector,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_sourceDefinition.IsBiState,
orderedPartitionProcessing: orderedPartitionProcessing,
emittedStreamsTracker: emittedStreamsTracker,
enableContentTypeValidation: _enableContentTypeValidation));
}
protected override IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher, IPublisher inputQueue, Guid projectionCorrelationId,
ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache, CoreProjection coreProjection, IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase)
{
var coreProjectionCheckpointWriter =
new CoreProjectionCheckpointWriter(
namingBuilder.MakeCheckpointStreamName(), ioDispatcher, _projectionVersion, _name);
var checkpointManager2 = new DefaultCheckpointManager(
publisher, projectionCorrelationId, _projectionVersion, SystemAccount.Principal, ioDispatcher,
_projectionConfig, _name, new PhasePositionTagger(1), namingBuilder, GetUseCheckpoints(), false,
_sourceDefinition.DefinesFold, coreProjectionCheckpointWriter);
IProjectionProcessingPhase writeResultsPhase;
if (GetProducesRunningResults())
{
writeResultsPhase = new WriteQueryEofProjectionProcessingPhase(
publisher,
1,
namingBuilder.GetResultStreamName(),
coreProjection,
partitionStateCache,
checkpointManager2,
checkpointManager2,
firstPhase.EmittedStreamsTracker);
}
else
{
writeResultsPhase = new WriteQueryResultProjectionProcessingPhase(
publisher,
1,
namingBuilder.GetResultStreamName(),
coreProjection,
partitionStateCache,
checkpointManager2,
checkpointManager2,
firstPhase.EmittedStreamsTracker);
}
return(new[] { firstPhase, writeResultsPhase });
}
public CoreProjection(
string name, Guid projectionCorrelationId, IPublisher publisher,
IProjectionStateHandler projectionStateHandler, ProjectionConfig projectionConfig, ILogger logger = null)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
if (name == "")
{
throw new ArgumentException("name");
}
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (projectionStateHandler == null)
{
throw new ArgumentNullException("projectionStateHandler");
}
_projectionCorrelationId = projectionCorrelationId;
_name = name;
_projectionConfig = projectionConfig;
_logger = logger;
_publisher = publisher;
_projectionStateHandler = projectionStateHandler;
_readDispatcher =
new RequestResponseDispatcher
<ClientMessage.ReadEventsBackwards, ClientMessage.ReadEventsBackwardsCompleted>(
_publisher, v => v.CorrelationId, v => v.CorrelationId);
_projectionCheckpointStreamId = ProjectionsStreamPrefix + _name + ProjectionCheckpointStreamSuffix;
var builder = new CheckpointStrategy.Builder();
_projectionStateHandler.ConfigureSourceProcessingStrategy(builder);
_checkpointStrategy = builder.Build(_projectionConfig.Mode);
_eventFilter = _checkpointStrategy.EventFilter;
_lastProcessedEventPosition = new PositionTracker(_checkpointStrategy.PositionTagger);
_partitionStateCache = new PartitionStateCache();
GoToState(State.Initial);
}
protected GetDataWorkItemBase(
IEnvelope envelope, Guid correlationId, Guid projectionId, CoreProjection projection,
PartitionStateCache partitionStateCache, string partition)
: base(projection, null)
{
if (envelope == null)
{
throw new ArgumentNullException("envelope");
}
if (partitionStateCache == null)
{
throw new ArgumentNullException("partitionStateCache");
}
if (partition == null)
{
throw new ArgumentNullException("partition");
}
_partition = partition;
_envelope = envelope;
_correlationId = correlationId;
_projectionId = projectionId;
_partitionStateCache = partitionStateCache;
}
public void setup()
{
//given
_cache = new PartitionStateCache(10);
_cachedAtCheckpointTag1 = CheckpointTag.FromPosition(0, 1000, 900);
for (var i = 0; i < 15; i++)
{
CheckpointTag at = CheckpointTag.FromPosition(0, 1000 + (i*100), 1000 + (i*100) - 50);
_cache.CacheAndLockPartitionState("partition1", new PartitionState("data1", null, at), at);
}
_cachedAtCheckpointTag2 = CheckpointTag.FromPosition(0, 20100, 20050);
_cachedAtCheckpointTag3 = CheckpointTag.FromPosition(0, 20200, 20150);
_cache.CacheAndLockPartitionState(
"partition1", new PartitionState("data1", null, _cachedAtCheckpointTag1), _cachedAtCheckpointTag1);
_cache.CacheAndLockPartitionState(
"partition2", new PartitionState("data2", null, _cachedAtCheckpointTag2), _cachedAtCheckpointTag2);
_cache.CacheAndLockPartitionState(
"partition3", new PartitionState("data3", null, _cachedAtCheckpointTag3), _cachedAtCheckpointTag3);
_cache.Unlock(_cachedAtCheckpointTag2);
// when
_cache.Initialize();
}
public GetStateWorkItem(IEnvelope envelope,
Guid projectionId,
CoreProjection projection,
PartitionStateCache partitionStateCache,
string partition)
: base(projection, string.Empty)
{
if (envelope == null)
{
throw new ArgumentNullException("envelope");
}
if (partitionStateCache == null)
{
throw new ArgumentNullException("partitionStateCache");
}
if (partition == null)
{
throw new ArgumentNullException("partition");
}
_partition = partition;
_envelope = envelope;
_projectionId = projectionId;
_partitionStateCache = partitionStateCache;
}
private void EmitStateUpdatedEventsIfAny(string partition, PartitionStateCache.State oldState, PartitionStateCache.State newState)
{
if (_emitStateUpdated && newState != null)
{
//TODO: move it our to writeoutput stage (currently in CheckpointManager, so it can decide if it needs to generate stateupdated)
var stateUpdatedEvents = CreateStateUpdatedEvents(
_namingBuilder, _positionTagger.MakeZeroCheckpointTag(), partition, oldState, newState);
if (stateUpdatedEvents != null)
EventsEmitted(stateUpdatedEvents.ToArray());
}
}
private void CaptureParitionStateUpdated(string partition, PartitionStateCache.State oldState, PartitionStateCache.State newState)
{
if (_partitionStateUpdateManager == null)
_partitionStateUpdateManager = new PartitionStateUpdateManager(_namingBuilder);
_partitionStateUpdateManager.StateUpdated(partition, newState.Data, oldState.CausedBy, newState.CausedBy);
}
public void StateUpdated(string partition, PartitionStateCache.State oldState, PartitionStateCache.State newState)
{
if (_emitStateUpdated)
EmitStateUpdatedEventsIfAny(partition, oldState, newState);
if (_emitPartitionCheckpoints && partition != "")
CaptureParitionStateUpdated(partition, oldState, newState);
if (partition == "" && newState != null) // ignore non-root partitions and non-changed states
_currentProjectionState = newState.Data;
EnsureStarted();
if (_stopping)
throw new InvalidOperationException("Stopping");
}
private static List<EmittedEvent> CreateStateUpdatedEvents(ProjectionNamesBuilder projectionNamesBuilder, CheckpointTag zeroCheckpointTag, string partition, PartitionStateCache.State oldState, PartitionStateCache.State newState)
{
var result = new List<EmittedEvent>();
if (!String.IsNullOrEmpty(partition)
&& (oldState.CausedBy == null || oldState.CausedBy == zeroCheckpointTag))
{
result.Add(
new EmittedEvent(
projectionNamesBuilder.GetPartitionCatalogStreamName(), Guid.NewGuid(), "PartitionCreated", partition, newState.CausedBy,
null));
}
result.Add(
new EmittedEvent(
projectionNamesBuilder.MakePartitionStateStreamName(partition), Guid.NewGuid(), "StateUpdated", newState.Data,
newState.CausedBy, oldState.CausedBy));
return result;
}
protected abstract IProjectionProcessingPhase[] CreateProjectionProcessingPhases(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
ProjectionNamesBuilder namingBuilder,
PartitionStateCache partitionStateCache,
CoreProjection coreProjection,
IODispatcher ioDispatcher,
IProjectionProcessingPhase firstPhase);
public GetResultWorkItem(
IEnvelope envelope, Guid correlationId, Guid projectionId, CoreProjection projection,
PartitionStateCache partitionStateCache, string partition)
: base(envelope, correlationId, projectionId, projection, partitionStateCache, partition)
{
}
protected abstract IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter);
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter)
{
var statePartitionSelector = CreateStatePartitionSelector();
var orderedPartitionProcessing = _sourceDefinition.ByStreams && _sourceDefinition.IsBiState;
return new EventProcessingProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_sourceDefinition.DefinesStateTransform,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
statePartitionSelector,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_sourceDefinition.IsBiState,
orderedPartitionProcessing: orderedPartitionProcessing);
}