protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
ExistingEvent("catalog", "$>", null, "0@test-stream");
ExistingEvent("catalog", "$>", null, "0@test-stream2");
ExistingEvent("catalog", "$>", null, "0@test-stream3");
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
CatalogStream = "catalog",
AllEvents = true,
ByStreams = true,
Options = new QuerySourcesDefinitionOptions { }
};
_readerStrategy = ReaderStrategy.Create(0, _sourceDefinition, _timeProvider, stopOnEof: true, runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
ExistingEvent("stream-a", "type1", "{}", "{Data: 1}");
ExistingEvent("stream-b", "type1", "{}", "{Data: 2}");
GivenOtherEvents();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
Streams = new []{"stream-a", "stream-b"},
AllEvents = true,
Options = new QuerySourcesDefinitionOptions {ReorderEvents = true, ProcessingLag = 100}
};
_readerStrategy = ReaderStrategy.Create(
"test",
0,
_sourceDefinition,
_timeProvider,
stopOnEof: false,
runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: false,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
_subscriptionId = Guid.NewGuid();
_readerStrategy = ReaderStrategy.CreateExternallyFedReaderStrategy(
"test",
0,
_timeProvider,
SystemAccount.Principal,
limitingCommitPosition: 10000);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
ExistingEvent("$$test-stream", "$metadata", "", "{Meta: 1}");
ExistingEvent("$$test-stream2", "$metadata", "", "{Meta: 2}");
ExistingEvent("$$test-stream3", "$metadata", "", "{Meta: 3}");
ExistingEvent("$streams", "$>", null, "0@test-stream");
ExistingEvent("$streams", "$>", null, "0@test-stream2");
ExistingEvent("$streams", "$>", null, "0@test-stream3");
ExistingEvent("$streams", "$>", null, "0@test-stream4");
NoOtherStreams();
_subscriptionId = Guid.NewGuid();
_readerStrategy = new ParallelQueryAllStreamsMasterReaderStrategy(
"test",
0,
SystemAccount.Principal,
_timeProvider);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
protected ReaderSubscriptionBase(
IPublisher publisher,
Guid subscriptionId,
CheckpointTag @from,
IReaderStrategy readerStrategy,
ITimeProvider timeProvider,
long? checkpointUnhandledBytesThreshold,
int? checkpointProcessedEventsThreshold,
bool stopOnEof,
int? stopAfterNEvents)
{
if (publisher == null) throw new ArgumentNullException("publisher");
if (readerStrategy == null) throw new ArgumentNullException("readerStrategy");
if (timeProvider == null) throw new ArgumentNullException("timeProvider");
if (checkpointProcessedEventsThreshold > 0 && stopAfterNEvents > 0)
throw new ArgumentException("checkpointProcessedEventsThreshold > 0 && stopAfterNEvents > 0");
_publisher = publisher;
_readerStrategy = readerStrategy;
_timeProvider = timeProvider;
_checkpointUnhandledBytesThreshold = checkpointUnhandledBytesThreshold;
_checkpointProcessedEventsThreshold = checkpointProcessedEventsThreshold;
_stopOnEof = stopOnEof;
_stopAfterNEvents = stopAfterNEvents;
_subscriptionId = subscriptionId;
_lastPassedOrCheckpointedEventPosition = null;
_eventFilter = readerStrategy.EventFilter;
_positionTagger = readerStrategy.PositionTagger;
_positionTracker = new PositionTracker(_positionTagger);
_positionTracker.UpdateByCheckpointTagInitial(@from);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream", "type2", "{}", "{Data: 3}");
GivenInitialIndexState();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
AllStreams = true,
Events = new[] {"type1", "type2"},
Options = new QuerySourcesDefinitionOptions {}
};
_readerStrategy = ReaderStrategy.Create(
"test",
0,
_sourceDefinition,
_timeProvider,
stopOnEof: false,
runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: false,
stopAfterNEvents: null);
}
public void setup()
{
_checkpointUnhandledBytesThreshold = 1000;
_checkpointProcessedEventsThreshold = 2000;
Given();
_bus = new InMemoryBus("bus");
_projectionCorrelationId = Guid.NewGuid();
_eventHandler = new TestHandler<EventReaderSubscriptionMessage.CommittedEventReceived>();
_checkpointHandler = new TestHandler<EventReaderSubscriptionMessage.CheckpointSuggested>();
_progressHandler = new TestHandler<EventReaderSubscriptionMessage.ProgressChanged>();
_subscriptionStartedHandler = new TestHandler<EventReaderSubscriptionMessage.SubscriptionStarted>();
_notAuthorizedHandler = new TestHandler<EventReaderSubscriptionMessage.NotAuthorized>();
_eofHandler = new TestHandler<EventReaderSubscriptionMessage.EofReached>();
_partitionEofHandler = new TestHandler<EventReaderSubscriptionMessage.PartitionEofReached>();
_partitionMeasuredHandler = new TestHandler<EventReaderSubscriptionMessage.PartitionMeasured>();
_bus.Subscribe(_eventHandler);
_bus.Subscribe(_checkpointHandler);
_bus.Subscribe(_progressHandler);
_bus.Subscribe(_eofHandler);
_bus.Subscribe(_partitionEofHandler);
_bus.Subscribe(_partitionMeasuredHandler);
_readerStrategy = CreateCheckpointStrategy();
_subscription = CreateProjectionSubscription();
When();
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
ExistingEvent("test-stream", "$>", "{}", "{Data: 1}");
ExistingEvent("test-stream", "$>", "{}", "{Data: 2}");
ExistingEvent("test-stream", "$>", "{}", "{Data: 3}");
ExistingEvent("test-stream", "eventType", "{}", "{Data: 4}");
ExistingEvent("test-stream", "eventType", "{}", "{Data: 5}");
ExistingEvent("test-stream", "eventType", "{}", "{Data: 6}");
ExistingEvent("test-stream", "eventType", "{}", "{Data: 7}");
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
ByStreams = true,
AllStreams = true,
AllEvents = true,
Options = new QuerySourcesDefinitionOptions
{
IncludeLinks = false
}
};
_readerStrategy = ReaderStrategy.Create(
"test",
0,
_sourceDefinition,
_timeProvider,
stopOnEof: true,
runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
public FakeProjectionProcessingPhase(int phase, specification_with_multi_phase_core_projection specification,
ICoreProjectionCheckpointManager checkpointManager, IReaderStrategy readerStrategy)
{
_phase = phase;
_specification = specification;
_checkpointManager = checkpointManager;
_readerStrategy = readerStrategy;
}
protected override ICoreProjectionCheckpointManager CreateCheckpointManager(
Guid projectionCorrelationId, IPublisher publisher, IODispatcher ioDispatcher, ProjectionNamesBuilder namingBuilder,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter, bool definesFold, IReaderStrategy readerStrategy)
{
return new NoopCheckpointManager(
publisher, projectionCorrelationId, _projectionConfig, _name, readerStrategy.PositionTagger,
namingBuilder);
}
public ReaderSubscription(
IPublisher publisher, Guid subscriptionId, CheckpointTag @from, IReaderStrategy readerStrategy,
long? checkpointUnhandledBytesThreshold, int? checkpointProcessedEventsThreshold, bool stopOnEof = false,
int? stopAfterNEvents = null)
: base(
publisher, subscriptionId, @from, readerStrategy, checkpointUnhandledBytesThreshold,
checkpointProcessedEventsThreshold, stopOnEof, stopAfterNEvents)
{
}
public ReaderSubscription(
IPublisher publisher, Guid subscriptionId, CheckpointTag @from, IReaderStrategy readerStrategy,
long?checkpointUnhandledBytesThreshold, int?checkpointProcessedEventsThreshold, bool stopOnEof = false,
int?stopAfterNEvents = null)
: base(
publisher, subscriptionId, @from, readerStrategy, checkpointUnhandledBytesThreshold,
checkpointProcessedEventsThreshold, stopOnEof, stopAfterNEvents)
{
}
public Subscribe(
Guid subscriptionId, CheckpointTag from,
IReaderStrategy readerStrategy, ReaderSubscriptionOptions readerSubscriptionOptions): base(subscriptionId)
{
if (@from == null) throw new ArgumentNullException("from");
if (readerStrategy == null) throw new ArgumentNullException("readerStrategy");
_fromPosition = @from;
_readerStrategy = readerStrategy;
_options = readerSubscriptionOptions;
}
protected override ICoreProjectionCheckpointManager CreateCheckpointManager(
Guid projectionCorrelationId, IPublisher publisher, IODispatcher ioDispatcher,
ProjectionNamesBuilder namingBuilder,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter, bool definesFold,
IReaderStrategy readerStrategy)
{
return(new NoopCheckpointManager(
publisher, projectionCorrelationId, _projectionConfig, _name, readerStrategy.PositionTagger,
namingBuilder));
}
protected override ProjectionProcessingStrategy GivenProjectionProcessingStrategy()
{
_phase1checkpointManager = new FakeCheckpointManager(_bus, _projectionCorrelationId);
_phase2checkpointManager = new FakeCheckpointManager(_bus, _projectionCorrelationId);
_phase1readerStrategy = GivenPhase1ReaderStrategy();
_phase2readerStrategy = GivenPhase2ReaderStrategy();
_phase1 = new FakeProjectionProcessingPhase(0, this, Phase1CheckpointManager, _phase1readerStrategy);
_phase2 = new FakeProjectionProcessingPhase(1, this, Phase2CheckpointManager, _phase2readerStrategy);
return(new FakeProjectionProcessingStrategy(
_projectionName, _version, new ConsoleLogger("logger"), Phase1, Phase2));
}
private CheckpointStrategy(
bool byStream, bool byCustomPartitions, bool useCheckpoints, bool definesStateTransform, IPrincipal runAs,
IReaderStrategy readerStrategy)
{
_readerStrategy = readerStrategy;
_byStream = byStream;
_byCustomPartitions = byCustomPartitions;
_useCheckpoints = useCheckpoints;
_definesStateTransform = definesStateTransform;
_runAs = runAs;
}
private CheckpointStrategy(
bool byStream, bool byCustomPartitions, bool useCheckpoints, bool definesStateTransform, IPrincipal runAs,
IReaderStrategy readerStrategy)
{
_readerStrategy = readerStrategy;
_byStream = byStream;
_byCustomPartitions = byCustomPartitions;
_useCheckpoints = useCheckpoints;
_definesStateTransform = definesStateTransform;
_runAs = runAs;
}
public FakeProjectionProcessingPhase(int phase,
specification_with_multi_phase_core_projection <TLogFormat, TStreamId> specification,
ICoreProjectionCheckpointManager checkpointManager, IReaderStrategy readerStrategy,
IEmittedStreamsTracker emittedStreamsTracker)
{
_phase = phase;
_specification = specification;
_checkpointManager = checkpointManager;
_readerStrategy = readerStrategy;
_emittedStreamsTracker = emittedStreamsTracker;
}
public EventProcessingProjectionProcessingPhase(
CoreProjection coreProjection,
Guid projectionCorrelationId,
IPublisher publisher,
IPublisher inputQueue,
ProjectionConfig projectionConfig,
Action updateStatistics,
IProjectionStateHandler projectionStateHandler,
PartitionStateCache partitionStateCache,
bool definesStateTransform,
string projectionName,
ILogger logger,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager coreProjectionCheckpointManager,
StatePartitionSelector statePartitionSelector,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
bool useCheckpoints,
bool stopOnEof,
bool isBiState,
bool orderedPartitionProcessing,
IEmittedStreamsTracker emittedStreamsTracker,
bool enableContentTypeValidation)
: base(
publisher,
inputQueue,
coreProjection,
projectionCorrelationId,
coreProjectionCheckpointManager,
projectionConfig,
projectionName,
logger,
zeroCheckpointTag,
partitionStateCache,
resultWriter,
updateStatistics,
subscriptionDispatcher,
readerStrategy,
useCheckpoints,
stopOnEof,
orderedPartitionProcessing,
isBiState,
emittedStreamsTracker,
enableContentTypeValidation)
{
_projectionStateHandler = projectionStateHandler;
_definesStateTransform = definesStateTransform;
_statePartitionSelector = statePartitionSelector;
_isBiState = isBiState;
_stopwatch = new Stopwatch();
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag, ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy, IResultWriter resultWriter)
{
return(new ParallelQueryMasterProjectionProcessingPhase(
coreProjection, projectionCorrelationId, publisher, _projectionConfig, updateStatistics, _stateHandler,
partitionStateCache, _name, _logger, zeroCheckpointTag, checkpointManager, subscriptionDispatcher,
readerStrategy, resultWriter, _projectionConfig.CheckpointsEnabled, this.GetStopOnEof(),
_spoolProcessingResponseDispatcher));
}
protected abstract IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter);
public ParallelQueryMasterProjectionProcessingPhase(
CoreProjection coreProjection, Guid projectionCorrelationId, IPublisher publisher,
ProjectionConfig projectionConfig, Action updateStatistics, PartitionStateCache partitionStateCache,
string name, ILogger logger, CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, IResultWriter resultWriter, bool checkpointsEnabled, bool stopOnEof,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher)
: base(
publisher, coreProjection, projectionCorrelationId, checkpointManager, projectionConfig, name, logger,
zeroCheckpointTag, partitionStateCache, resultWriter, updateStatistics, subscriptionDispatcher,
readerStrategy, checkpointsEnabled, stopOnEof)
{
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
}
protected override ProjectionProcessingStrategy GivenProjectionProcessingStrategy()
{
_phase1checkpointManager = new FakeCheckpointManager(_bus, _projectionCorrelationId);
_phase2checkpointManager = new FakeCheckpointManager(_bus, _projectionCorrelationId);
_emittedStreamsTracker = new FakeEmittedStreamsTracker();
_phase1readerStrategy = GivenPhase1ReaderStrategy();
_phase2readerStrategy = GivenPhase2ReaderStrategy();
_phase1 = new FakeProjectionProcessingPhase(0, this, Phase1CheckpointManager, _phase1readerStrategy,
_emittedStreamsTracker);
_phase2 = new FakeProjectionProcessingPhase(1, this, Phase2CheckpointManager, _phase2readerStrategy,
_emittedStreamsTracker);
return(new FakeProjectionProcessingStrategy(
_projectionName, _version, Log.Logger, Phase1, Phase2));
}
protected ReaderSubscriptionBase(
IPublisher publisher,
Guid subscriptionId,
CheckpointTag @from,
IReaderStrategy readerStrategy,
ITimeProvider timeProvider,
long?checkpointUnhandledBytesThreshold,
int?checkpointProcessedEventsThreshold,
int checkpointAfterMs,
bool stopOnEof,
int?stopAfterNEvents,
bool enableContentTypeValidation)
{
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (readerStrategy == null)
{
throw new ArgumentNullException("readerStrategy");
}
if (timeProvider == null)
{
throw new ArgumentNullException("timeProvider");
}
if (checkpointProcessedEventsThreshold > 0 && stopAfterNEvents > 0)
{
throw new ArgumentException("checkpointProcessedEventsThreshold > 0 && stopAfterNEvents > 0");
}
_publisher = publisher;
_readerStrategy = readerStrategy;
_timeProvider = timeProvider;
_checkpointUnhandledBytesThreshold = checkpointUnhandledBytesThreshold;
_checkpointProcessedEventsThreshold = checkpointProcessedEventsThreshold;
_checkpointAfter = TimeSpan.FromMilliseconds(checkpointAfterMs);
_stopOnEof = stopOnEof;
_stopAfterNEvents = stopAfterNEvents;
_subscriptionId = subscriptionId;
_lastPassedOrCheckpointedEventPosition = null;
_eventFilter = readerStrategy.EventFilter;
_positionTagger = readerStrategy.PositionTagger;
_positionTracker = new PositionTracker(_positionTagger);
_positionTracker.UpdateByCheckpointTagInitial(@from);
_lastCheckpointTag = _positionTracker.LastTag;
_enableContentTypeValidation = enableContentTypeValidation;
_logger = Serilog.Log.ForContext <ReaderSubscriptionBase>();
}
protected EventSubscriptionBasedProjectionProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
ICoreProjectionForProcessingPhase coreProjection,
Guid projectionCorrelationId,
ICoreProjectionCheckpointManager checkpointManager,
ProjectionConfig projectionConfig,
string projectionName,
ILogger logger,
CheckpointTag zeroCheckpointTag,
PartitionStateCache partitionStateCache,
IResultWriter resultWriter,
Action updateStatistics,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
bool useCheckpoints,
bool stopOnEof,
bool orderedPartitionProcessing,
bool isBiState,
IEmittedStreamsTracker emittedStreamsTracker,
bool enableContentTypeValidation)
{
_publisher = publisher;
_inputQueue = inputQueue;
_coreProjection = coreProjection;
_projectionCorrelationId = projectionCorrelationId;
_checkpointManager = checkpointManager;
_projectionConfig = projectionConfig;
_projectionName = projectionName;
_logger = logger;
_zeroCheckpointTag = zeroCheckpointTag;
_partitionStateCache = partitionStateCache;
_resultWriter = resultWriter;
_updateStatistics = updateStatistics;
_processingQueue = new CoreProjectionQueue(publisher,
projectionConfig.PendingEventsThreshold,
orderedPartitionProcessing);
_processingQueue.EnsureTickPending += EnsureTickPending;
_subscriptionDispatcher = subscriptionDispatcher;
_readerStrategy = readerStrategy;
_useCheckpoints = useCheckpoints;
_stopOnEof = stopOnEof;
_isBiState = isBiState;
_progressResultWriter = new ProgressResultWriter(this, _resultWriter);
_inutQueueEnvelope = new PublishEnvelope(_inputQueue);
_emittedStreamsTracker = emittedStreamsTracker;
_enableContentTypeValidation = enableContentTypeValidation;
}
public Subscribe(
Guid subscriptionId, CheckpointTag from,
IReaderStrategy readerStrategy, ReaderSubscriptionOptions readerSubscriptionOptions) : base(subscriptionId)
{
if (@from == null)
{
throw new ArgumentNullException("from");
}
if (readerStrategy == null)
{
throw new ArgumentNullException("readerStrategy");
}
_fromPosition = @from;
_readerStrategy = readerStrategy;
_options = readerSubscriptionOptions;
}
public ParallelQueryMasterProjectionProcessingPhase(
CoreProjection coreProjection,
Guid projectionCorrelationId,
IPublisher publisher,
IPublisher inputQueue,
ProjectionConfig projectionConfig,
Action updateStatistics,
IProjectionStateHandler stateHandler,
PartitionStateCache partitionStateCache,
string name,
ILogger logger,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
bool checkpointsEnabled,
bool stopOnEof,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
IEmittedStreamsTracker emittedStreamsTracker)
: base(
publisher,
inputQueue,
coreProjection,
projectionCorrelationId,
checkpointManager,
projectionConfig,
name,
logger,
zeroCheckpointTag,
partitionStateCache,
resultWriter,
updateStatistics,
subscriptionDispatcher,
readerStrategy,
checkpointsEnabled,
stopOnEof,
orderedPartitionProcessing: true,
isBiState: false,
emittedStreamsTracker: emittedStreamsTracker)
{
_stateHandler = stateHandler;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
_spoolProcessingWorkItems = new Dictionary <Guid, SpoolStreamProcessingWorkItem>();
}
public EventProcessingProjectionProcessingPhase(
CoreProjection coreProjection, Guid projectionCorrelationId, IPublisher publisher,
ProjectionConfig projectionConfig, Action updateStatistics, IProjectionStateHandler projectionStateHandler,
PartitionStateCache partitionStateCache, bool definesStateTransform, string projectionName, ILogger logger,
CheckpointTag zeroCheckpointTag, ICoreProjectionCheckpointManager coreProjectionCheckpointManager,
StatePartitionSelector statePartitionSelector, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, IResultWriter resultWriter, bool useCheckpoints, bool stopOnEof)
: base(
publisher, coreProjection, projectionCorrelationId, coreProjectionCheckpointManager, projectionConfig,
projectionName, logger, zeroCheckpointTag, partitionStateCache, resultWriter, updateStatistics,
subscriptionDispatcher, readerStrategy, useCheckpoints, stopOnEof)
{
_projectionStateHandler = projectionStateHandler;
_definesStateTransform = definesStateTransform;
_statePartitionSelector = statePartitionSelector;
_stopwatch = new Stopwatch();
}
protected ReaderSubscriptionBase(
IPublisher publisher,
Guid subscriptionId,
CheckpointTag @from,
IReaderStrategy readerStrategy,
ITimeProvider timeProvider,
long?checkpointUnhandledBytesThreshold,
int?checkpointProcessedEventsThreshold,
bool stopOnEof,
int?stopAfterNEvents)
{
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (readerStrategy == null)
{
throw new ArgumentNullException("readerStrategy");
}
if (timeProvider == null)
{
throw new ArgumentNullException("timeProvider");
}
if (checkpointProcessedEventsThreshold > 0 && stopAfterNEvents > 0)
{
throw new ArgumentException("checkpointProcessedEventsThreshold > 0 && stopAfterNEvents > 0");
}
_publisher = publisher;
_readerStrategy = readerStrategy;
_timeProvider = timeProvider;
_checkpointUnhandledBytesThreshold = checkpointUnhandledBytesThreshold;
_checkpointProcessedEventsThreshold = checkpointProcessedEventsThreshold;
_stopOnEof = stopOnEof;
_stopAfterNEvents = stopAfterNEvents;
_subscriptionId = subscriptionId;
_lastPassedOrCheckpointedEventPosition = null;
_eventFilter = readerStrategy.EventFilter;
_positionTagger = readerStrategy.PositionTagger;
_positionTracker = new PositionTracker(_positionTagger);
_positionTracker.UpdateByCheckpointTagInitial(@from);
}
private void ProcessLine(string line, IList result)
{
if (isBlank(line))
{
return;
}
if (isComment(line))
{
return;
}
string typeCode = GetTypeCode(line);
IReaderStrategy strategy = (IReaderStrategy)_strategies[typeCode];
if (null == strategy)
{
throw new Exception("Unable to find strategy");
}
result.Add(strategy.Process(line));
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter,
IEmittedStreamsTracker emittedStreamsTracker)
{
var statePartitionSelector = CreateStatePartitionSelector();
var orderedPartitionProcessing = _sourceDefinition.ByStreams && _sourceDefinition.IsBiState;
return(new EventProcessingProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_sourceDefinition.DefinesStateTransform,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
statePartitionSelector,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_sourceDefinition.IsBiState,
orderedPartitionProcessing: orderedPartitionProcessing,
emittedStreamsTracker: emittedStreamsTracker,
enableContentTypeValidation: _enableContentTypeValidation));
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
_subscriptionId = Guid.NewGuid();
_readerStrategy = ReaderStrategy.CreateExternallyFedReaderStrategy(
0, _timeProvider, SystemAccount.Principal, limitingCommitPosition: 10000);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type2", "{}", "{Data: 2}");
GivenInitialIndexState();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
AllStreams = true,
Events = new[] { "type1", "type2" },
Options = new QuerySourcesDefinitionOptions {
}
};
_readerStrategy = ReaderStrategy.Create(0, _sourceDefinition, _timeProvider, stopOnEof: false, runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: false,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
ExistingEvent("$$test-stream", "$metadata", "", "{Meta: 1}");
ExistingEvent("$$test-stream2", "$metadata", "", "{Meta: 2}");
ExistingEvent("$$test-stream3", "$metadata", "", "{Meta: 3}");
ExistingEvent("$streams", "$>", null, "0@test-stream");
ExistingEvent("$streams", "$>", null, "0@test-stream2");
ExistingEvent("$streams", "$>", null, "0@test-stream3");
ExistingEvent("$streams", "$>", null, "0@test-stream4");
NoOtherStreams();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
CatalogStream = "$all",
AllEvents = true,
ByStreams = true,
Options = new QuerySourcesDefinitionOptions {
}
};
_readerStrategy = new ParallelQueryAllStreamsMasterReaderStrategy(
0, SystemAccount.Principal, _timeProvider);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("stream-a", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("stream-b", "type1", "{}", "{Data: 2}");
GivenOtherEvents();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
Streams = new [] { "stream-a", "stream-b" },
AllEvents = true,
Options = new QuerySourcesDefinitionOptions {
ReorderEvents = true, ProcessingLag = 100
}
};
_readerStrategy = ReaderStrategy.Create(0, _sourceDefinition, _timeProvider, stopOnEof: false, runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: false,
stopAfterNEvents: null);
}
public void setup()
{
_checkpointUnhandledBytesThreshold = 1000;
_checkpointProcessedEventsThreshold = 2000;
Given();
_bus = new InMemoryBus("bus");
_projectionCorrelationId = Guid.NewGuid();
_eventHandler = new TestHandler <EventReaderSubscriptionMessage.CommittedEventReceived>();
_checkpointHandler = new TestHandler <EventReaderSubscriptionMessage.CheckpointSuggested>();
_progressHandler = new TestHandler <EventReaderSubscriptionMessage.ProgressChanged>();
_notAuthorizedHandler = new TestHandler <EventReaderSubscriptionMessage.NotAuthorized>();
_eofHandler = new TestHandler <EventReaderSubscriptionMessage.EofReached>();
_bus.Subscribe(_eventHandler);
_bus.Subscribe(_checkpointHandler);
_bus.Subscribe(_progressHandler);
_bus.Subscribe(_eofHandler);
_readerStrategy = CreateCheckpointStrategy().ReaderStrategy;
_subscription = CreateProjectionSubscription();
When();
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
ExistingEvent("catalog", "$>", null, "0@test-stream");
ExistingEvent("catalog", "$>", null, "0@test-stream2");
ExistingEvent("catalog", "$>", null, "0@test-stream3");
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
CatalogStream = "catalog",
AllEvents = true,
ByStreams = true,
Options = new QuerySourcesDefinitionOptions {
}
};
_readerStrategy = ReaderStrategy.Create(
"test",
0,
_sourceDefinition,
_timeProvider,
stopOnEof: true,
runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
public EventReorderingReaderSubscription(
IPublisher publisher,
Guid subscriptionId,
CheckpointTag @from,
IReaderStrategy readerStrategy,
ITimeProvider timeProvider,
long? checkpointUnhandledBytesThreshold,
int? checkpointProcessedEventsThreshold,
int processingLagMs,
bool stopOnEof = false,
int? stopAfterNEvents = null)
: base(
publisher,
subscriptionId,
@from,
readerStrategy,
timeProvider,
checkpointUnhandledBytesThreshold,
checkpointProcessedEventsThreshold,
stopOnEof,
stopAfterNEvents)
{
_processingLagMs = processingLagMs;
}
public EventReorderingReaderSubscription(
IPublisher publisher,
Guid subscriptionId,
CheckpointTag @from,
IReaderStrategy readerStrategy,
ITimeProvider timeProvider,
long?checkpointUnhandledBytesThreshold,
int?checkpointProcessedEventsThreshold,
int processingLagMs,
bool stopOnEof = false,
int?stopAfterNEvents = null)
: base(
publisher,
subscriptionId,
@from,
readerStrategy,
timeProvider,
checkpointUnhandledBytesThreshold,
checkpointProcessedEventsThreshold,
stopOnEof,
stopAfterNEvents)
{
_processingLagMs = processingLagMs;
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos = new TFPos[TailLength];
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream", "type1", "{}", "{Data: 3}");
for (var i = 0; i < TailLength; i++)
{
_tfPos[i] = ExistingEvent("test-stream", "type1", "{}", "{Data: " + i + "}");
}
GivenInitialIndexState();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition {
AllStreams = true,
Events = new[] { "type1", "type2" },
Options = new QuerySourcesDefinitionOptions {
}
};
_readerStrategy = ReaderStrategy.Create(
"test",
0,
_sourceDefinition,
_timeProvider,
stopOnEof: false,
runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100,
checkpointAfterMs: 10000, stopOnEof: false,
stopAfterNEvents: null,
enableContentTypeValidation: true);
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher, Guid projectionCorrelationId, PartitionStateCache partitionStateCache,
Action updateStatistics, CoreProjection coreProjection, ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag, ICoreProjectionCheckpointManager checkpointManager, IReaderStrategy readerStrategy, IResultWriter resultWriter)
{
var statePartitionSelector = CreateStatePartitionSelector(
_stateHandler, _sourceDefinition.ByCustomPartitions, _sourceDefinition.ByStreams);
return(new EventProcessingProjectionProcessingPhase(
coreProjection, projectionCorrelationId, publisher, _projectionConfig, updateStatistics,
_stateHandler, partitionStateCache, _sourceDefinition.DefinesStateTransform, _name, _logger,
zeroCheckpointTag, checkpointManager, statePartitionSelector, subscriptionDispatcher, readerStrategy,
resultWriter, _projectionConfig.CheckpointsEnabled, this.GetStopOnEof()));
}
protected override ProjectionProcessingStrategy GivenProjectionProcessingStrategy()
{
_phase1checkpointManager = new FakeCheckpointManager(_bus, _projectionCorrelationId);
_phase2checkpointManager = new FakeCheckpointManager(_bus, _projectionCorrelationId);
_phase1readerStrategy = GivenPhase1ReaderStrategy();
_phase2readerStrategy = GivenPhase2ReaderStrategy();
_phase1 = new FakeProjectionProcessingPhase(0, this, Phase1CheckpointManager, _phase1readerStrategy);
_phase2 = new FakeProjectionProcessingPhase(1, this, Phase2CheckpointManager, _phase2readerStrategy);
return new FakeProjectionProcessingStrategy(
_projectionName, _version, new ConsoleLogger(), Phase1, Phase2);
}
public FakeProjectionProcessingPhase(int phase, specification_with_multi_phase_core_projection specification,
ICoreProjectionCheckpointManager checkpointManager, IReaderStrategy readerStrategy)
{
_phase = phase;
_specification = specification;
_checkpointManager = checkpointManager;
_readerStrategy = readerStrategy;
}
public CheckpointStrategy Build(ProjectionConfig config, IPrincipal runAs, IReaderStrategy readerStrategy)
{
base.Validate();
return new CheckpointStrategy(
_byStream, _byCustomPartitions, config.CheckpointsEnabled, _definesStateTransform, runAs, readerStrategy);
}
public FileReader(IReaderStrategy reader)
{
_reader = reader;
}
public void AddStrategy(IReaderStrategy arg)
{
_strategies[arg.Code] = arg;
}
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter)
{
return new ParallelQueryMasterProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_spoolProcessingResponseDispatcher);
}
protected virtual ICoreProjectionCheckpointManager CreateCheckpointManager(
Guid projectionCorrelationId, IPublisher publisher, IODispatcher ioDispatcher,
ProjectionNamesBuilder namingBuilder, CoreProjectionCheckpointWriter coreProjectionCheckpointWriter,
bool definesFold, IReaderStrategy readerStrategy)
{
var emitAny = _projectionConfig.EmitEventEnabled;
//NOTE: not emitting one-time/transient projections are always handled by default checkpoint manager
// as they don't depend on stable event order
if (emitAny && !readerStrategy.IsReadingOrderRepeatable)
{
return new MultiStreamMultiOutputCheckpointManager(
publisher, projectionCorrelationId, _projectionVersion, _projectionConfig.RunAs, ioDispatcher,
_projectionConfig, _name, readerStrategy.PositionTagger, namingBuilder,
_projectionConfig.CheckpointsEnabled, GetProducesRunningResults(), definesFold,
coreProjectionCheckpointWriter);
}
else
{
return new DefaultCheckpointManager(
publisher, projectionCorrelationId, _projectionVersion, _projectionConfig.RunAs, ioDispatcher,
_projectionConfig, _name, readerStrategy.PositionTagger, namingBuilder,
_projectionConfig.CheckpointsEnabled, GetProducesRunningResults(), definesFold,
coreProjectionCheckpointWriter);
}
}
protected abstract IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter);
protected override IProjectionProcessingPhase CreateFirstProcessingPhase(
IPublisher publisher,
IPublisher inputQueue,
Guid projectionCorrelationId,
PartitionStateCache partitionStateCache,
Action updateStatistics,
CoreProjection coreProjection,
ReaderSubscriptionDispatcher subscriptionDispatcher,
CheckpointTag zeroCheckpointTag,
ICoreProjectionCheckpointManager checkpointManager,
IReaderStrategy readerStrategy,
IResultWriter resultWriter)
{
var statePartitionSelector = CreateStatePartitionSelector();
var orderedPartitionProcessing = _sourceDefinition.ByStreams && _sourceDefinition.IsBiState;
return new EventProcessingProjectionProcessingPhase(
coreProjection,
projectionCorrelationId,
publisher,
inputQueue,
_projectionConfig,
updateStatistics,
_stateHandler,
partitionStateCache,
_sourceDefinition.DefinesStateTransform,
_name,
_logger,
zeroCheckpointTag,
checkpointManager,
statePartitionSelector,
subscriptionDispatcher,
readerStrategy,
resultWriter,
_projectionConfig.CheckpointsEnabled,
this.GetStopOnEof(),
_sourceDefinition.IsBiState,
orderedPartitionProcessing: orderedPartitionProcessing);
}
