public CreateAndPrepareSlave(
IEnvelope envelope, Guid projectionId, string name, ProjectionVersion version, ProjectionConfig config,
IPublisher resultsPublisher, Guid masterCoreProjectionId, Func <IProjectionStateHandler> handlerFactory)
: base(projectionId)
{
if (envelope == null)
{
throw new ArgumentNullException("envelope");
}
if (name == null)
{
throw new ArgumentNullException("name");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
if (resultsPublisher == null)
{
throw new ArgumentNullException("resultsPublisher");
}
if (handlerFactory == null)
{
throw new ArgumentNullException("handlerFactory");
}
_envelope = envelope;
_name = name;
_version = version;
_config = config;
_resultsPublisher = resultsPublisher;
_masterCoreProjectionId = masterCoreProjectionId;
_handlerFactory = handlerFactory;
}
public ParallelQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
Func <IProjectionStateHandler> handlerFactory, ProjectionConfig projectionConfig,
IQueryDefinition sourceDefinition, ProjectionNamesBuilder namesBuilder, ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
public void setup()
{
_subscribeProjectionHandler = new TestHandler <ReaderSubscriptionManagement.Subscribe>();
_writeEventHandler = new TestHandler <ClientMessage.WriteEvents>();
_bus.Subscribe(_subscribeProjectionHandler);
_bus.Subscribe(_writeEventHandler);
_stateHandler = GivenProjectionStateHandler();
_firstWriteCorrelationId = Guid.NewGuid();
_projectionCorrelationId = Guid.NewGuid();
_projectionConfig = GivenProjectionConfig();
var projectionProcessingStrategy = GivenProjectionProcessingStrategy();
_coreProjection = GivenCoreProjection(projectionProcessingStrategy);
_bus.Subscribe <CoreProjectionProcessingMessage.CheckpointCompleted>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.CheckpointLoaded>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.PrerecordedEventsLoaded>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.RestartRequested>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.Failed>(_coreProjection);
_bus.Subscribe(new AdHocHandler <ProjectionCoreServiceMessage.CoreTick>(tick => tick.Action()));
_bus.Subscribe(new AdHocHandler <ReaderCoreServiceMessage.ReaderTick>(tick => tick.Action()));
PreWhen();
When();
}
public DefaultCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher <ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, IResultEmitter resultEmitter, bool useCheckpoints,
bool emitPartitionCheckpoints = false)
: base(
publisher, projectionCorrelationId, projectionConfig, name, positionTagger, namingBuilder, resultEmitter,
useCheckpoints, emitPartitionCheckpoints)
{
if (readDispatcher == null)
{
throw new ArgumentNullException("readDispatcher");
}
if (writeDispatcher == null)
{
throw new ArgumentNullException("writeDispatcher");
}
_projectionVersion = projectionVersion;
_runAs = runAs;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_positionTagger = positionTagger;
_projectionCheckpointStreamId = namingBuilder.MakeCheckpointStreamName();
_zeroTag = positionTagger.MakeZeroCheckpointTag();
}
public ICoreProjectionCheckpointManager CreateCheckpointManager(
Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPublisher publisher,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher <ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, ProjectionNamesBuilder namingBuilder)
{
var emitAny = projectionConfig.EmitEventEnabled;
var emitPartitionCheckpoints = UseCheckpoints && (_byCustomPartitions || _byStream);
var resultEmitter = _definesStateTransform
? new ResultEmitter(namingBuilder)
: (IResultEmitter) new NoopResultEmitter();
//NOTE: not emitting one-time/transient projections are always handled by default checkpoint manager
// as they don't depend on stable event order
if (emitAny && !ReaderStrategy.IsReadingOrderRepeatable)
{
return(new MultiStreamMultiOutputCheckpointManager(
publisher, projectionCorrelationId, projectionVersion, _runAs, readDispatcher, writeDispatcher,
projectionConfig, name, ReaderStrategy.PositionTagger, namingBuilder, resultEmitter, UseCheckpoints,
emitPartitionCheckpoints));
}
else
{
return(new DefaultCheckpointManager(
publisher, projectionCorrelationId, projectionVersion, _runAs, readDispatcher, writeDispatcher,
projectionConfig, name, ReaderStrategy.PositionTagger, namingBuilder, resultEmitter, UseCheckpoints,
emitPartitionCheckpoints));
}
}
protected CoreProjectionCheckpointManager(
ICoreProjection coreProjection, IPublisher publisher, Guid projectionCorrelationId,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger)
{
if (coreProjection == null) throw new ArgumentNullException("coreProjection");
if (publisher == null) throw new ArgumentNullException("publisher");
if (readDispatcher == null) throw new ArgumentNullException("readDispatcher");
if (writeDispatcher == null) throw new ArgumentNullException("writeDispatcher");
if (projectionConfig == null) throw new ArgumentNullException("projectionConfig");
if (name == null) throw new ArgumentNullException("name");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (name == "") throw new ArgumentException("name");
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_coreProjection = coreProjection;
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_projectionConfig = projectionConfig;
_logger = LogManager.GetLoggerFor<CoreProjectionCheckpointManager>();
_name = name;
_positionTagger = positionTagger;
}
public CreatePrepared(
IEnvelope envelope, Guid projectionId, string name, ProjectionVersion version, ProjectionConfig config,
IQuerySources sourceDefinition, string handlerType, string query)
: base(projectionId)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
if (sourceDefinition == null)
{
throw new ArgumentNullException("sourceDefinition");
}
if (handlerType == null)
{
throw new ArgumentNullException("handlerType");
}
if (query == null)
{
throw new ArgumentNullException("query");
}
_envelope = envelope;
_name = name;
_version = version;
_config = config;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
}
public void setup()
{
_subscribeProjectionHandler = new TestHandler <ReaderSubscriptionManagement.Subscribe>();
_writeEventHandler = new TestHandler <ClientMessage.WriteEvents>();
_bus.Subscribe(_subscribeProjectionHandler);
_bus.Subscribe(_writeEventHandler);
_stateHandler = _stateHandler
?? new FakeProjectionStateHandler(configureBuilder: _configureBuilderByQuerySource);
_firstWriteCorrelationId = Guid.NewGuid();
_projectionCorrelationId = Guid.NewGuid();
_projectionConfig = new ProjectionConfig(null,
_checkpointHandledThreshold, _checkpointUnhandledBytesThreshold, 1000, 250, true, true,
_createTempStreams, _stopOnEof);
_coreProjection = CoreProjection.CreateAndPrepare(
"projection", _version, _projectionCorrelationId, _bus, _stateHandler, _projectionConfig, _readDispatcher,
_writeDispatcher, _subscriptionDispatcher, null, _timeProvider);
_bus.Subscribe <CoreProjectionProcessingMessage.CheckpointCompleted>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.CheckpointLoaded>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.PrerecordedEventsLoaded>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.RestartRequested>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.Failed>(_coreProjection);
_bus.Subscribe <EventReaderSubscriptionMessage.CommittedEventReceived>(_coreProjection);
_bus.Subscribe <EventReaderSubscriptionMessage.CheckpointSuggested>(_coreProjection);
_bus.Subscribe <EventReaderSubscriptionMessage.EofReached>(_coreProjection);
_bus.Subscribe <EventReaderSubscriptionMessage.ProgressChanged>(_coreProjection);
_bus.Subscribe <EventReaderSubscriptionMessage.NotAuthorized>(_coreProjection);
_bus.Subscribe(new AdHocHandler <ProjectionCoreServiceMessage.CoreTick>(tick => tick.Action()));
_bus.Subscribe(new AdHocHandler <ReaderCoreServiceMessage.ReaderTick>(tick => tick.Action()));
PreWhen();
When();
}
public void TestFixtureSetUp()
{
_ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus), true);
_projectionVersion = new ProjectionVersion(3, 1, 2);
_projectionConfig = new ProjectionConfig(SystemAccounts.System, 10, 1000, 1000, 10, true, true, true,
false,
false, 5000, 10);
_positionTagger = new MultiStreamPositionTagger(3, _streams);
_positionTagger.AdjustTag(CheckpointTag.FromStreamPositions(3,
new Dictionary <string, long> {
{ "a", 0 }, { "b", 0 }, { "c", 0 }
}));
_namingBuilder = ProjectionNamesBuilder.CreateForTest("projection");
IODispatcherTestHelpers.SubscribeIODispatcher(_ioDispatcher, _bus);
_bus.Subscribe <ClientMessage.ReadStreamEventsBackward>(this);
_coreProjectionCheckpointWriter = new CoreProjectionCheckpointWriter(
_namingBuilder.MakeCheckpointStreamName(), _ioDispatcher,
_projectionVersion, _projectionName);
_checkpointManager = new MultiStreamMultiOutputCheckpointManager(_bus, _projectionId, _projectionVersion,
SystemAccounts.System,
_ioDispatcher, _projectionConfig, _projectionName, _positionTagger, _namingBuilder, true, true, false,
_coreProjectionCheckpointWriter);
When();
}
public ICoreProjectionCheckpointManager CreateCheckpointManager(
ICoreProjection coreProjection, Guid projectionCorrelationId, IPublisher publisher,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted>
requestResponseDispatcher,
RequestResponseDispatcher <ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> responseDispatcher,
ProjectionConfig projectionConfig, string name, string stateUpdatesStreamId)
{
if (_allStreams && _useEventIndexes && _events != null && _events.Count > 1)
{
string projectionStateUpdatesStreamId = stateUpdatesStreamId;
return(new MultiStreamCheckpointManager(
coreProjection, publisher, projectionCorrelationId, requestResponseDispatcher, responseDispatcher,
projectionConfig, name, PositionTagger, projectionStateUpdatesStreamId));
}
else if (_streams != null && _streams.Count > 1)
{
string projectionStateUpdatesStreamId = stateUpdatesStreamId;
return(new MultiStreamCheckpointManager(
coreProjection, publisher, projectionCorrelationId, requestResponseDispatcher, responseDispatcher,
projectionConfig, name, PositionTagger, projectionStateUpdatesStreamId));
}
else
{
string projectionCheckpointStreamId = CoreProjection.ProjectionsStreamPrefix + name
+ CoreProjection.ProjectionCheckpointStreamSuffix;
return(new DefaultCheckpointManager(
coreProjection, publisher, projectionCorrelationId, requestResponseDispatcher, responseDispatcher,
projectionConfig, projectionCheckpointStreamId, name, PositionTagger));
}
}
protected override void Given()
{
_projectionId = Guid.NewGuid();
_workerId = Guid.NewGuid();
_projectionName = "projection";
_handlerType = "JS";
_query = "from()";
_config = new ProjectionConfig(
new OpenGenericPrincipal("user", "a", "b"),
1000,
100000,
2000,
200,
true,
true,
true,
true,
true,
true,
10000,
1);
var builder = new SourceDefinitionBuilder();
builder.FromStream("s1");
builder.FromStream("s2");
builder.IncludeEvent("e1");
builder.IncludeEvent("e2");
builder.SetByStream();
builder.SetResultStreamNameOption("result-stream");
_definition = QuerySourcesDefinition.From(builder);
}
protected virtual IReaderStrategy CreateCheckpointStrategy()
{
var readerBuilder = new SourceDefinitionBuilder();
if (_source != null)
{
_source(readerBuilder);
}
else
{
readerBuilder.FromAll();
readerBuilder.AllEvents();
}
var config = ProjectionConfig.GetTest();
IQuerySources sources = readerBuilder.Build();
var readerStrategy = Core.Services.Processing.ReaderStrategy.Create(
"test",
0,
sources,
_timeProvider,
stopOnEof: false,
runAs: config.RunAs);
return(readerStrategy);
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionNamesBuilder namesBuilder,
IQueryDefinition sourceDefinition, ProjectionConfig projectionConfig,
Func<IProjectionStateHandler> handlerFactory, IProjectionStateHandler stateHandler)
{
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams
&& sourceDefinition.DefinesFold && !string.IsNullOrEmpty(sourceDefinition.CatalogStream))
{
return new ParallelQueryProcessingStrategy(
name, projectionVersion, stateHandler, handlerFactory, projectionConfig, sourceDefinition,
namesBuilder, _logger, _spoolProcessingResponseDispatcher, _subscriptionDispatcher);
}
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams
&& sourceDefinition.DefinesFold && sourceDefinition.HasCategories())
{
return new ParallelQueryProcessingStrategy(
name, projectionVersion, stateHandler, handlerFactory, projectionConfig, sourceDefinition,
namesBuilder, _logger, _spoolProcessingResponseDispatcher, _subscriptionDispatcher);
}
return projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, _logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, _logger,
_subscriptionDispatcher);
}
public EmittedStreamsTracker(IODispatcher ioDispatcher, ProjectionConfig projectionConfig,
ProjectionNamesBuilder projectionNamesBuilder)
{
_ioDispatcher = ioDispatcher;
_projectionConfig = projectionConfig;
_projectionNamesBuilder = projectionNamesBuilder;
}
protected override void Given()
{
_projectionId = Guid.NewGuid();
_workerId = Guid.NewGuid();
_masterWorkerId = Guid.NewGuid();
_masterCoreProjectionId = Guid.NewGuid();
_projectionName = "projection";
_handlerType = "JS";
_query = "from()";
_config = new ProjectionConfig(
new OpenGenericPrincipal("user", "a", "b"),
1000,
100000,
2000,
200,
true,
true,
true,
true,
true,
true,
10000,
1);
}
a_checkpoint_threshold_less_tan_checkpoint_handled_threshold_throws_argument_out_of_range_exception()
{
Assert.Throws <ArgumentException>(() => {
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
var version = new ProjectionVersion(1, 0, 0);
var projectionConfig = new ProjectionConfig(null, 10, 5, 1000, 250, true, true, false, false,
true, 10000, 1);
new ContinuousProjectionProcessingStrategy(
"projection",
version,
projectionStateHandler,
projectionConfig,
projectionStateHandler.GetSourceDefinition(),
null,
_subscriptionDispatcher,
true).Create(
Guid.NewGuid(),
new FakePublisher(),
Guid.NewGuid(),
SystemAccounts.System,
new FakePublisher(),
_ioDispatcher,
_subscriptionDispatcher,
new RealTimeProvider());
});
}
private ProjectionConfig CreateDefaultProjectionConfiguration()
{
var checkpointsEnabled = _persistedState.CheckpointsDisabled != true;
var checkpointHandledThreshold = checkpointsEnabled ? 4000 : 0;
var checkpointUnhandledBytesThreshold = checkpointsEnabled ? 10 * 1000 * 1000 : 0;
var pendingEventsThreshold = 5000;
var maxWriteBatchLength = 500;
var emitEventEnabled = _persistedState.EmitEnabled == true;
var createTempStreams = _persistedState.CreateTempStreams == true;
var stopOnEof = _persistedState.Mode <= ProjectionMode.OneTime;
var projectionConfig = new ProjectionConfig(
_runAs,
checkpointHandledThreshold,
checkpointUnhandledBytesThreshold,
pendingEventsThreshold,
maxWriteBatchLength,
emitEventEnabled,
checkpointsEnabled,
createTempStreams,
stopOnEof,
isSlaveProjection: false);
return(projectionConfig);
}
internal static FrameworkConfiguration GetTuioConfig(FrameworkElement uiParent)
{
AppConfig.LoadAppConfig();
ProjectionConfig proj = ProjectionConfig.Load(AppConfig.TrackingPath);
if (proj == null)
{
proj = ProjectionConfig.LoadDefaults();
}
AlignConfig align = new AlignConfig()
{
FlipX = AppConfig.FlipX,
FlipY = AppConfig.FlipY
};
TuioConfiguration conf = new TuioConfiguration
{
Owner = uiParent,
UIManagedThreadId = Thread.CurrentThread.ManagedThreadId,
CorrectProjection = AppConfig.CorrectProjection,
Alignment = align,
Projection = proj,
Port = AppConfig.Port,
};
return(conf);
}
public void a_negative_checkpoint_handled_interval_throws_argument_out_of_range_exception()
{
Assert.Throws <ArgumentOutOfRangeException>(() => {
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
var version = new ProjectionVersion(1, 0, 0);
var projectionConfig = new ProjectionConfig(null, -1, 10, 1000, 250, true, true, false, false, false,
true, 10000, 1);
new ContinuousProjectionProcessingStrategy(
"projection",
version,
projectionStateHandler,
projectionConfig,
projectionStateHandler.GetSourceDefinition(),
null,
_subscriptionDispatcher).Create(
Guid.NewGuid(),
new FakePublisher(),
Guid.NewGuid(),
SystemAccount.Principal,
new FakePublisher(),
_ioDispatcher,
_subscriptionDispatcher,
new RealTimeProvider());
});
}
public void setup()
{
_subscribeProjectionHandler = new TestHandler <ReaderSubscriptionManagement.Subscribe>();
_writeEventHandler = new TestHandler <ClientMessage.WriteEvents>();
_bus.Subscribe(_subscribeProjectionHandler);
_bus.Subscribe(_writeEventHandler);
_stateHandler = GivenProjectionStateHandler();
_firstWriteCorrelationId = Guid.NewGuid();
_workerId = Guid.NewGuid();
var dispatcher = new ProjectionManagerMessageDispatcher(new Dictionary <Guid, IPublisher> {
{ _workerId, GetInputQueue() }
});
_projectionCorrelationId = Guid.NewGuid();
_projectionConfig = GivenProjectionConfig();
var projectionProcessingStrategy = GivenProjectionProcessingStrategy();
_coreProjection = GivenCoreProjection(projectionProcessingStrategy);
_bus.Subscribe <CoreProjectionProcessingMessage.CheckpointCompleted>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.CheckpointLoaded>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.PrerecordedEventsLoaded>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.RestartRequested>(_coreProjection);
_bus.Subscribe <CoreProjectionProcessingMessage.Failed>(_coreProjection);
_bus.Subscribe(new AdHocHandler <ProjectionCoreServiceMessage.CoreTick>(tick => tick.Action()));
_bus.Subscribe(new AdHocHandler <ReaderCoreServiceMessage.ReaderTick>(tick => tick.Action()));
_bus.Subscribe <PartitionProcessingResultOutputBase>(dispatcher);
PreWhen();
When();
}
protected CoreProjectionCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger, ProjectionNamesBuilder namingBuilder, bool usePersistentCheckpoints,
bool producesRunningResults)
{
if (publisher == null) throw new ArgumentNullException("publisher");
if (projectionConfig == null) throw new ArgumentNullException("projectionConfig");
if (name == null) throw new ArgumentNullException("name");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (namingBuilder == null) throw new ArgumentNullException("namingBuilder");
if (name == "") throw new ArgumentException("name");
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_zeroTag = positionTagger.MakeZeroCheckpointTag();
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_projectionConfig = projectionConfig;
_logger = LogManager.GetLoggerFor<CoreProjectionCheckpointManager>();
_namingBuilder = namingBuilder;
_usePersistentCheckpoints = usePersistentCheckpoints;
_producesRunningResults = producesRunningResults;
_requestedCheckpointState = new PartitionState("", null, _zeroTag);
_currentProjectionState = new PartitionState("", null, _zeroTag);
}
public NoopCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger, ProjectionNamesBuilder namingBuilder)
{
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (projectionConfig == null)
{
throw new ArgumentNullException("projectionConfig");
}
if (name == null)
{
throw new ArgumentNullException("name");
}
if (positionTagger == null)
{
throw new ArgumentNullException("positionTagger");
}
if (namingBuilder == null)
{
throw new ArgumentNullException("namingBuilder");
}
if (name == "")
{
throw new ArgumentException("name");
}
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestHandler <ClientMessage.ReadStreamEventsBackward>();
_bus.Subscribe(_listEventsHandler);
_readDispatcher = new RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_writeDispatcher = new RequestResponseDispatcher <ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_subscriptionDispatcher =
new PublishSubscribeDispatcher
<ReaderSubscriptionManagement.Subscribe,
ReaderSubscriptionManagement.ReaderSubscriptionManagementMessage, EventReaderSubscriptionMessage>
(_bus, v => v.SubscriptionId, v => v.SubscriptionId);
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_readDispatcher);
_bus.Subscribe(_writeDispatcher);
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
_projectionConfig = new ProjectionConfig(null, 5, 10, 1000, 250, true, true, false, false);
_coreProjection = CoreProjection.CreateAndPrepare(
"projection", new ProjectionVersion(1, 0, 0), Guid.NewGuid(), _bus, projectionStateHandler, _projectionConfig, _readDispatcher,
_writeDispatcher, _subscriptionDispatcher, null, new RealTimeProvider());
_coreProjection.Start();
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
ProjectionNamesBuilder namesBuilder,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler, string handlerType, string query)
{
return(projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher));
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionNamesBuilder namesBuilder,
IQueryDefinition sourceDefinition, ProjectionConfig projectionConfig,
Func <IProjectionStateHandler> handlerFactory, IProjectionStateHandler stateHandler)
{
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams &&
sourceDefinition.DefinesFold && !string.IsNullOrEmpty(sourceDefinition.CatalogStream))
{
return(new ParallelQueryProcessingStrategy(
name, projectionVersion, stateHandler, handlerFactory, projectionConfig, sourceDefinition,
namesBuilder, _logger, _spoolProcessingResponseDispatcher, _subscriptionDispatcher));
}
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams &&
sourceDefinition.DefinesFold && sourceDefinition.HasCategories())
{
return(new ParallelQueryProcessingStrategy(
name, projectionVersion, stateHandler, handlerFactory, projectionConfig, sourceDefinition,
namesBuilder, _logger, _spoolProcessingResponseDispatcher, _subscriptionDispatcher));
}
return(projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, _logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, _logger,
_subscriptionDispatcher));
}
public ParallelQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
Func<IProjectionStateHandler> handlerFactory, ProjectionConfig projectionConfig,
IQueryDefinition sourceDefinition, ProjectionNamesBuilder namesBuilder, ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
private void BeginCreatePrepared(ProjectionConfig config, Action onPrepared)
{
_onPrepared = onPrepared;
if (config == null)
{
throw new ArgumentNullException("config");
}
//TODO: which states are allowed here?
if (_state >= ManagedProjectionState.Preparing)
{
DisposeCoreProjection();
_state = ManagedProjectionState.Loaded;
}
//TODO: load configuration from the definition
if (_persistedState.SourceDefinition == null)
{
throw new Exception(
"The projection cannot be loaded as stopped as it was stored in the old format. Update the projection query text to force prepare");
}
var createProjectionMessage =
new CoreProjectionManagementMessage.CreatePrepared(
new PublishEnvelope(_inputQueue), Id, _name,
new ProjectionVersion(_projectionId, _persistedState.Epoch ?? 0, _persistedState.Version ?? 1),
config, _persistedState.SourceDefinition, HandlerType, Query);
//note: set running before start as coreProjection.start() can respond with faulted
_state = ManagedProjectionState.Preparing;
_coreQueue.Publish(createProjectionMessage);
}
private Message CreateCreateAndPrepareProjectionMessage(ProjectionConfig config)
{
var createProjectionMessage = _isSlave
? (Message)
new CoreProjectionManagementMessage.CreateAndPrepareSlave(
Id,
_workerId,
_name,
new ProjectionVersion(_projectionId, _persistedState.Epoch ?? 0, _persistedState.Version ?? 0),
config,
_slaveMasterWorkerId,
_slaveMasterCorrelationId,
HandlerType,
Query)
: new CoreProjectionManagementMessage.CreateAndPrepare(
Id,
_workerId,
_name,
new ProjectionVersion(_projectionId, _persistedState.Epoch ?? 0, _persistedState.Version ?? 0),
config,
HandlerType,
Query);
return(createProjectionMessage);
}
protected EventSubscriptionBasedProjectionProcessingPhase(
IPublisher publisher, ICoreProjectionForProcessingPhase coreProjection, Guid projectionCorrelationId,
ICoreProjectionCheckpointManager checkpointManager, ProjectionConfig projectionConfig, string projectionName,
ILogger logger, CheckpointTag zeroCheckpointTag, PartitionStateCache partitionStateCache,
IResultWriter resultWriter, Action updateStatistics, ReaderSubscriptionDispatcher subscriptionDispatcher,
IReaderStrategy readerStrategy, bool useCheckpoints, bool stopOnEof)
{
_publisher = publisher;
_coreProjection = coreProjection;
_projectionCorrelationId = projectionCorrelationId;
_checkpointManager = checkpointManager;
_projectionConfig = projectionConfig;
_projectionName = projectionName;
_logger = logger;
_zeroCheckpointTag = zeroCheckpointTag;
_partitionStateCache = partitionStateCache;
_resultWriter = resultWriter;
_processingQueue = new CoreProjectionQueue(
projectionCorrelationId, publisher, projectionConfig.PendingEventsThreshold, updateStatistics);
_processingQueue.EnsureTickPending += EnsureTickPending;
_subscriptionDispatcher = subscriptionDispatcher;
_readerStrategy = readerStrategy;
_useCheckpoints = useCheckpoints;
_stopOnEof = stopOnEof;
}
public void it_can_be_created()
{
CheckpointStrategy.Create(
new QuerySourcesDefinition {
AllStreams = true, AllEvents = true
}, ProjectionConfig.GetTest(),
new RealTimeProvider());
}
protected DefaultProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
ReaderSubscriptionDispatcher subscriptionDispatcher, bool enableContentTypeValidation)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher, enableContentTypeValidation)
{
_stateHandler = stateHandler;
}
public static CheckpointStrategy Create(
ISourceDefinitionConfigurator sources, ProjectionConfig config, ITimeProvider timeProvider)
{
var builder = new Builder();
sources.ConfigureSourceProcessingStrategy(builder);
return builder.Build(
config, config.RunAs, Processing.ReaderStrategy.Create(sources, timeProvider, config.RunAs));
}
public CheckpointStrategy Build(ProjectionConfig config)
{
base.Validate(config);
return new CheckpointStrategy(
_allStreams, ToSet(_categories), ToSet(_streams), _allEvents, ToSet(_events), _byStream,
_byCustomPartitions, _options.UseEventIndexes, _options.ReorderEvents, _options.ProcessingLag,
_options.EmitStateUpdated, config.CheckpointsEnabled);
}
public void Validate(ProjectionConfig config)
{
base.Validate();
if (_definesStateTransform && !config.EmitEventEnabled)
{
throw new InvalidOperationException("transformBy/filterBy requires EmitEventEnabled mode");
}
}
protected CoreProjectionCheckpointManager(
ICoreProjection coreProjection, IPublisher publisher, Guid projectionCorrelationId,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher <ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string projectionCheckpointStreamId, string name,
PositionTagger positionTagger)
{
if (coreProjection == null)
{
throw new ArgumentNullException("coreProjection");
}
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (readDispatcher == null)
{
throw new ArgumentNullException("readDispatcher");
}
if (writeDispatcher == null)
{
throw new ArgumentNullException("writeDispatcher");
}
if (projectionConfig == null)
{
throw new ArgumentNullException("projectionConfig");
}
if (projectionCheckpointStreamId == null)
{
throw new ArgumentNullException("projectionCheckpointStreamId");
}
if (projectionCheckpointStreamId == "")
{
throw new ArgumentException("projectionCheckpointStreamId");
}
if (name == null)
{
throw new ArgumentNullException("name");
}
if (positionTagger == null)
{
throw new ArgumentNullException("positionTagger");
}
if (name == "")
{
throw new ArgumentException("name");
}
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_coreProjection = coreProjection;
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_projectionConfig = projectionConfig;
_logger = LogManager.GetLoggerFor <CoreProjectionCheckpointManager>();
_name = name;
}
protected EventReaderBasedProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionConfig projectionConfig,
IQuerySources sourceDefinition, ILogger logger, ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, logger)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_subscriptionDispatcher = subscriptionDispatcher;
}
public void setup()
{
Given();
_namingBuilder = ProjectionNamesBuilder.CreateForTest("projection");
_config = new ProjectionConfig(null, _checkpointHandledThreshold, _checkpointUnhandledBytesThreshold,
_pendingEventsThreshold, _maxWriteBatchLength, _emitEventEnabled,
_checkpointsEnabled, _createTempStreams, _stopOnEof, false, _trackEmittedStreams, _checkpointAfterMs, _maximumAllowedWritesInFlight);
When();
}
public QueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
}
public ProjectionProcessingStrategy CreateSlaveProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionSourceDefinition sourceDefinition,
ProjectionConfig projectionConfig, IProjectionStateHandler stateHandler, IPublisher resultsEnvelope,
Guid masterCoreProjectionId, ProjectionCoreService projectionCoreService)
{
return new SlaveQueryProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, projectionCoreService.Logger,
resultsEnvelope, masterCoreProjectionId, _subscriptionDispatcher);
}
public MultiStreamCheckpointManager(
ICoreProjection coreProjection, IPublisher publisher, Guid projectionCorrelationId,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger, string projectionStateUpdatesStreamId)
: base(coreProjection, publisher, projectionCorrelationId, readDispatcher, writeDispatcher, projectionConfig, name, positionTagger)
{
_projectionStateUpdatesStreamId = projectionStateUpdatesStreamId;
}
public MultiStreamMultiOutputCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
IODispatcher ioDispatcher, ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, bool useCheckpoints, bool producesRunningResults, bool definesFold,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
: base(
publisher, projectionCorrelationId, projectionVersion, runAs, ioDispatcher, projectionConfig, name,
positionTagger, namingBuilder, useCheckpoints, producesRunningResults, definesFold,
coreProjectionCheckpointWriter)
{
_positionTagger = positionTagger;
}
public SlaveQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
IPublisher resultsPublisher, Guid masterCoreProjectionId,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
_resultsPublisher = resultsPublisher;
_masterCoreProjectionId = masterCoreProjectionId;
}
public ParallelQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, Func<IProjectionStateHandler> handlerFactory,
ProjectionConfig projectionConfig, IQueryDefinition sourceDefinition, ProjectionNamesBuilder namesBuilder,
ILogger logger, SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
}
public DefaultCheckpointManager(
ICoreProjection coreProjection, IPublisher publisher, Guid projectionCorrelationId,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string projectionCheckpointStreamId, string name,
PositionTagger positionTagger)
: base(coreProjection, publisher, projectionCorrelationId, readDispatcher, writeDispatcher, projectionConfig, name, positionTagger)
{
if (projectionCheckpointStreamId == null) throw new ArgumentNullException("projectionCheckpointStreamId");
if (projectionCheckpointStreamId == "") throw new ArgumentException("projectionCheckpointStreamId");
_projectionCheckpointStreamId = projectionCheckpointStreamId;
}
public DefaultCheckpointManager(
ICoreProjection coreProjection, IPublisher publisher, Guid projectionCorrelationId,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger, ProjectionNamesBuilder namingBuilder, bool useCheckpoints,
bool emitStateUpdated, bool emitPartitionCheckpoints = false)
: base(
coreProjection, publisher, projectionCorrelationId, readDispatcher, writeDispatcher, projectionConfig,
name, positionTagger, namingBuilder, useCheckpoints, emitStateUpdated, emitPartitionCheckpoints)
{
_projectionCheckpointStreamId = namingBuilder.MakeCheckpointStreamName();
}
public MultiStreamMultiOutputCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, IResultEmitter resultEmitter, bool useCheckpoints,
bool emitPartitionCheckpoints = false)
: base(
publisher, projectionCorrelationId, projectionVersion, runAs, readDispatcher, writeDispatcher,
projectionConfig, name, positionTagger, namingBuilder, resultEmitter, useCheckpoints,
emitPartitionCheckpoints)
{
_positionTagger = positionTagger;
}
public NoopCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger, ProjectionNamesBuilder namingBuilder)
{
if (publisher == null) throw new ArgumentNullException("publisher");
if (projectionConfig == null) throw new ArgumentNullException("projectionConfig");
if (name == null) throw new ArgumentNullException("name");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (namingBuilder == null) throw new ArgumentNullException("namingBuilder");
if (name == "") throw new ArgumentException("name");
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
}
public DefaultCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, IResultEmitter resultEmitter, bool useCheckpoints,
bool emitPartitionCheckpoints = false)
: base(
publisher, projectionCorrelationId, projectionConfig, name, positionTagger, namingBuilder, resultEmitter,
useCheckpoints, emitPartitionCheckpoints)
{
if (readDispatcher == null) throw new ArgumentNullException("readDispatcher");
if (writeDispatcher == null) throw new ArgumentNullException("writeDispatcher");
_projectionVersion = projectionVersion;
_runAs = runAs;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_positionTagger = positionTagger;
_projectionCheckpointStreamId = namingBuilder.MakeCheckpointStreamName();
_zeroTag = positionTagger.MakeZeroCheckpointTag();
}
public ParallelQueryProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig,
IQuerySources sourceDefinition,
string handlerType,
string query,
ProjectionNamesBuilder namesBuilder,
ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
if (string.IsNullOrEmpty(handlerType)) throw new ArgumentNullException("handlerType");
if (string.IsNullOrEmpty(query)) throw new ArgumentNullException("query");
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
private void BeginCreatePrepared(ProjectionConfig config, Action onPrepared)
{
_onPrepared = onPrepared;
if (config == null) throw new ArgumentNullException("config");
//TODO: which states are allowed here?
if (_state >= ManagedProjectionState.Preparing)
{
DisposeCoreProjection();
_state = ManagedProjectionState.Loaded;
}
//TODO: load configuration from the definition
if (_persistedState.SourceDefinition == null)
throw new Exception(
"The projection cannot be loaded as stopped as it was stored in the old format. Update the projection query text to force prepare");
var createProjectionMessage =
new CoreProjectionManagementMessage.CreatePrepared(
new PublishEnvelope(_inputQueue), Id, _name,
new ProjectionVersion(_projectionId, _persistedState.Epoch ?? 0, _persistedState.Version ?? 1),
config, _persistedState.SourceDefinition, HandlerType, Query);
//note: set running before start as coreProjection.start() can respond with faulted
_state = ManagedProjectionState.Preparing;
_coreQueue.Publish(createProjectionMessage);
}
private ProjectionConfig CreateDefaultProjectionConfiguration()
{
var checkpointsEnabled = _persistedState.CheckpointsDisabled != true;
var checkpointHandledThreshold = checkpointsEnabled ? 4000 : 0;
var checkpointUnhandledBytesThreshold = checkpointsEnabled ? 10*1000*1000 : 0;
var pendingEventsThreshold = 5000;
var maxWriteBatchLength = 500;
var emitEventEnabled = _persistedState.EmitEnabled == true;
var createTempStreams = _persistedState.CreateTempStreams == true;
var stopOnEof = _persistedState.Mode <= ProjectionMode.OneTime;
var projectionConfig = new ProjectionConfig(
_runAs, checkpointHandledThreshold, checkpointUnhandledBytesThreshold, pendingEventsThreshold,
maxWriteBatchLength, emitEventEnabled, checkpointsEnabled, createTempStreams, stopOnEof,
isSlaveProjection: false);
return projectionConfig;
}
public void Validate(ProjectionConfig config)
{
base.Validate();
if (_definesStateTransform && !config.EmitEventEnabled)
throw new InvalidOperationException("transformBy/filterBy requires EmitEventEnabled mode");
}
private void BeginCreateAndPrepare(
ProjectionStateHandlerFactory handlerFactory, ProjectionConfig config,
Action onPrepared)
{
_onPrepared = onPrepared;
if (handlerFactory == null) throw new ArgumentNullException("handlerFactory");
if (config == null) throw new ArgumentNullException("config");
//TODO: which states are allowed here?
if (_state >= ManagedProjectionState.Preparing)
throw new InvalidOperationException("Already preparing or has been prepared");
//TODO: load configuration from the definition
var createProjectionMessage =
new CoreProjectionManagementMessage.CreateAndPrepare(
new PublishEnvelope(_inputQueue), _id, _name, config, delegate
{
// this delegate runs in the context of a projection core thread
// TODO: move this code to the projection core service as we may be in different processes in the future
IProjectionStateHandler stateHandler = null;
try
{
stateHandler = handlerFactory.Create(HandlerType, Query, Console.WriteLine);
var checkpointStrategyBuilder = new CheckpointStrategy.Builder();
stateHandler.ConfigureSourceProcessingStrategy(checkpointStrategyBuilder);
checkpointStrategyBuilder.Validate(Mode); // avoid future exceptions in coreprojection
return stateHandler;
}
catch (Exception ex)
{
SetFaulted(
string.Format(
"Cannot create a projection state handler.\r\n\r\nHandler type: {0}\r\nQuery:\r\n\r\n{1}\r\n\r\nMessage:\r\n\r\n{2}",
HandlerType, Query, ex.Message), ex);
if (stateHandler != null)
stateHandler.Dispose();
throw;
}
});
//note: set runnign before start as coreProjection.start() can respond with faulted
_state = ManagedProjectionState.Preparing;
_coreQueue.Publish(createProjectionMessage);
}
public ICoreProjectionCheckpointManager CreateCheckpointManager(
Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPublisher publisher,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, ProjectionNamesBuilder namingBuilder)
{
var emitAny = projectionConfig.EmitEventEnabled;
var emitPartitionCheckpoints = UseCheckpoints && (_byCustomPartitions || _byStream);
var resultEmitter = _definesStateTransform
? new ResultEmitter(namingBuilder)
: (IResultEmitter) new NoopResultEmitter();
//NOTE: not emitting one-time/transient projections are always handled by default checkpoint manager
// as they don't depend on stable event order
if (emitAny && !ReaderStrategy.IsReadingOrderRepeatable)
{
return new MultiStreamMultiOutputCheckpointManager(
publisher, projectionCorrelationId, projectionVersion, _runAs, readDispatcher, writeDispatcher,
projectionConfig, name, ReaderStrategy.PositionTagger, namingBuilder, resultEmitter, UseCheckpoints,
emitPartitionCheckpoints);
}
else
{
return new DefaultCheckpointManager(
publisher, projectionCorrelationId, projectionVersion, _runAs, readDispatcher, writeDispatcher,
projectionConfig, name, ReaderStrategy.PositionTagger, namingBuilder, resultEmitter, UseCheckpoints,
emitPartitionCheckpoints);
}
}
private CoreProjectionManagementMessage.CreatePrepared CreatePreparedMessage(ProjectionConfig config)
{
if (PersistedProjectionState.SourceDefinition == null)
throw new Exception(
"The projection cannot be loaded as stopped as it was stored in the old format. Update the projection query text to force prepare");
var createProjectionMessage = new CoreProjectionManagementMessage.CreatePrepared(
Id,
_workerId,
_name,
new ProjectionVersion(_projectionId, PersistedProjectionState.Epoch ?? 0, PersistedProjectionState.Version ?? 1),
config,
QuerySourcesDefinition.From(PersistedProjectionState.SourceDefinition),
HandlerType,
Query);
return createProjectionMessage;
}
private void BeginCreateAndPrepare(
ProjectionStateHandlerFactory handlerFactory, ProjectionConfig config, Action onPrepared)
{
_onPrepared = _onStopped = () =>
{
_onStopped = null;
_onPrepared = null;
if (onPrepared != null)
onPrepared();
};
if (handlerFactory == null) throw new ArgumentNullException("handlerFactory");
if (config == null) throw new ArgumentNullException("config");
//TODO: which states are allowed here?
if (_state >= ManagedProjectionState.Preparing)
{
DisposeCoreProjection();
_state = ManagedProjectionState.Loaded;
}
//TODO: load configuration from the definition
Func<IProjectionStateHandler> stateHandlerFactory = delegate
{
// this delegate runs in the context of a projection core thread
// TODO: move this code to the projection core service as we may be in different processes in the future
IProjectionStateHandler stateHandler = null;
try
{
stateHandler = handlerFactory.Create(
HandlerType, Query, logger: s => _logger.Trace(s),
cancelCallbackFactory:
_timeoutScheduler == null
? (Action<int, Action>) null
: _timeoutScheduler.Schedule);
return stateHandler;
}
catch (Exception ex)
{
SetFaulted(
string.Format(
"Cannot create a projection state handler.\r\n\r\nHandler type: {0}\r\nQuery:\r\n\r\n{1}\r\n\r\nMessage:\r\n\r\n{2}",
HandlerType, Query, ex.Message), ex);
if (stateHandler != null)
stateHandler.Dispose();
throw;
}
};
var createProjectionMessage = _isSlave ?
(Message) new CoreProjectionManagementMessage.CreateAndPrepareSlave(
new PublishEnvelope(_inputQueue), Id, _name,
new ProjectionVersion(_projectionId, _persistedState.Epoch ?? 0, _persistedState.Version ?? 0),
config, _slaveResultsPublisher, _slaveMasterCorrelationId, stateHandlerFactory) :
new CoreProjectionManagementMessage.CreateAndPrepare(
new PublishEnvelope(_inputQueue), Id, _name,
new ProjectionVersion(_projectionId, _persistedState.Epoch ?? 0, _persistedState.Version ?? 0),
config, HandlerType, Query, stateHandlerFactory);
//note: set runnign before start as coreProjection.start() can respond with faulted
_state = ManagedProjectionState.Preparing;
_coreQueue.Publish(createProjectionMessage);
}
public void PrepareOrWriteStartOrLoadStopped()
{
if (_state == ManagedProjectionState.Prepared)
{
WriteStartOrLoadStopped();
return;
}
if (Prepared && Created && !(Enabled && _enabledToRun))
{
WriteStartOrLoadStopped();
return;
}
_projectionConfig = CreateDefaultProjectionConfiguration();
var prepareMessage = !(Enabled && _enabledToRun) && !_pendingWritePersistedState
? CreatePreparedMessage(_projectionConfig)
: CreateCreateAndPrepareMessage(_projectionConfig);
Prepare(_projectionConfig, prepareMessage);
}
protected CoreProjectionCheckpointManager(
ICoreProjection coreProjection, IPublisher publisher, Guid projectionCorrelationId,
RequestResponseDispatcher<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, bool useCheckpoints, bool emitStateUpdated,
bool emitPartitionCheckpoints)
{
if (coreProjection == null) throw new ArgumentNullException("coreProjection");
if (publisher == null) throw new ArgumentNullException("publisher");
if (readDispatcher == null) throw new ArgumentNullException("readDispatcher");
if (writeDispatcher == null) throw new ArgumentNullException("writeDispatcher");
if (projectionConfig == null) throw new ArgumentNullException("projectionConfig");
if (name == null) throw new ArgumentNullException("name");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (namingBuilder == null) throw new ArgumentNullException("namingBuilder");
if (name == "") throw new ArgumentException("name");
if (emitPartitionCheckpoints && emitStateUpdated)
throw new InvalidOperationException("EmitPartitionCheckpoints && EmitStateUpdated cannot be both set");
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_coreProjection = coreProjection;
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_projectionConfig = projectionConfig;
_logger = LogManager.GetLoggerFor<CoreProjectionCheckpointManager>();
_name = name;
_positionTagger = positionTagger;
_namingBuilder = namingBuilder;
_useCheckpoints = useCheckpoints;
_emitStateUpdated = emitStateUpdated;
_emitPartitionCheckpoints = emitPartitionCheckpoints;
}
public CheckpointStrategy Build(ProjectionConfig config, IPrincipal runAs, IReaderStrategy readerStrategy)
{
base.Validate();
return new CheckpointStrategy(
_byStream, _byCustomPartitions, config.CheckpointsEnabled, _definesStateTransform, runAs, readerStrategy);
}
private static ProjectionConfig CreateDefaultProjectionConfiguration(ProjectionMode mode)
{
var projectionConfig = new ProjectionConfig(
mode, mode > ProjectionMode.AdHoc ? 2000 : 0, mode > ProjectionMode.AdHoc ? 10*1000*1000 : 0, 1000, 500,
publishStateUpdates: mode == ProjectionMode.Persistent, checkpointsEnabled: mode > ProjectionMode.AdHoc,
emitEventEnabled: mode == ProjectionMode.Persistent); //TODO: allow emit in continuous
return projectionConfig;
}
