public static CheckpointTagVersion ParseCheckpointTagVersionExtraJson(this string source, ProjectionVersion current)
{
if (string.IsNullOrEmpty(source))
return new CheckpointTagVersion { Version = new ProjectionVersion(current.ProjectionId, 0, 0), Tag = null };
var reader = new JsonTextReader(new StringReader(source));
return CheckpointTag.FromJson(reader, current);
}
public ParallelQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
Func<IProjectionStateHandler> handlerFactory, ProjectionConfig projectionConfig,
IQueryDefinition sourceDefinition, ProjectionNamesBuilder namesBuilder, ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
public ProjectionCheckpoint(
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionVersion projectionVersion, IPrincipal runAs, IProjectionCheckpointManager readyHandler,
CheckpointTag from, PositionTagger positionTagger, CheckpointTag zero, int maxWriteBatchLength,
ILogger logger = null)
{
if (readDispatcher == null) throw new ArgumentNullException("readDispatcher");
if (writeDispatcher == null) throw new ArgumentNullException("writeDispatcher");
if (readyHandler == null) throw new ArgumentNullException("readyHandler");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (zero == null) throw new ArgumentNullException("zero");
if (from.CommitPosition <= from.PreparePosition) throw new ArgumentException("from");
//NOTE: fromCommit can be equal fromPrepare on 0 position. Is it possible anytime later? Ignoring for now.
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_projectionVersion = projectionVersion;
_runAs = runAs;
_readyHandler = readyHandler;
_positionTagger = positionTagger;
_zero = zero;
_from = _last = from;
_maxWriteBatchLength = maxWriteBatchLength;
_logger = logger;
}
public static CheckpointTagVersion ParseCheckpointTagVersionExtraJson(this byte[] source, ProjectionVersion current)
{
if (source == null || source.Length == 0)
return new CheckpointTagVersion { Version = new ProjectionVersion(current.ProjectionId, 0, 0), Tag = null };
var reader = new JsonTextReader(new StreamReader(new MemoryStream(source)));
return CheckpointTag.FromJson(reader, current);
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionNamesBuilder namesBuilder,
IQueryDefinition sourceDefinition, ProjectionConfig projectionConfig,
Func<IProjectionStateHandler> handlerFactory, IProjectionStateHandler stateHandler)
{
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams
&& sourceDefinition.DefinesFold && !string.IsNullOrEmpty(sourceDefinition.CatalogStream))
{
return new ParallelQueryProcessingStrategy(
name, projectionVersion, stateHandler, handlerFactory, projectionConfig, sourceDefinition,
namesBuilder, _logger, _spoolProcessingResponseDispatcher, _subscriptionDispatcher);
}
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams
&& sourceDefinition.DefinesFold && sourceDefinition.HasCategories())
{
return new ParallelQueryProcessingStrategy(
name, projectionVersion, stateHandler, handlerFactory, projectionConfig, sourceDefinition,
namesBuilder, _logger, _spoolProcessingResponseDispatcher, _subscriptionDispatcher);
}
return projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, _logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, _logger,
_subscriptionDispatcher);
}
public void equal()
{
var v1 = new ProjectionVersion(10, 5, 6);
var v2 = new ProjectionVersion(10, 5, 6);
Assert.AreEqual(v1, v2);
}
public void not_equal_version()
{
var v1 = new ProjectionVersion(10, 5, 6);
var v2 = new ProjectionVersion(10, 5, 7);
Assert.AreNotEqual(v1, v2);
}
public FakeProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ILogger logger, FakeProjectionProcessingPhase phase1,
FakeProjectionProcessingPhase phase2)
: base(name, projectionVersion, logger)
{
_phase1 = phase1;
_phase2 = phase2;
}
public CheckpointTag AdjustBy(PositionTagger tagger, ProjectionVersion version)
{
if (SystemVersion == Projections.VERSION && Version.Version == version.Version
&& Version.ProjectionId == version.ProjectionId)
return Tag;
return tagger.AdjustTag(Tag);
}
public ProjectionProcessingStrategy CreateSlaveProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionSourceDefinition sourceDefinition,
ProjectionConfig projectionConfig, IProjectionStateHandler stateHandler, IPublisher resultsEnvelope,
Guid masterCoreProjectionId, ProjectionCoreService projectionCoreService)
{
return new SlaveQueryProcessingStrategy(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, projectionCoreService.Logger,
resultsEnvelope, masterCoreProjectionId, _subscriptionDispatcher);
}
public QueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
}
protected EventReaderBasedProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionConfig projectionConfig,
IQuerySources sourceDefinition, ILogger logger, ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, logger)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_subscriptionDispatcher = subscriptionDispatcher;
}
public CoreProjectionCheckpointReader(
IPublisher publisher, Guid projectionCorrelationId, IODispatcher ioDispatcher, string projectionCheckpointStreamId, ProjectionVersion projectionVersion, bool useCheckpoints)
{
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_ioDispatcher = ioDispatcher;
_projectionCheckpointStreamId = projectionCheckpointStreamId;
_projectionVersion = projectionVersion;
_useCheckpoints = useCheckpoints;
}
public void a_null_name_throws_argument_null_excveption()
{
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
var version = new ProjectionVersion(1, 0, 0);
new ContinuousProjectionProcessingStrategy(
null, version, projectionStateHandler, _defaultProjectionConfig,
projectionStateHandler.GetSourceDefinition(), null, _subscriptionDispatcher).Create(
Guid.NewGuid(), new FakePublisher(), SystemAccount.Principal, new FakePublisher(), _ioDispatcher,
_subscriptionDispatcher, new RealTimeProvider());
}
public CoreProjectionCheckpointWriter(
string projectionCheckpointStreamId, IODispatcher ioDispatcher, ProjectionVersion projectionVersion,
string name)
{
_projectionCheckpointStreamId = projectionCheckpointStreamId;
_logger = LogManager.GetLoggerFor<CoreProjectionCheckpointWriter>();
_ioDispatcher = ioDispatcher;
_projectionVersion = projectionVersion;
_name = name;
}
public void a_negative_checkpoint_handled_interval_throws_argument_out_of_range_exception()
{
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
var version = new ProjectionVersion(1, 0, 0);
var projectionConfig = new ProjectionConfig(null, -1, 10, 1000, 250, true, true, false, false, false);
new ContinuousProjectionProcessingStrategy(
"projection", version, projectionStateHandler, projectionConfig,
projectionStateHandler.GetSourceDefinition(), null, _subscriptionDispatcher).Create(
Guid.NewGuid(), new FakePublisher(), SystemAccount.Principal, new FakePublisher(), _ioDispatcher,
_subscriptionDispatcher, new RealTimeProvider());
}
protected new virtual void When()
{
_projectionVersion = new ProjectionVersion(1, 0, 0);
_projectionName = "projection";
_checkpointWriter = new CoreProjectionCheckpointWriter(
_namingBuilder.MakeCheckpointStreamName(), _ioDispatcher, _projectionVersion, _projectionName);
_checkpointReader = new CoreProjectionCheckpointReader(
GetInputQueue(), _projectionCorrelationId, _ioDispatcher, _projectionCheckpointStreamId,
_projectionVersion, _checkpointsEnabled);
_manager = GivenCheckpointManager();
}
public MultiStreamMultiOutputCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
IODispatcher ioDispatcher, ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, bool useCheckpoints, bool producesRunningResults, bool definesFold,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter)
: base(
publisher, projectionCorrelationId, projectionVersion, runAs, ioDispatcher, projectionConfig, name,
positionTagger, namingBuilder, useCheckpoints, producesRunningResults, definesFold,
coreProjectionCheckpointWriter)
{
_positionTagger = positionTagger;
}
public SlaveQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
IPublisher resultsPublisher, Guid masterCoreProjectionId,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
_resultsPublisher = resultsPublisher;
_masterCoreProjectionId = masterCoreProjectionId;
}
public ParallelQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, Func<IProjectionStateHandler> handlerFactory,
ProjectionConfig projectionConfig, IQueryDefinition sourceDefinition, ProjectionNamesBuilder namesBuilder,
ILogger logger, SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
}
protected override void When()
{
_projectionVersion = new ProjectionVersion(1, 2, 3);
_sut.Handle(
new CoreProjectionManagementMessage.CreateAndPrepare(
_projectionId,
_workerId,
_projectionName,
_projectionVersion,
_config,
_handlerType,
_query));
}
public MultiStreamMultiOutputCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, IResultEmitter resultEmitter, bool useCheckpoints,
bool emitPartitionCheckpoints = false)
: base(
publisher, projectionCorrelationId, projectionVersion, runAs, readDispatcher, writeDispatcher,
projectionConfig, name, positionTagger, namingBuilder, resultEmitter, useCheckpoints,
emitPartitionCheckpoints)
{
_positionTagger = positionTagger;
}
protected override void Given()
{
_version = new ProjectionVersion(1, 2, 2);
ExistingEvent(
"$projections-projection-result", "Result",
@"{""v"":1, ""c"": 100, ""p"": 50}", _testProjectionState);
ExistingEvent(
"$projections-projection-checkpoint", "$ProjectionCheckpoint",
@"{""v"":1, ""c"": 100, ""p"": 50}", _testProjectionState);
ExistingEvent(
"$projections-projection-result", "Result",
@"{""v"":1, ""c"": 200, ""p"": 150}", _testProjectionState);
ExistingEvent(
"$projections-projection-result", "Result",
@"{""v"":1, ""c"": 300, ""p"": 250}", _testProjectionState);
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestHandler<ClientMessage.ReadStreamEventsBackward>();
_bus.Subscribe(_listEventsHandler);
_ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
_subscriptionDispatcher =
new ReaderSubscriptionDispatcher
(_bus);
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionEofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionMeasured>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionDeleted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.SubscriptionStarted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ReaderAssignedReader>());
_bus.Subscribe(_ioDispatcher.BackwardReader);
_bus.Subscribe(_ioDispatcher.ForwardReader);
_bus.Subscribe(_ioDispatcher.Writer);
_bus.Subscribe(_ioDispatcher);
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
_projectionConfig = new ProjectionConfig(null, 5, 10, 1000, 250, true, true, false, false, false);
var version = new ProjectionVersion(1, 0, 0);
var projectionProcessingStrategy = new ContinuousProjectionProcessingStrategy(
"projection", version, projectionStateHandler, _projectionConfig,
projectionStateHandler.GetSourceDefinition(), null, _subscriptionDispatcher);
_coreProjection = projectionProcessingStrategy.Create(
Guid.NewGuid(),
_bus,
Guid.NewGuid(),
SystemAccount.Principal,
_bus,
_ioDispatcher,
_subscriptionDispatcher,
new RealTimeProvider());
_coreProjection.Start();
}
public ProjectionCheckpoint(
IODispatcher ioDispatcher, ProjectionVersion projectionVersion, IPrincipal runAs,
IProjectionCheckpointManager readyHandler, CheckpointTag from, PositionTagger positionTagger,
CheckpointTag zero, int maxWriteBatchLength, ILogger logger = null)
{
if (ioDispatcher == null) throw new ArgumentNullException("ioDispatcher");
if (readyHandler == null) throw new ArgumentNullException("readyHandler");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (zero == null) throw new ArgumentNullException("zero");
if (from.CommitPosition < from.PreparePosition) throw new ArgumentException("from");
//NOTE: fromCommit can be equal fromPrepare on 0 position. Is it possible anytime later? Ignoring for now.
_ioDispatcher = ioDispatcher;
_projectionVersion = projectionVersion;
_runAs = runAs;
_readyHandler = readyHandler;
_positionTagger = positionTagger;
_zero = zero;
_from = _last = from;
_maxWriteBatchLength = maxWriteBatchLength;
_logger = logger;
}
public DefaultCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionVersion projectionVersion, IPrincipal runAs,
RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted> readDispatcher,
RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted> writeDispatcher,
ProjectionConfig projectionConfig, string name, PositionTagger positionTagger,
ProjectionNamesBuilder namingBuilder, IResultEmitter resultEmitter, bool useCheckpoints,
bool emitPartitionCheckpoints = false)
: base(
publisher, projectionCorrelationId, projectionConfig, name, positionTagger, namingBuilder, resultEmitter,
useCheckpoints, emitPartitionCheckpoints)
{
if (readDispatcher == null) throw new ArgumentNullException("readDispatcher");
if (writeDispatcher == null) throw new ArgumentNullException("writeDispatcher");
_projectionVersion = projectionVersion;
_runAs = runAs;
_readDispatcher = readDispatcher;
_writeDispatcher = writeDispatcher;
_positionTagger = positionTagger;
_projectionCheckpointStreamId = namingBuilder.MakeCheckpointStreamName();
_zeroTag = positionTagger.MakeZeroCheckpointTag();
}
//private readonly CheckpointTag _tag3 =CheckpointTag.FromStreamPositions(new Dictionary<string, int> {{"a", 1}, {"b", 1}, {"c", 1}});
protected override void Given()
{
base.Given();
_projectionVersion = new ProjectionVersion(1, 2, 2);
ExistingEvent(
"$projections-projection-checkpoint", "$ProjectionCheckpoint", @"{""v"":2, ""s"": {""a"": 0, ""b"": 0, ""c"": 0}}",
"{}");
ExistingEvent("a", "StreamCreated", "", "");
ExistingEvent("b", "StreamCreated", "", "");
ExistingEvent("c", "StreamCreated", "", "");
ExistingEvent("a", "Event", "", @"{""data"":""a""");
ExistingEvent("b", "Event", "", @"{""data"":""b""");
ExistingEvent("c", "Event", "", @"{""data"":""c""");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":1, ""s"": {""a"": 0, ""b"": 0, ""c"": 0}}", "0@c");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":1, ""s"": {""a"": 0, ""b"": 0, ""c"": 1}}", "1@c");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":1, ""s"": {""a"": 1, ""b"": 0, ""c"": 1}}", "1@a");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":1, ""s"": {""a"": 1, ""b"": 1, ""c"": 1}}", "1@b");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":2, ""s"": {""a"": 0, ""b"": 0, ""c"": 0}}", "0@c");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":2, ""s"": {""a"": 0, ""b"": 0, ""c"": 1}}", "1@c");
ExistingEvent("$projections-projection-order", "$>", @"{""v"":2, ""s"": {""a"": 1, ""b"": 0, ""c"": 1}}", "1@a");
}
public ParallelQueryProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig,
IQuerySources sourceDefinition,
string handlerType,
string query,
ProjectionNamesBuilder namesBuilder,
ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
if (string.IsNullOrEmpty(handlerType)) throw new ArgumentNullException("handlerType");
if (string.IsNullOrEmpty(query)) throw new ArgumentNullException("query");
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
protected ProjectionProcessingStrategy(string name, ProjectionVersion projectionVersion, ILogger logger)
{
_name = name;
_projectionVersion = projectionVersion;
_logger = logger;
}
protected override void Given1()
{
_version = new ProjectionVersion(1, 0, 0);
_projectionName = "projection";
}
public static CheckpointTagVersion FromJson(JsonReader reader, ProjectionVersion current)
{
Check(reader.Read(), reader);
Check(JsonToken.StartObject, reader);
long?commitPosition = null;
long?preparePosition = null;
Dictionary <string, int> streams = null;
Dictionary <string, JToken> extra = null;
var projectionId = current.ProjectionId;
var projectionEpoch = 0;
var projectionVersion = 0;
var projectionSystemVersion = 0;
while (true)
{
Check(reader.Read(), reader);
if (reader.TokenType == JsonToken.EndObject)
{
break;
}
Check(JsonToken.PropertyName, reader);
var name = (string)reader.Value;
switch (name)
{
case "$v":
case "v":
Check(reader.Read(), reader);
if (reader.ValueType == typeof(long))
{
var v = (int)(long)reader.Value;
if (v > 0) // TODO: remove this if with time
{
projectionVersion = v;
}
}
else
{
//TODO: better handle errors
var v = (string)reader.Value;
string[] parts = v.Split(':');
if (parts.Length == 2)
{
projectionVersion = Int32.Parse(parts[1]);
}
else
{
projectionId = Int32.Parse(parts[0]);
projectionEpoch = Int32.Parse(parts[1]);
projectionVersion = Int32.Parse(parts[2]);
if (parts.Length >= 4)
{
projectionSystemVersion = Int32.Parse(parts[3]);
}
}
}
break;
case "$c":
case "c":
case "commitPosition":
Check(reader.Read(), reader);
commitPosition = (long)reader.Value;
break;
case "$p":
case "p":
case "preparePosition":
Check(reader.Read(), reader);
preparePosition = (long)reader.Value;
break;
case "$s":
case "s":
case "streams":
Check(reader.Read(), reader);
Check(JsonToken.StartObject, reader);
streams = new Dictionary <string, int>();
while (true)
{
Check(reader.Read(), reader);
if (reader.TokenType == JsonToken.EndObject)
{
break;
}
Check(JsonToken.PropertyName, reader);
var streamName = (string)reader.Value;
Check(reader.Read(), reader);
var position = (int)(long)reader.Value;
streams.Add(streamName, position);
}
break;
default:
if (extra == null)
{
extra = new Dictionary <string, JToken>();
}
Check(reader.Read(), reader);
var jToken = JToken.ReadFrom(reader);
extra.Add(name, jToken);
break;
}
}
return(new CheckpointTagVersion
{
Tag =
new CheckpointTag(
new TFPos(commitPosition ?? Int64.MinValue, preparePosition ?? Int64.MinValue), streams),
Version = new ProjectionVersion(projectionId, projectionEpoch, projectionVersion),
SystemVersion = projectionSystemVersion,
ExtraMetadata = extra,
});
}
//NOTE: this is only for slave projections (TBD)
public CoreProjection(
ProjectionProcessingStrategy projectionProcessingStrategy,
ProjectionVersion version,
Guid projectionCorrelationId,
IPublisher inputQueue,
Guid workerId,
IPrincipal runAs,
IPublisher publisher,
IODispatcher ioDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher,
ILogger logger,
ProjectionNamesBuilder namingBuilder,
CoreProjectionCheckpointWriter coreProjectionCheckpointWriter,
PartitionStateCache partitionStateCache,
string effectiveProjectionName,
ITimeProvider timeProvider)
{
if (publisher == null)
{
throw new ArgumentNullException("publisher");
}
if (ioDispatcher == null)
{
throw new ArgumentNullException("ioDispatcher");
}
if (subscriptionDispatcher == null)
{
throw new ArgumentNullException("subscriptionDispatcher");
}
_projectionProcessingStrategy = projectionProcessingStrategy;
_projectionCorrelationId = projectionCorrelationId;
_inputQueue = inputQueue;
_workerId = workerId;
_runAs = runAs;
_name = effectiveProjectionName;
_version = version;
_stopOnEof = projectionProcessingStrategy.GetStopOnEof();
_logger = logger ?? LogManager.GetLoggerFor <CoreProjection>();
_publisher = publisher;
_ioDispatcher = ioDispatcher;
_partitionStateCache = partitionStateCache;
_requiresRootPartition = projectionProcessingStrategy.GetRequiresRootPartition();
var useCheckpoints = projectionProcessingStrategy.GetUseCheckpoints();
_coreProjectionCheckpointWriter = coreProjectionCheckpointWriter;
_projectionProcessingPhases = projectionProcessingStrategy.CreateProcessingPhases(
publisher,
inputQueue,
projectionCorrelationId,
partitionStateCache,
UpdateStatistics,
this,
namingBuilder,
timeProvider,
ioDispatcher,
coreProjectionCheckpointWriter);
//NOTE: currently assuming the first checkpoint manager to be able to load any state
_checkpointReader = new CoreProjectionCheckpointReader(
publisher,
_projectionCorrelationId,
ioDispatcher,
namingBuilder.MakeCheckpointStreamName(),
_version,
useCheckpoints);
_enrichStatistics = projectionProcessingStrategy.EnrichStatistics;
GoToState(State.Initial);
}
public static CheckpointTagVersion ParseCheckpointTagVersionExtraJson(this byte[] source, ProjectionVersion current)
{
if (source == null || source.Length == 0)
{
return new CheckpointTagVersion {
Version = new ProjectionVersion(current.ProjectionId, 0, 0), Tag = null
}
}
;
var reader = new JsonTextReader(new StreamReader(new MemoryStream(source)));
return(CheckpointTag.FromJson(reader, current));
}
private static void WriteVersion(ProjectionVersion projectionVersion, JsonWriter jsonWriter)
{
jsonWriter.WriteValue(
projectionVersion.ProjectionId + ":" + projectionVersion.Epoch + ":" + projectionVersion.Version + ":"
+ ProjectionsSubsystem.VERSION);
}
public static CheckpointTagVersion FromJson(JsonReader reader, ProjectionVersion current,
bool skipStartObject = false)
{
if (!skipStartObject)
{
Check(reader.Read(), reader);
}
Check(JsonToken.StartObject, reader);
long? commitPosition = null;
long? preparePosition = null;
string catalogStream = null;
string dataStream = null;
long? catalogPosition = null;
long? dataPosition = null;
bool byStreamMode = false;
Dictionary <string, long> streams = null;
Dictionary <string, JToken> extra = null;
var projectionId = current.ProjectionId;
var projectionEpoch = 0;
int projectionVersion = 0;
var projectionSystemVersion = 0;
int projectionPhase = 0;
while (true)
{
Check(reader.Read(), reader);
if (reader.TokenType == JsonToken.EndObject)
{
break;
}
Check(JsonToken.PropertyName, reader);
var name = (string)reader.Value;
switch (name)
{
case "$cp":
Check(reader.Read(), reader);
var completed = (bool)reader.Value;
commitPosition = completed ? Int64.MaxValue : Int64.MinValue;
preparePosition = completed ? Int64.MaxValue : Int64.MinValue;
break;
case "$v":
case "v":
Check(reader.Read(), reader);
if (reader.ValueType == typeof(long))
{
var v = (int)(long)reader.Value;
if (v > 0) // TODO: remove this if with time
{
projectionVersion = v;
}
}
else
{
//TODO: better handle errors
var v = (string)reader.Value;
string[] parts = v.Split(':');
if (parts.Length == 2)
{
projectionVersion = Int32.Parse(parts[1]);
}
else
{
projectionId = Int32.Parse(parts[0]);
projectionEpoch = Int32.Parse(parts[1]);
projectionVersion = Int32.Parse(parts[2]);
if (parts.Length >= 4)
{
projectionSystemVersion = Int32.Parse(parts[3]);
}
}
}
break;
case "$c":
case "c":
case "commitPosition":
Check(reader.Read(), reader);
commitPosition = (long)reader.Value;
break;
case "$p":
case "p":
case "preparePosition":
Check(reader.Read(), reader);
preparePosition = (long)reader.Value;
break;
case "$s":
case "s":
case "streams":
Check(reader.Read(), reader);
if (reader.TokenType == JsonToken.StartArray)
{
Check(reader.Read(), reader);
Check(JsonToken.StartObject, reader);
Check(reader.Read(), reader);
Check(JsonToken.PropertyName, reader);
catalogStream = (string)reader.Value;
Check(reader.Read(), reader);
catalogPosition = (long)reader.Value;
Check(reader.Read(), reader);
Check(JsonToken.EndObject, reader);
Check(reader.Read(), reader);
if (reader.TokenType == JsonToken.StartObject)
{
Check(reader.Read(), reader);
Check(JsonToken.PropertyName, reader);
dataStream = (string)reader.Value;
Check(reader.Read(), reader);
dataPosition = (long)reader.Value;
Check(reader.Read(), reader);
Check(JsonToken.EndObject, reader);
Check(reader.Read(), reader);
}
Check(JsonToken.EndArray, reader);
}
else
{
Check(JsonToken.StartObject, reader);
streams = new Dictionary <string, long>();
while (true)
{
Check(reader.Read(), reader);
if (reader.TokenType == JsonToken.EndObject)
{
break;
}
Check(JsonToken.PropertyName, reader);
var streamName = (string)reader.Value;
Check(reader.Read(), reader);
long position = (long)reader.Value;
streams.Add(streamName, position);
}
}
break;
case "$ph":
Check(reader.Read(), reader);
projectionPhase = (int)(long)reader.Value;
break;
case "$m":
Check(reader.Read(), reader);
var readMode = (string)reader.Value;
if (readMode != "bs")
{
throw new ApplicationException("Unknown checkpoint tag mode: " + readMode);
}
byStreamMode = true;
break;
default:
if (extra == null)
{
extra = new Dictionary <string, JToken>();
}
Check(reader.Read(), reader);
var jToken = JToken.ReadFrom(reader);
extra.Add(name, jToken);
break;
}
}
return(new CheckpointTagVersion {
Tag =
byStreamMode
? new CheckpointTag(
projectionPhase, catalogStream, catalogPosition.GetValueOrDefault(), dataStream,
dataPosition ?? -1, commitPosition.GetValueOrDefault())
: new CheckpointTag(
projectionPhase,
new TFPos(commitPosition ?? Int64.MinValue, preparePosition ?? Int64.MinValue), streams),
Version = new ProjectionVersion(projectionId, projectionEpoch, projectionVersion),
SystemVersion = projectionSystemVersion,
ExtraMetadata = extra,
});
}
public void WriteTo(ProjectionVersion projectionVersion,
IEnumerable <KeyValuePair <string, JToken> > extraMetaData, JsonWriter jsonWriter)
{
jsonWriter.WriteStartObject();
if (projectionVersion.ProjectionId > 0)
{
jsonWriter.WritePropertyName("$v");
WriteVersion(projectionVersion, jsonWriter);
}
if (Phase != 0)
{
jsonWriter.WritePropertyName("$ph");
jsonWriter.WriteValue(Phase);
}
switch (Mode_)
{
case Mode.Phase:
jsonWriter.WritePropertyName("$cp");
jsonWriter.WriteValue(Completed);
break;
case Mode.Position:
case Mode.EventTypeIndex:
jsonWriter.WritePropertyName("$c");
jsonWriter.WriteValue(CommitPosition.GetValueOrDefault());
jsonWriter.WritePropertyName("$p");
jsonWriter.WriteValue(PreparePosition.GetValueOrDefault());
if (Mode_ == Mode.EventTypeIndex)
{
goto case Mode.MultiStream;
}
break;
case Mode.PreparePosition:
jsonWriter.WritePropertyName("$p");
jsonWriter.WriteValue(PreparePosition.GetValueOrDefault());
break;
case Mode.Stream:
case Mode.MultiStream:
jsonWriter.WritePropertyName("$s");
jsonWriter.WriteStartObject();
foreach (var stream in Streams)
{
jsonWriter.WritePropertyName(stream.Key);
jsonWriter.WriteValue(stream.Value);
}
jsonWriter.WriteEndObject();
break;
case Mode.ByStream:
jsonWriter.WritePropertyName("$m");
jsonWriter.WriteValue("bs");
jsonWriter.WritePropertyName("$c");
jsonWriter.WriteValue(CommitPosition.GetValueOrDefault());
jsonWriter.WritePropertyName("$s");
jsonWriter.WriteStartArray();
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName(CatalogStream);
jsonWriter.WriteValue(CatalogPosition);
jsonWriter.WriteEndObject();
if (!string.IsNullOrEmpty(DataStream))
{
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName(DataStream);
jsonWriter.WriteValue(DataPosition);
jsonWriter.WriteEndObject();
}
jsonWriter.WriteEndArray();
break;
}
if (extraMetaData != null)
{
foreach (var pair in extraMetaData)
{
jsonWriter.WritePropertyName(pair.Key);
pair.Value.WriteTo(jsonWriter);
}
}
jsonWriter.WriteEndObject();
}
public static CheckpointTagVersion ParseCheckpointTagVersionExtraJson(this string source, ProjectionVersion current)
{
if (string.IsNullOrEmpty(source))
{
return new CheckpointTagVersion {
Version = new ProjectionVersion(current.ProjectionId, 0, 0), Tag = null
}
}
;
var reader = new JsonTextReader(new StringReader(source));
return(CheckpointTag.FromJson(reader, current));
}
public CoreProjectionCheckpointReader(
IPublisher publisher, Guid projectionCorrelationId, IODispatcher ioDispatcher, string projectionCheckpointStreamId, ProjectionVersion projectionVersion, bool useCheckpoints)
{
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_ioDispatcher = ioDispatcher;
_projectionCheckpointStreamId = projectionCheckpointStreamId;
_projectionVersion = projectionVersion;
_useCheckpoints = useCheckpoints;
}
protected ProjectionProcessingStrategy(string name, ProjectionVersion projectionVersion, ILogger logger)
{
_name = name;
_projectionVersion = projectionVersion;
_logger = logger;
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
ProjectionNamesBuilder namesBuilder,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler, string handlerType, string query)
{
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams &&
sourceDefinition.DefinesFold && !string.IsNullOrEmpty(sourceDefinition.CatalogStream))
{
return(new ParallelQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
handlerType,
query,
namesBuilder,
_logger,
_spoolProcessingResponseDispatcher,
_subscriptionDispatcher));
}
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams &&
sourceDefinition.DefinesFold && sourceDefinition.HasCategories())
{
return(new ParallelQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
handlerType,
query,
namesBuilder,
_logger,
_spoolProcessingResponseDispatcher,
_subscriptionDispatcher));
}
return(projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher));
}