public void Setup()
{
_state = null;
_projection = null;
Given();
_logged = new List <string>();
_stateHandlerFactory = new ProjectionStateHandlerFactory(TimeSpan.FromMilliseconds(1000), TimeSpan.FromMilliseconds(500), JavascriptProjectionRuntime.Legacy);
_stateHandler = CreateStateHandler();
_source = _stateHandler.GetSourceDefinition();
if (_state != null)
{
_stateHandler.Load(_state);
}
else
{
_stateHandler.Initialize();
}
if (_sharedState != null)
{
_stateHandler.LoadShared(_sharedState);
}
When();
}
public void Setup()
{
_state = null;
_projection = null;
Given();
_logged = new List <string>();
_stateHandlerFactory =
new ProjectionStateHandlerFactory(CompilationTimeout, ExecutionTimeout, JavascriptProjectionRuntime.Interpreted);
_stateHandler = CreateStateHandler();
_source = _stateHandler.GetSourceDefinition();
if (_state != null)
{
_stateHandler.Load(_state);
}
else
{
_stateHandler.Initialize();
}
if (_sharedState != null)
{
_stateHandler.LoadShared(_sharedState);
}
When();
}
public void Setup()
{
_state = null;
_projection = null;
Given();
_logged = new List <string>();
_stateHandlerFactory = new ProjectionStateHandlerFactory();
_stateHandler = CreateStateHandler();
_source = _stateHandler.GetSourceDefinition();
if (_state != null)
{
_stateHandler.Load(_state);
}
else
{
_stateHandler.Initialize();
}
if (_sharedState != null)
{
_stateHandler.LoadShared(_sharedState);
}
When();
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
ProjectionNamesBuilder namesBuilder,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler, string handlerType, string query)
{
return(projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher));
}
public void Setup()
{
_state = null;
_projection = null;
Given();
_logged = new List<string>();
_stateHandlerFactory = new ProjectionStateHandlerFactory();
_stateHandler = _stateHandlerFactory.Create(
"JS", _projection, logger: s =>
{
if (s.StartsWith("P:"))
Console.WriteLine(s);
else
_logged.Add(s);
}); // skip prelude debug output
_source = _stateHandler.GetSourceDefinition();
if (_state != null)
_stateHandler.Load(_state);
else
_stateHandler.Initialize();
if (_sharedState != null)
_stateHandler.LoadShared(_sharedState);
When();
}
protected virtual IReaderStrategy CreateCheckpointStrategy()
{
var readerBuilder = new SourceDefinitionBuilder();
if (_source != null)
{
_source(readerBuilder);
}
else
{
readerBuilder.FromAll();
readerBuilder.AllEvents();
}
var config = ProjectionConfig.GetTest();
IQuerySources sources = readerBuilder.Build();
var readerStrategy = Core.Services.Processing.ReaderStrategy.Create(
"test",
0,
sources,
_timeProvider,
stopOnEof: false,
runAs: config.RunAs);
return(readerStrategy);
}
public static QuerySourcesDefinition From(IQuerySources sources)
{
return(new QuerySourcesDefinition {
AllEvents = sources.AllEvents,
AllStreams = sources.AllStreams,
ByStreams = sources.ByStreams,
ByCustomPartitions = sources.ByCustomPartitions,
Categories = (sources.Categories ?? new string[0]).ToArray(),
Events = (sources.Events ?? new string[0]).ToArray(),
Streams = (sources.Streams ?? new string[0]).ToArray(),
LimitingCommitPosition = sources.LimitingCommitPosition,
Options =
new QuerySourcesDefinitionOptions {
DefinesStateTransform = sources.DefinesStateTransform,
ProducesResults = sources.ProducesResults,
DefinesFold = sources.DefinesFold,
HandlesDeletedNotifications = sources.HandlesDeletedNotifications,
IncludeLinks = sources.IncludeLinksOption,
PartitionResultStreamNamePattern = sources.PartitionResultStreamNamePatternOption,
ProcessingLag = sources.ProcessingLagOption.GetValueOrDefault(),
IsBiState = sources.IsBiState,
ReorderEvents = sources.ReorderEventsOption,
ResultStreamName = sources.ResultStreamNameOption,
}
});
}
public CreatePrepared(
IEnvelope envelope, Guid projectionId, string name, ProjectionVersion version, ProjectionConfig config,
IQuerySources sourceDefinition, string handlerType, string query)
: base(projectionId)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
if (sourceDefinition == null)
{
throw new ArgumentNullException("sourceDefinition");
}
if (handlerType == null)
{
throw new ArgumentNullException("handlerType");
}
if (query == null)
{
throw new ArgumentNullException("query");
}
_envelope = envelope;
_name = name;
_version = version;
_config = config;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
}
public void Setup()
{
_state = null;
_projection = null;
Given();
_logged = new List <string>();
_stateHandlerFactory = new ProjectionStateHandlerFactory();
_stateHandler = _stateHandlerFactory.Create(
"JS", _projection, logger: (s, _) => {
if (s.StartsWith("P:"))
{
Console.WriteLine(s);
}
else
{
_logged.Add(s);
}
}); // skip prelude debug output
_source = _stateHandler.GetSourceDefinition();
if (_state != null)
{
_stateHandler.Load(_state);
}
else
{
_stateHandler.Initialize();
}
if (_sharedState != null)
{
_stateHandler.LoadShared(_sharedState);
}
When();
}
protected DefaultProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
ReaderSubscriptionDispatcher subscriptionDispatcher, bool enableContentTypeValidation)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher, enableContentTypeValidation)
{
_stateHandler = stateHandler;
}
public QueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
}
public QueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
}
protected EventReaderBasedProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionConfig projectionConfig,
IQuerySources sourceDefinition, ILogger logger, ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, logger)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_subscriptionDispatcher = subscriptionDispatcher;
}
protected EventReaderBasedProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionConfig projectionConfig,
IQuerySources sourceDefinition, ILogger logger, ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, logger)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_subscriptionDispatcher = subscriptionDispatcher;
}
protected EventReaderBasedProjectionProcessingStrategy(
string name, ProjectionVersion projectionVersion, ProjectionConfig projectionConfig,
IQuerySources sourceDefinition, Serilog.ILogger logger, ReaderSubscriptionDispatcher subscriptionDispatcher, bool enableContentTypeValidation)
: base(name, projectionVersion, logger)
{
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_subscriptionDispatcher = subscriptionDispatcher;
_isBiState = sourceDefinition.IsBiState;
_enableContentTypeValidation = enableContentTypeValidation;
}
public ProjectionNamesBuilder(string name, IQuerySources sources)
{
if (sources == null) throw new ArgumentNullException("sources");
_name = name;
_sources = sources;
_resultStreamName = _sources.ResultStreamNameOption
?? ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionsStateStreamSuffix;
_partitionCatalogStreamName = ProjectionsStreamPrefix + EffectiveProjectionName
+ ProjectionPartitionCatalogStreamSuffix;
_checkpointStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionCheckpointStreamSuffix;
_orderStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionOrderStreamSuffix;
}
public SlaveQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
IPublisher resultsPublisher, Guid masterCoreProjectionId,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
_resultsPublisher = resultsPublisher;
_masterCoreProjectionId = masterCoreProjectionId;
}
public SlaveQueryProcessingStrategy(
string name, ProjectionVersion projectionVersion, IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig, IQuerySources sourceDefinition, ILogger logger,
IPublisher resultsPublisher, Guid masterCoreProjectionId,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(
name, projectionVersion, stateHandler, projectionConfig, sourceDefinition, logger,
subscriptionDispatcher)
{
_resultsPublisher = resultsPublisher;
_masterCoreProjectionId = masterCoreProjectionId;
}
public static IReaderStrategy Create(int phase, IQuerySources sources, ITimeProvider timeProvider, bool stopOnEof, IPrincipal runAs)
{
if (!sources.AllStreams && !sources.HasCategories() && !sources.HasStreams()
&& string.IsNullOrEmpty(sources.CatalogStream))
throw new InvalidOperationException("None of streams and categories are included");
if (!sources.AllEvents && !sources.HasEvents())
throw new InvalidOperationException("None of events are included");
if (sources.HasStreams() && sources.HasCategories())
throw new InvalidOperationException(
"Streams and categories cannot be included in a filter at the same time");
if (sources.AllStreams && (sources.HasCategories() || sources.HasStreams()))
throw new InvalidOperationException("Both FromAll and specific categories/streams cannot be set");
if (sources.AllEvents && sources.HasEvents())
throw new InvalidOperationException("Both AllEvents and specific event filters cannot be set");
if (sources.ByStreams && sources.HasStreams())
throw new InvalidOperationException("foreachStream projections are not supported on stream based sources");
if ((sources.HasStreams() || sources.AllStreams) && !string.IsNullOrEmpty(sources.CatalogStream))
throw new InvalidOperationException("catalogStream cannot be used with streams or allStreams");
if (!string.IsNullOrEmpty(sources.CatalogStream) && !sources.ByStreams)
throw new InvalidOperationException("catalogStream is only supported in the byStream mode");
if (!string.IsNullOrEmpty(sources.CatalogStream) && !stopOnEof)
throw new InvalidOperationException("catalogStream is not supported in the projections mode");
if (sources.ReorderEventsOption)
{
if (!string.IsNullOrEmpty(sources.CatalogStream))
throw new InvalidOperationException("Event reordering cannot be used with stream catalogs");
if (sources.AllStreams)
throw new InvalidOperationException("Event reordering cannot be used with fromAll()");
if (!(sources.HasStreams() && sources.Streams.Length > 1))
{
throw new InvalidOperationException(
"Event reordering is only available in fromStreams([]) projections");
}
if (sources.ProcessingLagOption < 50)
throw new InvalidOperationException("Event reordering requires processing lag at least of 50ms");
}
if (sources.HandlesDeletedNotifications && !sources.ByStreams)
throw new InvalidOperationException(
"Deleted stream notifications are only supported with foreachStream()");
var readerStrategy = new ReaderStrategy(
phase, sources.AllStreams, sources.Categories, sources.Streams, sources.AllEvents,
sources.IncludeLinksOption, sources.Events, sources.HandlesDeletedNotifications, sources.CatalogStream,
sources.ProcessingLagOption, sources.ReorderEventsOption, runAs, timeProvider);
return readerStrategy;
}
public ProjectionNamesBuilder(string name, IQuerySources sources)
{
_name = name;
_sources = sources;
_partitionResultStreamNamePattern = _sources.PartitionResultStreamNamePatternOption
?? ProjectionsStreamPrefix + EffectiveProjectionName + "-{0}"
+ ProjectionsStateStreamSuffix;
_resultStreamName = _sources.ResultStreamNameOption
?? ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionsStateStreamSuffix;
_partitionCatalogStreamName = ProjectionsStreamPrefix + EffectiveProjectionName
+ ProjectionPartitionCatalogStreamSuffix;
_checkpointStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionCheckpointStreamSuffix;
_orderStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionOrderStreamSuffix;
}
public ProjectionNamesBuilder(string name, IQuerySources sources)
{
_name = name;
_sources = sources;
_partitionResultStreamNamePattern = _sources.PartitionResultStreamNamePatternOption
?? ProjectionsStreamPrefix + EffectiveProjectionName + "-{0}"
+ ProjectionsStateStreamSuffix;
_resultStreamName = _sources.ResultStreamNameOption
?? ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionsStateStreamSuffix;
_partitionCatalogStreamName = ProjectionsStreamPrefix + EffectiveProjectionName
+ ProjectionPartitionCatalogStreamSuffix;
_checkpointStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionCheckpointStreamSuffix;
_orderStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionOrderStreamSuffix;
}
public ProjectionNamesBuilder(string name, IQuerySources sources)
{
if (sources == null)
{
throw new ArgumentNullException("sources");
}
_name = name;
_sources = sources;
_resultStreamName = _sources.ResultStreamNameOption
?? ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionsStateStreamSuffix;
_partitionCatalogStreamName = ProjectionsStreamPrefix + EffectiveProjectionName
+ ProjectionPartitionCatalogStreamSuffix;
_checkpointStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionCheckpointStreamSuffix;
_orderStreamName = ProjectionsStreamPrefix + EffectiveProjectionName + ProjectionOrderStreamSuffix;
}
public ParallelQueryProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig,
IQuerySources sourceDefinition,
string handlerType,
string query,
ProjectionNamesBuilder namesBuilder,
ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
if (string.IsNullOrEmpty(handlerType))
{
throw new ArgumentNullException("handlerType");
}
if (string.IsNullOrEmpty(query))
{
throw new ArgumentNullException("query");
}
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
public static ProjectionSourceDefinition From(
string name, IQuerySources sources, string handlerType, string query)
{
var namingBuilder = new ProjectionNamesBuilder(name, sources);
return(new ProjectionSourceDefinition
{
AllEvents = sources.AllEvents,
AllStreams = sources.AllStreams,
ByStream = sources.ByStreams,
ByCustomPartitions = sources.ByCustomPartitions,
Categories = (sources.Categories ?? new string[0]).ToArray(),
Events = (sources.Events ?? new string[0]).ToArray(),
Streams = (sources.Streams ?? new string[0]).ToArray(),
CatalogStream = sources.CatalogStream,
LimitingCommitPosition = sources.LimitingCommitPosition,
Options =
new QuerySourceOptions
{
DefinesStateTransform = sources.DefinesStateTransform,
DefinesCatalogTransform = sources.DefinesCatalogTransform,
ProducesResults = sources.ProducesResults,
DefinesFold = sources.DefinesFold,
HandlesDeletedNotifications = sources.HandlesDeletedNotifications,
ForceProjectionName = sources.ForceProjectionNameOption,
IncludeLinks = sources.IncludeLinksOption,
DisableParallelism = sources.DisableParallelismOption,
PartitionResultStreamNamePattern = sources.PartitionResultStreamNamePatternOption,
ProcessingLag = sources.ProcessingLagOption.GetValueOrDefault(),
IsBiState = sources.IsBiState,
ReorderEvents = sources.ReorderEventsOption,
ResultStreamName = sources.ResultStreamNameOption,
},
ResultStreamName = namingBuilder.GetResultStreamName(),
PartitionResultStreamNamePattern = namingBuilder.GetPartitionResultStreamNamePattern(),
PartitionResultCatalogStream = namingBuilder.GetPartitionResultCatalogStreamName(),
PartitionCatalogStream = namingBuilder.GetPartitionCatalogStreamName(),
HandlerType = handlerType,
Query = query
});
}
public static void AreEqual(IQuerySources expected, IQuerySources actual)
{
Assert.AreEqual(expected.AllEvents, actual.AllEvents, $"Expected {nameof(expected.AllEvents)} to be {expected.AllEvents} but was {actual.AllEvents}");
Assert.AreEqual(expected.AllStreams, actual.AllStreams, $"Expected {nameof(expected.AllStreams)} to be {expected.AllStreams} but was {actual.AllStreams}");
Assert.AreEqual(expected.ByCustomPartitions, actual.ByCustomPartitions, $"Expected {nameof(expected.ByCustomPartitions)} to be {expected.ByCustomPartitions} but was {actual.ByCustomPartitions}");
Assert.AreEqual(expected.ByStreams, actual.ByStreams, $"Expected {nameof(expected.ByStreams)} to be {expected.ByStreams} but was {actual.ByStreams}");
Assert.AreEqual(expected.DefinesFold, actual.DefinesFold, $"Expected {nameof(expected.DefinesFold)} to be {expected.DefinesFold} but was {actual.DefinesFold}");
Assert.AreEqual(expected.DefinesStateTransform, actual.DefinesStateTransform, $"Expected {nameof(expected.DefinesStateTransform)} to be {expected.DefinesStateTransform} but was {actual.DefinesStateTransform}");
Assert.AreEqual(expected.IncludeLinksOption, actual.IncludeLinksOption, $"Expected {nameof(expected.IncludeLinksOption)} to be {expected.IncludeLinksOption} but was {actual.IncludeLinksOption}");
Assert.AreEqual(expected.IsBiState, actual.IsBiState, $"Expected {nameof(expected.IsBiState)} to be {expected.IsBiState} but was {actual.IsBiState}");
Assert.AreEqual(expected.LimitingCommitPosition, actual.LimitingCommitPosition, $"Expected {nameof(expected.LimitingCommitPosition)} to be {expected.LimitingCommitPosition} but was {actual.LimitingCommitPosition}");
Assert.AreEqual(expected.PartitionResultStreamNamePatternOption, actual.PartitionResultStreamNamePatternOption, $"Expected {nameof(expected.PartitionResultStreamNamePatternOption)} to be {expected.PartitionResultStreamNamePatternOption} but was {actual.PartitionResultStreamNamePatternOption}");
Assert.AreEqual(expected.ProcessingLagOption, actual.ProcessingLagOption, $"Expected {nameof(expected.ProcessingLagOption)} to be {expected.ProcessingLagOption} but was {actual.ProcessingLagOption}");
Assert.AreEqual(expected.ProducesResults, actual.ProducesResults, $"Expected {nameof(expected.ProducesResults)} to be {expected.ProducesResults} but was {actual.ProducesResults}");
Assert.AreEqual(expected.ReorderEventsOption, actual.ReorderEventsOption, $"Expected {nameof(expected.ReorderEventsOption)} to be {expected.ReorderEventsOption} but was {actual.ReorderEventsOption}");
Assert.AreEqual(expected.ResultStreamNameOption, actual.ResultStreamNameOption, $"Expected {nameof(expected.ResultStreamNameOption)} to be {expected.ResultStreamNameOption} but was {actual.ResultStreamNameOption}");
Assert.AreEqual(expected.HandlesDeletedNotifications, actual.HandlesDeletedNotifications, $"Expected {nameof(expected.HandlesDeletedNotifications)} to be {expected.HandlesDeletedNotifications} but was {actual.HandlesDeletedNotifications}");
AssertWithName(nameof(expected.Categories), expected.Categories, actual.Categories);
AssertWithName(nameof(expected.Events), expected.Events, actual.Events);
AssertWithName(nameof(expected.Streams), expected.Streams, actual.Streams);
public ProjectionProcessingStrategy CreateSlaveProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler,
Guid workerId,
IPublisher publisher,
Guid masterCoreProjectionId,
ProjectionCoreService projectionCoreService)
{
return(new SlaveQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
projectionCoreService.Logger,
workerId,
publisher,
masterCoreProjectionId,
_subscriptionDispatcher));
}
public ParallelQueryProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
IProjectionStateHandler stateHandler,
ProjectionConfig projectionConfig,
IQuerySources sourceDefinition,
string handlerType,
string query,
ProjectionNamesBuilder namesBuilder,
ILogger logger,
SpooledStreamReadingDispatcher spoolProcessingResponseDispatcher,
ReaderSubscriptionDispatcher subscriptionDispatcher)
: base(name, projectionVersion, projectionConfig, sourceDefinition, logger, subscriptionDispatcher)
{
if (string.IsNullOrEmpty(handlerType)) throw new ArgumentNullException("handlerType");
if (string.IsNullOrEmpty(query)) throw new ArgumentNullException("query");
_stateHandler = stateHandler;
_projectionConfig = projectionConfig;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
_namesBuilder = namesBuilder;
_spoolProcessingResponseDispatcher = spoolProcessingResponseDispatcher;
if (_sourceDefinition.CatalogStream == SystemStreams.AllStream)
{
_catalogStreamName = SystemStreams.AllStream;
}
else if (_sourceDefinition.HasCategories())
{
_catalogStreamName = _namesBuilder.GetCategoryCatalogStreamName(_sourceDefinition.Categories[0]);
}
else
{
_catalogStreamName = _sourceDefinition.CatalogStream;
}
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
ProjectionNamesBuilder namesBuilder,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler, string handlerType, string query)
{
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams
&& sourceDefinition.DefinesFold && !string.IsNullOrEmpty(sourceDefinition.CatalogStream))
{
return new ParallelQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
handlerType,
query,
namesBuilder,
_logger,
_spoolProcessingResponseDispatcher,
_subscriptionDispatcher);
}
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams
&& sourceDefinition.DefinesFold && sourceDefinition.HasCategories())
{
return new ParallelQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
handlerType,
query,
namesBuilder,
_logger,
_spoolProcessingResponseDispatcher,
_subscriptionDispatcher);
}
return projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher);
}
public ProjectionProcessingStrategy CreateSlaveProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler,
Guid workerId,
IPublisher publisher,
Guid masterCoreProjectionId,
ProjectionCoreService projectionCoreService)
{
return new SlaveQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
projectionCoreService.Logger,
workerId,
publisher,
masterCoreProjectionId,
_subscriptionDispatcher);
}
public ProjectionProcessingStrategy CreateProjectionProcessingStrategy(
string name,
ProjectionVersion projectionVersion,
ProjectionNamesBuilder namesBuilder,
IQuerySources sourceDefinition,
ProjectionConfig projectionConfig,
IProjectionStateHandler stateHandler, string handlerType, string query)
{
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams &&
sourceDefinition.DefinesFold && !string.IsNullOrEmpty(sourceDefinition.CatalogStream))
{
return(new ParallelQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
handlerType,
query,
namesBuilder,
_logger,
_spoolProcessingResponseDispatcher,
_subscriptionDispatcher));
}
if (!sourceDefinition.DisableParallelismOption && projectionConfig.StopOnEof && sourceDefinition.ByStreams &&
sourceDefinition.DefinesFold && sourceDefinition.HasCategories())
{
return(new ParallelQueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
handlerType,
query,
namesBuilder,
_logger,
_spoolProcessingResponseDispatcher,
_subscriptionDispatcher));
}
return(projectionConfig.StopOnEof
? (ProjectionProcessingStrategy)
new QueryProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher)
: new ContinuousProjectionProcessingStrategy(
name,
projectionVersion,
stateHandler,
projectionConfig,
sourceDefinition,
_logger,
_subscriptionDispatcher));
}
public CreatePrepared(
IEnvelope envelope, Guid projectionId, string name, ProjectionVersion version, ProjectionConfig config,
IQuerySources sourceDefinition, string handlerType, string query)
: base(projectionId)
{
if (name == null) throw new ArgumentNullException("name");
if (config == null) throw new ArgumentNullException("config");
if (sourceDefinition == null) throw new ArgumentNullException("sourceDefinition");
if (handlerType == null) throw new ArgumentNullException("handlerType");
if (query == null) throw new ArgumentNullException("query");
_envelope = envelope;
_name = name;
_version = version;
_config = config;
_sourceDefinition = sourceDefinition;
_handlerType = handlerType;
_query = query;
}
public FakeProjectionStateHandler(string source, Action<string, object[]> logger)
{
_definition = source.ParseJson<QuerySourcesDefinition>();
}
public static ProjectionSourceDefinition From(
string name, IQuerySources sources, string handlerType, string query)
{
var namingBuilder = new ProjectionNamesBuilder(name, sources);
return new ProjectionSourceDefinition
{
AllEvents = sources.AllEvents,
AllStreams = sources.AllStreams,
ByStream = sources.ByStreams,
ByCustomPartitions = sources.ByCustomPartitions,
Categories = (sources.Categories ?? new string[0]).ToArray(),
Events = (sources.Events ?? new string[0]).ToArray(),
Streams = (sources.Streams ?? new string[0]).ToArray(),
CatalogStream = sources.CatalogStream,
LimitingCommitPosition = sources.LimitingCommitPosition,
Options =
new QuerySourceOptions
{
DefinesStateTransform = sources.DefinesStateTransform,
ProducesResults = sources.ProducesResults,
DefinesFold = sources.DefinesFold,
ForceProjectionName = sources.ForceProjectionNameOption,
IncludeLinks = sources.IncludeLinksOption,
PartitionResultStreamNamePattern = sources.PartitionResultStreamNamePatternOption,
ProcessingLag = sources.ProcessingLagOption.GetValueOrDefault(),
ReorderEvents = sources.ReorderEventsOption,
ResultStreamName = sources.ResultStreamNameOption,
},
ResultStreamName = namingBuilder.GetResultStreamName(),
PartitionResultStreamNamePattern = namingBuilder.GetPartitionResultStreamNamePattern(),
PartitionResultCatalogStream = namingBuilder.GetPartitionResultCatalogStreamName(),
PartitionCatalogStream = namingBuilder.GetPartitionCatalogStreamName(),
HandlerType = handlerType,
Query = query
};
}
public static IReaderStrategy Create(
string tag,
int phase,
IQuerySources sources,
ITimeProvider timeProvider,
bool stopOnEof,
IPrincipal runAs)
{
if (!sources.AllStreams && !sources.HasCategories() && !sources.HasStreams() &&
string.IsNullOrEmpty(sources.CatalogStream))
{
throw new InvalidOperationException("None of streams and categories are included");
}
if (!sources.AllEvents && !sources.HasEvents())
{
throw new InvalidOperationException("None of events are included");
}
if (sources.HasStreams() && sources.HasCategories())
{
throw new InvalidOperationException(
"Streams and categories cannot be included in a filter at the same time");
}
if (sources.AllStreams && (sources.HasCategories() || sources.HasStreams()))
{
throw new InvalidOperationException("Both FromAll and specific categories/streams cannot be set");
}
if (sources.AllEvents && sources.HasEvents())
{
throw new InvalidOperationException("Both AllEvents and specific event filters cannot be set");
}
if (sources.ByStreams && sources.HasStreams())
{
throw new InvalidOperationException(
"foreachStream projections are not supported on stream based sources");
}
if ((sources.HasStreams() || sources.AllStreams) && !string.IsNullOrEmpty(sources.CatalogStream))
{
throw new InvalidOperationException("catalogStream cannot be used with streams or allStreams");
}
if (!string.IsNullOrEmpty(sources.CatalogStream) && !sources.ByStreams)
{
throw new InvalidOperationException("catalogStream is only supported in the byStream mode");
}
if (!string.IsNullOrEmpty(sources.CatalogStream) && !stopOnEof)
{
throw new InvalidOperationException("catalogStream is not supported in the projections mode");
}
if (sources.ReorderEventsOption)
{
if (!string.IsNullOrEmpty(sources.CatalogStream))
{
throw new InvalidOperationException("Event reordering cannot be used with stream catalogs");
}
if (sources.AllStreams)
{
throw new InvalidOperationException("Event reordering cannot be used with fromAll()");
}
if (!(sources.HasStreams() && sources.Streams.Length > 1))
{
throw new InvalidOperationException(
"Event reordering is only available in fromStreams([]) projections");
}
if (sources.ProcessingLagOption < 50)
{
throw new InvalidOperationException("Event reordering requires processing lag at least of 50ms");
}
}
if (sources.HandlesDeletedNotifications && !sources.ByStreams)
{
throw new InvalidOperationException(
"Deleted stream notifications are only supported with foreachStream()");
}
var readerStrategy = new ReaderStrategy(
tag,
phase,
sources.AllStreams,
sources.Categories,
sources.Streams,
sources.AllEvents,
sources.IncludeLinksOption,
sources.Events,
sources.HandlesDeletedNotifications,
sources.CatalogStream,
sources.ProcessingLagOption,
sources.ReorderEventsOption,
runAs,
timeProvider);
return(readerStrategy);
}
public static IEnumerable <object[]> GetTestCases()
{
var assembly = typeof(SpecRunner).Assembly;
var specs = assembly.GetManifestResourceNames().Where(x => x.EndsWith("-spec.json"));
foreach (var spec in specs)
{
JsonDocument doc;
using (var stream = assembly.GetManifestResourceStream(spec)) {
doc = JsonDocument.Parse(stream !);
}
var projection = doc.RootElement.GetProperty("projection").GetString();
Assert.False(string.IsNullOrWhiteSpace(projection));
var projectionSourceName = assembly.GetManifestResourceNames()
.SingleOrDefault(x => x.EndsWith($"{projection}.js"));
Assert.NotNull(projectionSourceName);
string source;
using (var stream = assembly.GetManifestResourceStream(projectionSourceName !)) {
Assert.NotNull(stream);
using (var sr = new StreamReader(stream !)) {
source = sr.ReadToEnd();
}
}
List <InputEventSequence> sequences = new();
foreach (var inputEvent in doc.RootElement.GetProperty("input").EnumerateArray())
{
var stream = inputEvent.GetProperty("streamId").GetString();
Assert.NotNull(stream);
var sequence = new InputEventSequence(stream !);
sequences.Add(sequence);
foreach (var e in inputEvent.GetProperty("events").EnumerateArray())
{
var et = e.GetProperty("eventType").GetString();
Assert.NotNull(et);
bool skip = false;
if (e.TryGetProperty("skip", out var skipElement))
{
skip = skipElement.GetBoolean();
}
var initializedPartitions = new List <string>();
if (e.TryGetProperty("initializedPartitions", out var partitions))
{
foreach (var element in partitions.EnumerateArray())
{
initializedPartitions.Add(element.GetString() !);
}
}
var expectedStates = new Dictionary <string, string?>();
int stateCount = 0;
if (e.TryGetProperty("states", out var states))
{
foreach (var state in states.EnumerateArray())
{
stateCount++;
var expectedStateNode = state.GetProperty("state");
var expectedState = expectedStateNode.ValueKind == JsonValueKind.Null ? null : expectedStateNode.GetRawText();
if (!expectedStates.TryAdd(state.GetProperty("partition").GetString() !,
expectedState))
{
throw new InvalidOperationException("Duplicate state");
}
}
}
if (stateCount > 2)
{
throw new InvalidOperationException("Cannot specify more than 2 states");
}
sequence.Events.Add(new InputEvent(et !, e.GetProperty("data").GetRawText(), e.TryGetProperty("metadata", out var metadata) ? metadata.GetRawText() : null, initializedPartitions, expectedStates, skip, e.TryGetProperty("eventId", out var idElement) && idElement.TryGetGuid(out var id) ? id: Guid.NewGuid()));
}
}
var output = doc.RootElement.GetProperty("output");
var sdb = new SourceDefinitionBuilder();
var config = output.GetProperty("config");
foreach (var item in config.EnumerateObject())
{
switch (item.Name)
{
case "definesStateTransform":
if (item.Value.GetBoolean())
{
sdb.SetDefinesStateTransform();
}
break;
case "handlesDeletedNotifications":
sdb.SetHandlesStreamDeletedNotifications(item.Value.GetBoolean());
break;
case "producesResults":
if (item.Value.GetBoolean())
{
sdb.SetOutputState();
}
break;
case "definesFold":
if (item.Value.GetBoolean())
{
sdb.SetDefinesFold();
}
break;
case "resultStreamName":
sdb.SetResultStreamNameOption(item.Value.GetString());
break;
case "partitionResultStreamNamePattern":
sdb.SetPartitionResultStreamNamePatternOption(item.Value.GetString());
break;
case "$includeLinks":
sdb.SetIncludeLinks(item.Value.GetBoolean());
break;
case "reorderEvents":
sdb.SetReorderEvents(item.Value.GetBoolean());
break;
case "processingLag":
sdb.SetProcessingLag(item.Value.GetInt32());
break;
case "biState":
sdb.SetIsBiState(item.Value.GetBoolean());
break;
case "categories":
foreach (var c in item.Value.EnumerateArray())
{
sdb.FromCategory(c.GetString());
}
break;
case "partitioned":
if (item.Value.GetBoolean())
{
sdb.SetByCustomPartitions();
}
break;
case "events":
foreach (var e in item.Value.EnumerateArray())
{
sdb.IncludeEvent(e.GetString());
}
break;
case "allEvents":
if (item.Value.GetBoolean())
{
sdb.AllEvents();
}
else
{
sdb.NotAllEvents();
}
break;
case "allStreams":
if (item.Value.GetBoolean())
{
sdb.FromAll();
}
break;
default:
throw new Exception($"unexpected property in expected config {item.Name}");
}
}
#nullable disable
List <OutputEvent> expectedEmittedEvents = new();
if (output.TryGetProperty("emitted", out var expectedEmittedElement))
{
foreach (var element in expectedEmittedElement.EnumerateObject())
{
var stream = element.Name;
foreach (var eventElement in element.Value.EnumerateArray())
{
if (eventElement.ValueKind == JsonValueKind.String)
{
expectedEmittedEvents.Add(
new OutputEvent(stream, "$>", eventElement.GetString(), null));
}
}
}
}
JintProjectionStateHandler runner = null;
IQuerySources definition = null;
IQuerySources expectedDefinition = sdb.Build();
yield return(WithOutput($"{projection} compiles", o => {
runner = new JintProjectionStateHandler(source, true,
TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100));
}));
yield return(For($"{projection} getDefinition", () => {
definition = runner.GetSourceDefinition();
}));
yield return(For($"{projection} qs.AllStreams", () => Assert.Equal(expectedDefinition.AllStreams, definition.AllStreams)));
yield return(For($"{projection} qs.Categories", () => Assert.Equal(expectedDefinition.Categories, definition.Categories)));
yield return(For($"{projection} qs.Streams", () => Assert.Equal(expectedDefinition.Streams, definition.Streams)));
yield return(For($"{projection} qs.AllEvents", () => Assert.Equal(expectedDefinition.AllEvents, definition.AllEvents)));
yield return(For($"{projection} qs.Events", () => Assert.Equal(expectedDefinition.Events, definition.Events)));
yield return(For($"{projection} qs.ByStreams", () => Assert.Equal(expectedDefinition.ByStreams, definition.ByStreams)));
yield return(For($"{projection} qs.ByCustomPartitions", () => Assert.Equal(expectedDefinition.ByCustomPartitions, definition.ByCustomPartitions)));
yield return(For($"{projection} qs.DefinesStateTransform", () => Assert.Equal(expectedDefinition.DefinesStateTransform, definition.DefinesStateTransform)));
yield return(For($"{projection} qs.DefinesFold", () => Assert.Equal(expectedDefinition.DefinesFold, definition.DefinesFold)));
yield return(For($"{projection} qs.HandlesDeletedNotifications", () => Assert.Equal(expectedDefinition.HandlesDeletedNotifications, definition.HandlesDeletedNotifications)));
yield return(For($"{projection} qs.ProducesResults", () => Assert.Equal(expectedDefinition.ProducesResults, definition.ProducesResults)));
yield return(For($"{projection} qs.IsBiState", () => Assert.Equal(expectedDefinition.IsBiState, definition.IsBiState)));
yield return(For($"{projection} qs.IncludeLinksOption", () => Assert.Equal(expectedDefinition.IncludeLinksOption, definition.IncludeLinksOption)));
yield return(For($"{projection} qs.ResultStreamNameOption", () => Assert.Equal(expectedDefinition.ResultStreamNameOption, definition.ResultStreamNameOption)));
yield return(For($"{projection} qs.PartitionResultStreamNamePatternOption", () => Assert.Equal(expectedDefinition.PartitionResultStreamNamePatternOption,
definition.PartitionResultStreamNamePatternOption)));
yield return(For($"{projection} qs.ReorderEventsOption", () => Assert.Equal(expectedDefinition.ReorderEventsOption, definition.ReorderEventsOption)));
yield return(For($"{projection} qs.ProcessingLagOption", () => Assert.Equal(expectedDefinition.ProcessingLagOption, definition.ProcessingLagOption)));
yield return(For($"{projection} qs.LimitingCommitPosition", () => Assert.Equal(expectedDefinition.LimitingCommitPosition, definition.LimitingCommitPosition)));
var partitionedState = new Dictionary <string, string>();
var sharedStateInitialized = false;
var revision = new Dictionary <string, long>();
List <EmittedEventEnvelope> actualEmittedEvents = new();
for (int i = 0; i < sequences.Count; i++)
{
var sequence = sequences[i];
if (!revision.TryGetValue(sequence.Stream, out _))
{
revision[sequence.Stream] = 0;
}
for (int j = 0; j < sequences[i].Events.Count; j++)
{
var logPosition = i * 100 + j;
var flags = PrepareFlags.IsJson | PrepareFlags.Data;
if (j == 0)
{
flags |= PrepareFlags.TransactionBegin;
}
if (j == sequence.Events.Count - 1)
{
flags |= PrepareFlags.TransactionEnd;
}
/*Sequence:
* Get partition if bycustom partition or by stream
* if the partition is null or an empty string skip this event (NB: need to indicate that an event should be skipped)
* load the state if it doesn't exist or tell the projection to init state
* init shared if it doesn't exist
* if init was state was called, call created
* if processing a delete, call partition deleted (NB: bistate partitions do not support deletes)
* process the event
* save any shared state if it isn't null
* save any state
* save emitted events to verify later
*/
var @event = sequence.Events[j];
var body = JObject.Parse(@event.Body).ToString(Formatting.Indented);
var metadata = Array.Empty <byte>();
if (@event.Metadata != null)
{
metadata = Utf8NoBom.GetBytes(JObject.Parse(@event.Metadata).ToString(Formatting.Indented));
}
var er = new EventRecord(
revision[sequence.Stream], logPosition, Guid.NewGuid(), @event.EventId, i, j,
sequence.Stream, i, DateTime.Now, flags, @event.EventType,
Utf8NoBom.GetBytes(body), metadata);
var e = new ResolvedEvent(EventStore.Core.Data.ResolvedEvent.ForUnresolvedEvent(er, logPosition), Array.Empty <byte>());
if (@event.Skip)
{
yield return(For($"{projection} skips {er.EventNumber}@{sequence.Stream}",
() => Assert.Null(runner.GetStatePartition(CheckpointTag.Empty, "", e))));
}
else
{
var expectedPartition = "";
if (expectedDefinition.DefinesFold)
{
if (@event.ExpectedStates.Any())
{
expectedPartition = @event.ExpectedStates
.SingleOrDefault(x => string.Empty != x.Key).Key ?? string.Empty;
if (expectedPartition != string.Empty)
{
yield return(For(
$"{projection} {er.EventNumber}@{sequence.Stream} returns expected partition",
() => Assert.Equal(expectedPartition,
runner.GetStatePartition(CheckpointTag.Empty, "", e))));
foreach (var initializedState in @event.InitializedPartitions)
{
yield return(For(
$"should not have already initialized \"{initializedState}\" at {er.EventNumber}@{sequence.Stream}",
() => Assert.DoesNotContain(initializedState,
(IReadOnlyDictionary <string, string>)partitionedState)));
}
}
if (expectedDefinition.IsBiState && !sharedStateInitialized)
{
sharedStateInitialized = true;
yield return(For("initializes shared state without error", () => {
runner.InitializeShared();
}));
}
if ([email protected](expectedPartition))
{
yield return(For(
$"can load current state at {er.EventNumber}@{sequence.Stream}",
() => {
Assert.True(
partitionedState.TryGetValue(expectedPartition, out var value),
$"did not find expected state for partition{expectedPartition}");
runner.Load(value);
}));
public static QuerySourcesDefinition From(IQuerySources sources)
{
return new QuerySourcesDefinition
{
AllEvents = sources.AllEvents,
AllStreams = sources.AllStreams,
ByStreams = sources.ByStreams,
ByCustomPartitions = sources.ByCustomPartitions,
Categories = (sources.Categories ?? new string[0]).ToArray(),
Events = (sources.Events ?? new string[0]).ToArray(),
Streams = (sources.Streams ?? new string[0]).ToArray(),
CatalogStream = sources.CatalogStream,
LimitingCommitPosition = sources.LimitingCommitPosition,
Options =
new QuerySourcesDefinitionOptions
{
DefinesStateTransform = sources.DefinesStateTransform,
ProducesResults = sources.ProducesResults,
DefinesFold = sources.DefinesFold,
ForceProjectionName = sources.ForceProjectionNameOption,
IncludeLinks = sources.IncludeLinksOption,
PartitionResultStreamNamePattern = sources.PartitionResultStreamNamePatternOption,
ProcessingLag = sources.ProcessingLagOption.GetValueOrDefault(),
ReorderEvents = sources.ReorderEventsOption,
ResultStreamName = sources.ResultStreamNameOption,
}
};
}
public FakeProjectionStateHandler(string source, Action <string> logger)
{
_definition = source.ParseJson <QuerySourcesDefinition>();
}
public static bool HasEvents(this IQuerySources sources)
{
var events = sources.Events;
return(events != null && events.Length > 0);
}
public static bool HasStreams(this IQuerySources sources)
{
var streams = sources.Streams;
return(streams != null && streams.Length > 0);
}
public static bool HasCategories(this IQuerySources sources)
{
var categories = sources.Categories;
return(categories != null && categories.Length > 0);
}
public static ProjectionSourceDefinition From(IQuerySources sources)
{
return new ProjectionSourceDefinition
{
AllEvents = sources.AllEvents,
AllStreams = sources.AllStreams,
ByStream = sources.ByStreams,
ByCustomPartitions = sources.ByCustomPartitions,
Categories = (sources.Categories ?? new string[0]).ToArray(),
Events = (sources.Events ?? new string[0]).ToArray(),
Streams = (sources.Streams ?? new string[0]).ToArray(),
CatalogStream = sources.CatalogStream,
LimitingCommitPosition = sources.LimitingCommitPosition,
Options =
new QuerySourceOptions
{
DefinesStateTransform = sources.DefinesStateTransform,
DefinesCatalogTransform = sources.DefinesCatalogTransform,
ProducesResults = sources.ProducesResults,
DefinesFold = sources.DefinesFold,
HandlesDeletedNotifications = sources.HandlesDeletedNotifications,
IncludeLinks = sources.IncludeLinksOption,
DisableParallelism = sources.DisableParallelismOption,
PartitionResultStreamNamePattern = sources.PartitionResultStreamNamePatternOption,
ProcessingLag = sources.ProcessingLagOption.GetValueOrDefault(),
IsBiState = sources.IsBiState,
ReorderEvents = sources.ReorderEventsOption,
ResultStreamName = sources.ResultStreamNameOption,
},
};
}
