public EstablishTombstoneStream(EventGraph events)
{
var pkFields = events.TenancyStyle == TenancyStyle.Conjoined
? "id, tenant_id"
: "id";
_sql = $@"
insert into {events.DatabaseSchemaName}.mt_streams (id, tenant_id, version)
values (?, '*DEFAULT*', 0)
ON CONFLICT ({pkFields})
DO NOTHING
";
if (events.StreamIdentity == StreamIdentity.AsGuid)
{
_configureParameter = p =>
{
p.Value = StreamId;
p.NpgsqlDbType = NpgsqlDbType.Uuid;
};
}
else
{
_configureParameter = p =>
{
p.Value = StreamKey;
p.NpgsqlDbType = NpgsqlDbType.Varchar;
};
}
}
public void AddEventsAndEdges(IEnumerable <EventEdge <AbstractEvent> > edges)
{
foreach (EventEdge <AbstractEvent> edge in edges)
{
EventGraph.AddVerticesAndEdge(edge);
}
}
public StoreOptions()
{
Events = new EventGraph(this);
Schema = new MartenRegistry();
Transforms = new Transforms.Transforms(this);
Storage = new StorageFeatures(this);
}
public void GenerateAppendCode(GeneratedMethod method, EventGraph graph, int index)
{
method.Frames.Code($"parameters[{index}].{nameof(NpgsqlParameter.NpgsqlDbType)} = {{0}};",
NpgsqlDbType);
method.Frames.Code(
$"parameters[{index}].{nameof(NpgsqlParameter.Value)} = {{0}}.{_member.Name};", Use.Type <IEvent>());
}
public void GenerateSelectorCodeAsync(GeneratedMethod method, EventGraph graph, int index)
{
method.IfDbReaderValueIsNotNull(index, () =>
{
method.AssignMemberFromReader(null, index, _eventMemberExpression);
});
}
public HighWaterDetector(ITenant tenant, EventGraph graph)
{
_tenant = tenant;
_findSafeSequence = new NpgsqlCommand($@"select min(seq_id) from {graph.DatabaseSchemaName}.mt_events where mt_events.timestamp >= :timestamp");
_safeTimestamp = _findSafeSequence.AddNamedParameter("timestamp", DateTimeOffset.MinValue);
_gapDetection = new NpgsqlCommand($@"
select seq_id
from (select
seq_id,
lead(seq_id)
over (order by seq_id) as no
from
{graph.DatabaseSchemaName}.mt_events where seq_id > :start) ct
where no is not null
and no - seq_id > 1
LIMIT 1;
select max(seq_id) from {graph.DatabaseSchemaName}.mt_events where seq_id > :start;
".Trim());
_start = _gapDetection.AddNamedParameter("start", 0L);
_stateDetection = new NpgsqlCommand($@"
select last_value from {graph.DatabaseSchemaName}.mt_events_sequence;
select last_seq_id, last_updated, transaction_timestamp() as timestamp from {graph.DatabaseSchemaName}.mt_event_progression where name = '{ShardState.HighWaterMark}';
".Trim());
_updateStatus =
new NpgsqlCommand($"select {graph.DatabaseSchemaName}.mt_mark_event_progression('{ShardState.HighWaterMark}', :seq);");
_newSeq = _updateStatus.AddNamedParameter("seq", 0L);
}
public void GenerateSelectorCodeSync(GeneratedMethod method, EventGraph graph, int index)
{
method.IfDbReaderValueIsNotNull(index, () =>
{
method.AssignMemberFromReader <IEvent>(null, index, x => x.CorrelationId);
});
}
private static GeneratedType buildAppendEventOperation(EventGraph graph, GeneratedAssembly assembly)
{
var operationType = assembly.AddType("AppendEventOperation", typeof(AppendEventOperationBase));
var configure = operationType.MethodFor(nameof(AppendEventOperationBase.ConfigureCommand));
configure.DerivedVariables.Add(new Variable(typeof(IEvent), nameof(AppendEventOperationBase.Event)));
configure.DerivedVariables.Add(new Variable(typeof(StreamAction), nameof(AppendEventOperationBase.Stream)));
var columns = new EventsTable(graph).SelectColumns()
// Hokey, use an explicit model for writeable vs readable columns some day
.Where(x => !(x is IsArchivedColumn)).ToList();
var sql =
$"insert into {graph.DatabaseSchemaName}.mt_events ({columns.Select(x => x.Name).Join(", ")}) values ({columns.Select(_ => "?").Join(", ")})";
configure.Frames.Code($"var parameters = {{0}}.{nameof(CommandBuilder.AppendWithParameters)}(\"{sql}\");",
Use.Type <CommandBuilder>());
for (var i = 0; i < columns.Count; i++)
{
columns[i].GenerateAppendCode(configure, graph, i);
}
return(operationType);
}
private static void buildConfigureCommandMethodForStreamState(EventGraph graph, GeneratedType streamQueryHandlerType)
{
var sql =
$"select id, version, type, timestamp, created as timestamp, is_archived from {graph.DatabaseSchemaName}.mt_streams where id = ?";
if (graph.TenancyStyle == TenancyStyle.Conjoined)
{
streamQueryHandlerType.AllInjectedFields.Add(new InjectedField(typeof(string), "tenantId"));
sql += $" and {TenantIdColumn.Name} = ?";
}
var configureCommand = streamQueryHandlerType.MethodFor("ConfigureCommand");
configureCommand.Frames.Call <CommandBuilder>(x => x.AppendWithParameters(""), @call =>
{
@call.Arguments[0] = Constant.ForString(sql);
@call.ReturnAction = ReturnAction.Initialize;
});
var idDbType = graph.StreamIdentity == StreamIdentity.AsGuid ? DbType.Guid : DbType.String;
configureCommand.Frames.Code("{0}[0].Value = _streamId;", Use.Type <NpgsqlParameter[]>());
configureCommand.Frames.Code("{0}[0].DbType = {1};", Use.Type <NpgsqlParameter[]>(), idDbType);
if (graph.TenancyStyle == TenancyStyle.Conjoined)
{
configureCommand.Frames.Code("{0}[1].Value = _tenantId;", Use.Type <NpgsqlParameter[]>());
configureCommand.Frames.Code("{0}[1].DbType = {1};", Use.Type <NpgsqlParameter[]>(), DbType.String);
}
}
private static GeneratedType buildInsertStream(GeneratedType builderType, GeneratedAssembly generatedAssembly,
EventGraph graph)
{
var operationType = generatedAssembly.AddType(InsertStreamOperationName, typeof(InsertStreamBase));
var columns = new StreamsTable(graph)
.Columns
.OfType <IStreamTableColumn>()
.Where(x => x.Writes)
.ToArray();
var sql = $"insert into {graph.DatabaseSchemaName}.mt_streams ({columns.Select(x => x.Name).Join(", ")}) values ({columns.Select(_ => "?").Join(", ")})";
var configureCommand = operationType.MethodFor("ConfigureCommand");
configureCommand.DerivedVariables.Add(new Variable(typeof(StreamAction), nameof(InsertStreamBase.Stream)));
configureCommand.Frames.Code($"var parameters = {{0}}.{nameof(CommandBuilder.AppendWithParameters)}(\"{sql}\");",
Use.Type <CommandBuilder>());
for (var i = 0; i < columns.Length; i++)
{
columns[i].GenerateAppendCode(configureCommand, i);
}
builderType.MethodFor(nameof(EventDocumentStorage.InsertStream))
.Frames.Code($"return new Marten.Generated.{InsertStreamOperationName}(stream);");
return(operationType);
}
public HighWaterStatisticsDetector(EventGraph graph)
{
_stateDetection = new NpgsqlCommand($@"
select last_value from {graph.DatabaseSchemaName}.mt_events_sequence;
select last_seq_id, last_updated, transaction_timestamp() as timestamp from {graph.DatabaseSchemaName}.mt_event_progression where name = '{ShardState.HighWaterMark}';
".Trim());
}
public StreamIdColumn(EventGraph graph) : base("stream_id", "varchar")
{
Type = graph.GetStreamIdDBType();
Directive = graph.TenancyStyle != TenancyStyle.Conjoined
? $"REFERENCES {graph.DatabaseSchemaName}.mt_streams ON DELETE CASCADE"
: null;
}
public void GenerateSelectorCodeAsync(GeneratedMethod method, EventGraph graph, int index)
{
method.IfDbReaderValueIsNotNullAsync(index, () =>
{
method.AssignMemberFromReaderAsync <IEvent>(null, index, x => x.TenantId);
});
}
private static GeneratedType buildStreamQueryHandlerType(EventGraph graph, GeneratedAssembly assembly)
{
var streamQueryHandlerType =
assembly.AddType(StreamStateSelectorTypeName, typeof(StreamStateQueryHandler));
streamQueryHandlerType.AllInjectedFields.Add(graph.StreamIdentity == StreamIdentity.AsGuid
? new InjectedField(typeof(Guid), "streamId")
: new InjectedField(typeof(string), "streamId"));
buildConfigureCommandMethodForStreamState(graph, streamQueryHandlerType);
var sync = streamQueryHandlerType.MethodFor("Resolve");
var async = streamQueryHandlerType.MethodFor("ResolveAsync");
sync.Frames.Add(new ConstructorFrame <StreamState>(() => new StreamState()));
async.Frames.Add(new ConstructorFrame <StreamState>(() => new StreamState()));
if (graph.StreamIdentity == StreamIdentity.AsGuid)
{
sync.AssignMemberFromReader <StreamState>(streamQueryHandlerType, 0, x => x.Id);
async.AssignMemberFromReaderAsync <StreamState>(streamQueryHandlerType, 0, x => x.Id);
}
else
{
sync.AssignMemberFromReader <StreamState>(streamQueryHandlerType, 0, x => x.Key);
async.AssignMemberFromReaderAsync <StreamState>(streamQueryHandlerType, 0, x => x.Key);
}
sync.AssignMemberFromReader <StreamState>(streamQueryHandlerType, 1, x => x.Version);
async.AssignMemberFromReaderAsync <StreamState>(streamQueryHandlerType, 1, x => x.Version);
sync.Frames.Call <StreamStateQueryHandler>(x => x.SetAggregateType(null, null, null), call =>
{
call.IsLocal = true;
});
#pragma warning disable 4014
async.Frames.Call <StreamStateQueryHandler>(
x => x.SetAggregateTypeAsync(null, null, null, CancellationToken.None), call =>
#pragma warning restore 4014
{
call.IsLocal = true;
});
sync.AssignMemberFromReader <StreamState>(streamQueryHandlerType, 3, x => x.LastTimestamp);
async.AssignMemberFromReaderAsync <StreamState>(streamQueryHandlerType, 3, x => x.LastTimestamp);
sync.AssignMemberFromReader <StreamState>(streamQueryHandlerType, 4, x => x.Created);
async.AssignMemberFromReaderAsync <StreamState>(streamQueryHandlerType, 4, x => x.Created);
sync.AssignMemberFromReader <StreamState>(streamQueryHandlerType, 5, x => x.IsArchived);
async.AssignMemberFromReaderAsync <StreamState>(streamQueryHandlerType, 5, x => x.IsArchived);
sync.Frames.Return(typeof(StreamState));
async.Frames.Return(typeof(StreamState));
return(streamQueryHandlerType);
}
public EventAppender(EventGraph graph, EventDocumentStorage builder, IInlineProjection[] projections)
{
_graph = graph;
_builder = builder;
_projections = projections;
// TODO -- track the event streams separately within the DocumentSessionBase
}
public StreamActionTester()
{
theSession = Substitute.For <IMartenSession>();
theSession.TenantId.Returns("TX");
theSession.Database.Returns(Substitute.For <IMartenDatabase>());
theEvents = new EventGraph(new StoreOptions());
}
public EventsTable(EventGraph events) : base(new DbObjectName(events.DatabaseSchemaName, "mt_events"))
{
AddColumn(new EventTableColumn("seq_id", x => x.Sequence)).AsPrimaryKey();
AddColumn(new EventTableColumn("id", x => x.Id)).NotNull();
AddColumn(new StreamIdColumn(events));
AddColumn(new EventTableColumn("version", x => x.Version)).NotNull();
AddColumn <EventJsonDataColumn>();
AddColumn <EventTypeColumn>();
AddColumn(new EventTableColumn("timestamp", x => x.Timestamp))
.NotNull().DefaultValueByString("(now())");
AddColumn <TenantIdColumn>();
AddColumn <DotNetTypeColumn>().AllowNulls();
AddIfActive(events.Metadata.CorrelationId);
AddIfActive(events.Metadata.CausationId);
AddIfActive(events.Metadata.Headers);
if (events.TenancyStyle == TenancyStyle.Conjoined)
{
ForeignKeys.Add(new ForeignKey("fkey_mt_events_stream_id_tenant_id")
{
ColumnNames = new string[] { "stream_id", TenantIdColumn.Name },
LinkedNames = new string[] { "id", TenantIdColumn.Name },
LinkedTable = new DbObjectName(events.DatabaseSchemaName, "mt_streams")
});
Indexes.Add(new IndexDefinition("pk_mt_events_stream_and_version")
{
IsUnique = true,
Columns = new string[] { "stream_id", TenantIdColumn.Name, "version" }
});
}
else
{
ForeignKeys.Add(new ForeignKey("fkey_mt_events_stream_id")
{
ColumnNames = new string[] { "stream_id" },
LinkedNames = new string[] { "id" },
LinkedTable = new DbObjectName(events.DatabaseSchemaName, "mt_streams"),
OnDelete = CascadeAction.Cascade
});
Indexes.Add(new IndexDefinition("pk_mt_events_stream_and_version")
{
IsUnique = true,
Columns = new string[] { "stream_id", "version" }
});
}
Indexes.Add(new IndexDefinition("pk_mt_events_id_unique")
{
Columns = new string[] { "id" },
IsUnique = true
});
}
internal IStorageOperation BuildProgressionOperation(EventGraph events)
{
if (SequenceFloor == 0)
{
return(new InsertProjectionProgress(events, this));
}
return(new UpdateProjectionProgress(events, this));
}
public StreamActionTester()
{
theSession = Substitute.For <IMartenSession>();
theTenant = Substitute.For <ITenant>();
theSession.Tenant.Returns(theTenant);
theTenant.TenantId.Returns("TX");
theEvents = new EventGraph(new StoreOptions());
}
public DocumentSchema(StoreOptions options, IConnectionFactory factory, IDocumentSchemaCreation creation)
{
_factory = factory;
_creation = creation;
StoreOptions = options;
Sequences = new SequenceFactory(this, _factory, _creation);
Events = new EventGraph();
}
public DocumentSchema(StoreOptions options, ICommandRunner runner, IDocumentSchemaCreation creation)
{
_creation = creation;
_runner = runner;
StoreOptions = options;
Sequences = new SequenceFactory(this, _runner, _creation);
Events = new EventGraph();
}
public HighWaterDetector(ISingleQueryRunner runner, EventGraph graph)
{
_runner = runner;
_gapDetector = new GapDetector(graph);
_safeSequenceFinder = new SafeSequenceFinder(graph);
_highWaterStatisticsDetector = new HighWaterStatisticsDetector(graph);
_updateStatus =
new NpgsqlCommand($"select {graph.DatabaseSchemaName}.mt_mark_event_progression('{ShardState.HighWaterMark}', :seq);");
_newSeq = _updateStatus.AddNamedParameter("seq", 0L);
}
public void GenerateSelectorCodeAsync(GeneratedMethod method, EventGraph graph, int index)
{
if (graph.StreamIdentity == StreamIdentity.AsGuid)
{
method.AssignMemberFromReaderAsync <IEvent>(null, index, x => x.StreamId);
}
else
{
method.AssignMemberFromReaderAsync <IEvent>(null, index, x => x.StreamKey);
}
}
public void GenerateAppendCode(GeneratedMethod method, EventGraph graph, int index)
{
if (graph.StreamIdentity == StreamIdentity.AsGuid)
{
method.SetParameterFromMember <StreamAction>(index, x => x.Id);
}
else
{
method.SetParameterFromMember <StreamAction>(index, x => x.Key);
}
}
public ProjectionTrack(IFetcher fetcher, IDocumentStore store, IProjection projection, IDaemonLogger logger, IDaemonErrorHandler errorHandler)
{
_fetcher = fetcher;
_projection = projection;
_logger = logger;
_errorHandler = errorHandler;
_store = store;
_events = store.Schema.Events;
ViewType = _projection.Produces;
}
public ProjectionTrack(IFetcher fetcher, IDocumentStore store, IProjection projection, IDaemonLogger logger, IDaemonErrorHandler errorHandler)
{
_fetcher = fetcher;
_projection = projection;
_logger = logger;
_errorHandler = errorHandler;
_store = store;
_events = store.Schema.Events;
ViewType = _projection.Produces;
}
public void use_string_id_if_as_string_identifiers()
{
var events = new EventGraph(new StoreOptions())
{
StreamIdentity = StreamIdentity.AsString
};
var table = new StreamsTable(events);
table.PrimaryKey.Type.ShouldBe("varchar");
table.First().Name.ShouldBe("id");
}
public StoreOptions()
{
Events = new EventGraph(this);
Schema = new MartenRegistry(this);
Transforms = new Transforms.Transforms(this);
var patching = new TransformFunction(this, PatchDoc, SchemaBuilder.GetJavascript(this, "mt_patching"));
patching.OtherArgs.Add("patch");
Transforms.Load(patching);
}
public void use_the_plain_append_event_function_when_async_and_js_are_disabled()
{
var graph = new EventGraph(new StoreOptions())
{
AsyncProjectionsEnabled = false,
JavascriptProjectionsEnabled = false
};
var storage = new EventStreamStorage(graph);
storage.AppendEventFunction.Name.ShouldBe("mt_append_event");
}
public ProjectionTrack(IFetcher fetcher, DocumentStore store, IProjection projection, IDaemonLogger logger, IDaemonErrorHandler errorHandler, ITenant tenant)
{
_fetcher = fetcher;
_projection = projection;
_logger = logger;
_errorHandler = errorHandler;
_tenant = tenant;
_store = store;
_events = store.Events;
ViewType = _projection.ProjectedType();
}
public void use_rolling_buffer_version_of_append_event_function_when_async_is_enabled()
{
var graph = new EventGraph(new StoreOptions())
{
AsyncProjectionsEnabled = true,
JavascriptProjectionsEnabled = false
};
var storage = new EventStreamStorage(graph);
storage.AppendEventFunction.Name.ShouldBe("mt_append_event_with_buffering");
}
public EventProgressWrite(EventGraph events, string key, long number)
{
_sproc = new FunctionName(events.DatabaseSchemaName, "mt_mark_event_progression");
_key = key;
_number = number;
}