public async Task Trigger(
ScheduledCommand scheduled,
SchedulerAdvancedResult result,
CommandSchedulerDbContext db)
{
await deliver(scheduled, result, db);
}
internal async Task Trigger(
ScheduledCommand scheduled,
SchedulerAdvancedResult result,
CommandSchedulerDbContext db)
{
var deliver = commandDispatchers.IfContains(scheduled.AggregateType)
.ElseDefault();
if (deliver == null)
{
// QUESTION: (Trigger) is this worth raising a warning for or is there a reasonable chance that not registering a handler was deliberate?
// var error = ScheduledCommandFailure();
//
// activity.OnNext(new CommandSchedulerActivity(scheduled, error));
//
// result.Add(error);
// db.Errors.Add(error);
}
else
{
await deliver(scheduled);
result.Add(scheduled.Result);
}
scheduled.Attempts++;
await db.SaveChangesAsync();
}
private static void RescheduleIfAppropriate(
ScheduledCommand storedCommand,
ScheduledCommandResult result,
CommandSchedulerDbContext db)
{
var failure = (CommandFailed)result;
var now = Domain.Clock.Now();
if (failure.IsCanceled || failure.RetryAfter == null)
{
// no further retries
storedCommand.FinalAttemptTime = now;
}
else
{
storedCommand.DueTime = now + failure.RetryAfter;
}
db.Errors.Add(new CommandExecutionError
{
ScheduledCommand = storedCommand,
Error = result.IfTypeIs <CommandFailed>()
.Then(f => f.Exception.ToJson())
.ElseDefault()
});
}
private static async Task SaveScheduledCommandToDatabase <TAggregate>(
CommandSchedulerDbContext db,
ScheduledCommand storedScheduledCommand,
IScheduledCommand <TAggregate> scheduledCommand)
{
var etag = new ETag
{
Scope = scheduledCommand.TargetId,
ETagValue = scheduledCommand.Command.ETag,
CreatedDomainTime = Domain.Clock.Now(),
CreatedRealTime = DateTimeOffset.UtcNow
};
while (true)
{
try
{
await InsertScheduledCommandAndETag(db, storedScheduledCommand, etag);
break;
}
catch (Exception exception)
{
if (!exception.IsConcurrencyException())
{
throw;
}
if (exception.ToString().Contains(@"object 'Scheduler.ScheduledCommand'"))
{
if (storedScheduledCommand.SequenceNumber < 0)
{
// for scheduler-assigned sequence numbers, decrement and retry
storedScheduledCommand.SequenceNumber--;
}
else
{
// this is not a scheduler-assigned sequence number
break;
}
}
else if (exception.ToString().Contains(@"object 'Scheduler.ETag'"))
{
scheduledCommand.Result = new CommandDeduplicated(
scheduledCommand,
"Schedule");
return;
}
else
{
throw;
}
}
}
scheduledCommand.Result = new CommandScheduled(
scheduledCommand,
storedScheduledCommand.Clock);
}
/// <summary>
/// Deserializes a scheduled command from SQL storage and delivers it to the target.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <param name="serializedCommand">The serialized command.</param>
/// <param name="db">The command scheduler database context.</param>
public static async Task DeserializeAndDeliver(
this Configuration configuration,
ScheduledCommand serializedCommand,
CommandSchedulerDbContext db) =>
await DeserializeAndDeliver(
configuration.Container.Resolve<CommandSchedulerResolver>(),
serializedCommand,
db);
public async Task Deliver(ScheduledCommand scheduled)
{
var command = scheduled.ToScheduledCommand <TAggregate>();
await scheduler.Deliver(command);
scheduled.Result = command.Result;
}
/// <summary>
/// Deserializes a scheduled command from the database model to the domain model.
/// </summary>
internal static CommandScheduled <TAggregate> ToScheduledCommand <TAggregate>(
this ScheduledCommand scheduled) where TAggregate : IEventSourced
{
var json = scheduled.SerializedCommand;
var command = json.FromJsonTo <CommandScheduled <TAggregate> >();
command.Metadata.NumberOfPreviousAttempts = scheduled.Attempts;
return(command);
}
/// <summary>
/// Deserializes a scheduled command from the database model to the domain model.
/// </summary>
internal static IScheduledCommand <TAggregate> ToScheduledCommand <TAggregate>(
this ScheduledCommand scheduled)
{
var json = scheduled.SerializedCommand;
var command = json.FromJsonTo <ScheduledCommand <TAggregate> >();
command.Clock = scheduled.Clock;
command.DueTime = scheduled.DueTime;
command.SequenceNumber = scheduled.SequenceNumber;
command.NumberOfPreviousAttempts = scheduled.Attempts;
return(command);
}
public async Task Deliver(ScheduledCommand scheduled)
{
var command = scheduled.ToScheduledCommand <TAggregate>();
//here we are setting the command.SequenceNumber to the scheduled.SequenceNumber because when
//multiple commands are scheduled simultaniously against the same aggregate we were decrementing the
//scheduled.SequenceNumber correctly, however we were not updating the command.SequenceNumber.
//this is to prevent any side effects that may have been caused by that bug
command.SequenceNumber = scheduled.SequenceNumber;
await scheduler.Deliver(command);
scheduled.Result = command.Result;
}
internal static async Task DeserializeAndDeliver(
CommandSchedulerResolver schedulerResolver,
ScheduledCommand serializedCommand,
CommandSchedulerDbContext db)
{
dynamic scheduler = schedulerResolver.ResolveSchedulerForAggregateTypeNamed(serializedCommand.AggregateType);
await Storage.DeserializeAndDeliverScheduledCommand(
serializedCommand,
scheduler);
serializedCommand.Attempts++;
await db.SaveChangesAsync();
}
internal static async Task DeserializeAndDeliverScheduledCommand <TAggregate>(
ScheduledCommand scheduled,
ICommandScheduler <TAggregate> scheduler)
{
try
{
var command = scheduled.ToScheduledCommand <TAggregate>();
await scheduler.Deliver(command);
scheduled.Result = command.Result;
}
catch (Exception exception)
{
scheduled.Result = new CommandFailed(scheduled, exception);
}
}
private static ScheduledCommand CreateStoredScheduledCommand <TAggregate>(
IScheduledCommand <TAggregate> scheduledCommand,
DateTimeOffset domainTime,
Clock schedulerClock)
where TAggregate : class =>
new ScheduledCommand
{
AggregateId = ScheduledCommand <TAggregate> .TargetGuid(scheduledCommand),
SequenceNumber = scheduledCommand
.IfTypeIs <IEvent>()
.Then(e => e.SequenceNumber)
.Else(() => - DateTimeOffset.UtcNow.Ticks),
AggregateType = Command.TargetNameFor(scheduledCommand.Command.GetType()),
SerializedCommand = scheduledCommand.ToJson(),
CreatedTime = domainTime,
DueTime = scheduledCommand.DueTime ?? schedulerClock.Now(),
Clock = schedulerClock
};
internal static async Task UpdateScheduledCommand <TAggregate>(
IScheduledCommand <TAggregate> scheduledCommand,
Func <CommandSchedulerDbContext> createDbContext) where TAggregate : class
{
using (var db = createDbContext())
{
var scheduledCommandGuid = ScheduledCommand <TAggregate> .TargetGuid(scheduledCommand);
var storedCommand = await GetStoredScheduledCommand(
scheduledCommand,
db,
scheduledCommandGuid);
if (storedCommand == null)
{
if (!scheduledCommand.Command.RequiresDurableScheduling())
{
return;
}
throw new InvalidOperationException("Scheduled command not found");
}
storedCommand.Attempts++;
var result = scheduledCommand.Result;
if (result is CommandSucceeded)
{
storedCommand.AppliedTime = Domain.Clock.Now();
}
else
{
RescheduleIfAppropriate(storedCommand, result, db);
}
await db.SaveChangesAsync();
}
}
public override async Task When_a_clock_is_set_on_a_command_then_it_takes_precedence_over_GetClockName()
{
// arrange
var clockName = Any.CamelCaseName();
var targetId = Any.CamelCaseName();
var command = new CreateCommandTarget(targetId);
var clock = new CommandScheduler.Clock
{
Name = clockName,
UtcNow = DateTimeOffset.Parse("2016-03-01 02:00:00 AM")
};
using (var commandScheduler = Configuration.Current.CommandSchedulerDbContext())
{
commandScheduler.Clocks.Add(clock);
commandScheduler.SaveChanges();
}
var scheduledCommand = new ScheduledCommand<CommandTarget>(
targetId: targetId,
command: command,
dueTime: DateTimeOffset.Parse("2016-03-20 09:00:00 AM"))
{
Clock = clock
};
// act
await scheduler.Schedule(scheduledCommand);
await Configuration.Current.SchedulerClockTrigger()
.AdvanceClock(clockName, by: 30.Days());
//assert
var target = await store.Get(targetId);
target.Should().NotBeNull();
}
public async Task Trigger(ScheduledCommand scheduled, SchedulerAdvancedResult result, CommandSchedulerDbContext db)
{
await ClockTrigger.Trigger(scheduled, result, db);
}
public override async Task When_a_clock_is_set_on_a_command_then_it_takes_precedence_over_GetClockName()
{
// arrange
var clockName = Any.CamelCaseName();
var create = new CreateOrder(Any.FullName())
{
AggregateId = Any.Guid()
};
var clock = new CommandScheduler.Clock
{
Name = clockName,
UtcNow = DateTimeOffset.Parse("2016-03-01 02:00:00 AM")
};
using (var commandScheduler = new CommandSchedulerDbContext())
{
commandScheduler.Clocks.Add(clock);
commandScheduler.SaveChanges();
}
var scheduledCommand = new ScheduledCommand<Order>(
aggregateId: create.AggregateId,
command: create,
dueTime: DateTimeOffset.Parse("2016-03-20 09:00:00 AM"))
{
Clock = clock
};
// act
var configuration = Configuration.Current;
await configuration.CommandScheduler<Order>().Schedule(scheduledCommand);
await configuration.SchedulerClockTrigger()
.AdvanceClock(clockName,
by: 30.Days());
//assert
var target = await configuration.Repository<Order>().GetLatest(create.AggregateId);
target.Should().NotBeNull();
}
public async Task Trigger(
ScheduledCommand scheduled,
SchedulerAdvancedResult result,
CommandSchedulerDbContext db) =>
await deliver(scheduled, result, db);
private async Task <ScheduledCommand> StoreScheduledCommand(IScheduledCommand <TAggregate> scheduledCommandEvent)
{
ScheduledCommand storedScheduledCommand;
using (var db = createCommandSchedulerDbContext())
{
var domainTime = Domain.Clock.Now();
// store the scheduled command
var clockName = await ClockNameForEvent(scheduledCommandEvent, db);
var schedulerClock = await db.Clocks.SingleOrDefaultAsync(c => c.Name == clockName);
if (schedulerClock == null)
{
Debug.WriteLine(string.Format("SqlCommandScheduler: Creating clock '{0}' @ {1}", clockName, domainTime));
schedulerClock = new Clock
{
Name = clockName,
UtcNow = domainTime,
StartTime = domainTime
};
db.Clocks.Add(schedulerClock);
await db.SaveChangesAsync();
}
storedScheduledCommand = new ScheduledCommand
{
AggregateId = scheduledCommandEvent.AggregateId,
SequenceNumber = scheduledCommandEvent.SequenceNumber,
AggregateType = eventStreamName,
SerializedCommand = scheduledCommandEvent.ToJson(),
CreatedTime = domainTime,
DueTime = scheduledCommandEvent.DueTime,
Clock = schedulerClock
};
if (storedScheduledCommand.ShouldBeDeliveredImmediately() &&
!scheduledCommandEvent.Command.RequiresDurableScheduling)
{
storedScheduledCommand.NonDurable = true;
return(storedScheduledCommand);
}
Debug.WriteLine(string.Format("SqlCommandScheduler: Storing command '{0}' ({1}:{2}) on clock '{3}'",
scheduledCommandEvent.Command.CommandName,
scheduledCommandEvent.AggregateId,
scheduledCommandEvent.SequenceNumber,
clockName));
db.ScheduledCommands.Add(storedScheduledCommand);
while (true)
{
try
{
db.SaveChanges();
break;
}
catch (DbUpdateException exception)
{
if (exception.IsConcurrencyException())
{
if (storedScheduledCommand.SequenceNumber < 0)
{
// for scheduler-assigned sequence numbers, decrement and retry
storedScheduledCommand.SequenceNumber--;
}
else
{
// this is not a scheduler-assigned sequence number, so the concurrency exception indicates an actual issue
break;
}
}
else
{
throw;
}
}
}
}
return(storedScheduledCommand);
}
private static async Task InsertScheduledCommandAndETag(CommandSchedulerDbContext db, ScheduledCommand storedScheduledCommand, ETag etag)
{
var sql = @"
INSERT [Scheduler].[ScheduledCommand]
(
[AggregateId],
[SequenceNumber],
[AggregateType],
[CommandName],
[CreatedTime],
[DueTime],
[SerializedCommand],
[Attempts],
[Clock_Id]
)
VALUES
(
@aggregateId,
@sequenceNumber,
@aggregateType,
@commandName,
@createdTime,
@dueTime,
@serializedCommand,
@attempts,
@clock_Id
)
INSERT [Scheduler].[ETag]
(
[Scope],
[ETagValue],
[CreatedDomainTime],
[CreatedRealTime]
)
VALUES
(
@scope,
@eTagValue,
@createdDomainTime,
@createdRealTime
)";
var parameters = new[]
{
// ScheduledCommand
new SqlParameter("@aggregateId", storedScheduledCommand.AggregateId),
new SqlParameter("@sequenceNumber", storedScheduledCommand.SequenceNumber),
new SqlParameter("@aggregateType", storedScheduledCommand.AggregateType),
new SqlParameter("@commandName", storedScheduledCommand.CommandName),
new SqlParameter("@createdTime", storedScheduledCommand.CreatedTime),
new SqlParameter("@dueTime", storedScheduledCommand.DueTime),
new SqlParameter("@serializedCommand", storedScheduledCommand.SerializedCommand),
new SqlParameter("@attempts", storedScheduledCommand.Attempts),
new SqlParameter("@clock_Id", storedScheduledCommand.Clock.Id),
// ETag
new SqlParameter("@scope", etag.Scope),
new SqlParameter("@eTagValue", etag.ETagValue),
new SqlParameter("@createdDomainTime", etag.CreatedDomainTime),
new SqlParameter("@createdRealTime", etag.CreatedRealTime)
};
await db.Database.ExecuteSqlCommandAsync(
TransactionalBehavior.EnsureTransaction,
sql,
parameters);
}
private static async Task SaveScheduledCommandToDatabase <TAggregate>(
CommandSchedulerDbContext db,
ScheduledCommand storedScheduledCommand,
IScheduledCommand <TAggregate> scheduledCommand)
{
db.ScheduledCommands.Add(storedScheduledCommand);
db.ETags.Add(new ETag
{
Scope = scheduledCommand.TargetId,
ETagValue = scheduledCommand.Command.ETag,
CreatedDomainTime = Domain.Clock.Now(),
CreatedRealTime = DateTimeOffset.UtcNow
});
while (true)
{
try
{
using (var transaction = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
await db.SaveChangesAsync();
transaction.Complete();
}
break;
}
catch (DbUpdateException exception)
{
if (!exception.IsConcurrencyException())
{
throw;
}
if (exception.ToString().Contains(@"object 'Scheduler.ScheduledCommand'"))
{
if (storedScheduledCommand.SequenceNumber < 0)
{
// for scheduler-assigned sequence numbers, decrement and retry
storedScheduledCommand.SequenceNumber--;
}
else
{
// this is not a scheduler-assigned sequence number
break;
}
}
else if (exception.ToString().Contains(@"object 'Scheduler.ETag'"))
{
scheduledCommand.Result = new CommandDeduplicated(
scheduledCommand,
"Schedule");
return;
}
else
{
throw;
}
}
}
scheduledCommand.Result = new CommandScheduled(
scheduledCommand,
storedScheduledCommand.Clock);
}
private ScheduledCommand StoreScheduledCommand(IScheduledCommand <TAggregate> scheduledCommandEvent)
{
ScheduledCommand storedScheduledCommand;
using (var db = createCommandSchedulerDbContext())
{
var domainTime = Domain.Clock.Now();
// store the scheduled command
var clockName = ClockNameForEvent(scheduledCommandEvent, db);
var schedulerClock = db.Clocks.SingleOrDefault(c => c.Name == clockName);
if (schedulerClock == null)
{
schedulerClock = new Clock
{
Name = clockName,
UtcNow = domainTime,
StartTime = domainTime
};
db.Clocks.Add(schedulerClock);
db.SaveChanges();
}
storedScheduledCommand = new ScheduledCommand
{
AggregateId = scheduledCommandEvent.AggregateId,
SequenceNumber = scheduledCommandEvent.SequenceNumber,
AggregateType = eventStreamName,
SerializedCommand = scheduledCommandEvent.ToJson(),
CreatedTime = domainTime,
DueTime = scheduledCommandEvent.DueTime,
Clock = schedulerClock
};
if (storedScheduledCommand.ShouldBeDeliveredImmediately() &&
!scheduledCommandEvent.Command.RequiresDurableScheduling)
{
storedScheduledCommand.NonDurable = true;
return(storedScheduledCommand);
}
db.ScheduledCommands.Add(storedScheduledCommand);
while (true)
{
try
{
db.SaveChanges();
break;
}
catch (DbUpdateException exception)
{
if (exception.IsConcurrencyException())
{
if (storedScheduledCommand.SequenceNumber < 0)
{
// for scheduler-assigned sequence numbers, decrement and retry
storedScheduledCommand.SequenceNumber--;
}
else
{
// this is not a scheduler-assigned sequence number, so the concurrency exception indicates
break;
}
}
else
{
throw;
}
}
}
}
return(storedScheduledCommand);
}
internal static async Task <ScheduledCommand> StoreScheduledCommand <TAggregate>(
IScheduledCommand <TAggregate> scheduledCommand,
Func <CommandSchedulerDbContext> createDbContext,
Func <IScheduledCommand <TAggregate>, CommandSchedulerDbContext, Task <string> > clockNameForEvent) where TAggregate : class, IEventSourced
{
ScheduledCommand storedScheduledCommand;
using (var db = createDbContext())
{
var domainTime = Domain.Clock.Now();
// get or create a clock to schedule the command on
var clockName = await clockNameForEvent(scheduledCommand, db);
var schedulerClock = await db.Clocks.SingleOrDefaultAsync(c => c.Name == clockName);
if (schedulerClock == null)
{
Debug.WriteLine(String.Format("Creating clock '{0}' @ {1}", clockName, domainTime));
schedulerClock = new Clock
{
Name = clockName,
UtcNow = domainTime,
StartTime = domainTime
};
db.Clocks.Add(schedulerClock);
await db.SaveChangesAsync();
}
storedScheduledCommand = new ScheduledCommand
{
AggregateId = scheduledCommand.AggregateId,
SequenceNumber = scheduledCommand.SequenceNumber,
AggregateType = AggregateType <TAggregate> .EventStreamName,
SerializedCommand = scheduledCommand.ToJson(),
CreatedTime = domainTime,
DueTime = scheduledCommand.DueTime,
Clock = schedulerClock
};
if (scheduledCommand.IsDue(storedScheduledCommand.Clock) &&
!scheduledCommand.Command.RequiresDurableScheduling)
{
storedScheduledCommand.NonDurable = true;
return(storedScheduledCommand);
}
Debug.WriteLine(String.Format("Storing command '{0}' ({1}:{2}) on clock '{3}'",
scheduledCommand.Command.CommandName,
scheduledCommand.AggregateId,
scheduledCommand.SequenceNumber,
clockName));
db.ScheduledCommands.Add(storedScheduledCommand);
while (true)
{
try
{
db.SaveChanges();
scheduledCommand.Result = new CommandScheduled(scheduledCommand)
{
ClockName = schedulerClock.Name
};
break;
}
catch (DbUpdateException exception)
{
if (exception.IsConcurrencyException())
{
if (storedScheduledCommand.SequenceNumber < 0)
{
// for scheduler-assigned sequence numbers, decrement and retry
storedScheduledCommand.SequenceNumber--;
}
else
{
// this is not a scheduler-assigned sequence number, so the concurrency exception indicates an actual issue
break;
}
}
else
{
throw;
}
}
}
}
return(storedScheduledCommand);
}
public async Task Multiple_scheduled_commands_having_the_some_causative_command_etag_have_repeatable_and_unique_etags()
{
var senderId = Any.Word();
await store.Put(new CommandTarget(senderId));
var targetIds = new[] { Any.Word(), Any.Word(), Any.Word() };
var results = new ConcurrentBag<RequestReply>();
configuration.TraceScheduledCommands(
onScheduling: cmd =>
{
var requestReply = ((dynamic) cmd).Command as RequestReply;
if (requestReply != null)
{
results.Add(requestReply);
}
});
var initialEtag = "initial".ToETag();
var firstCommand = new SendRequests(targetIds)
{
ETag = initialEtag
};
var scheduledCommand = new ScheduledCommand<CommandTarget>(
firstCommand,
senderId);
await scheduler.Deliver(scheduledCommand);
var secondCommand = new SendRequests(targetIds)
{
ETag = initialEtag
};
scheduledCommand = new ScheduledCommand<CommandTarget>(
secondCommand,
senderId);
// redeliver
await scheduler.Deliver(scheduledCommand);
Console.WriteLine(results.ToJson());
results.Should().HaveCount(6);
results.Select(r => r.ETag)
.Distinct()
.Should()
.HaveCount(3);
}
public async Task Deliver(ScheduledCommand scheduled)
{
await Storage.DeserializeAndDeliverScheduledCommand <TAggregate>(
scheduled,
scheduler);
}
