public bool RebuildIndex()
{
var indexState = index.GetIndexState();
if (indexState.IsPresent())
{
return(true);
}
var indexBuilder = index.GetIndexBuilder();
foreach (var aggregateCommit in eventStorePlayer.LoadAggregateCommits())
{
foreach (var @event in aggregateCommit.Events)
{
try
{
var unwrapedEvent = @event.Unwrap();
var rootId = System.Text.Encoding.UTF8.GetString(aggregateCommit.AggregateRootId);
var eventOrigin = new EventOrigin(rootId, aggregateCommit.Revision, aggregateCommit.Events.IndexOf(@event), aggregateCommit.Timestamp);
indexBuilder.Feed(unwrapedEvent, eventOrigin);
}
catch (Exception ex)
{
log.ErrorException(ex.Message, ex);
}
}
}
indexBuilder.Complete();
return(true);
}
public bool Rebuild(Type projectionType, ProjectionVersion version, DateTime replayUntil)
{
DateTime startRebuildTimestamp = DateTime.UtcNow;
int progressCounter = 0;
log.Info(() => $"Start rebuilding projection `{projectionType.Name}` for version {version}. Deadline is {replayUntil}");
var projection = FastActivator.CreateInstance(projectionType) as IProjectionDefinition;
var projectionEventTypes = GetInvolvedEvents(projectionType).ToList();
projectionStore.InitializeProjectionStore(version);
snapshotStore.InitializeProjectionSnapshotStore(version);
var indexState = index.GetIndexState();
if (indexState.IsPresent() == false)
{
return(false);
}
foreach (var eventType in projectionEventTypes)
{
log.Debug(() => $"Rebuilding projection `{projectionType.Name}` for version {version} using eventType `{eventType}`. Deadline is {replayUntil}");
var indexId = new EventStoreIndexEventTypeId(eventType);
IEnumerable <ProjectionCommit> indexCommits = index.EnumerateCommitsByEventType(indexId);
foreach (var indexCommit in indexCommits)
{
progressCounter++;
if (progressCounter % 1000 == 0)
{
log.Trace(() => $"Rebuilding projection {projectionType.Name} => PROGRESS:{progressCounter} Version:{version} EventType:{eventType} Deadline:{replayUntil} Total minutes working:{(DateTime.UtcNow - startRebuildTimestamp).TotalMinutes}. logId:{Guid.NewGuid().ToString()}");
}
// if the replay did not finish in time (specified by the AR) we need to abort.
if (DateTime.UtcNow >= replayUntil)
{
log.Info(() => $"Rebuilding projection `{projectionType.Name}` stopped bacause the deadline has been reached. PROGRESS:{progressCounter} Version:{version} EventType:`{eventType}` Deadline:{replayUntil}.");
return(false);
}
IAggregateRootId arId = GetAggregateRootId(indexCommit.EventOrigin.AggregateRootId);
IEventStore eventStore = eventStoreFactory.GetEventStore(tenantResolver.Resolve(arId));
EventStream stream = eventStore.Load(arId, theId => projectionType.GetBoundedContext().BoundedContextName);
foreach (AggregateCommit arCommit in stream.Commits)
{
for (int i = 0; i < arCommit.Events.Count; i++)
{
IEvent theEvent = arCommit.Events[i].Unwrap();
if (projectionEventTypes.Contains(theEvent.GetType().GetContractId()))
{
var origin = new EventOrigin(Convert.ToBase64String(arCommit.AggregateRootId), arCommit.Revision, i, arCommit.Timestamp);
projectionRepository.Save(projectionType, theEvent, origin); // overwrite
}
}
}
}
}
log.Info(() => $"Finish rebuilding projection `{projectionType.Name}` for version {version}. Deadline was {replayUntil}");
return(true);
}
public ProjectionCommit(IBlobId projectionId, string contractId, IEvent @event, int snapshotMarker, EventOrigin eventOrigin, DateTime timeStamp)
{
ProjectionId = projectionId;
ContractId = contractId;
Event = @event;
SnapshotMarker = snapshotMarker;
EventOrigin = eventOrigin;
TimeStamp = timeStamp;
}
public ProjectionCommit(IBlobId projectionId, ProjectionVersion version, IEvent @event, int snapshotMarker, EventOrigin eventOrigin, DateTime timeStamp)
{
ProjectionId = projectionId;
ProjectionName = version.ProjectionName;
Event = @event;
SnapshotMarker = snapshotMarker;
EventOrigin = eventOrigin;
TimeStamp = timeStamp;
Version = version;
}
public void Save(Type projectionType, CronusMessage cronusMessage)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, cronusMessage))
{
throw new ArgumentNullException(nameof(cronusMessage));
}
EventOrigin eventOrigin = cronusMessage.GetEventOrigin();
IEvent @event = cronusMessage.Payload as IEvent;
Save(projectionType, @event, eventOrigin);
}
public async Task SaveAsync(Type projectionType, CronusMessage cronusMessage)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, cronusMessage))
{
throw new ArgumentNullException(nameof(cronusMessage));
}
EventOrigin eventOrigin = cronusMessage.GetEventOrigin();
IEvent @event = cronusMessage.Payload as IEvent;
await SaveAsync(projectionType, @event, eventOrigin).ConfigureAwait(false);;
}
public void Save(Type projectionType, IEvent @event, EventOrigin eventOrigin)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, @event))
{
throw new ArgumentNullException(nameof(@event));
}
if (ReferenceEquals(null, eventOrigin))
{
throw new ArgumentNullException(nameof(eventOrigin));
}
string contractId = projectionType.GetContractId();
var instance = FastActivator.CreateInstance(projectionType);
var statefullProjection = instance as IProjectionDefinition;
if (statefullProjection != null)
{
var projectionIds = statefullProjection.GetProjectionIds(@event);
foreach (var version in GetProjectionVersions(contractId))
{
foreach (var projectionId in projectionIds)
{
ISnapshot snapshot = snapshotStore.Load(contractId, projectionId, version);
ProjectionStream projectionStream = LoadProjectionStream(projectionType, version, projectionId, snapshot);
int snapshotMarker = snapshotStrategy.GetSnapshotMarker(projectionStream.Commits, snapshot.Revision);
var commit = new ProjectionCommit(projectionId, version, @event, snapshotMarker, eventOrigin, DateTime.UtcNow);
projectionStore.Save(commit);
}
}
}
}
public void Save(Type projectionType, CronusMessage cronusMessage)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, cronusMessage))
{
throw new ArgumentNullException(nameof(cronusMessage));
}
var projection = FastActivator.CreateInstance(projectionType) as IProjectionDefinition;
if (projection != null)
{
var projectionIds = projection.GetProjectionIds(cronusMessage.Payload as IEvent);
string contractId = projectionType.GetContractId();
foreach (var projectionId in projectionIds)
{
foreach (var version in GetProjectionVersions(contractId))
{
ISnapshot snapshot = snapshotStore.Load(contractId, projectionId, version);
ProjectionStream projectionStream = LoadProjectionStream(projectionType, version, projectionId, snapshot);
int snapshotMarker = snapshotStrategy.GetSnapshotMarker(projectionStream.Commits, snapshot.Revision);
EventOrigin eventOrigin = cronusMessage.GetEventOrigin();
DateTime timestamp = DateTime.UtcNow;
IEvent @event = cronusMessage.Payload as IEvent;
var commit = new ProjectionCommit(projectionId, version, @event, snapshotMarker, eventOrigin, timestamp);
projectionStore.Save(commit);
}
}
}
}
public void Save(Type projectionType, IEvent @event, EventOrigin eventOrigin)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, @event))
{
throw new ArgumentNullException(nameof(@event));
}
if (ReferenceEquals(null, eventOrigin))
{
throw new ArgumentNullException(nameof(eventOrigin));
}
string projectionName = projectionType.GetContractId();
var handlerInstance = handlerFactory.Create(projectionType);
var projection = handlerInstance.Current as IProjectionDefinition;
if (projection != null)
{
var projectionIds = projection.GetProjectionIds(@event);
foreach (var projectionId in projectionIds)
{
ReadResult <ProjectionVersions> result = GetProjectionVersions(projectionName);
if (result.IsSuccess)
{
foreach (ProjectionVersion version in result.Data)
{
if (version.Status == ProjectionStatus.Building || version.Status == ProjectionStatus.Live)
{
try
{
SnapshotMeta snapshotMeta = null;
if (projectionType.IsSnapshotable())
{
snapshotMeta = snapshotStore.LoadMeta(projectionName, projectionId, version);
}
else
{
snapshotMeta = new NoSnapshot(projectionId, projectionName).GetMeta();
}
ProjectionStream projectionStream = LoadProjectionStream(projectionType, version, projectionId, snapshotMeta);
int snapshotMarker = snapshotStrategy.GetSnapshotMarker(projectionStream.Commits, snapshotMeta.Revision);
var commit = new ProjectionCommit(projectionId, version, @event, snapshotMarker, eventOrigin, DateTime.UtcNow);
projectionStore.Save(commit);
}
catch (Exception ex)
{
log.ErrorException("Failed to persist event." + Environment.NewLine + $"\tProjectionVersion:{version}" + Environment.NewLine + $"\tEvent:{@event}", ex);
}
}
else if (version.Status == ProjectionStatus.NotPresent)
{
var commit = new ProjectionCommit(projectionId, version, @event, 1, eventOrigin, DateTime.UtcNow);
projectionStore.Save(commit);
}
}
}
}
}
}
public void Save(Type projectionType, IEvent @event, EventOrigin eventOrigin, ProjectionVersion version)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, @event))
{
throw new ArgumentNullException(nameof(@event));
}
if (ReferenceEquals(null, eventOrigin))
{
throw new ArgumentNullException(nameof(eventOrigin));
}
if (ReferenceEquals(null, version))
{
throw new ArgumentNullException(nameof(version));
}
if ((version.Status == ProjectionStatus.Building || version.Status == ProjectionStatus.Live) == false)
{
throw new ArgumentException("Invalid version. Only versions in `Building` and `Live` status are eligable for persistence.", nameof(version));
}
string projectionName = projectionType.GetContractId();
if (projectionName.Equals(version.ProjectionName, StringComparison.OrdinalIgnoreCase) == false)
{
throw new ArgumentException($"Invalid version. The version `{version}` does not match projection `{projectionName}`", nameof(version));
}
var handlerInstance = handlerFactory.Create(projectionType);
var projection = handlerInstance.Current as IProjectionDefinition;
if (projection != null)
{
var projectionIds = projection.GetProjectionIds(@event);
foreach (var projectionId in projectionIds)
{
try
{
SnapshotMeta snapshotMeta = null;
if (projectionType.IsSnapshotable())
{
snapshotMeta = snapshotStore.LoadMeta(projectionName, projectionId, version);
}
else
{
snapshotMeta = new NoSnapshot(projectionId, projectionName).GetMeta();
}
ProjectionStream projectionStream = LoadProjectionStream(projectionType, version, projectionId, snapshotMeta);
int snapshotMarker = snapshotStrategy.GetSnapshotMarker(projectionStream.Commits, snapshotMeta.Revision);
var commit = new ProjectionCommit(projectionId, version, @event, snapshotMarker, eventOrigin, DateTime.UtcNow);
projectionStore.Save(commit);
}
catch (Exception ex)
{
log.ErrorException("Failed to persist event." + Environment.NewLine + $"\tProjectionVersion:{version}" + Environment.NewLine + $"\tEvent:{@event}", ex);
}
}
}
}
public ReplayResult Rebuild(ProjectionVersion version, DateTime rebuildUntil)
{
if (ReferenceEquals(null, version))
{
throw new ArgumentNullException(nameof(version));
}
if (index.Status.IsNotPresent())
{
RebuildIndex(); //TODO (2)
}
Type projectionType = version.ProjectionName.GetTypeByContract();
try
{
if (IsVersionTrackerMissing() && IsNotSystemProjection(projectionType))
{
return(ReplayResult.RetryLater($"Projection `{version}` still don't have present index.")); //WHEN TO RETRY AGAIN
}
if (HasReplayTimeout(rebuildUntil))
{
return(ReplayResult.Timeout($"Rebuild of projection `{version}` has expired. Version:{version} Deadline:{rebuildUntil}."));
}
var allVersions = GetAllVersions(version);
if (allVersions.IsOutdatad(version))
{
return(new ReplayResult($"Version `{version}` is outdated. There is a newer one which is already live."));
}
if (allVersions.IsCanceled(version))
{
return(new ReplayResult($"Version `{version}` was canceled."));
}
if (index.Status.IsNotPresent())
{
return(ReplayResult.RetryLater($"Projection `{version}` still don't have present index.")); //WHEN TO RETRY AGAIN
}
DateTime startRebuildTimestamp = DateTime.UtcNow;
int progressCounter = 0;
log.Info(() => $"Start rebuilding projection `{version.ProjectionName}` for version {version}. Deadline is {rebuildUntil}");
Dictionary <int, string> processedAggregates = new Dictionary <int, string>();
projectionStoreInitializer.Initialize(version);
var projectionHandledEventTypes = GetInvolvedEvents(projectionType);
foreach (var eventType in projectionHandledEventTypes)
{
log.Info(() => $"Rebuilding projection `{version.ProjectionName}` for version {version} using eventType `{eventType}`. Deadline is {rebuildUntil}");
IEnumerable <IndexRecord> indexRecords = index.EnumerateRecords(eventType);
foreach (IndexRecord indexRecord in indexRecords)
{
// TODO: (5) Decorator pattern which will give us the tracking
progressCounter++;
if (progressCounter % 1000 == 0)
{
log.Info(() => $"Rebuilding projection {version.ProjectionName} => PROGRESS:{progressCounter} Version:{version} EventType:{eventType} Deadline:{rebuildUntil} Total minutes working:{(DateTime.UtcNow - startRebuildTimestamp).TotalMinutes}. logId:{Guid.NewGuid().ToString()} ProcessedAggregatesSize:{processedAggregates.Count}");
}
int aggreagteRootIdHash = indexRecord.AggregateRootId.GetHashCode();
if (processedAggregates.ContainsKey(aggreagteRootIdHash))
{
continue;
}
processedAggregates.Add(aggreagteRootIdHash, null);
string mess = Encoding.UTF8.GetString(indexRecord.AggregateRootId);
IAggregateRootId arId = GetAggregateRootId(mess);
EventStream stream = eventStore.Load(arId);
foreach (AggregateCommit arCommit in stream.Commits)
{
for (int i = 0; i < arCommit.Events.Count; i++)
{
IEvent theEvent = arCommit.Events[i].Unwrap();
if (projectionHandledEventTypes.Contains(theEvent.GetType().GetContractId())) // filter out the events which are not part of the projection
{
var origin = new EventOrigin(mess, arCommit.Revision, i, arCommit.Timestamp);
projectionWriter.Save(projectionType, theEvent, origin, version);
}
}
}
}
}
log.Info(() => $"Finish rebuilding projection `{projectionType.Name}` for version {version}. Deadline was {rebuildUntil}");
return(new ReplayResult());
}
catch (Exception ex)
{
string message = $"Unable to replay projection. Version:{version} ProjectionType:{projectionType.FullName}";
log.ErrorException(message, ex);
return(new ReplayResult(message + Environment.NewLine + ex.Message + Environment.NewLine + ex.StackTrace));
}
}
public async Task SaveAsync(Type projectionType, IEvent @event, EventOrigin eventOrigin)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, @event))
{
throw new ArgumentNullException(nameof(@event));
}
if (ReferenceEquals(null, eventOrigin))
{
throw new ArgumentNullException(nameof(eventOrigin));
}
string projectionName = projectionType.GetContractId();
var handlerInstance = handlerFactory.Create(projectionType);
var projection = handlerInstance.Current as IProjectionDefinition;
if (projection != null)
{
var projectionIds = projection.GetProjectionIds(@event);
foreach (var projectionId in projectionIds)
{
using (log.BeginScope(s => s.AddScope("cronus_projection_id", projectionId)))
{
ReadResult <ProjectionVersions> result = await GetProjectionVersionsAsync(projectionName).ConfigureAwait(false);
if (result.IsSuccess)
{
foreach (ProjectionVersion version in result.Data)
{
using (log.BeginScope(s => s.AddScope("cronus_projection_version", version)))
{
if (ShouldSaveEventForVersion(version))
{
try
{
SnapshotMeta snapshotMeta = null;
if (projectionType.IsSnapshotable())
{
snapshotMeta = await snapshotStore.LoadMetaAsync(projectionName, projectionId, version).ConfigureAwait(false);
}
else
{
snapshotMeta = new NoSnapshot(projectionId, projectionName).GetMeta();
}
int snapshotMarker = snapshotMeta.Revision + 2;
var commit = new ProjectionCommit(projectionId, version, @event, snapshotMarker, eventOrigin, DateTime.UtcNow);
await projectionStore.SaveAsync(commit).ConfigureAwait(false);
}
catch (Exception ex) when(LogMessageWhenFailToUpdateProjection(ex, version))
{
}
}
}
}
if (result.HasError)
{
log.Error(() => "Failed to update projection because the projection version failed to load. Please replay the projection to restore the state. Self-heal hint!" + Environment.NewLine + result.Error + Environment.NewLine + $"\tProjectionName:{projectionName}" + Environment.NewLine + $"\tEvent:{@event}");
}
}
}
}
}
}
// Used by replay projections only
public async Task SaveAsync(Type projectionType, IEvent @event, EventOrigin eventOrigin, ProjectionVersion version)
{
if (ReferenceEquals(null, projectionType))
{
throw new ArgumentNullException(nameof(projectionType));
}
if (ReferenceEquals(null, @event))
{
throw new ArgumentNullException(nameof(@event));
}
if (ReferenceEquals(null, eventOrigin))
{
throw new ArgumentNullException(nameof(eventOrigin));
}
if (ReferenceEquals(null, version))
{
throw new ArgumentNullException(nameof(version));
}
if (ShouldSaveEventForVersion(version) == false)
{
throw new ArgumentException("Invalid version. Only versions in `Building` and `Live` status are eligable for persistence.", nameof(version));
}
string projectionName = projectionType.GetContractId();
if (projectionName.Equals(version.ProjectionName, StringComparison.OrdinalIgnoreCase) == false)
{
throw new ArgumentException($"Invalid version. The version `{version}` does not match projection `{projectionName}`", nameof(version));
}
var handlerInstance = handlerFactory.Create(projectionType);
var projection = handlerInstance.Current as IProjectionDefinition;
if (projection != null)
{
var projectionIds = projection.GetProjectionIds(@event);
foreach (var projectionId in projectionIds)
{
try
{
SnapshotMeta snapshotMeta = await GetSnapshotMeta(projectionType, projectionName, projectionId, version).ConfigureAwait(false);
int snapshotMarker = snapshotMeta.Revision == 0 ? 1 : snapshotMeta.Revision + 2;
var commit = new ProjectionCommit(projectionId, version, @event, snapshotMarker, eventOrigin, DateTime.UtcNow);
await projectionStore.SaveAsync(commit).ConfigureAwait(false);
}
catch (Exception ex)
{
log.ErrorException(ex, () => "Failed to persist event." + Environment.NewLine + $"\tProjectionVersion:{version}" + Environment.NewLine + $"\tEvent:{@event}");
}
}
}
else if (handlerInstance.Current is IAmEventSourcedProjection eventSourcedProjection)
{
try
{
var projectionId = Urn.Parse($"urn:cronus:{projectionName}");
var commit = new ProjectionCommit(projectionId, version, @event, 1, eventOrigin, DateTime.UtcNow); // Snapshotting is disable till we test it => hardcoded 1
await projectionStore.SaveAsync(commit).ConfigureAwait(false);
}
catch (Exception ex)
{
log.ErrorException(ex, () => "Failed to persist event." + Environment.NewLine + $"\tProjectionVersion:{version}" + Environment.NewLine + $"\tEvent:{@event}");
}
}
}