private string SequenceNumbersLogString(ReplicationEndpointInfo info)
{
var sb = new StringBuilder();
foreach (var(logName, sequenceNr) in info.LogSequenceNumbers)
{
sb.Append(logName).Append(':').Append(sequenceNr).Append(", ");
}
return(sb.ToString());
}
private async Task <long> SynchronizeReplicationProgress(ReplicationEndpointInfo info, string name)
{
var logActor = Endpoint.Logs[name];
var logId = info.LogId(name);
var remoteSequenceNr = info.LogSequenceNumbers[name];
var currentProgress = await ReadReplicationProgress(logActor, logId);
if (currentProgress > remoteSequenceNr)
{
return(await UpdateReplicationMetadata(logActor, logId, remoteSequenceNr));
}
else
{
return(currentProgress);
}
}
/// <summary>
/// Delete events from a local log identified by <paramref name="logName"/> with a sequence number less than or equal to
/// <paramref name="toSequenceNr"/>. Deletion is split into logical deletion and physical deletion. Logical deletion is
/// supported by any storage backend and ensures that deleted events are not replayed any more. It has
/// immediate effect. Logically deleted events can still be replicated to remote <see cref="ReplicationEndpoint"/>s.
/// They are only physically deleted if the storage backend supports that (currently LevelDB only). Furthermore,
/// physical deletion only starts after all remote replication endpoints identified by <paramref name="remoteEndpointIds"/>
/// have successfully replicated these events. Physical deletion is implemented as reliable background
/// process that survives event log restarts.
///
/// Use with care! When events are physically deleted they cannot be replicated any more to new replication
/// endpoints (i.e. those that were unknown at the time of deletion). Also, a location with deleted events
/// may not be suitable any more for disaster recovery of other locations.
/// </summary>
/// <param name="logName">Events are deleted from the local log with this name.</param>
/// <param name="toSequenceNr">Sequence number up to which events shall be deleted (inclusive).</param>
/// <param name="remoteEndpointIds">
/// A set of remote <see cref="ReplicationEndpoint"/> ids that must have replicated events
/// to their logs before they are allowed to be physically deleted at this endpoint.
/// </param>
/// <returns>
/// The sequence number up to which events have been logically deleted. When the returned task
/// completes logical deletion is effective. The returned sequence number can differ from the requested
/// one, if:
///
/// - the log's current sequence number is smaller than the requested number. In this case the current
/// sequence number is returned.
/// - there was a previous successful deletion request with a higher sequence number. In this case that
/// number is returned.
/// </returns>
public async Task <long> Delete(string logName, long toSequenceNr, ImmutableHashSet <string> remoteEndpointIds)
{
var remoteLogIds = remoteEndpointIds
.Select(id => ReplicationEndpointInfo.LogId(id, logName))
.ToImmutableHashSet();
var response = await Logs[logName].Ask(new Delete(toSequenceNr, remoteEndpointIds), timeout: Settings.WriteTimeout);
switch (response)
{
case DeleteSuccess s: return(s.DeletedTo);
case DeleteFailure f: throw f.Cause;
default: throw new InvalidOperationException($"Expected either [{nameof(DeleteSuccess)}] or [{nameof(DeleteFailure)}] but got [{response.GetType().FullName}]");
}
}
private async Task <ReplicationEndpointInfo> SynchronizeReplicationProgressWithRemote(ActorSelection remoteAcceptor, ReplicationEndpointInfo info)
{
var i = 0;
while (true)
{
try
{
i++;
var result = await remoteAcceptor.Ask(new SynchronizeReplicationProgress(info), timeout : settings.RemoteOperationTimeout);
switch (result)
{
case SynchronizeReplicationProgressSuccess s: return(s.Info);
case SynchronizeReplicationProgressFailure f: throw f.Cause;
default: throw new InvalidOperationException($"Expected either [{nameof(SynchronizeReplicationProgressSuccess)}] or [{nameof(SynchronizeReplicationProgressFailure)}] but got [{result.GetType().FullName}]");
}
}
catch (AskTimeoutException) when(i <= settings.RemoteOperationRetryMax)
{
await Task.Delay(settings.RemoteOperationRetryDelay);
}
}
}
private RecoveryLink ToRecoveryLink(ReplicationLink link, ReplicationEndpointInfo localInfo, ReplicationEndpointInfo remoteInfo) => new RecoveryLink(link, localInfo.LogSequenceNumbers[link.Target.LogName], remoteInfo.LogSequenceNumbers[link.Target.LogName]);
/// <summary>
/// Synchronize sequence numbers of local logs with replication progress stored in remote replicas.
/// </summary>
/// <returns>A set of <see cref="RecoveryLink"/>s indicating the events that need to be recovered</returns>
public async Task <ImmutableHashSet <RecoveryLink> > SynchronizeReplicationProgressesWithRemote(ReplicationEndpointInfo info)
{
var results = Endpoint.connectors
.Select(async connector =>
{
var remoteInfo = await SynchronizeReplicationProgressWithRemote(connector.RemoteAcceptor, info);
var links = connector.Links(remoteInfo);
return(links.Select(link => ToRecoveryLink(link, info, remoteInfo)));
})
.ToArray();
return((await Task.WhenAll(results)).SelectMany(x => x).ToImmutableHashSet());
}
internal IImmutableSet <string> CommonLogNames(ReplicationEndpointInfo info)
{
return(LogNames.Intersect(info.LogNames));
}
/// <summary> /// Returns all log names this endpoint and <paramref name="endpointInfo"/> have in common. /// </summary> internal ImmutableHashSet <string> CommonLogNames(ReplicationEndpointInfo endpointInfo) => this.LogNames.Intersect(endpointInfo.LogNames);
/// <summary> /// Returns the unique log id for given <paramref name="logName"/>. /// </summary> public string LogId(string logName) => ReplicationEndpointInfo.LogId(Id, logName);