private bool CanContinueBatch(ReplicationState state, ref long next)
{
if (MissingAttachmentsInLastBatch)
{
state.MissingTxMarkers.Remove(state.LastTransactionMarker);
if (state.MissingTxMarkers.Count != 0)
{
return(true);
}
}
if (state.Delay.Ticks > 0)
{
var nextReplication = state.Item.LastModifiedTicks + state.Delay.Ticks;
if (_parent._database.Time.GetUtcNow().Ticks < nextReplication)
{
if (Interlocked.CompareExchange(ref next, nextReplication, state.CurrentNext) == state.CurrentNext)
{
return(false);
}
}
}
if (_parent.SupportedFeatures.Replication.TimeSeries == false)
{
AssertNotTimeSeriesForLegacyReplication(state.Item);
}
if (_parent.SupportedFeatures.Replication.CountersBatch == false)
{
AssertNotCounterForLegacyReplication(state.Item);
}
if (_parent.SupportedFeatures.Replication.ClusterTransaction == false)
{
AssertNotClusterTransactionDocumentForLegacyReplication(state.Item);
}
// We want to limit batch sizes to reasonable limits.
var totalSize =
state.Size + state.Context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);
if (state.MaxSizeToSend.HasValue && totalSize > state.MaxSizeToSend.Value.GetValue(SizeUnit.Bytes) ||
state.BatchSize.HasValue && state.NumberOfItemsSent > state.BatchSize.Value)
{
return(false);
}
if (_stats.Storage.CurrentStats.InputCount % 16384 == 0)
{
// ReSharper disable once PossibleLossOfFraction
if ((_parent._parent.MinimalHeartbeatInterval / 2) < _stats.Storage.Duration.TotalMilliseconds)
{
return(false);
}
}
return(true);
}
public static ReplicationStateCommand Create
(ReplicationState databaseToLastModified)
{
return(new ReplicationStateCommand()
{
DatabaseToLastModified = databaseToLastModified,
Completion = new TaskCompletionSource <object>()
});
}
private bool CanContinueBatch(ReplicationState state, ref long next)
{
if (MissingAttachmentsInLastBatch)
{
// we do have missing attachments but we haven't gathered yet any of the missing hashes
if (state.MissingAttachmentBase64Hashes == null)
{
return(true);
}
// we do have missing attachments but we haven't included all of them in the batch yet
if (state.MissingAttachmentBase64Hashes.Count > 0)
{
return(true);
}
}
if (state.Delay.Ticks > 0)
{
var nextReplication = state.Item.LastModifiedTicks + state.Delay.Ticks;
if (_parent._database.Time.GetUtcNow().Ticks < nextReplication)
{
if (Interlocked.CompareExchange(ref next, nextReplication, state.CurrentNext) == state.CurrentNext)
{
return(false);
}
}
}
// We want to limit batch sizes to reasonable limits.
var totalSize =
state.Size + state.Context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);
if (state.MaxSizeToSend.HasValue && totalSize >= state.MaxSizeToSend.Value.GetValue(SizeUnit.Bytes) ||
state.BatchSize.HasValue && state.NumberOfItemsSent >= state.BatchSize.Value)
{
return(false);
}
var currentCount = _stats.Storage.CurrentStats.InputCount;
if (currentCount > 0 && currentCount % 16384 == 0)
{
// ReSharper disable once PossibleLossOfFraction
if ((_parent._parent.MinimalHeartbeatInterval / 2) < _stats.Storage.Duration.TotalMilliseconds)
{
return(false);
}
}
return(true);
}
public Task SendReplicationStateAsync(ReplicationState replicationState)
{
try
{
var command = ReplicationStateCommand.Create(replicationState);
raftEngine.AppendCommand(command);
return(command.Completion.Task);
}
catch (NotLeadingException)
{
return(SendReplicationStateAsync(raftEngine.GetLeaderNode(WaitForLeaderTimeoutInSeconds), replicationState));
}
}
private async Task SendReplicationStateAsync(NodeConnectionInfo node, ReplicationState replicationState)
{
var url = node.GetAbsoluteUri() + "cluster/replicationState";
using (var request = CreateRequest(node, url, HttpMethods.Post))
{
var response = await request.WriteAsync(
() => new JsonContent(RavenJToken.FromObject(replicationState)))
.ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return;
}
throw await CreateErrorResponseExceptionAsync(response).ConfigureAwait(false);
}
}
PrintReplicationRelationshipObject(
ManagementObject relationshipObject)
{
DateTime lastReplicationTime = ManagementDateTimeConverter.ToDateTime(
relationshipObject.GetPropertyValue("LastReplicationTime").ToString());
DateTime lastApplyTime = ManagementDateTimeConverter.ToDateTime(
relationshipObject.GetPropertyValue("LastApplyTime").ToString());
ReplicationHealth replicationHealth =
(ReplicationHealth)((UInt16)relationshipObject.GetPropertyValue("ReplicationHealth"));
ReplicationState replicationState =
(ReplicationState)((UInt16)relationshipObject.GetPropertyValue("ReplicationState"));
bool isPrimaryRelationship =
relationshipObject.GetPropertyValue("InstanceID").ToString().EndsWith("0", StringComparison.CurrentCulture);
Console.WriteLine("Name \t: {0}",
relationshipObject.GetPropertyValue("ElementName").ToString());
Console.WriteLine("RelationshipType \t: {0}",
isPrimaryRelationship ? "Primary" : "Extended");
Console.WriteLine("ReplicationHealth \t: {0}", replicationHealth.ToString());
Console.WriteLine("ReplicationState \t: {0}", replicationState.ToString());
Console.WriteLine("LastReplicationTime\t: {0}", lastReplicationTime.ToString());
Console.WriteLine("LastApplyTime \t: {0}", lastApplyTime.ToString());
}
public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next)
{
EnsureValidStats(stats);
var wasInterrupted = false;
var delay = GetDelayReplication();
var currentNext = next;
using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
try
{
// we scan through the documents to send to the other side, we need to be careful about
// filtering a lot of documents, because we need to let the other side know about this, and
// at the same time, we need to send a heartbeat to keep the tcp connection alive
_lastEtag = _parent._lastSentDocumentEtag;
_parent.CancellationToken.ThrowIfCancellationRequested();
var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo();
long prevLastEtag = _lastEtag;
var replicationState = new ReplicationState
{
BatchSize = _parent._database.Configuration.Replication.MaxItemsCount,
MaxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend,
CurrentNext = currentNext,
Delay = delay,
Context = documentsContext,
LastTransactionMarker = -1,
NumberOfItemsSent = 0,
Size = 0L,
MissingTxMarkers = new HashSet <short>()
};
using (_stats.Storage.Start())
{
foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters))
{
_parent.CancellationToken.ThrowIfCancellationRequested();
if (replicationState.LastTransactionMarker != item.TransactionMarker)
{
replicationState.Item = item;
if (CanContinueBatch(replicationState, ref next) == false)
{
wasInterrupted = true;
break;
}
replicationState.LastTransactionMarker = item.TransactionMarker;
}
_stats.Storage.RecordInputAttempt();
// here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary
if (MissingAttachmentsInLastBatch &&
item.Type == ReplicationBatchItem.ReplicationItemType.Document &&
item is DocumentReplicationItem docItem &&
docItem.Flags.Contain(DocumentFlags.HasAttachments))
{
var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document;
foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector))
{
// we need to filter attachments that are been sent in the same batch as the document
if (attachment.Etag >= prevLastEtag)
{
if (attachment.TransactionMarker != item.TransactionMarker)
{
replicationState.MissingTxMarkers.Add(attachment.TransactionMarker);
}
continue;
}
var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash);
attachment.Stream = stream;
var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment);
AddReplicationItemToBatch(attachmentItem, _stats.Storage, skippedReplicationItemsInfo);
replicationState.Size += attachmentItem.Size;
}
}
_lastEtag = item.Etag;
if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false)
{
// this item won't be needed anymore
item.Dispose();
continue;
}
replicationState.Size += item.Size;
replicationState.NumberOfItemsSent++;
}
}
if (_log.IsInfoEnabled)
{
if (skippedReplicationItemsInfo.SkippedItems > 0)
{
var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector);
_log.Info(message);
}
var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " +
$"and {_replicaAttachmentStreams.Count} attachment's streams " +
$"to replicate to {_parent.Node.FromString()}, ";
var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);
if (encryptionSize > 0)
{
msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, ";
}
msg += $"total size: {new Size(replicationState.Size + encryptionSize, SizeUnit.Bytes)}";
_log.Info(msg);
}
if (_orderedReplicaItems.Count == 0)
{
var hasModification = _lastEtag != _parent._lastSentDocumentEtag;
// ensure that the other server is aware that we skipped
// on (potentially a lot of) documents to send, and we update
// the last etag they have from us on the other side
_parent._lastSentDocumentEtag = _lastEtag;
_parent._lastDocumentSentTime = DateTime.UtcNow;
var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext);
_parent.SendHeartbeat(changeVector);
return(hasModification);
}
_parent.CancellationToken.ThrowIfCancellationRequested();
try
{
using (_stats.Network.Start())
{
SendDocumentsBatch(documentsContext, _stats.Network);
tcpConnectionOptions._lastEtagSent = _lastEtag;
tcpConnectionOptions.RegisterBytesSent(replicationState.Size);
if (MissingAttachmentsInLastBatch)
{
return(false);
}
}
}
catch (OperationCanceledException)
{
if (_log.IsInfoEnabled)
{
_log.Info("Received cancellation notification while sending document replication batch.");
}
throw;
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Failed to send document replication batch", e);
}
throw;
}
MissingAttachmentsInLastBatch = false;
return(true);
}
finally
{
foreach (var item in _orderedReplicaItems)
{
item.Value.Dispose();
}
_orderedReplicaItems.Clear();
_replicaAttachmentStreams.Clear();
}
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="source">The state that the replication was in before the trigger.</param>
/// <param name="destination">The state the replication was in after the trigger.</param>
/// <param name="trigger">The trigger that caused the state change.</param>
public ReplicationStateTransition(ReplicationState source, ReplicationState destination, ReplicationTrigger trigger)
{
Source = source;
Destination = destination;
Trigger = trigger;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="source">The state that the replication was in before the trigger.</param>
/// <param name="destination">The state the replication was in after the trigger.</param>
/// <param name="trigger">The trigger that caused the state change.</param>
public ReplicationStateTransition(ReplicationState source, ReplicationState destination, ReplicationTrigger trigger)
{
Source = source;
Destination = destination;
Trigger = trigger;
}
internal static ReplicationLinkData DeserializeReplicationLinkData(JsonElement element)
{
ResourceIdentifier id = default;
string name = default;
ResourceType type = default;
Optional <string> partnerServer = default;
Optional <string> partnerDatabase = default;
Optional <string> partnerLocation = default;
Optional <ReplicationRole> role = default;
Optional <ReplicationRole> partnerRole = default;
Optional <string> replicationMode = default;
Optional <DateTimeOffset> startTime = default;
Optional <int> percentComplete = default;
Optional <ReplicationState> replicationState = default;
Optional <bool> isTerminationAllowed = default;
Optional <ReplicationLinkType> linkType = default;
foreach (var property in element.EnumerateObject())
{
if (property.NameEquals("id"))
{
id = new ResourceIdentifier(property.Value.GetString());
continue;
}
if (property.NameEquals("name"))
{
name = property.Value.GetString();
continue;
}
if (property.NameEquals("type"))
{
type = property.Value.GetString();
continue;
}
if (property.NameEquals("properties"))
{
if (property.Value.ValueKind == JsonValueKind.Null)
{
property.ThrowNonNullablePropertyIsNull();
continue;
}
foreach (var property0 in property.Value.EnumerateObject())
{
if (property0.NameEquals("partnerServer"))
{
partnerServer = property0.Value.GetString();
continue;
}
if (property0.NameEquals("partnerDatabase"))
{
partnerDatabase = property0.Value.GetString();
continue;
}
if (property0.NameEquals("partnerLocation"))
{
partnerLocation = property0.Value.GetString();
continue;
}
if (property0.NameEquals("role"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
role = property0.Value.GetString().ToReplicationRole();
continue;
}
if (property0.NameEquals("partnerRole"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
partnerRole = property0.Value.GetString().ToReplicationRole();
continue;
}
if (property0.NameEquals("replicationMode"))
{
replicationMode = property0.Value.GetString();
continue;
}
if (property0.NameEquals("startTime"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
startTime = property0.Value.GetDateTimeOffset("O");
continue;
}
if (property0.NameEquals("percentComplete"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
percentComplete = property0.Value.GetInt32();
continue;
}
if (property0.NameEquals("replicationState"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
replicationState = new ReplicationState(property0.Value.GetString());
continue;
}
if (property0.NameEquals("isTerminationAllowed"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
isTerminationAllowed = property0.Value.GetBoolean();
continue;
}
if (property0.NameEquals("linkType"))
{
if (property0.Value.ValueKind == JsonValueKind.Null)
{
property0.ThrowNonNullablePropertyIsNull();
continue;
}
linkType = new ReplicationLinkType(property0.Value.GetString());
continue;
}
}
continue;
}
}
return(new ReplicationLinkData(id, name, type, partnerServer.Value, partnerDatabase.Value, partnerLocation.Value, Optional.ToNullable(role), Optional.ToNullable(partnerRole), replicationMode.Value, Optional.ToNullable(startTime), Optional.ToNullable(percentComplete), Optional.ToNullable(replicationState), Optional.ToNullable(isTerminationAllowed), Optional.ToNullable(linkType)));
}
/// <summary> /// Converts the <see cref="sourceValue" /> parameter to the <see cref="destinationType" /> parameter using <see cref="formatProvider" /// /> and <see cref="ignoreCase" /> /// </summary> /// <param name="sourceValue">the <see cref="System.Object"/> to convert from</param> /// <param name="destinationType">the <see cref="System.Type" /> to convert to</param> /// <param name="formatProvider">not used by this TypeConverter.</param> /// <param name="ignoreCase">when set to <c>true</c>, will ignore the case when converting.</param> /// <returns> /// an instance of <see cref="ReplicationState" />, or <c>null</c> if there is no suitable conversion. /// </returns> public override object ConvertFrom(object sourceValue, global::System.Type destinationType, global::System.IFormatProvider formatProvider, bool ignoreCase) => ReplicationState.CreateFrom(sourceValue);
private bool AddReplicationItemToBatch(ReplicationBatchItem item, OutgoingReplicationStatsScope stats, ReplicationState state, SkippedReplicationItemsInfo skippedReplicationItemsInfo)
{
if (ShouldSkip(item, stats, skippedReplicationItemsInfo))
{
return(false);
}
if (skippedReplicationItemsInfo.SkippedItems > 0)
{
if (_log.IsInfoEnabled)
{
var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector);
_log.Info(message);
}
skippedReplicationItemsInfo.Reset();
}
if (item is AttachmentReplicationItem attachment)
{
_replicaAttachmentStreams[attachment.Base64Hash] = attachment;
if (MissingAttachmentsInLastBatch)
{
state.MissingAttachmentBase64Hashes?.Remove(attachment.Base64Hash);
}
}
_orderedReplicaItems.Add(item.Etag, item);
return(true);
}
