public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next) { EnsureValidStats(stats); var wasInterrupted = false; var delay = GetDelayReplication(); var currentNext = next; using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext)) using (documentsContext.OpenReadTransaction()) { try { // we scan through the documents to send to the other side, we need to be careful about // filtering a lot of documents, because we need to let the other side know about this, and // at the same time, we need to send a heartbeat to keep the tcp connection alive _lastEtag = _parent._lastSentDocumentEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var batchSize = _parent._database.Configuration.Replication.MaxItemsCount; var maxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend; long size = 0; var numberOfItemsSent = 0; var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo(); short lastTransactionMarker = -1; long prevLastEtag = _lastEtag; using (_stats.Storage.Start()) { foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters)) { _parent.CancellationToken.ThrowIfCancellationRequested(); if (lastTransactionMarker != item.TransactionMarker) { if (delay.Ticks > 0) { var nextReplication = item.LastModifiedTicks + delay.Ticks; if (_parent._database.Time.GetUtcNow().Ticks < nextReplication) { if (Interlocked.CompareExchange(ref next, nextReplication, currentNext) == currentNext) { wasInterrupted = true; break; } } } lastTransactionMarker = item.TransactionMarker; if (_parent.SupportedFeatures.Replication.TimeSeries == false) { AssertNotTimeSeriesForLegacyReplication(item); } if (_parent.SupportedFeatures.Replication.CountersBatch == false) { AssertNotCounterForLegacyReplication(item); } if (_parent.SupportedFeatures.Replication.ClusterTransaction == false) { AssertNotClusterTransactionDocumentForLegacyReplication(item); } // We want to limit batch sizes to reasonable limits. var totalSize = size + documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes); if (maxSizeToSend.HasValue && totalSize > maxSizeToSend.Value.GetValue(SizeUnit.Bytes) || batchSize.HasValue && numberOfItemsSent > batchSize.Value) { wasInterrupted = true; break; } if (_stats.Storage.CurrentStats.InputCount % 16384 == 0) { // ReSharper disable once PossibleLossOfFraction if ((_parent._parent.MinimalHeartbeatInterval / 2) < _stats.Storage.Duration.TotalMilliseconds) { wasInterrupted = true; break; } } } _stats.Storage.RecordInputAttempt(); // here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary if (MissingAttachmentsInLastBatch && item.Type == ReplicationBatchItem.ReplicationItemType.Document && item is DocumentReplicationItem docItem && docItem.Flags.Contain(DocumentFlags.HasAttachments)) { var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document; foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector)) { // we need to filter attachments that are been sent in the same batch as the document if (attachment.Etag >= prevLastEtag) { continue; } var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash); attachment.Stream = stream; var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment); AddReplicationItemToBatch(attachmentItem, _stats.Storage, skippedReplicationItemsInfo); size += attachmentItem.Size; } } _lastEtag = item.Etag; if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false) { // this item won't be needed anymore item.Dispose(); continue; } size += item.Size; numberOfItemsSent++; } } if (_log.IsInfoEnabled) { if (skippedReplicationItemsInfo.SkippedItems > 0) { var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector); _log.Info(message); } var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " + $"and {_replicaAttachmentStreams.Count} attachment's streams " + $"to replicate to {_parent.Node.FromString()}, "; var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes); if (encryptionSize > 0) { msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, "; } msg += $"total size: {new Size(size + encryptionSize, SizeUnit.Bytes)}"; _log.Info(msg); } if (_orderedReplicaItems.Count == 0) { var hasModification = _lastEtag != _parent._lastSentDocumentEtag; // ensure that the other server is aware that we skipped // on (potentially a lot of) documents to send, and we update // the last etag they have from us on the other side _parent._lastSentDocumentEtag = _lastEtag; _parent._lastDocumentSentTime = DateTime.UtcNow; var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext); _parent.SendHeartbeat(changeVector); return(hasModification); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { using (_stats.Network.Start()) { SendDocumentsBatch(documentsContext, _stats.Network); tcpConnectionOptions._lastEtagSent = _lastEtag; tcpConnectionOptions.RegisterBytesSent(size); if (MissingAttachmentsInLastBatch) { return(false); } } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info("Received cancellation notification while sending document replication batch."); } throw; } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send document replication batch", e); } throw; } MissingAttachmentsInLastBatch = false; return(true); } finally { foreach (var item in _orderedReplicaItems) { item.Value.Dispose(); } _orderedReplicaItems.Clear(); _replicaAttachmentStreams.Clear(); } } }
public bool ExecuteReplicationOnce() { var readTx = _parent._documentsContext.OpenReadTransaction(); try { // we scan through the documents to send to the other side, we need to be careful about // filtering a lot of documents, because we need to let the other side know about this, and // at the same time, we need to send a heartbeat to keep the tcp connection alive long maxEtag; maxEtag = _lastEtag = _parent._lastSentDocumentEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var docs = _parent._database.DocumentsStorage.GetDocumentsFrom(_parent._documentsContext, _lastEtag + 1, 0, 1024) .ToList(); var tombstones = _parent._database.DocumentsStorage.GetTombstonesFrom(_parent._documentsContext, _lastEtag + 1, 0, 1024) .ToList(); if (docs.Count > 0) { maxEtag = docs[docs.Count - 1].Etag; } if (tombstones.Count > 0) { maxEtag = Math.Max(maxEtag, tombstones[tombstones.Count - 1].Etag); } long size = 0; foreach (var doc in docs) { if (doc.Etag > maxEtag) { break; } size += doc.Data.Size; AddReplicationItemToBatch(new ReplicationBatchDocumentItem { Etag = doc.Etag, ChangeVector = doc.ChangeVector, Data = doc.Data, Key = doc.Key }); if (size > 16 * 1024 * 1024) { break; // we want to limit batch sizes to reasonable limits } } foreach (var tombstone in tombstones) { if (tombstone.Etag > maxEtag) { break; } AddReplicationItemToBatch(new ReplicationBatchDocumentItem { Etag = tombstone.Etag, ChangeVector = tombstone.ChangeVector, Collection = tombstone.Collection, Key = tombstone.Key }); } if (_log.IsInfoEnabled) { _log.Info($"Found {_orderedReplicaItems.Count:#,#;;0} documents to replicate to {_parent.Destination.Database} @ {_parent.Destination.Url}"); } if (_orderedReplicaItems.Count == 0) { var hasModification = _lastEtag != _parent._lastSentDocumentEtag; if (hasModification == false) { _parent._lastSentDocumentEtag = maxEtag; } else { _parent._lastSentDocumentEtag = _lastEtag; } // ensure that the other server is aware that we skipped // on (potentially a lot of) documents to send, and we update // the last etag they have from us on the other side using (_parent._configurationContext.OpenReadTransaction()) { _parent._lastDocumentSentTime = DateTime.UtcNow; _parent.SendHeartbeat(); } return(hasModification); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { SendDocumentsBatch(); } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send document replication batch", e); } throw; } return(true); } finally { foreach (var item in _orderedReplicaItems) { item.Value.Data?.Dispose(); //item.Value.Data is null if tombstone } _orderedReplicaItems.Clear(); if (readTx.Disposed == false) { readTx.Dispose(); } } }