Example #1
0
        public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next)
        {
            EnsureValidStats(stats);
            var wasInterrupted = false;
            var delay          = GetDelayReplication();
            var currentNext    = next;

            using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
                using (documentsContext.OpenReadTransaction())
                {
                    try
                    {
                        // we scan through the documents to send to the other side, we need to be careful about
                        // filtering a lot of documents, because we need to let the other side know about this, and
                        // at the same time, we need to send a heartbeat to keep the tcp connection alive
                        _lastEtag = _parent._lastSentDocumentEtag;
                        _parent.CancellationToken.ThrowIfCancellationRequested();

                        var   batchSize                   = _parent._database.Configuration.Replication.MaxItemsCount;
                        var   maxSizeToSend               = _parent._database.Configuration.Replication.MaxSizeToSend;
                        long  size                        = 0;
                        var   numberOfItemsSent           = 0;
                        var   skippedReplicationItemsInfo = new SkippedReplicationItemsInfo();
                        short lastTransactionMarker       = -1;
                        long  prevLastEtag                = _lastEtag;

                        using (_stats.Storage.Start())
                        {
                            foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters))
                            {
                                _parent.CancellationToken.ThrowIfCancellationRequested();

                                if (lastTransactionMarker != item.TransactionMarker)
                                {
                                    if (delay.Ticks > 0)
                                    {
                                        var nextReplication = item.LastModifiedTicks + delay.Ticks;
                                        if (_parent._database.Time.GetUtcNow().Ticks < nextReplication)
                                        {
                                            if (Interlocked.CompareExchange(ref next, nextReplication, currentNext) == currentNext)
                                            {
                                                wasInterrupted = true;
                                                break;
                                            }
                                        }
                                    }
                                    lastTransactionMarker = item.TransactionMarker;

                                    if (_parent.SupportedFeatures.Replication.TimeSeries == false)
                                    {
                                        AssertNotTimeSeriesForLegacyReplication(item);
                                    }

                                    if (_parent.SupportedFeatures.Replication.CountersBatch == false)
                                    {
                                        AssertNotCounterForLegacyReplication(item);
                                    }

                                    if (_parent.SupportedFeatures.Replication.ClusterTransaction == false)
                                    {
                                        AssertNotClusterTransactionDocumentForLegacyReplication(item);
                                    }

                                    // We want to limit batch sizes to reasonable limits.
                                    var totalSize =
                                        size + documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);

                                    if (maxSizeToSend.HasValue && totalSize > maxSizeToSend.Value.GetValue(SizeUnit.Bytes) ||
                                        batchSize.HasValue && numberOfItemsSent > batchSize.Value)
                                    {
                                        wasInterrupted = true;
                                        break;
                                    }

                                    if (_stats.Storage.CurrentStats.InputCount % 16384 == 0)
                                    {
                                        // ReSharper disable once PossibleLossOfFraction
                                        if ((_parent._parent.MinimalHeartbeatInterval / 2) < _stats.Storage.Duration.TotalMilliseconds)
                                        {
                                            wasInterrupted = true;
                                            break;
                                        }
                                    }
                                }

                                _stats.Storage.RecordInputAttempt();

                                // here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary
                                if (MissingAttachmentsInLastBatch &&
                                    item.Type == ReplicationBatchItem.ReplicationItemType.Document &&
                                    item is DocumentReplicationItem docItem &&
                                    docItem.Flags.Contain(DocumentFlags.HasAttachments))
                                {
                                    var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document;
                                    foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector))
                                    {
                                        // we need to filter attachments that are been sent in the same batch as the document
                                        if (attachment.Etag >= prevLastEtag)
                                        {
                                            continue;
                                        }

                                        var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash);
                                        attachment.Stream = stream;
                                        var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment);
                                        AddReplicationItemToBatch(attachmentItem, _stats.Storage, skippedReplicationItemsInfo);
                                        size += attachmentItem.Size;
                                    }
                                }

                                _lastEtag = item.Etag;

                                if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false)
                                {
                                    // this item won't be needed anymore
                                    item.Dispose();
                                    continue;
                                }

                                size += item.Size;

                                numberOfItemsSent++;
                            }
                        }

                        if (_log.IsInfoEnabled)
                        {
                            if (skippedReplicationItemsInfo.SkippedItems > 0)
                            {
                                var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector);
                                _log.Info(message);
                            }

                            var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " +
                                      $"and {_replicaAttachmentStreams.Count} attachment's streams " +
                                      $"to replicate to {_parent.Node.FromString()}, ";

                            var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);
                            if (encryptionSize > 0)
                            {
                                msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, ";
                            }
                            msg += $"total size: {new Size(size + encryptionSize, SizeUnit.Bytes)}";

                            _log.Info(msg);
                        }

                        if (_orderedReplicaItems.Count == 0)
                        {
                            var hasModification = _lastEtag != _parent._lastSentDocumentEtag;

                            // ensure that the other server is aware that we skipped
                            // on (potentially a lot of) documents to send, and we update
                            // the last etag they have from us on the other side
                            _parent._lastSentDocumentEtag = _lastEtag;
                            _parent._lastDocumentSentTime = DateTime.UtcNow;
                            var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext);
                            _parent.SendHeartbeat(changeVector);
                            return(hasModification);
                        }

                        _parent.CancellationToken.ThrowIfCancellationRequested();

                        try
                        {
                            using (_stats.Network.Start())
                            {
                                SendDocumentsBatch(documentsContext, _stats.Network);
                                tcpConnectionOptions._lastEtagSent = _lastEtag;
                                tcpConnectionOptions.RegisterBytesSent(size);
                                if (MissingAttachmentsInLastBatch)
                                {
                                    return(false);
                                }
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            if (_log.IsInfoEnabled)
                            {
                                _log.Info("Received cancellation notification while sending document replication batch.");
                            }
                            throw;
                        }
                        catch (Exception e)
                        {
                            if (_log.IsInfoEnabled)
                            {
                                _log.Info("Failed to send document replication batch", e);
                            }
                            throw;
                        }

                        MissingAttachmentsInLastBatch = false;

                        return(true);
                    }
                    finally
                    {
                        foreach (var item in _orderedReplicaItems)
                        {
                            item.Value.Dispose();
                        }
                        _orderedReplicaItems.Clear();
                        _replicaAttachmentStreams.Clear();
                    }
                }
        }
Example #2
0
        public bool ExecuteReplicationOnce()
        {
            var readTx = _parent._documentsContext.OpenReadTransaction();

            try
            {
                // we scan through the documents to send to the other side, we need to be careful about
                // filtering a lot of documents, because we need to let the other side know about this, and
                // at the same time, we need to send a heartbeat to keep the tcp connection alive
                long maxEtag;
                maxEtag = _lastEtag = _parent._lastSentDocumentEtag;
                _parent.CancellationToken.ThrowIfCancellationRequested();

                var docs =
                    _parent._database.DocumentsStorage.GetDocumentsFrom(_parent._documentsContext,
                                                                        _lastEtag + 1, 0, 1024)
                    .ToList();
                var tombstones =
                    _parent._database.DocumentsStorage.GetTombstonesFrom(_parent._documentsContext,
                                                                         _lastEtag + 1, 0, 1024)
                    .ToList();


                if (docs.Count > 0)
                {
                    maxEtag = docs[docs.Count - 1].Etag;
                }

                if (tombstones.Count > 0)
                {
                    maxEtag = Math.Max(maxEtag, tombstones[tombstones.Count - 1].Etag);
                }

                long size = 0;

                foreach (var doc in docs)
                {
                    if (doc.Etag > maxEtag)
                    {
                        break;
                    }


                    size += doc.Data.Size;

                    AddReplicationItemToBatch(new ReplicationBatchDocumentItem
                    {
                        Etag         = doc.Etag,
                        ChangeVector = doc.ChangeVector,
                        Data         = doc.Data,
                        Key          = doc.Key
                    });

                    if (size > 16 * 1024 * 1024)
                    {
                        break; // we want to limit batch sizes to reasonable limits
                    }
                }

                foreach (var tombstone in tombstones)
                {
                    if (tombstone.Etag > maxEtag)
                    {
                        break;
                    }

                    AddReplicationItemToBatch(new ReplicationBatchDocumentItem
                    {
                        Etag         = tombstone.Etag,
                        ChangeVector = tombstone.ChangeVector,
                        Collection   = tombstone.Collection,
                        Key          = tombstone.Key
                    });
                }


                if (_log.IsInfoEnabled)
                {
                    _log.Info($"Found {_orderedReplicaItems.Count:#,#;;0} documents to replicate to {_parent.Destination.Database} @ {_parent.Destination.Url}");
                }

                if (_orderedReplicaItems.Count == 0)
                {
                    var hasModification = _lastEtag != _parent._lastSentDocumentEtag;
                    if (hasModification == false)
                    {
                        _parent._lastSentDocumentEtag = maxEtag;
                    }
                    else
                    {
                        _parent._lastSentDocumentEtag = _lastEtag;
                    }
                    // ensure that the other server is aware that we skipped
                    // on (potentially a lot of) documents to send, and we update
                    // the last etag they have from us on the other side
                    using (_parent._configurationContext.OpenReadTransaction())
                    {
                        _parent._lastDocumentSentTime = DateTime.UtcNow;
                        _parent.SendHeartbeat();
                    }
                    return(hasModification);
                }

                _parent.CancellationToken.ThrowIfCancellationRequested();
                try
                {
                    SendDocumentsBatch();
                }
                catch (Exception e)
                {
                    if (_log.IsInfoEnabled)
                    {
                        _log.Info("Failed to send document replication batch", e);
                    }
                    throw;
                }
                return(true);
            }
            finally
            {
                foreach (var item in _orderedReplicaItems)
                {
                    item.Value.Data?.Dispose(); //item.Value.Data is null if tombstone
                }
                _orderedReplicaItems.Clear();

                if (readTx.Disposed == false)
                {
                    readTx.Dispose();
                }
            }
        }