private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder, out int replicatedDocuments) { replicatedDocuments = 0; JsonDocumentsToReplicate documentsToReplicate = null; var sp = Stopwatch.StartNew(); IDisposable removeBatch = null; var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url, x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner, string.Format("Replication for URL: {0}" ,destination.ConnectionStringOptions.DefaultDatabase) )); prefetchingBehavior.AdditionalInfo = string.Format("For destination: {0}. Last replicated etag: {1}", destination.ConnectionStringOptions.Url, destinationsReplicationInformationForSource.LastDocumentEtag); try { using (var scope = recorder.StartRecording("Get")) { documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555 { using (scope.StartRecording("Notify")) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); scope.Record(new RavenJObject { { "LastDocEtag", documentsToReplicate.LastEtag.ToString() } }); } } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } } // if the db is idling in all respect except sending out replication, let us keep it that way. docDb.WorkContext.UpdateFoundWork(); removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs); using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } // if we had an error sending to this endpoint, it might be because we are sending too much data, or because // the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts // and we'll be much more conservative with increasing the sizes prefetchingBehavior.OutOfMemoryExceptionHappened(); scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } } finally { if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null) { prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed); replicatedDocuments = documentsToReplicate.LoadedDocs.Count; } if (removeBatch != null) removeBatch.Dispose(); } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats)) { SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource; using (var scope = stats.StartRecording("Destination")) { try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) { log.Error("Failed to replicate documents to destination {0}, because was not able to receive last Etag", destination.ConnectionStringOptions.Url); return false; } } scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource)); if (destinationsReplicationInformationForSource.LastDocumentEtag == Etag.InvalidEtag && destinationsReplicationInformationForSource.LastAttachmentEtag == Etag.InvalidEtag) { DateTime lastSent; // todo: move lastModifiedDate after the condition var lastModifiedDate = destinationsReplicationInformationForSource.LastModified.HasValue ? destinationsReplicationInformationForSource.LastModified.Value.ToLocalTime() : DateTime.MinValue; if (destinationAlertSent.TryGetValue(destination.ConnectionStringOptions.Url, out lastSent) && (SystemTime.UtcNow - lastSent).TotalMinutes < 1) { // todo: remove this log line log.Debug(string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source)); return false; } docDb.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source), Title = string.Format("Replication error. Multiple databases replicating at the same time with same DatabaseId ('{0}') detected.", docDb.TransactionalStorage.Id), UniqueKey = "Replication to " + destination.ConnectionStringOptions.Url + " errored. Wrong DatabaseId: " + docDb.TransactionalStorage.Id }); destinationAlertSent.AddOrUpdate(destination.ConnectionStringOptions.Url, SystemTime.UtcNow, (_, __) => SystemTime.UtcNow); return false; } } catch (Exception e) { scope.RecordError(e); log.WarnException("Failed to replicate to: " + destination, e); return false; } } bool? replicated = null; int replicatedDocuments; using (var scope = stats.StartRecording("Documents")) { switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope, out replicatedDocuments)) { case true: replicated = true; break; case false: return false; } } using (var scope = stats.StartRecording("Attachments")) { switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } var elapsedMicroseconds = (long)(stats.ElapsedTime.Ticks * SystemTime.MicroSecPerTick); docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update(elapsedMicroseconds); UpdateReplicationPerformance(destination, stats.Started, stats.ElapsedTime, replicatedDocuments); return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0, 1)); holder.Release(); } }
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { Tuple<RavenJArray, Etag> tuple; RavenJArray attachments; using (var scope = recorder.StartRecording("Get")) { tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope); attachments = tuple.Item1; if (attachments == null || attachments.Length == 0) { if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag) { SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2); } return null; } } using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments:false); return true; } } scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments: false); return true; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds); var duration = Stopwatch.StartNew(); var result = new JsonDocumentsToReplicate { LastEtag = Etag.Empty, }; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch; docDb.TransactionalStorage.Batch(actions => { var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested(); docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(result.LastEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag); return false; } } string reason; return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); result.CountOfFilteredDocumentsWhichOriginFromDestination += docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); if (duration.Elapsed > timeout) break; } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); scope.Record(new RavenJObject { {"StartEtag", lastEtag.ToString()}, {"EndEtag", result.LastEtag.ToString()}, {"Count", docsSinceLastReplEtag}, {"FilteredCount", filteredDocsToReplicate.Count} }); result.LoadedDocs = filteredDocsToReplicate; docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag); docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { JsonDocument.EnsureIdInMetadata(x); EnsureReplicationInformationInMetadata(x.Metadata, docDb); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { scope.RecordError(e); log.WarnException( "Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds); var duration = Stopwatch.StartNew(); RavenJArray attachments = null; Etag lastAttachmentEtag = Etag.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch; docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List<AttachmentInformation> attachmentsToReplicate; List<AttachmentInformation> filteredAttachmentsToReplicate; var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; lastAttachmentEtag = startEtag; while (true) { attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag, maxNumberOfItemsToReceiveInSingleBatch); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Etag attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; if (duration.Elapsed > timeout) break; } log.Debug(() => { if (attachmentSinceLastEtag == 0) return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastAttachmentEtag); if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) return string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastAttachmentEtag, destination); var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastAttachmentEtag); }); scope.Record(new RavenJObject { {"StartEtag", startEtag.ToString()}, {"EndEtag", lastAttachmentEtag.ToString()}, {"Count", attachmentSinceLastEtag}, {"FilteredCount", filteredAttachmentsToReplicate.Count} }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } EnsureReplicationInformationInMetadata(x.Metadata, docDb); return new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", data} }; })); }); } catch (InvalidDataException e) { RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e)); scope.RecordError(e); log.ErrorException("Data is corrupted, could not proceed with replication", e); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return Tuple.Create(attachments, lastAttachmentEtag); }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { JsonDocumentsToReplicate documentsToReplicate; using (var scope = recorder.StartRecording("Get")) { documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, scope); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555 { using (scope.StartRecording("Notify")) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); scope.Record(new RavenJObject { { "LastDocEtag", documentsToReplicate.LastEtag.ToString() } }); } } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } } using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats)) { SourceReplicationInformation destinationsReplicationInformationForSource; using (var scope = stats.StartRecording("Destination")) { try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) return false; scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource)); } catch (Exception e) { scope.RecordError(e); log.WarnException("Failed to replicate to: " + destination, e); return false; } } bool? replicated = null; using (var scope = stats.StartRecording("Documents")) { switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } using (var scope = stats.StartRecording("Attachments")) { switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } docDb.WorkContext.MetricsCounters.GetReplicationDurationMetric(destination).Mark((long)stats.ElapsedTime.TotalMilliseconds); docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update((long)stats.ElapsedTime.TotalMilliseconds); return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0,1)); holder.Release(); } }
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { RavenJArray attachments = null; Etag lastAttachmentEtag = Etag.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List<AttachmentInformation> attachmentsToReplicate; List<AttachmentInformation> filteredAttachmentsToReplicate; var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; lastAttachmentEtag = startEtag; while (true) { attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Etag attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; } log.Debug(() => { if (attachmentSinceLastEtag == 0) return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastAttachmentEtag); if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) return string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastAttachmentEtag, destination); var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastAttachmentEtag); }); scope.Record(new RavenJObject { {"StartEtag", startEtag.ToString()}, {"EndEtag", lastAttachmentEtag.ToString()}, {"Count", attachmentSinceLastEtag}, {"FilteredCount", filteredAttachmentsToReplicate.Count} }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } return new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", data} }; })); }); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return Tuple.Create(attachments, lastAttachmentEtag); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(result.LastEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag); return false; } } return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); result.CountOfFilteredDocumentsWhichOriginFromDestination += docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); scope.Record(new RavenJObject { {"StartEtag", lastEtag.ToString()}, {"EndEtag", result.LastEtag.ToString()}, {"Count", docsSinceLastReplEtag}, {"FilteredCount", filteredDocsToReplicate.Count} }); result.LoadedDocs = filteredDocsToReplicate; result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { scope.RecordError(e); log.WarnException( "Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats)) { SourceReplicationInformation destinationsReplicationInformationForSource; using (var scope = stats.StartRecording("Destination")) { try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) return false; scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource)); } catch (Exception e) { scope.RecordError(e); log.WarnException("Failed to replicate to: " + destination, e); return false; } } bool? replicated = null; using (var scope = stats.StartRecording("Documents")) { switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } using (var scope = stats.StartRecording("Attachments")) { switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder()); Thread.VolatileWrite(ref holder.Value, 0); } }