private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder, out int replicatedDocuments)
{
replicatedDocuments = 0;
JsonDocumentsToReplicate documentsToReplicate = null;
var sp = Stopwatch.StartNew();
IDisposable removeBatch = null;
var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url,
x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner, string.Format("Replication for URL: {0}" ,destination.ConnectionStringOptions.DefaultDatabase) ));
prefetchingBehavior.AdditionalInfo = string.Format("For destination: {0}. Last replicated etag: {1}", destination.ConnectionStringOptions.Url, destinationsReplicationInformationForSource.LastDocumentEtag);
try
{
using (var scope = recorder.StartRecording("Get"))
{
documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
|| documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
|| documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
{
using (scope.StartRecording("Notify"))
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
scope.Record(new RavenJObject
{
{ "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
});
}
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
return null;
}
}
// if the db is idling in all respect except sending out replication, let us keep it that way.
docDb.WorkContext.UpdateFoundWork();
removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs);
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
// if we had an error sending to this endpoint, it might be because we are sending too much data, or because
// the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts
// and we'll be much more conservative with increasing the sizes
prefetchingBehavior.OutOfMemoryExceptionHappened();
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
}
finally
{
if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null)
{
prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed);
replicatedDocuments = documentsToReplicate.LoadedDocs.Count;
}
if (removeBatch != null)
removeBatch.Dispose();
}
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats))
{
SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource;
using (var scope = stats.StartRecording("Destination"))
{
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
log.Error("Failed to replicate documents to destination {0}, because was not able to receive last Etag", destination.ConnectionStringOptions.Url);
return false;
}
}
scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource));
if (destinationsReplicationInformationForSource.LastDocumentEtag == Etag.InvalidEtag && destinationsReplicationInformationForSource.LastAttachmentEtag == Etag.InvalidEtag)
{
DateTime lastSent;
// todo: move lastModifiedDate after the condition
var lastModifiedDate = destinationsReplicationInformationForSource.LastModified.HasValue ? destinationsReplicationInformationForSource.LastModified.Value.ToLocalTime() : DateTime.MinValue;
if (destinationAlertSent.TryGetValue(destination.ConnectionStringOptions.Url, out lastSent) && (SystemTime.UtcNow - lastSent).TotalMinutes < 1)
{
// todo: remove this log line
log.Debug(string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source));
return false;
}
docDb.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source),
Title = string.Format("Replication error. Multiple databases replicating at the same time with same DatabaseId ('{0}') detected.", docDb.TransactionalStorage.Id),
UniqueKey = "Replication to " + destination.ConnectionStringOptions.Url + " errored. Wrong DatabaseId: " + docDb.TransactionalStorage.Id
});
destinationAlertSent.AddOrUpdate(destination.ConnectionStringOptions.Url, SystemTime.UtcNow, (_, __) => SystemTime.UtcNow);
return false;
}
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
}
bool? replicated = null;
int replicatedDocuments;
using (var scope = stats.StartRecording("Documents"))
{
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope, out replicatedDocuments))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
using (var scope = stats.StartRecording("Attachments"))
{
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
var elapsedMicroseconds = (long)(stats.ElapsedTime.Ticks * SystemTime.MicroSecPerTick);
docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update(elapsedMicroseconds);
UpdateReplicationPerformance(destination, stats.Started, stats.ElapsedTime, replicatedDocuments);
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0, 1));
holder.Release();
}
}
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
Tuple<RavenJArray, Etag> tuple;
RavenJArray attachments;
using (var scope = recorder.StartRecording("Get"))
{
tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope);
attachments = tuple.Item1;
if (attachments == null || attachments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag)
{
SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2);
}
return null;
}
}
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination);
if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments:false);
return true;
}
}
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
RecordSuccess(destination.ConnectionStringOptions.Url,
lastReplicatedEtag: tuple.Item2, forDocuments: false);
return true;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
var duration = Stopwatch.StartNew();
var result = new JsonDocumentsToReplicate
{
LastEtag = Etag.Empty,
};
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;
docDb.TransactionalStorage.Batch(actions =>
{
var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested();
docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
{
log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, destinationId, info.TouchedEtag);
return false;
}
}
string reason;
return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) &&
prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments +=
docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
result.CountOfFilteredDocumentsWhichOriginFromDestination +=
docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
if (duration.Elapsed > timeout)
break;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", lastEtag.ToString()},
{"EndEtag", result.LastEtag.ToString()},
{"Count", docsSinceLastReplEtag},
{"FilteredCount", filteredDocsToReplicate.Count}
});
result.LoadedDocs = filteredDocsToReplicate;
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag);
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag);
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
EnsureReplicationInformationInMetadata(x.Metadata, docDb);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException(
"Could not get documents to replicate after: " +
destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
var duration = Stopwatch.StartNew();
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
lastAttachmentEtag = startEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag, maxNumberOfItemsToReceiveInSingleBatch);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
if (duration.Elapsed > timeout)
break;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastAttachmentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastAttachmentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastAttachmentEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", startEtag.ToString()},
{"EndEtag", lastAttachmentEtag.ToString()},
{"Count", attachmentSinceLastEtag},
{"FilteredCount", filteredAttachmentsToReplicate.Count}
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
EnsureReplicationInformationInMetadata(x.Metadata, docDb);
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (InvalidDataException e)
{
RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e));
scope.RecordError(e);
log.ErrorException("Data is corrupted, could not proceed with replication", e);
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
JsonDocumentsToReplicate documentsToReplicate;
using (var scope = recorder.StartRecording("Get"))
{
documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, scope);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
|| documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
|| documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
{
using (scope.StartRecording("Notify"))
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
scope.Record(new RavenJObject
{
{ "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
});
}
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
return null;
}
}
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats))
{
SourceReplicationInformation destinationsReplicationInformationForSource;
using (var scope = stats.StartRecording("Destination"))
{
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
return false;
scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource));
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
}
bool? replicated = null;
using (var scope = stats.StartRecording("Documents"))
{
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
using (var scope = stats.StartRecording("Attachments"))
{
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
docDb.WorkContext.MetricsCounters.GetReplicationDurationMetric(destination).Mark((long)stats.ElapsedTime.TotalMilliseconds);
docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update((long)stats.ElapsedTime.TotalMilliseconds);
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0,1));
holder.Release();
}
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
lastAttachmentEtag = startEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastAttachmentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastAttachmentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastAttachmentEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", startEtag.ToString()},
{"EndEtag", lastAttachmentEtag.ToString()},
{"Count", attachmentSinceLastEtag},
{"FilteredCount", filteredAttachmentsToReplicate.Count}
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
{
log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, destinationId, info.TouchedEtag);
return false;
}
}
return destination.FilterDocuments(destinationId, document.Key, document.Metadata) &&
prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments +=
docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
result.CountOfFilteredDocumentsWhichOriginFromDestination +=
docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", lastEtag.ToString()},
{"EndEtag", result.LastEtag.ToString()},
{"Count", docsSinceLastReplEtag},
{"FilteredCount", filteredDocsToReplicate.Count}
});
result.LoadedDocs = filteredDocsToReplicate;
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException(
"Could not get documents to replicate after: " +
destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats))
{
SourceReplicationInformation destinationsReplicationInformationForSource;
using (var scope = stats.StartRecording("Destination"))
{
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
return false;
scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource));
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
}
bool? replicated = null;
using (var scope = stats.StartRecording("Documents"))
{
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
using (var scope = stats.StartRecording("Attachments"))
{
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder());
Thread.VolatileWrite(ref holder.Value, 0);
}
}
