private void OnGet(IHttpContext context, string src)
{
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
Guid serverInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote serve
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
Source = src
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
}
var currentEtag = context.Request.QueryString["currentEtag"];
log.Debug("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src,
sourceReplicationInformation.LastDocumentEtag, currentEtag);
context.WriteJson(sourceReplicationInformation);
}
}
private ReplicationTopologySourceNode HandleSource(SourceReplicationInformation source)
{
if (from.Contains(source.Source))
{
var state = CheckSourceConnectionState(source.Source);
switch (state)
{
case ReplicatonNodeState.Online:
return ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag);
case ReplicatonNodeState.Offline:
return ReplicationTopologySourceNode.Offline(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag);
default:
throw new NotSupportedException(state.ToString());
}
}
string error;
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(source.Source, new RavenConnectionStringOptions(), out rootNode, out error))
{
var node = ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return node;
}
var offline = ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag);
if (string.IsNullOrEmpty(error) == false)
offline.Errors.Add(error);
return offline;
}
public override void Respond(IHttpContext context)
{
var src = context.Request.QueryString["from"];
var currentEtag = context.Request.QueryString["currentEtag"];
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
while (src.EndsWith("/"))
src = src.Substring(0, src.Length - 1);// remove last /, because that has special meaning for Raven
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Get(ReplicationConstants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = Database.TransactionalStorage.Id
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = Database.TransactionalStorage.Id;
}
log.Debug("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src,
sourceReplicationInformation.LastDocumentEtag, currentEtag);
context.WriteJson(sourceReplicationInformation);
}
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 ||
documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15)
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag);
return null;
}
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError))// success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var tuple = GetAttachments(destinationsReplicationInformationForSource, destination);
var attachments = tuple.Item1;
if (attachments == null || attachments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag)
{
SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2);
}
return null;
}
string lastError;
if (TryReplicationAttachments(destination, attachments, out lastError) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationAttachments(destination, attachments, out lastError))// success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2);
return true;
}
}
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
RecordSuccess(destination.ConnectionStringOptions.Url,
lastReplicatedEtag: tuple.Item2);
return true;
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
Guid lastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(lastDocumentEtag, 100).ToList();
filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(document, destinationId)).ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
if (docsToReplicate.Count == 0 ||
filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", docsToReplicate);
lastDocumentEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("Nothing to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag);
if(docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag);
});
jsonDocuments = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return jsonDocuments;
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x=>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return jsonDocuments;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
{
log.Debug("Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag);
return false;
}
}
return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
result.CountOfFilteredDocumentsWhichOriginFromDestination +=
docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", lastEtag.ToString()},
{"EndEtag", result.LastEtag.ToString()},
{"Count", docsSinceLastReplEtag},
{"FilteredCount", filteredDocsToReplicate.Count}
});
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag);
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag);
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024*1024*10)
.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList();
filteredDocsToReplicate =
docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)).
ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.AllFilteredDocumentsAreSystemDocuments =
docsToReplicate.All(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag);
result.LastEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag);
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag)
.Where(destination.FilterAttachments)
// we don't replicate stuff that was created there
.Where(x=>x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId)
.Take(100)
.Select(x => new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", actions.Attachments.GetAttachment(x.Key).Data().ReadData()}
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return jsonAttachments;
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
.Where(destination.FilterDocuments)
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Select(x=>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return jsonDocuments;
}
private bool? ReplicateAttachments(string destination, SourceReplicationInformation sourceReplicationInformation)
{
var attachments = GetAttachments(sourceReplicationInformation.LastAttachmentEtag);
if (attachments == null || attachments.Count == 0)
return null;
if (TryReplicationAttachments(destination, attachments) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.InfoFormat(
"This is the first failure for {0}, assuming transinet failure and trying again",
destination);
if (TryReplicationAttachments(destination, attachments))// success on second faile
return true;
}
IncrementFailureCount(destination);
return false;
}
return true;
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag,100)
.Where(information => destination.FilterAttachments(information, destinationInstanceId))
.Select(x => new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", actions.Attachments.GetAttachment(x.Key).Data().ReadData()}
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return jsonAttachments;
}
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
Tuple<RavenJArray, Etag> tuple;
RavenJArray attachments;
using (var scope = recorder.StartRecording("Get"))
{
tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope);
attachments = tuple.Item1;
if (attachments == null || attachments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag)
{
SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2);
}
return null;
}
}
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination);
if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2);
return true;
}
}
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
RecordSuccess(destination.ConnectionStringOptions.Url,
lastReplicatedEtag: tuple.Item2);
return true;
}
private Tuple<RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Guid lastAttachmentEtag = Guid.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = actions.Attachments.GetAttachmentsAfter(lastAttachmentEtag, 100, 1024 * 1024 * 10)
.Concat(actions.Lists.Read(Constants.RavenReplicationAttachmentsTombstones, lastAttachmentEtag, 100)
.Select(x => new AttachmentInformation
{
Key = x.Key,
Etag = x.Etag,
Metadata = x.Data,
Size = 0,
}))
.OrderBy(x => x.Etag)
.ToList();
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Guid attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag);
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
JsonDocumentsToReplicate documentsToReplicate;
using (var scope = recorder.StartRecording("Get"))
{
documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, scope);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
|| documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
|| documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
{
using (scope.StartRecording("Notify"))
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
scope.Record(new RavenJObject
{
{ "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
});
}
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
return null;
}
}
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag();
var lastEtag = etagSynchronizer.CalculateSynchronizationEtag(
synchronizationEtag,
destinationsReplicationInformationForSource.LastDocumentEtag);
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.PreTouchEtag.CompareTo(result.LastEtag) >= 0)
return false;
}
return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
lastAttachmentEtag = startEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastAttachmentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastAttachmentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastAttachmentEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", startEtag.ToString()},
{"EndEtag", lastAttachmentEtag.ToString()},
{"Count", attachmentSinceLastEtag},
{"FilteredCount", filteredAttachmentsToReplicate.Count}
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (InvalidDataException e)
{
RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e));
scope.RecordError(e);
log.ErrorException("Data is corrupted, could not proceed with replication", e);
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
public HttpResponseMessage ReplicationLastEtagGet()
{
string src;
string dbid;
var result = GetValuesForLastEtag(out src, out dbid);
if (result != null)
return result;
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
var serverInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote serve
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
Source = src,
ServerInstanceId = serverInstanceId
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
}
var currentEtag = GetQueryStringValue("currentEtag");
Log.Debug(() => string.Format("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src, sourceReplicationInformation.LastDocumentEtag, currentEtag));
return GetMessageWithObject(sourceReplicationInformation);
}
}
private void OnPut(IHttpContext context, string src)
{
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
Etag docEtag = null, attachmentEtag = null;
try
{
docEtag = Etag.Parse(context.Request.QueryString["docEtag"]);
}
catch
{
}
try
{
attachmentEtag = Etag.Parse(context.Request.QueryString["attachmentEtag"]);
}
catch
{
}
Guid serverInstanceId;
if (Guid.TryParse(context.Request.QueryString["dbid"], out serverInstanceId) == false)
serverInstanceId = Database.TransactionalStorage.Id;
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = serverInstanceId,
LastAttachmentEtag = attachmentEtag ?? Etag.Empty,
LastDocumentEtag = docEtag ?? Etag.Empty,
Source = src
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag;
sourceReplicationInformation.LastAttachmentEtag = attachmentEtag ?? sourceReplicationInformation.LastAttachmentEtag;
}
var etag = document == null ? Etag.Empty : document.Etag;
var metadata = document == null ? new RavenJObject() : document.Metadata;
var newDoc = RavenJObject.FromObject(sourceReplicationInformation);
log.Debug("Updating replication last etags from {0}: [doc: {1} attachment: {2}]", src,
sourceReplicationInformation.LastDocumentEtag,
sourceReplicationInformation.LastAttachmentEtag);
Database.Put(Constants.RavenReplicationSourcesBasePath + "/" + src, etag, newDoc, metadata, null);
}
}
public HttpResponseMessage ReplicationLastEtagPut()
{
string src;
string dbid;
var result = GetValuesForLastEtag(out src, out dbid);
if (result != null)
return result;
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
Etag docEtag = null, attachmentEtag = null;
try
{
docEtag = Etag.Parse(GetQueryStringValue("docEtag"));
}
catch
{
}
try
{
attachmentEtag = Etag.Parse(GetQueryStringValue("attachmentEtag"));
}
catch
{
}
Guid serverInstanceId;
if (Guid.TryParse(dbid, out serverInstanceId) == false)
serverInstanceId = Database.TransactionalStorage.Id;
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = serverInstanceId,
LastAttachmentEtag = attachmentEtag ?? Etag.Empty,
LastDocumentEtag = docEtag ?? Etag.Empty,
Source = src
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag;
sourceReplicationInformation.LastAttachmentEtag = attachmentEtag ?? sourceReplicationInformation.LastAttachmentEtag;
}
var etag = document == null ? Etag.Empty : document.Etag;
var metadata = document == null ? new RavenJObject() : document.Metadata;
var newDoc = RavenJObject.FromObject(sourceReplicationInformation);
log.Debug("Updating replication last etags from {0}: [doc: {1} attachment: {2}]", src,
sourceReplicationInformation.LastDocumentEtag,
sourceReplicationInformation.LastAttachmentEtag);
Database.Documents.Put(Constants.RavenReplicationSourcesBasePath + "/" + src, etag, newDoc, metadata, null);
}
return GetEmptyMessage();
}
private bool? ReplicateDocuments(RavenConnectionStringOptions destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var jsonDocuments = GetJsonDocuments(destinationsReplicationInformationForSource);
if (jsonDocuments == null || jsonDocuments.Length == 0)
return null;
if (TryReplicationDocuments(destination, jsonDocuments) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transinet failure and trying again",
destination);
if (TryReplicationDocuments(destination, jsonDocuments))// success on second faile
return true;
}
IncrementFailureCount(destination);
return false;
}
return true;
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
JsonDocumentsToReplicate documentsToReplicate = null;
Stopwatch sp = Stopwatch.StartNew();
IDisposable removeBatch = null;
var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url,
x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner));
try
{
using (var scope = recorder.StartRecording("Get"))
{
documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
|| documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
|| documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
{
using (scope.StartRecording("Notify"))
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
scope.Record(new RavenJObject
{
{ "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
});
}
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
return null;
}
}
// if the db is idling in all respect except sending out replication, let us keep it that way.
docDb.WorkContext.UpdateFoundWork();
removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs);
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
// if we had an error sending to this endpoint, it might be because we are sending too much data, or because
// the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts
// and we'll be much more conservative with increasing the sizes
prefetchingBehavior.OutOfMemoryExceptionHappened();
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
}
finally
{
if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null)
prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed);
if(removeBatch != null)
removeBatch.Dispose();
}
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId) // prevent replicating back to source
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Take(100)
.Select(x => new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", actions.Attachments.GetAttachment(x.Key).Data}
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return jsonAttachments;
}
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var attachments = GetAttachments(destinationsReplicationInformationForSource, destination);
if (attachments == null || attachments.Length == 0)
return null;
if (TryReplicationAttachments(destination, attachments) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationAttachments(destination, attachments))// success on second fail
return true;
}
IncrementFailureCount(destination);
return false;
}
ResetFailureCount(destination);
return true;
}
