private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag, 100)
.Where(information => destination.FilterAttachments(information, destinationInstanceId))
.Select(x => new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", actions.Attachments.GetAttachment(x.Key).Data().ReadData() }
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(jsonAttachments);
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
private bool?ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var attachments = GetAttachments(destinationsReplicationInformationForSource, destination);
if (attachments == null || attachments.Length == 0)
{
return(null);
}
if (TryReplicationAttachments(destination, attachments) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationAttachments(destination, attachments)) // success on second fail
{
return(true);
}
}
IncrementFailureCount(destination);
return(false);
}
ResetFailureCount(destination);
return(true);
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId) // prevent replicating back to source
.Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Take(100)
.Select(x => new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", actions.Attachments.GetAttachment(x.Key).Data }
}));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(jsonAttachments);
}
private void OnGet(IHttpContext context, string src)
{
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
Guid serverInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote serve
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
Source = src,
ServerInstanceId = serverInstanceId
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
}
var currentEtag = context.Request.QueryString["currentEtag"];
log.Debug("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src,
sourceReplicationInformation.LastDocumentEtag, currentEtag);
context.WriteJson(sourceReplicationInformation);
}
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag)
.Where(destination.FilterAttachments)
// we don't replicate stuff that was created there
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId)
.Take(100)
.Select(x => new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", actions.Attachments.GetAttachment(x.Key).Data().ReadData() }
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(jsonAttachments);
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag, 100)
.Where(destination.FilterDocuments)
.Where(x => x.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var tuple = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
var jsonDocuments = tuple.Item1;
if (jsonDocuments == null || jsonDocuments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastDocumentEtag)
{
SetLastReplicatedEtagForDocuments(destination, lastDocEtag: tuple.Item2);
}
return(null);
}
string lastError;
if (TryReplicationDocuments(destination, jsonDocuments, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, jsonDocuments, out lastError)) // success on second fail
{
ResetFailureCount(destination.ConnectionStringOptions.Url, lastError);
return(true);
}
}
IncrementFailureCount(destination, lastError);
return(false);
}
ResetFailureCount(destination.ConnectionStringOptions.Url, lastError);
return(true);
}
private bool?ReplicateDocuments(RavenConnectionStringOptions destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var jsonDocuments = GetJsonDocuments(destinationsReplicationInformationForSource);
if (jsonDocuments == null || jsonDocuments.Length == 0)
{
return(null);
}
if (TryReplicationDocuments(destination, jsonDocuments) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transinet failure and trying again",
destination);
if (TryReplicationDocuments(destination, jsonDocuments)) // success on second faile
{
return(true);
}
}
IncrementFailureCount(destination);
return(false);
}
return(true);
}
private bool?ReplicateAttachments(string destination, SourceReplicationInformation sourceReplicationInformation)
{
var attachments = GetAttachments(sourceReplicationInformation.LastAttachmentEtag);
if (attachments == null || attachments.Count == 0)
{
return(null);
}
if (TryReplicationAttachments(destination, attachments) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.InfoFormat(
"This is the first failure for {0}, assuming transinet failure and trying again",
destination);
if (TryReplicationAttachments(destination, attachments))// success on second faile
{
return(true);
}
}
IncrementFailureCount(destination);
return(false);
}
return(true);
}
private void OnPut(IHttpContext context, string src)
{
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
Etag docEtag = null, attachmentEtag = null;
try
{
docEtag = Etag.Parse(context.Request.QueryString["docEtag"]);
}
catch
{
}
try
{
attachmentEtag = Etag.Parse(context.Request.QueryString["attachmentEtag"]);
}
catch
{
}
Guid serverInstanceId;
if (Guid.TryParse(context.Request.QueryString["dbid"], out serverInstanceId) == false)
{
serverInstanceId = Database.TransactionalStorage.Id;
}
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = serverInstanceId,
LastAttachmentEtag = attachmentEtag ?? Etag.Empty,
LastDocumentEtag = docEtag ?? Etag.Empty,
Source = src
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag;
sourceReplicationInformation.LastAttachmentEtag = attachmentEtag ?? sourceReplicationInformation.LastAttachmentEtag;
}
var etag = document == null ? Etag.Empty : document.Etag;
var metadata = document == null ? new RavenJObject() : document.Metadata;
var newDoc = RavenJObject.FromObject(sourceReplicationInformation);
log.Debug("Updating replication last etags from {0}: [doc: {1} attachment: {2}]", src,
sourceReplicationInformation.LastDocumentEtag,
sourceReplicationInformation.LastAttachmentEtag);
Database.Put(Constants.RavenReplicationSourcesBasePath + "/" + src, etag, newDoc, metadata, null);
}
}
private ReplicationTopologySourceNode HandleSource(SourceReplicationInformation source)
{
if (from.Contains(source.Source))
{
var state = CheckSourceConnectionState(source.Source);
switch (state)
{
case ReplicatonNodeState.Online:
return(ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId,
currentServerId, source.LastDocumentEtag, source.LastAttachmentEtag));
case ReplicatonNodeState.Offline:
return(ReplicationTopologySourceNode.Offline(source.Source, source.ServerInstanceId,
currentServerId, source.LastDocumentEtag, source.LastAttachmentEtag));
default:
throw new NotSupportedException(state.ToString());
}
}
string error;
ReplicationTopologyRootNode rootNode;
if (TryGetSchema(source.Source, new RavenConnectionStringOptions(), out rootNode, out error))
{
var node = ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId,
currentServerId, source.LastDocumentEtag, source.LastAttachmentEtag);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId,
currentServerId, source.LastDocumentEtag, source.LastAttachmentEtag);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
public override void Respond(IHttpContext context)
{
var src = context.Request.QueryString["from"];
var currentEtag = context.Request.QueryString["currentEtag"];
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
while (src.EndsWith("/"))
{
src = src.Substring(0, src.Length - 1); // remove last /, because that has special meaning for Raven
}
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Get(ReplicationConstants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = Database.TransactionalStorage.Id
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = Database.TransactionalStorage.Id;
}
log.Debug("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src,
sourceReplicationInformation.LastDocumentEtag, currentEtag);
context.WriteJson(sourceReplicationInformation);
}
}
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 ||
documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15)
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag);
return(null);
}
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return(true);
}
}
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return(false);
}
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return(true);
}
private List <ReplicationTopologySourceNode> HandleSources(IEnumerable <RavenJToken> sources, ReplicationTopologyRootNode root)
{
var nodes = new List <ReplicationTopologySourceNode>();
foreach (var sourceAsJson in sources.Cast <RavenJObject>())
{
SourceReplicationInformation source = null;
try
{
source = sourceAsJson.JsonDeserialization <SourceReplicationInformation>();
}
catch (Exception)
{
root.Errors.Add(string.Format("Could not deserialize source node."));
}
var node = HandleSource(source);
nodes.Add(node);
}
return(nodes);
}
public HttpResponseMessage ReplicationLastEtagGet()
{
string src;
string dbid;
var result = GetValuesForLastEtag(out src, out dbid);
if (result != null)
{
return(result);
}
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
var serverInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote serve
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
Source = src,
ServerInstanceId = serverInstanceId
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
}
var currentEtag = GetQueryStringValue("currentEtag");
Log.Debug(() => string.Format("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src, sourceReplicationInformation.LastDocumentEtag, currentEtag));
return(GetMessageWithObject(sourceReplicationInformation));
}
}
public HttpResponseMessage ReplicationLastEtagPut()
{
string src;
string dbid;
var result = GetValuesForLastEtag(out src, out dbid);
if (result != null)
{
return(result);
}
using (Database.DisableAllTriggersForCurrentThread())
{
var document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null);
SourceReplicationInformation sourceReplicationInformation;
Etag docEtag = null, attachmentEtag = null;
try
{
docEtag = Etag.Parse(GetQueryStringValue("docEtag"));
}
catch
{
}
try
{
attachmentEtag = Etag.Parse(GetQueryStringValue("attachmentEtag"));
}
catch
{
}
Guid serverInstanceId;
if (Guid.TryParse(dbid, out serverInstanceId) == false)
{
serverInstanceId = Database.TransactionalStorage.Id;
}
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = serverInstanceId,
LastAttachmentEtag = attachmentEtag ?? Etag.Empty,
LastDocumentEtag = docEtag ?? Etag.Empty,
Source = src
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag;
sourceReplicationInformation.LastAttachmentEtag = attachmentEtag ?? sourceReplicationInformation.LastAttachmentEtag;
}
var etag = document == null ? Etag.Empty : document.Etag;
var metadata = document == null ? new RavenJObject() : document.Metadata;
var newDoc = RavenJObject.FromObject(sourceReplicationInformation);
log.Debug("Updating replication last etags from {0}: [doc: {1} attachment: {2}]", src,
sourceReplicationInformation.LastDocumentEtag,
sourceReplicationInformation.LastAttachmentEtag);
Database.Documents.Put(Constants.RavenReplicationSourcesBasePath + "/" + src, etag, newDoc, metadata, null);
}
return(GetEmptyMessage());
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
Guid lastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(lastDocumentEtag, 100).ToList();
filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(document, destinationId)).ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
if (docsToReplicate.Count == 0 ||
filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", docsToReplicate);
lastDocumentEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("Nothing to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
jsonDocuments = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024 * 1024 * 10)
.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList();
filteredDocsToReplicate =
docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)).
ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag);
result.LastEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(result);
}
public HttpResponseMessage ReplicationLastEtagPut()
{
string src;
string dbid;
string collections;
var result = GetValuesForLastEtag(out src, out dbid, out collections);
if (result != null)
{
return(result);
}
using (Database.DisableAllTriggersForCurrentThread())
{
var key = Constants.RavenReplicationSourcesBasePath + "/" + dbid;
if (!String.IsNullOrEmpty(collections))
{
key += ("/" + collections);
}
var document = Database.Documents.Get(key);
SourceReplicationInformation sourceReplicationInformation;
Etag docEtag = null;
try
{
docEtag = Etag.Parse(GetQueryStringValue("docEtag"));
}
catch
{
}
Guid serverInstanceId = Guid.Parse(dbid);
if (document == null)
{
sourceReplicationInformation = new SourceReplicationInformation()
{
ServerInstanceId = serverInstanceId,
LastDocumentEtag = docEtag ?? Etag.Empty,
Source = src,
LastModified = SystemTime.UtcNow
};
}
else
{
sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
sourceReplicationInformation.ServerInstanceId = serverInstanceId;
sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag;
sourceReplicationInformation.LastModified = SystemTime.UtcNow;
}
var etag = document == null ? Etag.Empty : document.Etag;
var metadata = document == null ? new RavenJObject() : document.Metadata;
var newDoc = RavenJObject.FromObject(sourceReplicationInformation);
if (log.IsDebugEnabled)
{
log.Debug("Updating replication last etags from {0}: [doc: {1}]", src,
sourceReplicationInformation.LastDocumentEtag);
}
Database.Documents.Put(key, etag, newDoc, metadata, null);
}
return(GetEmptyMessage());
}
private Tuple <RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Guid lastAttachmentEtag = Guid.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List <AttachmentInformation> attachmentsToReplicate;
List <AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Guid attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
{
return(string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
{
return(string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return(new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", data }
});
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(Tuple.Create(attachments, lastAttachmentEtag));
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false)
{
return(false);
}
}
return(destination.FilterDocuments(destinationId, document.Key, document.Metadata));
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag.Value;
if (lastDoc.LastModified.HasValue)
{
result.LastLastModified = lastDoc.LastModified.Value;
}
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(result);
}
