private SourceReplicationInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl());
request.Credentials = destination.ConnectionStringOptions.Credentials ?? CredentialCache.DefaultNetworkCredentials;
request.PreAuthenticate = true;
request.UseDefaultCredentials = true;
request.Timeout = replicationRequestTimeoutInMs;
using (var response = request.GetResponse())
using (var stream = response.GetResponseStream())
{
var etagFromServer = (SourceReplicationInformation) new JsonSerializer().Deserialize(new StreamReader(stream), typeof(SourceReplicationInformation));
return(etagFromServer);
}
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
{
log.WarnException("Replication is not enabled on: " + destination, e);
}
else
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
return(null);
}
private bool IsNotFailing(ReplicationStrategy dest, int currentReplicationAttempts)
{
var jsonDocument = docDb.Get(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(dest.ConnectionStringOptions.Url), null);
if (jsonDocument == null)
{
return(true);
}
var failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>();
if (failureInformation.FailureCount > 1000)
{
var shouldReplicateTo = currentReplicationAttempts % 10 == 0;
log.Debug("Failure count for {0} is {1}, skipping replication: {2}",
dest, failureInformation.FailureCount, shouldReplicateTo == false);
return(shouldReplicateTo);
}
if (failureInformation.FailureCount > 100)
{
var shouldReplicateTo = currentReplicationAttempts % 5 == 0;
log.Debug("Failure count for {0} is {1}, skipping replication: {2}",
dest, failureInformation.FailureCount, shouldReplicateTo == false);
return(shouldReplicateTo);
}
if (failureInformation.FailureCount > 10)
{
var shouldReplicateTo = currentReplicationAttempts % 2 == 0;
log.Debug("Failure count for {0} is {1}, skipping replication: {2}",
dest, failureInformation.FailureCount, shouldReplicateTo == false);
return(shouldReplicateTo);
}
return(true);
}
private SourceReplicationInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination)
{
try
{
var currentEtag = Guid.Empty;
docDb.TransactionalStorage.Batch(accessor => currentEtag = accessor.Staleness.GetMostRecentDocumentEtag());
var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() +
"¤tEtag=" + currentEtag + "&dbid=" + docDb.TransactionalStorage.Id;
var request = httpRavenRequestFactory.Create(url, "GET", destination.ConnectionStringOptions);
return(request.ExecuteRequest <SourceReplicationInformation>());
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
{
log.WarnException("Replication is not enabled on: " + destination, e);
}
else
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
RecordFailure(destination.ConnectionStringOptions.Url, e.Message);
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
RecordFailure(destination.ConnectionStringOptions.Url, e.Message);
}
return(null);
}
private SourceReplicationInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl();
var request = httpRavenRequestFactory.Create(url, "GET", destination.ConnectionStringOptions);
return(request.ExecuteRequest <SourceReplicationInformation>());
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
{
log.WarnException("Replication is not enabled on: " + destination, e);
}
else
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
return(null);
}
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var jsonDocuments = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
if (jsonDocuments == null || jsonDocuments.Length == 0)
{
return(null);
}
if (TryReplicationDocuments(destination, jsonDocuments) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, jsonDocuments)) // success on second fail
{
return(true);
}
}
IncrementFailureCount(destination);
return(false);
}
ResetFailureCount(destination);
return(true);
}
private void SetLastReplicatedEtagForDocuments(ReplicationStrategy destination, Guid?lastDocEtag = null, Guid?lastAttachmentEtag = null)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() +
"&dbid=" + docDb.TransactionalStorage.Id;
if (lastDocEtag != null)
{
url += "&docEtag=" + lastDocEtag.Value;
}
if (lastAttachmentEtag != null)
{
url += "&attachmentEtag=" + lastAttachmentEtag.Value;
}
var request = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions);
request.ExecuteRequest();
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
{
log.WarnException("Replication is not enabled on: " + destination, e);
}
else
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
}
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl();
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions);
request.Write(jsonDocuments);
request.ExecuteRequest();
log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
return(true);
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStream()))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
return(false);
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
return(false);
}
}
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var tuple = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
var jsonDocuments = tuple.Item1;
if (jsonDocuments == null || jsonDocuments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastDocumentEtag)
{
SetLastReplicatedEtagForDocuments(destination, lastDocEtag: tuple.Item2);
}
return(null);
}
string lastError;
if (TryReplicationDocuments(destination, jsonDocuments, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, jsonDocuments, out lastError)) // success on second fail
{
ResetFailureCount(destination.ConnectionStringOptions.Url, lastError);
return(true);
}
}
IncrementFailureCount(destination, lastError);
return(false);
}
ResetFailureCount(destination.ConnectionStringOptions.Url, lastError);
return(true);
}
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition)
{
var url = string.Format("{0}/replication/side-by-side?{1}", destination.ConnectionStringOptions.Url, GetDebugInformation());
IndexReplaceDocument indexReplaceDocument;
try
{
indexReplaceDocument = database.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name).DataAsJson.JsonDeserialization <IndexReplaceDocument>();
}
catch (Exception e)
{
Log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e);
return;
}
var sideBySideReplicationInfo = new SideBySideReplicationInfo
{
Index = indexDefinition,
SideBySideIndex = sideBySideIndexDefinition,
OriginDatabaseId = destination.CurrentDatabaseId,
IndexReplaceDocument = indexReplaceDocument
};
var replicationRequest = httpRavenRequestFactory.Create(url, HttpMethod.Post, destination.ConnectionStringOptions, replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo));
replicationRequest.ExecuteRequest();
}
private bool IsFirstFailue(ReplicationStrategy destination)
{
FailureCount value;
if (replicationFailureStats.TryGetValue(destination.ConnectionStringOptions.Url, out value))
{
return(value.Count == 0);
}
return(false);
}
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString()
};
return(CreateReplicationStrategyFromDocument(x, replicationStrategy));
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
{
return(false);
}
using (docDb.DisableAllTriggersForCurrentThread())
{
SourceReplicationInformation destinationsReplicationInformationForSource;
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
return(false);
}
}
catch (Exception e)
{
log.WarnException("Failed to replicate to: " + destination, e);
return(false);
}
bool?replicated = null;
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource))
{
case true:
replicated = true;
break;
case false:
return(false);
}
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource))
{
case true:
replicated = true;
break;
case false:
return(false);
}
return(replicated ?? false);
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder());
Thread.VolatileWrite(ref holder.Value, 0);
}
}
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments, out string errorMessage)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" +
UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id;
var sp = Stopwatch.StartNew();
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions);
request.WebRequest.Headers.Add("Attachment-Ids", string.Join(", ", jsonAttachments.Select(x => x.Value <string>("@id"))));
request.WriteBson(jsonAttachments);
request.ExecuteRequest();
log.Info("Replicated {0} attachments to {1} in {2:#,#;;0} ms", jsonAttachments.Length, destination, sp.ElapsedMilliseconds);
errorMessage = "";
return(true);
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression()))
{
var error = streamReader.ReadToEnd();
try
{
var ravenJObject = RavenJObject.Parse(error);
log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value <string>("Error"), e);
errorMessage = error;
return(false);
}
catch (Exception)
{
}
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
errorMessage = error;
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
errorMessage = e.Message;
}
return(false);
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
errorMessage = e.Message;
return(false);
}
}
private bool IsFirstFailue(ReplicationStrategy destination)
{
var jsonDocument = docDb.Get(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null);
if (jsonDocument == null)
{
return(true);
}
var failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>();
return(failureInformation.FailureCount == 0);
}
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments, out string lastError)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl()
+ "&dbid=" + docDb.TransactionalStorage.Id;
var sp = Stopwatch.StartNew();
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions);
request.Write(jsonDocuments);
request.ExecuteRequest();
log.Info("Replicated {0} documents to {1} in {2:#,#;;0} ms", jsonDocuments.Length, destination, sp.ElapsedMilliseconds);
lastError = "";
return(true);
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
Stream responseStream = response.GetResponseStream();
if (responseStream != null)
{
using (var streamReader = new StreamReader(responseStream))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
lastError = e.Message;
return(false);
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
lastError = e.Message;
return(false);
}
}
private void IncrementFailureCount(ReplicationStrategy destination)
{
var jsonDocument = docDb.Get(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null);
var failureInformation = new DestinationFailureInformation {
Destination = destination.ConnectionStringOptions.Url
};
if (jsonDocument != null)
{
failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>();
}
failureInformation.FailureCount += 1;
docDb.Put(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null,
RavenJObject.FromObject(failureInformation), new RavenJObject(), null);
}
private void ReplicateIndexesMultiPut(ReplicationStrategy destination, List <Tuple <IndexDefinition, IndexingPriority> > candidatesForReplication)
{
var indexesToAdd = candidatesForReplication
.Select(x => new IndexToAdd {
Definition = x.Item1, Name = x.Item1.Name, Priority = x.Item2
})
.ToArray();
var serializedIndexDefinitions = RavenJToken.FromObject(indexesToAdd);
var url = string.Format("{0}/indexes?{1}", destination.ConnectionStringOptions.Url, GetDebugInformation());
var replicationRequest = httpRavenRequestFactory.Create(url, HttpMethods.Put, destination.ConnectionStringOptions, replication.GetRequestBuffering(destination));
replicationRequest.Write(serializedIndexDefinitions);
replicationRequest.ExecuteRequest();
}
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var request = (HttpWebRequest)WebRequest.Create(destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl());
request.UseDefaultCredentials = true;
request.PreAuthenticate = true;
request.ContentType = "application/json; charset=utf-8";
request.Credentials = destination.ConnectionStringOptions.Credentials ?? CredentialCache.DefaultNetworkCredentials;
request.Method = "POST";
using (var stream = request.GetRequestStream())
using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
{
jsonDocuments.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
stream.Flush();
}
using (request.GetResponse())
{
log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
}
return(true);
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStream()))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
return(false);
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
return(false);
}
}
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" + UrlEncodedServerUrl();
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions);
request.WriteBson(jsonAttachments);
request.ExecuteRequest();
log.Info("Replicated {0} attachments to {1}", jsonAttachments.Length, destination);
return(true);
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression()))
{
var error = streamReader.ReadToEnd();
try
{
var ravenJObject = RavenJObject.Parse(error);
log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value <string>("Error"), e);
return(false);
}
catch (Exception)
{
}
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
return(false);
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
return(false);
}
}
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition)
{
try
{
var clonedTransformer = definition.Clone();
clonedTransformer.TransfomerId = 0;
var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInformation();
var replicationRequest = httpRavenRequestFactory.Create(url, HttpMethod.Put, destination.ConnectionStringOptions, replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(clonedTransformer));
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
replication.HandleRequestBufferingErrors(e, destination);
Log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e);
}
}
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 ||
documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15)
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag);
return(null);
}
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return(true);
}
}
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return(false);
}
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return(true);
}
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString()
};
if (string.IsNullOrEmpty(x.ConnectionStringName))
{
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = x.Url
};
return(replicationStrategy);
}
var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
{
throw new InvalidOperationException("Could not figure out what the replication URL is");
}
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
if (options.Url.EndsWith("/") == false)
{
options.Url += "/";
}
options.Url += "databases/" + options.DefaultDatabase;
}
replicationStrategy.ConnectionStringOptions = options;
return(replicationStrategy);
}
private void IncrementFailureCount(ReplicationStrategy destination, string lastError)
{
var failureCount = replicationFailureStats.GetOrAdd(destination.ConnectionStringOptions.Url);
Interlocked.Increment(ref failureCount.Count);
failureCount.Timestamp = SystemTime.UtcNow;
if (string.IsNullOrWhiteSpace(lastError) == false)
{
failureCount.LastError = lastError;
}
var jsonDocument = docDb.Get(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination.ConnectionStringOptions.Url), null);
var failureInformation = new DestinationFailureInformation {
Destination = destination.ConnectionStringOptions.Url
};
if (jsonDocument != null)
{
failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>();
}
failureInformation.FailureCount += 1;
docDb.Put(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination.ConnectionStringOptions.Url), null,
RavenJObject.FromObject(failureInformation), new RavenJObject(), null);
}
private bool IsNotFailing(ReplicationStrategy dest, int currentReplicationAttempts)
{
var jsonDocument = docDb.Documents.Get(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(dest.ConnectionStringOptions.Url), null);
if (jsonDocument == null)
return true;
var failureInformation = jsonDocument.DataAsJson.JsonDeserialization<DestinationFailureInformation>();
if (failureInformation.FailureCount > 1000)
{
var shouldReplicateTo = currentReplicationAttempts % 10 == 0;
log.Debug("Failure count for {0} is {1}, skipping replication: {2}",
dest, failureInformation.FailureCount, shouldReplicateTo == false);
return shouldReplicateTo;
}
if (failureInformation.FailureCount > 100)
{
var shouldReplicateTo = currentReplicationAttempts % 5 == 0;
log.Debug("Failure count for {0} is {1}, skipping replication: {2}",
dest, failureInformation.FailureCount, shouldReplicateTo == false);
return shouldReplicateTo;
}
if (failureInformation.FailureCount > 10)
{
var shouldReplicateTo = currentReplicationAttempts % 2 == 0;
log.Debug("Failure count for {0} is {1}, skipping replication: {2}",
dest, failureInformation.FailureCount, shouldReplicateTo == false);
return shouldReplicateTo;
}
return true;
}
private static ReplicationStrategy CreateReplicationStrategyFromDocument(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var url = x.Url;
if (string.IsNullOrEmpty(x.Database) == false)
{
url = url + "/databases/" + x.Database;
}
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = url,
ApiKey = x.ApiKey,
AuthenticationScheme = x.AuthenticationScheme
};
if (string.IsNullOrEmpty(x.Username) == false)
{
replicationStrategy.ConnectionStringOptions.Credentials = string.IsNullOrEmpty(x.Domain)
? new NetworkCredential(x.Username, x.Password)
: new NetworkCredential(x.Username, x.Password, x.Domain);
}
return replicationStrategy;
}
internal SourceReplicationInformationWithBatchInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination)
{
try
{
Etag currentEtag = Etag.Empty;
docDb.TransactionalStorage.Batch(accessor => currentEtag = accessor.Staleness.GetMostRecentDocumentEtag());
var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() +
"¤tEtag=" + currentEtag + "&dbid=" + docDb.TransactionalStorage.Id;
var request = httpRavenRequestFactory.Create(url, "GET", destination.ConnectionStringOptions);
var lastReplicatedEtagFrom = request.ExecuteRequest<SourceReplicationInformationWithBatchInformation>();
log.Debug("Received last replicated document Etag {0} from server {1}", lastReplicatedEtagFrom.LastDocumentEtag, destination.ConnectionStringOptions.Url);
return lastReplicatedEtagFrom;
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
log.WarnException("Replication is not enabled on: " + destination, e);
else
log.WarnException("Failed to contact replication destination: " + destination, e);
RecordFailure(destination.ConnectionStringOptions.Url, e.Message);
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
RecordFailure(destination.ConnectionStringOptions.Url, e.Message);
}
return null;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
var duration = Stopwatch.StartNew();
var result = new JsonDocumentsToReplicate
{
LastEtag = Etag.Empty,
};
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;
docDb.TransactionalStorage.Batch(actions =>
{
var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested();
docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
{
log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, destinationId, info.TouchedEtag);
return false;
}
}
string reason;
return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) &&
prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments +=
docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
result.CountOfFilteredDocumentsWhichOriginFromDestination +=
docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
if (duration.Elapsed > timeout)
break;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", lastEtag.ToString()},
{"EndEtag", result.LastEtag.ToString()},
{"Count", docsSinceLastReplEtag},
{"FilteredCount", filteredDocsToReplicate.Count}
});
result.LoadedDocs = filteredDocsToReplicate;
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag);
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag);
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
EnsureReplicationInformationInMetadata(x.Metadata, docDb);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException(
"Could not get documents to replicate after: " +
destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private void HandleRequestBufferingErrors(Exception e, ReplicationStrategy destination)
{
if (destination.ConnectionStringOptions.Credentials != null && string.Equals(e.Message, "This request requires buffering data to succeed.", StringComparison.OrdinalIgnoreCase))
destinationForceBuffering.AddOrUpdate(destination.ConnectionStringOptions.Url, true, (s, b) => true);
}
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition)
{
try
{
var clonedTransformer = definition.Clone();
clonedTransformer.TransfomerId = 0;
var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInfomration();
var replicationRequest = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions, GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(clonedTransformer));
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
HandleRequestBufferingErrors(e, destination);
log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e);
}
}
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments, out string errorMessage)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" +
UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id;
var sp = Stopwatch.StartNew();
using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url))
{
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));
request.WriteBson(jsonAttachments);
request.ExecuteRequest(docDb.WorkContext.CancellationToken);
log.Info("Replicated {0} attachments to {1} in {2:#,#;;0} ms", jsonAttachments.Length, destination, sp.ElapsedMilliseconds);
errorMessage = "";
return true;
}
}
catch (WebException e)
{
HandleRequestBufferingErrors(e, destination);
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression()))
{
var error = streamReader.ReadToEnd();
try
{
var ravenJObject = RavenJObject.Parse(error);
log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value<string>("Error"), e);
errorMessage = error;
return false;
}
catch (Exception)
{
}
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
errorMessage = error;
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
errorMessage = e.Message;
}
return false;
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
errorMessage = e.Message;
return false;
}
}
private Tuple <RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Guid lastAttachmentEtag = Guid.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List <AttachmentInformation> attachmentsToReplicate;
List <AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Guid attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
{
return(string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
{
return(string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return(new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", data }
});
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(Tuple.Create(attachments, lastAttachmentEtag));
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false)
{
return(false);
}
}
return(destination.FilterDocuments(destinationId, document.Key, document.Metadata));
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag.Value;
if (lastDoc.LastModified.HasValue)
{
result.LastLastModified = lastDoc.LastModified.Value;
}
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(result);
}
private void ResetFailureCount(ReplicationStrategy destination)
{
docDb.Delete(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null,
null);
}
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonDocuments = null;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
Guid lastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(lastDocumentEtag, 100).ToList();
filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(document, destinationId)).ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
if (docsToReplicate.Count == 0 ||
filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", docsToReplicate);
lastDocumentEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("Nothing to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
jsonDocuments = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(jsonDocuments);
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag, 100)
.Where(information => destination.FilterAttachments(information, destinationInstanceId))
.Select(x => new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", actions.Attachments.GetAttachment(x.Key).Data().ReadData() }
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(jsonAttachments);
}
private void UpdateReplicationPerformance(ReplicationStrategy destination, DateTime startTime, TimeSpan elapsed, int batchSize)
{
if (batchSize > 0)
{
var queue = docDb.WorkContext.MetricsCounters.GetReplicationPerformanceStats(destination);
queue.Enqueue(new ReplicationPerformanceStats
{
Duration = elapsed,
Started = startTime,
BatchSize = batchSize
});
while (queue.Count() > 25)
{
ReplicationPerformanceStats _;
queue.TryDequeue(out _);
}
}
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder, out int replicatedDocuments)
{
replicatedDocuments = 0;
JsonDocumentsToReplicate documentsToReplicate = null;
var sp = Stopwatch.StartNew();
IDisposable removeBatch = null;
var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url,
x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner, string.Format("Replication for URL: {0}" ,destination.ConnectionStringOptions.DefaultDatabase) ));
prefetchingBehavior.AdditionalInfo = string.Format("For destination: {0}. Last replicated etag: {1}", destination.ConnectionStringOptions.Url, destinationsReplicationInformationForSource.LastDocumentEtag);
try
{
using (var scope = recorder.StartRecording("Get"))
{
documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
|| documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
|| documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
{
using (scope.StartRecording("Notify"))
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
scope.Record(new RavenJObject
{
{ "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
});
}
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
return null;
}
}
// if the db is idling in all respect except sending out replication, let us keep it that way.
docDb.WorkContext.UpdateFoundWork();
removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs);
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
// if we had an error sending to this endpoint, it might be because we are sending too much data, or because
// the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts
// and we'll be much more conservative with increasing the sizes
prefetchingBehavior.OutOfMemoryExceptionHappened();
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
}
finally
{
if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null)
{
prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed);
replicatedDocuments = documentsToReplicate.LoadedDocs.Count;
}
if (removeBatch != null)
removeBatch.Dispose();
}
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments, out string lastError)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl()
+ "&dbid=" + docDb.TransactionalStorage.Id +
"&count=" + jsonDocuments.Length;
var sp = Stopwatch.StartNew();
using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url))
{
var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));
request.Write(jsonDocuments);
request.ExecuteRequest(docDb.WorkContext.CancellationToken);
log.Info("Replicated {0} documents to {1} in {2:#,#;;0} ms", jsonDocuments.Length, destination, sp.ElapsedMilliseconds);
lastError = "";
return true;
}
}
catch (WebException e)
{
HandleRequestBufferingErrors(e, destination);
var response = e.Response as HttpWebResponse;
if (response != null)
{
var responseStream = response.GetResponseStream();
if (responseStream != null)
{
using (var streamReader = new StreamReader(responseStream))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
lastError = e.Message;
return false;
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
lastError = e.Message;
return false;
}
}
private static ReplicationStrategy CreateReplicationStrategyFromDocument(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var url = x.Url;
if (string.IsNullOrEmpty(x.Database) == false)
{
url = url + "/databases/" + x.Database;
}
replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions
{
Url = url,
ApiKey = x.ApiKey,
};
if (string.IsNullOrEmpty(x.Username) == false)
{
replicationStrategy.ConnectionStringOptions.Credentials = string.IsNullOrEmpty(x.Domain)
? new NetworkCredential(x.Username, x.Password)
: new NetworkCredential(x.Username, x.Password, x.Domain);
}
return(replicationStrategy);
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats))
{
SourceReplicationInformation destinationsReplicationInformationForSource;
using (var scope = stats.StartRecording("Destination"))
{
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
return false;
scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource));
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
}
bool? replicated = null;
using (var scope = stats.StartRecording("Documents"))
{
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
using (var scope = stats.StartRecording("Attachments"))
{
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder());
Thread.VolatileWrite(ref holder.Value, 0);
}
}
private void ReplicateSingleIndex(ReplicationStrategy destination, IndexDefinition definition)
{
try
{
var url = string.Format("{0}/indexes/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(definition.Name), GetDebugInfomration());
var replicationRequest = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions, GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(definition));
replicationRequest.ExecuteRequest();
}
catch (Exception e)
{
HandleRequestBufferingErrors(e, destination);
log.WarnException("Could not replicate index " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e);
}
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
{
SourceReplicationInformation destinationsReplicationInformationForSource;
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
return false;
}
catch (Exception e)
{
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
bool? replicated = null;
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource))
{
case true:
replicated = true;
break;
case false:
return false;
}
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource))
{
case true:
replicated = true;
break;
case false:
return false;
}
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder());
Thread.VolatileWrite(ref holder.Value, 0);
}
}
private void ReplicateTransformerDeletionIfNeeded(List<JsonDocument> transformerTombstones,
ReplicationStrategy destination,
Dictionary<string, int> replicatedTransformerTombstones)
{
if (transformerTombstones.Count == 0)
return;
foreach (var tombstone in transformerTombstones)
{
try
{
int value;
if (docDb.Transformers.GetTransformerDefinition(tombstone.Key) != null) //if in the meantime the transformer was recreated under the same name
{
replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value);
replicatedTransformerTombstones[tombstone.Key] = value + 1;
continue;
}
var url = string.Format("{0}/transformers/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInfomration());
var replicationRequest = httpRavenRequestFactory.Create(url, "DELETE", destination.ConnectionStringOptions, GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(emptyRequestBody));
replicationRequest.ExecuteRequest();
log.Info("Replicated transformer deletion (transformer name = {0})", tombstone.Key);
replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value);
replicatedTransformerTombstones[tombstone.Key] = value + 1;
}
catch (Exception e)
{
HandleRequestBufferingErrors(e, destination);
log.ErrorException(string.Format("Failed to replicate transformer deletion (transformer name = {0})", tombstone.Key), e);
}
}
}
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var tuple = GetAttachments(destinationsReplicationInformationForSource, destination);
var attachments = tuple.Item1;
if (attachments == null || attachments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag)
{
SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2);
}
return null;
}
string lastError;
if (TryReplicationAttachments(destination, attachments, out lastError) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationAttachments(destination, attachments, out lastError))// success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2);
return true;
}
}
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
RecordSuccess(destination.ConnectionStringOptions.Url,
lastReplicatedEtag: tuple.Item2);
return true;
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
var duration = Stopwatch.StartNew();
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
lastAttachmentEtag = startEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag, maxNumberOfItemsToReceiveInSingleBatch);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
if (duration.Elapsed > timeout)
break;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastAttachmentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastAttachmentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastAttachmentEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", startEtag.ToString()},
{"EndEtag", lastAttachmentEtag.ToString()},
{"Count", attachmentSinceLastEtag},
{"FilteredCount", filteredAttachmentsToReplicate.Count}
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
EnsureReplicationInformationInMetadata(x.Metadata, docDb);
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (InvalidDataException e)
{
RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e));
scope.RecordError(e);
log.ErrorException("Data is corrupted, could not proceed with replication", e);
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource)
{
var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 ||
documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15)
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag);
return null;
}
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError))// success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
RecordSuccess(destination.ConnectionStringOptions.Url,
documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
public static ReplicationStrategy GetConnectionOptions(ReplicationDestination x, DocumentDatabase database)
{
var replicationStrategy = new ReplicationStrategy
{
ReplicationOptionsBehavior = x.TransitiveReplicationBehavior,
CurrentDatabaseId = database.TransactionalStorage.Id.ToString()
};
return CreateReplicationStrategyFromDocument(x, replicationStrategy);
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag();
var lastEtag = etagSynchronizer.CalculateSynchronizationEtag(
synchronizationEtag,
destinationsReplicationInformationForSource.LastDocumentEtag);
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.PreTouchEtag.CompareTo(result.LastEtag) <= 0)
{
log.Debug("Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.PreTouchEtag);
return false;
}
}
return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
public ReplicationStatisticsRecorder(ReplicationStrategy destination, ConcurrentDictionary<string, DestinationStats> destinationStats)
{
this.destination = destination;
this.destinationStats = destinationStats;
watch = Stopwatch.StartNew();
Started = SystemTime.UtcNow;
records = new RavenJArray();
record = new RavenJObject
{
{ "Url", destination.ConnectionStringOptions.Url },
{ "StartTime", SystemTime.UtcNow},
{ "Records", records }
};
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag);
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats))
{
SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource;
using (var scope = stats.StartRecording("Destination"))
{
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
{
log.Error("Failed to replicate documents to destination {0}, because was not able to receive last Etag", destination.ConnectionStringOptions.Url);
return false;
}
}
scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource));
if (destinationsReplicationInformationForSource.LastDocumentEtag == Etag.InvalidEtag && destinationsReplicationInformationForSource.LastAttachmentEtag == Etag.InvalidEtag)
{
DateTime lastSent;
// todo: move lastModifiedDate after the condition
var lastModifiedDate = destinationsReplicationInformationForSource.LastModified.HasValue ? destinationsReplicationInformationForSource.LastModified.Value.ToLocalTime() : DateTime.MinValue;
if (destinationAlertSent.TryGetValue(destination.ConnectionStringOptions.Url, out lastSent) && (SystemTime.UtcNow - lastSent).TotalMinutes < 1)
{
// todo: remove this log line
log.Debug(string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source));
return false;
}
docDb.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source),
Title = string.Format("Replication error. Multiple databases replicating at the same time with same DatabaseId ('{0}') detected.", docDb.TransactionalStorage.Id),
UniqueKey = "Replication to " + destination.ConnectionStringOptions.Url + " errored. Wrong DatabaseId: " + docDb.TransactionalStorage.Id
});
destinationAlertSent.AddOrUpdate(destination.ConnectionStringOptions.Url, SystemTime.UtcNow, (_, __) => SystemTime.UtcNow);
return false;
}
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
}
bool? replicated = null;
int replicatedDocuments;
using (var scope = stats.StartRecording("Documents"))
{
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope, out replicatedDocuments))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
using (var scope = stats.StartRecording("Attachments"))
{
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
var elapsedMicroseconds = (long)(stats.ElapsedTime.Ticks * SystemTime.MicroSecPerTick);
docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update(elapsedMicroseconds);
UpdateReplicationPerformance(destination, stats.Started, stats.ElapsedTime, replicatedDocuments);
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0, 1));
holder.Release();
}
}
private bool ReplicateTo(ReplicationStrategy destination)
{
try
{
if (docDb.Disposed)
return false;
using (docDb.DisableAllTriggersForCurrentThread())
using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats))
{
SourceReplicationInformation destinationsReplicationInformationForSource;
using (var scope = stats.StartRecording("Destination"))
{
try
{
destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination);
if (destinationsReplicationInformationForSource == null)
return false;
scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource));
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException("Failed to replicate to: " + destination, e);
return false;
}
}
bool? replicated = null;
using (var scope = stats.StartRecording("Documents"))
{
switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
using (var scope = stats.StartRecording("Attachments"))
{
switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope))
{
case true:
replicated = true;
break;
case false:
return false;
}
}
docDb.WorkContext.MetricsCounters.GetReplicationDurationMetric(destination).Mark((long)stats.ElapsedTime.TotalMilliseconds);
docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update((long)stats.ElapsedTime.TotalMilliseconds);
return replicated ?? false;
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0,1));
holder.Release();
}
}
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
Tuple<RavenJArray, Etag> tuple;
RavenJArray attachments;
using (var scope = recorder.StartRecording("Get"))
{
tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope);
attachments = tuple.Item1;
if (attachments == null || attachments.Length == 0)
{
if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag)
{
SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2);
}
return null;
}
}
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination);
if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments:false);
return true;
}
}
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
RecordSuccess(destination.ConnectionStringOptions.Url,
lastReplicatedEtag: tuple.Item2, forDocuments: false);
return true;
}
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder)
{
JsonDocumentsToReplicate documentsToReplicate;
using (var scope = recorder.StartRecording("Get"))
{
documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, scope);
if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0)
{
if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag)
{
// we don't notify remote server about updates to system docs, see: RavenDB-715
if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0
|| documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate
|| documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555
{
using (scope.StartRecording("Notify"))
{
SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag);
scope.Record(new RavenJObject
{
{ "LastDocEtag", documentsToReplicate.LastEtag.ToString() }
});
}
}
}
RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag);
return null;
}
}
using (var scope = recorder.StartRecording("Send"))
{
string lastError;
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy
{
if (IsFirstFailure(destination.ConnectionStringOptions.Url))
{
log.Info(
"This is the first failure for {0}, assuming transient failure and trying again",
destination);
if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail
{
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
}
scope.RecordError(lastError);
RecordFailure(destination.ConnectionStringOptions.Url, lastError);
return false;
}
}
RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified);
return true;
}
private void SetLastReplicatedEtagForServer(ReplicationStrategy destination, Etag lastDocEtag = null, Etag lastAttachmentEtag = null)
{
try
{
var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() +
"&dbid=" + docDb.TransactionalStorage.Id;
if (lastDocEtag != null)
url += "&docEtag=" + lastDocEtag;
if (lastAttachmentEtag != null)
url += "&attachmentEtag=" + lastAttachmentEtag;
var request = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions, GetRequestBuffering(destination));
request.Write(new byte[0]);
request.ExecuteRequest();
log.Debug("Sent last replicated document Etag {0} to server {1}", lastDocEtag, destination.ConnectionStringOptions.Url);
}
catch (WebException e)
{
HandleRequestBufferingErrors(e, destination);
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
log.WarnException("Replication is not enabled on: " + destination, e);
else
log.WarnException("Failed to contact replication destination: " + destination, e);
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
}
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition)
{
var url = string.Format("{0}/replication/side-by-side?{1}", destination.ConnectionStringOptions.Url, GetDebugInfomration());
IndexReplaceDocument indexReplaceDocument;
try
{
indexReplaceDocument = docDb.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name, null).DataAsJson.JsonDeserialization<IndexReplaceDocument>();
}
catch (Exception e)
{
log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e);
return;
}
var sideBySideReplicationInfo = new SideBySideReplicationInfo
{
Index = indexDefinition,
SideBySideIndex = sideBySideIndexDefinition,
OriginDatabaseId = destination.CurrentDatabaseId,
IndexReplaceDocument = indexReplaceDocument
};
var replicationRequest = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo));
replicationRequest.ExecuteRequest();
}
private bool GetRequestBuffering(ReplicationStrategy destination)
{
return destinationForceBuffering.GetOrAdd(destination.ConnectionStringOptions.Url, docDb.Configuration.Replication.ForceReplicationRequestBuffering);
}
private static ReplicationStrategy CreateReplicationStrategyFromConnectionString(ReplicationDestination x, ReplicationStrategy replicationStrategy)
{
var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName);
connectionStringParser.Parse();
var options = connectionStringParser.ConnectionStringOptions;
if (string.IsNullOrEmpty(options.Url))
{
throw new InvalidOperationException("Could not figure out what the replication URL is");
}
if (string.IsNullOrEmpty(options.DefaultDatabase) == false)
{
options.Url += "/databases/" + options.DefaultDatabase;
}
replicationStrategy.ConnectionStringOptions = options;
return(replicationStrategy);
}
private ReplicatonNodeState CheckDestinationConnectionState(ReplicationStrategy destination)
{
return CheckConnectionState(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions);
}
private void ReplicateIndexDeletionIfNeeded(List<JsonDocument> indexTombstones, ReplicationStrategy destination, Dictionary<string, int> replicatedIndexTombstones)
{
if (indexTombstones.Count == 0)
return;
foreach (var tombstone in indexTombstones)
{
try
{
int value;
//In case the index was recreated under the same name we will increase the destination count for this tombstone
//As if we sent the delete request but without actually sending the request, ending with a NOOP and deleting the index tombstone.
if (Database.IndexStorage.HasIndex(tombstone.Key))
{
replicatedIndexTombstones.TryGetValue(tombstone.Key, out value);
replicatedIndexTombstones[tombstone.Key] = value + 1;
continue;
}
var url = string.Format("{0}/indexes/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInformation());
var replicationRequest = HttpRavenRequestFactory.Create(url, "DELETE", destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination));
replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody));
replicationRequest.ExecuteRequest();
Log.Info("Replicated index deletion (index name = {0})", tombstone.Key);
replicatedIndexTombstones.TryGetValue(tombstone.Key, out value);
replicatedIndexTombstones[tombstone.Key] = value + 1;
}
catch (Exception e)
{
Replication.HandleRequestBufferingErrors(e, destination);
Log.ErrorException(string.Format("Failed to replicate index deletion (index name = {0})", tombstone.Key), e);
}
}
}