private SourceReplicationInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination) { try { var request = (HttpWebRequest)WebRequest.Create(destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl()); request.Credentials = destination.ConnectionStringOptions.Credentials ?? CredentialCache.DefaultNetworkCredentials; request.PreAuthenticate = true; request.UseDefaultCredentials = true; request.Timeout = replicationRequestTimeoutInMs; using (var response = request.GetResponse()) using (var stream = response.GetResponseStream()) { var etagFromServer = (SourceReplicationInformation) new JsonSerializer().Deserialize(new StreamReader(stream), typeof(SourceReplicationInformation)); return(etagFromServer); } } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) { log.WarnException("Replication is not enabled on: " + destination, e); } else { log.WarnException("Failed to contact replication destination: " + destination, e); } } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); } return(null); }
private bool IsNotFailing(ReplicationStrategy dest, int currentReplicationAttempts) { var jsonDocument = docDb.Get(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(dest.ConnectionStringOptions.Url), null); if (jsonDocument == null) { return(true); } var failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>(); if (failureInformation.FailureCount > 1000) { var shouldReplicateTo = currentReplicationAttempts % 10 == 0; log.Debug("Failure count for {0} is {1}, skipping replication: {2}", dest, failureInformation.FailureCount, shouldReplicateTo == false); return(shouldReplicateTo); } if (failureInformation.FailureCount > 100) { var shouldReplicateTo = currentReplicationAttempts % 5 == 0; log.Debug("Failure count for {0} is {1}, skipping replication: {2}", dest, failureInformation.FailureCount, shouldReplicateTo == false); return(shouldReplicateTo); } if (failureInformation.FailureCount > 10) { var shouldReplicateTo = currentReplicationAttempts % 2 == 0; log.Debug("Failure count for {0} is {1}, skipping replication: {2}", dest, failureInformation.FailureCount, shouldReplicateTo == false); return(shouldReplicateTo); } return(true); }
private SourceReplicationInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination) { try { var currentEtag = Guid.Empty; docDb.TransactionalStorage.Batch(accessor => currentEtag = accessor.Staleness.GetMostRecentDocumentEtag()); var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() + "¤tEtag=" + currentEtag + "&dbid=" + docDb.TransactionalStorage.Id; var request = httpRavenRequestFactory.Create(url, "GET", destination.ConnectionStringOptions); return(request.ExecuteRequest <SourceReplicationInformation>()); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) { log.WarnException("Replication is not enabled on: " + destination, e); } else { log.WarnException("Failed to contact replication destination: " + destination, e); } RecordFailure(destination.ConnectionStringOptions.Url, e.Message); } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); RecordFailure(destination.ConnectionStringOptions.Url, e.Message); } return(null); }
private SourceReplicationInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination) { try { var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl(); var request = httpRavenRequestFactory.Create(url, "GET", destination.ConnectionStringOptions); return(request.ExecuteRequest <SourceReplicationInformation>()); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) { log.WarnException("Replication is not enabled on: " + destination, e); } else { log.WarnException("Failed to contact replication destination: " + destination, e); } } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); } return(null); }
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var jsonDocuments = GetJsonDocuments(destinationsReplicationInformationForSource, destination); if (jsonDocuments == null || jsonDocuments.Length == 0) { return(null); } if (TryReplicationDocuments(destination, jsonDocuments) == false) // failed to replicate, start error handling strategy { if (IsFirstFailue(destination)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, jsonDocuments)) // success on second fail { return(true); } } IncrementFailureCount(destination); return(false); } ResetFailureCount(destination); return(true); }
private void SetLastReplicatedEtagForDocuments(ReplicationStrategy destination, Guid?lastDocEtag = null, Guid?lastAttachmentEtag = null) { try { var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id; if (lastDocEtag != null) { url += "&docEtag=" + lastDocEtag.Value; } if (lastAttachmentEtag != null) { url += "&attachmentEtag=" + lastAttachmentEtag.Value; } var request = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions); request.ExecuteRequest(); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) { log.WarnException("Replication is not enabled on: " + destination, e); } else { log.WarnException("Failed to contact replication destination: " + destination, e); } } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); } }
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments) { try { log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination); var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl(); var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions); request.Write(jsonDocuments); request.ExecuteRequest(); log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination); return(true); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null) { using (var streamReader = new StreamReader(response.GetResponseStream())) { var error = streamReader.ReadToEnd(); log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } return(false); } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); return(false); } }
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var tuple = GetJsonDocuments(destinationsReplicationInformationForSource, destination); var jsonDocuments = tuple.Item1; if (jsonDocuments == null || jsonDocuments.Length == 0) { if (tuple.Item2 != destinationsReplicationInformationForSource.LastDocumentEtag) { SetLastReplicatedEtagForDocuments(destination, lastDocEtag: tuple.Item2); } return(null); } string lastError; if (TryReplicationDocuments(destination, jsonDocuments, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailue(destination)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, jsonDocuments, out lastError)) // success on second fail { ResetFailureCount(destination.ConnectionStringOptions.Url, lastError); return(true); } } IncrementFailureCount(destination, lastError); return(false); } ResetFailureCount(destination.ConnectionStringOptions.Url, lastError); return(true); }
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition) { var url = string.Format("{0}/replication/side-by-side?{1}", destination.ConnectionStringOptions.Url, GetDebugInformation()); IndexReplaceDocument indexReplaceDocument; try { indexReplaceDocument = database.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name).DataAsJson.JsonDeserialization <IndexReplaceDocument>(); } catch (Exception e) { Log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e); return; } var sideBySideReplicationInfo = new SideBySideReplicationInfo { Index = indexDefinition, SideBySideIndex = sideBySideIndexDefinition, OriginDatabaseId = destination.CurrentDatabaseId, IndexReplaceDocument = indexReplaceDocument }; var replicationRequest = httpRavenRequestFactory.Create(url, HttpMethod.Post, destination.ConnectionStringOptions, replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo)); replicationRequest.ExecuteRequest(); }
private bool IsFirstFailue(ReplicationStrategy destination) { FailureCount value; if (replicationFailureStats.TryGetValue(destination.ConnectionStringOptions.Url, out value)) { return(value.Count == 0); } return(false); }
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x) { var replicationStrategy = new ReplicationStrategy { ReplicationOptionsBehavior = x.TransitiveReplicationBehavior, CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString() }; return(CreateReplicationStrategyFromDocument(x, replicationStrategy)); }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) { return(false); } using (docDb.DisableAllTriggersForCurrentThread()) { SourceReplicationInformation destinationsReplicationInformationForSource; try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) { return(false); } } catch (Exception e) { log.WarnException("Failed to replicate to: " + destination, e); return(false); } bool?replicated = null; switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource)) { case true: replicated = true; break; case false: return(false); } switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource)) { case true: replicated = true; break; case false: return(false); } return(replicated ?? false); } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder()); Thread.VolatileWrite(ref holder.Value, 0); } }
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments, out string errorMessage) { try { var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" + UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id; var sp = Stopwatch.StartNew(); var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions); request.WebRequest.Headers.Add("Attachment-Ids", string.Join(", ", jsonAttachments.Select(x => x.Value <string>("@id")))); request.WriteBson(jsonAttachments); request.ExecuteRequest(); log.Info("Replicated {0} attachments to {1} in {2:#,#;;0} ms", jsonAttachments.Length, destination, sp.ElapsedMilliseconds); errorMessage = ""; return(true); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null) { using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression())) { var error = streamReader.ReadToEnd(); try { var ravenJObject = RavenJObject.Parse(error); log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value <string>("Error"), e); errorMessage = error; return(false); } catch (Exception) { } log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); errorMessage = error; } } else { log.WarnException("Replication to " + destination + " had failed", e); errorMessage = e.Message; } return(false); } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); errorMessage = e.Message; return(false); } }
private bool IsFirstFailue(ReplicationStrategy destination) { var jsonDocument = docDb.Get(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null); if (jsonDocument == null) { return(true); } var failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>(); return(failureInformation.FailureCount == 0); }
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments, out string lastError) { try { log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination); var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id; var sp = Stopwatch.StartNew(); var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions); request.Write(jsonDocuments); request.ExecuteRequest(); log.Info("Replicated {0} documents to {1} in {2:#,#;;0} ms", jsonDocuments.Length, destination, sp.ElapsedMilliseconds); lastError = ""; return(true); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null) { Stream responseStream = response.GetResponseStream(); if (responseStream != null) { using (var streamReader = new StreamReader(responseStream)) { var error = streamReader.ReadToEnd(); log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } lastError = e.Message; return(false); } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); lastError = e.Message; return(false); } }
private void IncrementFailureCount(ReplicationStrategy destination) { var jsonDocument = docDb.Get(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null); var failureInformation = new DestinationFailureInformation { Destination = destination.ConnectionStringOptions.Url }; if (jsonDocument != null) { failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>(); } failureInformation.FailureCount += 1; docDb.Put(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null, RavenJObject.FromObject(failureInformation), new RavenJObject(), null); }
private void ReplicateIndexesMultiPut(ReplicationStrategy destination, List <Tuple <IndexDefinition, IndexingPriority> > candidatesForReplication) { var indexesToAdd = candidatesForReplication .Select(x => new IndexToAdd { Definition = x.Item1, Name = x.Item1.Name, Priority = x.Item2 }) .ToArray(); var serializedIndexDefinitions = RavenJToken.FromObject(indexesToAdd); var url = string.Format("{0}/indexes?{1}", destination.ConnectionStringOptions.Url, GetDebugInformation()); var replicationRequest = httpRavenRequestFactory.Create(url, HttpMethods.Put, destination.ConnectionStringOptions, replication.GetRequestBuffering(destination)); replicationRequest.Write(serializedIndexDefinitions); replicationRequest.ExecuteRequest(); }
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments) { try { log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination); var request = (HttpWebRequest)WebRequest.Create(destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl()); request.UseDefaultCredentials = true; request.PreAuthenticate = true; request.ContentType = "application/json; charset=utf-8"; request.Credentials = destination.ConnectionStringOptions.Credentials ?? CredentialCache.DefaultNetworkCredentials; request.Method = "POST"; using (var stream = request.GetRequestStream()) using (var streamWriter = new StreamWriter(stream, Encoding.UTF8)) { jsonDocuments.WriteTo(new JsonTextWriter(streamWriter)); streamWriter.Flush(); stream.Flush(); } using (request.GetResponse()) { log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination); } return(true); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null) { using (var streamReader = new StreamReader(response.GetResponseStream())) { var error = streamReader.ReadToEnd(); log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } return(false); } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); return(false); } }
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments) { try { var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" + UrlEncodedServerUrl(); var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions); request.WriteBson(jsonAttachments); request.ExecuteRequest(); log.Info("Replicated {0} attachments to {1}", jsonAttachments.Length, destination); return(true); } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null) { using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression())) { var error = streamReader.ReadToEnd(); try { var ravenJObject = RavenJObject.Parse(error); log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value <string>("Error"), e); return(false); } catch (Exception) { } log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } return(false); } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); return(false); } }
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition) { try { var clonedTransformer = definition.Clone(); clonedTransformer.TransfomerId = 0; var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInformation(); var replicationRequest = httpRavenRequestFactory.Create(url, HttpMethod.Put, destination.ConnectionStringOptions, replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(clonedTransformer)); replicationRequest.ExecuteRequest(); } catch (Exception e) { replication.HandleRequestBufferingErrors(e, destination); Log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e); } }
private bool?ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return(null); } string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return(true); } } RecordFailure(destination.ConnectionStringOptions.Url, lastError); return(false); } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return(true); }
private ReplicationStrategy GetConnectionOptions(ReplicationDestination x) { var replicationStrategy = new ReplicationStrategy { ReplicationOptionsBehavior = x.TransitiveReplicationBehavior, CurrentDatabaseId = docDb.TransactionalStorage.Id.ToString() }; if (string.IsNullOrEmpty(x.ConnectionStringName)) { replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions { Url = x.Url }; return(replicationStrategy); } var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName); connectionStringParser.Parse(); var options = connectionStringParser.ConnectionStringOptions; if (string.IsNullOrEmpty(options.Url)) { throw new InvalidOperationException("Could not figure out what the replication URL is"); } if (string.IsNullOrEmpty(options.DefaultDatabase) == false) { if (options.Url.EndsWith("/") == false) { options.Url += "/"; } options.Url += "databases/" + options.DefaultDatabase; } replicationStrategy.ConnectionStringOptions = options; return(replicationStrategy); }
private void IncrementFailureCount(ReplicationStrategy destination, string lastError) { var failureCount = replicationFailureStats.GetOrAdd(destination.ConnectionStringOptions.Url); Interlocked.Increment(ref failureCount.Count); failureCount.Timestamp = SystemTime.UtcNow; if (string.IsNullOrWhiteSpace(lastError) == false) { failureCount.LastError = lastError; } var jsonDocument = docDb.Get(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination.ConnectionStringOptions.Url), null); var failureInformation = new DestinationFailureInformation { Destination = destination.ConnectionStringOptions.Url }; if (jsonDocument != null) { failureInformation = jsonDocument.DataAsJson.JsonDeserialization <DestinationFailureInformation>(); } failureInformation.FailureCount += 1; docDb.Put(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination.ConnectionStringOptions.Url), null, RavenJObject.FromObject(failureInformation), new RavenJObject(), null); }
private bool IsNotFailing(ReplicationStrategy dest, int currentReplicationAttempts) { var jsonDocument = docDb.Documents.Get(Constants.RavenReplicationDestinationsBasePath + EscapeDestinationName(dest.ConnectionStringOptions.Url), null); if (jsonDocument == null) return true; var failureInformation = jsonDocument.DataAsJson.JsonDeserialization<DestinationFailureInformation>(); if (failureInformation.FailureCount > 1000) { var shouldReplicateTo = currentReplicationAttempts % 10 == 0; log.Debug("Failure count for {0} is {1}, skipping replication: {2}", dest, failureInformation.FailureCount, shouldReplicateTo == false); return shouldReplicateTo; } if (failureInformation.FailureCount > 100) { var shouldReplicateTo = currentReplicationAttempts % 5 == 0; log.Debug("Failure count for {0} is {1}, skipping replication: {2}", dest, failureInformation.FailureCount, shouldReplicateTo == false); return shouldReplicateTo; } if (failureInformation.FailureCount > 10) { var shouldReplicateTo = currentReplicationAttempts % 2 == 0; log.Debug("Failure count for {0} is {1}, skipping replication: {2}", dest, failureInformation.FailureCount, shouldReplicateTo == false); return shouldReplicateTo; } return true; }
private static ReplicationStrategy CreateReplicationStrategyFromDocument(ReplicationDestination x, ReplicationStrategy replicationStrategy) { var url = x.Url; if (string.IsNullOrEmpty(x.Database) == false) { url = url + "/databases/" + x.Database; } replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions { Url = url, ApiKey = x.ApiKey, AuthenticationScheme = x.AuthenticationScheme }; if (string.IsNullOrEmpty(x.Username) == false) { replicationStrategy.ConnectionStringOptions.Credentials = string.IsNullOrEmpty(x.Domain) ? new NetworkCredential(x.Username, x.Password) : new NetworkCredential(x.Username, x.Password, x.Domain); } return replicationStrategy; }
internal SourceReplicationInformationWithBatchInformation GetLastReplicatedEtagFrom(ReplicationStrategy destination) { try { Etag currentEtag = Etag.Empty; docDb.TransactionalStorage.Batch(accessor => currentEtag = accessor.Staleness.GetMostRecentDocumentEtag()); var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() + "¤tEtag=" + currentEtag + "&dbid=" + docDb.TransactionalStorage.Id; var request = httpRavenRequestFactory.Create(url, "GET", destination.ConnectionStringOptions); var lastReplicatedEtagFrom = request.ExecuteRequest<SourceReplicationInformationWithBatchInformation>(); log.Debug("Received last replicated document Etag {0} from server {1}", lastReplicatedEtagFrom.LastDocumentEtag, destination.ConnectionStringOptions.Url); return lastReplicatedEtagFrom; } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) log.WarnException("Replication is not enabled on: " + destination, e); else log.WarnException("Failed to contact replication destination: " + destination, e); RecordFailure(destination.ConnectionStringOptions.Url, e.Message); } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); RecordFailure(destination.ConnectionStringOptions.Url, e.Message); } return null; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds); var duration = Stopwatch.StartNew(); var result = new JsonDocumentsToReplicate { LastEtag = Etag.Empty, }; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch; docDb.TransactionalStorage.Batch(actions => { var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested(); docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(result.LastEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag); return false; } } string reason; return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); result.CountOfFilteredDocumentsWhichOriginFromDestination += docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); if (duration.Elapsed > timeout) break; } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); scope.Record(new RavenJObject { {"StartEtag", lastEtag.ToString()}, {"EndEtag", result.LastEtag.ToString()}, {"Count", docsSinceLastReplEtag}, {"FilteredCount", filteredDocsToReplicate.Count} }); result.LoadedDocs = filteredDocsToReplicate; docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag); docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { JsonDocument.EnsureIdInMetadata(x); EnsureReplicationInformationInMetadata(x.Metadata, docDb); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { scope.RecordError(e); log.WarnException( "Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private void HandleRequestBufferingErrors(Exception e, ReplicationStrategy destination) { if (destination.ConnectionStringOptions.Credentials != null && string.Equals(e.Message, "This request requires buffering data to succeed.", StringComparison.OrdinalIgnoreCase)) destinationForceBuffering.AddOrUpdate(destination.ConnectionStringOptions.Url, true, (s, b) => true); }
private void ReplicateSingleTransformer(ReplicationStrategy destination, TransformerDefinition definition) { try { var clonedTransformer = definition.Clone(); clonedTransformer.TransfomerId = 0; var url = destination.ConnectionStringOptions.Url + "/transformers/" + Uri.EscapeUriString(definition.Name) + "?" + GetDebugInfomration(); var replicationRequest = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions, GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(clonedTransformer)); replicationRequest.ExecuteRequest(); } catch (Exception e) { HandleRequestBufferingErrors(e, destination); log.WarnException("Could not replicate transformer " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e); } }
private bool TryReplicationAttachments(ReplicationStrategy destination, RavenJArray jsonAttachments, out string errorMessage) { try { var url = destination.ConnectionStringOptions.Url + "/replication/replicateAttachments?from=" + UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id; var sp = Stopwatch.StartNew(); using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url)) { var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination)); request.WriteBson(jsonAttachments); request.ExecuteRequest(docDb.WorkContext.CancellationToken); log.Info("Replicated {0} attachments to {1} in {2:#,#;;0} ms", jsonAttachments.Length, destination, sp.ElapsedMilliseconds); errorMessage = ""; return true; } } catch (WebException e) { HandleRequestBufferingErrors(e, destination); var response = e.Response as HttpWebResponse; if (response != null) { using (var streamReader = new StreamReader(response.GetResponseStreamWithHttpDecompression())) { var error = streamReader.ReadToEnd(); try { var ravenJObject = RavenJObject.Parse(error); log.WarnException("Replication to " + destination + " had failed\r\n" + ravenJObject.Value<string>("Error"), e); errorMessage = error; return false; } catch (Exception) { } log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); errorMessage = error; } } else { log.WarnException("Replication to " + destination + " had failed", e); errorMessage = e.Message; } return false; } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); errorMessage = e.Message; return false; } }
private Tuple <RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray attachments = null; Guid lastAttachmentEtag = Guid.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List <AttachmentInformation> attachmentsToReplicate; List <AttachmentInformation> filteredAttachmentsToReplicate; lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; while (true) { attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Guid attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; } log.Debug(() => { if (attachmentSinceLastEtag == 0) { return(string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag)); } if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) { return(string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination)); } var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag)); }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } return(new RavenJObject { { "@metadata", x.Metadata }, { "@id", x.Key }, { "@etag", x.Etag.ToByteArray() }, { "data", data } }); })); }); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return(Tuple.Create(attachments, lastAttachmentEtag)); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List <JsonDocument> docsToReplicate; List <JsonDocument> filteredDocsToReplicate; result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false) { return(false); } } return(destination.FilterDocuments(destinationId, document.Key, document.Metadata)); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag.Value; if (lastDoc.LastModified.HasValue) { result.LastLastModified = lastDoc.LastModified.Value; } } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) { return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag)); } if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) { return(string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination)); } var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag)); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(result); }
private void ResetFailureCount(ReplicationStrategy destination) { docDb.Delete(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null, null); }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List <JsonDocument> docsToReplicate; List <JsonDocument> filteredDocsToReplicate; Guid lastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = actions.Documents.GetDocumentsAfter(lastDocumentEtag, 100).ToList(); filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(document, destinationId)).ToList(); docsSinceLastReplEtag += docsToReplicate.Count; if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } JsonDocument jsonDocument = docsToReplicate.Last(); Debug.Assert(jsonDocument.Etag != null); Guid documentEtag = jsonDocument.Etag.Value; log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", docsToReplicate); lastDocumentEtag = documentEtag; } log.Debug(() => { if (docsSinceLastReplEtag == 0) { return(string.Format("Nothing to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag)); } if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) { return(string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination)); } var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag)); }); jsonDocuments = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(jsonDocuments); }
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonAttachments = null; try { string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag, 100) .Where(information => destination.FilterAttachments(information, destinationInstanceId)) .Select(x => new RavenJObject { { "@metadata", x.Metadata }, { "@id", x.Key }, { "@etag", x.Etag.ToByteArray() }, { "data", actions.Attachments.GetAttachment(x.Key).Data().ReadData() } })); }); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return(jsonAttachments); }
private void UpdateReplicationPerformance(ReplicationStrategy destination, DateTime startTime, TimeSpan elapsed, int batchSize) { if (batchSize > 0) { var queue = docDb.WorkContext.MetricsCounters.GetReplicationPerformanceStats(destination); queue.Enqueue(new ReplicationPerformanceStats { Duration = elapsed, Started = startTime, BatchSize = batchSize }); while (queue.Count() > 25) { ReplicationPerformanceStats _; queue.TryDequeue(out _); } } }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder, out int replicatedDocuments) { replicatedDocuments = 0; JsonDocumentsToReplicate documentsToReplicate = null; var sp = Stopwatch.StartNew(); IDisposable removeBatch = null; var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url, x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner, string.Format("Replication for URL: {0}" ,destination.ConnectionStringOptions.DefaultDatabase) )); prefetchingBehavior.AdditionalInfo = string.Format("For destination: {0}. Last replicated etag: {1}", destination.ConnectionStringOptions.Url, destinationsReplicationInformationForSource.LastDocumentEtag); try { using (var scope = recorder.StartRecording("Get")) { documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555 { using (scope.StartRecording("Notify")) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); scope.Record(new RavenJObject { { "LastDocEtag", documentsToReplicate.LastEtag.ToString() } }); } } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } } // if the db is idling in all respect except sending out replication, let us keep it that way. docDb.WorkContext.UpdateFoundWork(); removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs); using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } // if we had an error sending to this endpoint, it might be because we are sending too much data, or because // the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts // and we'll be much more conservative with increasing the sizes prefetchingBehavior.OutOfMemoryExceptionHappened(); scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } } finally { if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null) { prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed); replicatedDocuments = documentsToReplicate.LoadedDocs.Count; } if (removeBatch != null) removeBatch.Dispose(); } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private bool TryReplicationDocuments(ReplicationStrategy destination, RavenJArray jsonDocuments, out string lastError) { try { log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination); var url = destination.ConnectionStringOptions.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id + "&count=" + jsonDocuments.Length; var sp = Stopwatch.StartNew(); using (HttpRavenRequestFactory.Expect100Continue(destination.ConnectionStringOptions.Url)) { var request = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination)); request.Write(jsonDocuments); request.ExecuteRequest(docDb.WorkContext.CancellationToken); log.Info("Replicated {0} documents to {1} in {2:#,#;;0} ms", jsonDocuments.Length, destination, sp.ElapsedMilliseconds); lastError = ""; return true; } } catch (WebException e) { HandleRequestBufferingErrors(e, destination); var response = e.Response as HttpWebResponse; if (response != null) { var responseStream = response.GetResponseStream(); if (responseStream != null) { using (var streamReader = new StreamReader(responseStream)) { var error = streamReader.ReadToEnd(); log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } lastError = e.Message; return false; } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); lastError = e.Message; return false; } }
private static ReplicationStrategy CreateReplicationStrategyFromDocument(ReplicationDestination x, ReplicationStrategy replicationStrategy) { var url = x.Url; if (string.IsNullOrEmpty(x.Database) == false) { url = url + "/databases/" + x.Database; } replicationStrategy.ConnectionStringOptions = new RavenConnectionStringOptions { Url = url, ApiKey = x.ApiKey, }; if (string.IsNullOrEmpty(x.Username) == false) { replicationStrategy.ConnectionStringOptions.Credentials = string.IsNullOrEmpty(x.Domain) ? new NetworkCredential(x.Username, x.Password) : new NetworkCredential(x.Username, x.Password, x.Domain); } return(replicationStrategy); }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats)) { SourceReplicationInformation destinationsReplicationInformationForSource; using (var scope = stats.StartRecording("Destination")) { try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) return false; scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource)); } catch (Exception e) { scope.RecordError(e); log.WarnException("Failed to replicate to: " + destination, e); return false; } } bool? replicated = null; using (var scope = stats.StartRecording("Documents")) { switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } using (var scope = stats.StartRecording("Attachments")) { switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder()); Thread.VolatileWrite(ref holder.Value, 0); } }
private void ReplicateSingleIndex(ReplicationStrategy destination, IndexDefinition definition) { try { var url = string.Format("{0}/indexes/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(definition.Name), GetDebugInfomration()); var replicationRequest = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions, GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(definition)); replicationRequest.ExecuteRequest(); } catch (Exception e) { HandleRequestBufferingErrors(e, destination); log.WarnException("Could not replicate index " + definition.Name + " to " + destination.ConnectionStringOptions.Url, e); } }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) { SourceReplicationInformation destinationsReplicationInformationForSource; try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) return false; } catch (Exception e) { log.WarnException("Failed to replicate to: " + destination, e); return false; } bool? replicated = null; switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource)) { case true: replicated = true; break; case false: return false; } switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource)) { case true: replicated = true; break; case false: return false; } return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, new IntHolder()); Thread.VolatileWrite(ref holder.Value, 0); } }
private void ReplicateTransformerDeletionIfNeeded(List<JsonDocument> transformerTombstones, ReplicationStrategy destination, Dictionary<string, int> replicatedTransformerTombstones) { if (transformerTombstones.Count == 0) return; foreach (var tombstone in transformerTombstones) { try { int value; if (docDb.Transformers.GetTransformerDefinition(tombstone.Key) != null) //if in the meantime the transformer was recreated under the same name { replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value); replicatedTransformerTombstones[tombstone.Key] = value + 1; continue; } var url = string.Format("{0}/transformers/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInfomration()); var replicationRequest = httpRavenRequestFactory.Create(url, "DELETE", destination.ConnectionStringOptions, GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(emptyRequestBody)); replicationRequest.ExecuteRequest(); log.Info("Replicated transformer deletion (transformer name = {0})", tombstone.Key); replicatedTransformerTombstones.TryGetValue(tombstone.Key, out value); replicatedTransformerTombstones[tombstone.Key] = value + 1; } catch (Exception e) { HandleRequestBufferingErrors(e, destination); log.ErrorException(string.Format("Failed to replicate transformer deletion (transformer name = {0})", tombstone.Key), e); } } }
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var tuple = GetAttachments(destinationsReplicationInformationForSource, destination); var attachments = tuple.Item1; if (attachments == null || attachments.Length == 0) { if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag) { SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2); } return null; } string lastError; if (TryReplicationAttachments(destination, attachments, out lastError) == false)// failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationAttachments(destination, attachments, out lastError))// success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2); return true; } } RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2); return true; }
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds); var duration = Stopwatch.StartNew(); RavenJArray attachments = null; Etag lastAttachmentEtag = Etag.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch; docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List<AttachmentInformation> attachmentsToReplicate; List<AttachmentInformation> filteredAttachmentsToReplicate; var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; lastAttachmentEtag = startEtag; while (true) { attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag, maxNumberOfItemsToReceiveInSingleBatch); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Etag attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; if (duration.Elapsed > timeout) break; } log.Debug(() => { if (attachmentSinceLastEtag == 0) return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastAttachmentEtag); if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) return string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastAttachmentEtag, destination); var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastAttachmentEtag); }); scope.Record(new RavenJObject { {"StartEtag", startEtag.ToString()}, {"EndEtag", lastAttachmentEtag.ToString()}, {"Count", attachmentSinceLastEtag}, {"FilteredCount", filteredAttachmentsToReplicate.Count} }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } EnsureReplicationInformationInMetadata(x.Metadata, docDb); return new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", data} }; })); }); } catch (InvalidDataException e) { RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e)); scope.RecordError(e); log.ErrorException("Data is corrupted, could not proceed with replication", e); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return Tuple.Create(attachments, lastAttachmentEtag); }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false)// failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError))// success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
public static ReplicationStrategy GetConnectionOptions(ReplicationDestination x, DocumentDatabase database) { var replicationStrategy = new ReplicationStrategy { ReplicationOptionsBehavior = x.TransitiveReplicationBehavior, CurrentDatabaseId = database.TransactionalStorage.Id.ToString() }; return CreateReplicationStrategyFromDocument(x, replicationStrategy); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag(); var lastEtag = etagSynchronizer.CalculateSynchronizationEtag( synchronizationEtag, destinationsReplicationInformationForSource.LastDocumentEtag); int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (info.PreTouchEtag.CompareTo(result.LastEtag) <= 0) { log.Debug("Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.PreTouchEtag); return false; } } return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
public ReplicationStatisticsRecorder(ReplicationStrategy destination, ConcurrentDictionary<string, DestinationStats> destinationStats) { this.destination = destination; this.destinationStats = destinationStats; watch = Stopwatch.StartNew(); Started = SystemTime.UtcNow; records = new RavenJArray(); record = new RavenJObject { { "Url", destination.ConnectionStringOptions.Url }, { "StartTime", SystemTime.UtcNow}, { "Records", records } }; }
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray attachments = null; Etag lastAttachmentEtag = Etag.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List<AttachmentInformation> attachmentsToReplicate; List<AttachmentInformation> filteredAttachmentsToReplicate; lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; while (true) { attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Etag attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; } log.Debug(() => { if (attachmentSinceLastEtag == 0) return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag); if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) return string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination); var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag); }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } return new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", data} }; })); }); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return Tuple.Create(attachments, lastAttachmentEtag); }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats)) { SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource; using (var scope = stats.StartRecording("Destination")) { try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) { log.Error("Failed to replicate documents to destination {0}, because was not able to receive last Etag", destination.ConnectionStringOptions.Url); return false; } } scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource)); if (destinationsReplicationInformationForSource.LastDocumentEtag == Etag.InvalidEtag && destinationsReplicationInformationForSource.LastAttachmentEtag == Etag.InvalidEtag) { DateTime lastSent; // todo: move lastModifiedDate after the condition var lastModifiedDate = destinationsReplicationInformationForSource.LastModified.HasValue ? destinationsReplicationInformationForSource.LastModified.Value.ToLocalTime() : DateTime.MinValue; if (destinationAlertSent.TryGetValue(destination.ConnectionStringOptions.Url, out lastSent) && (SystemTime.UtcNow - lastSent).TotalMinutes < 1) { // todo: remove this log line log.Debug(string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source)); return false; } docDb.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = string.Format(@"Destination server is forbidding replication due to a possibility of having multiple instances with same DatabaseId replicating to it. After 10 minutes from '{2}' another instance will start replicating. Destination Url: {0}. DatabaseId: {1}. Current source: {3}. Stored source on destination: {4}.", destination.ConnectionStringOptions.Url, docDb.TransactionalStorage.Id, lastModifiedDate, docDb.ServerUrl, destinationsReplicationInformationForSource.Source), Title = string.Format("Replication error. Multiple databases replicating at the same time with same DatabaseId ('{0}') detected.", docDb.TransactionalStorage.Id), UniqueKey = "Replication to " + destination.ConnectionStringOptions.Url + " errored. Wrong DatabaseId: " + docDb.TransactionalStorage.Id }); destinationAlertSent.AddOrUpdate(destination.ConnectionStringOptions.Url, SystemTime.UtcNow, (_, __) => SystemTime.UtcNow); return false; } } catch (Exception e) { scope.RecordError(e); log.WarnException("Failed to replicate to: " + destination, e); return false; } } bool? replicated = null; int replicatedDocuments; using (var scope = stats.StartRecording("Documents")) { switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope, out replicatedDocuments)) { case true: replicated = true; break; case false: return false; } } using (var scope = stats.StartRecording("Attachments")) { switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } var elapsedMicroseconds = (long)(stats.ElapsedTime.Ticks * SystemTime.MicroSecPerTick); docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update(elapsedMicroseconds); UpdateReplicationPerformance(destination, stats.Started, stats.ElapsedTime, replicatedDocuments); return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0, 1)); holder.Release(); } }
private bool ReplicateTo(ReplicationStrategy destination) { try { if (docDb.Disposed) return false; using (docDb.DisableAllTriggersForCurrentThread()) using (var stats = new ReplicationStatisticsRecorder(destination, destinationStats)) { SourceReplicationInformation destinationsReplicationInformationForSource; using (var scope = stats.StartRecording("Destination")) { try { destinationsReplicationInformationForSource = GetLastReplicatedEtagFrom(destination); if (destinationsReplicationInformationForSource == null) return false; scope.Record(RavenJObject.FromObject(destinationsReplicationInformationForSource)); } catch (Exception e) { scope.RecordError(e); log.WarnException("Failed to replicate to: " + destination, e); return false; } } bool? replicated = null; using (var scope = stats.StartRecording("Documents")) { switch (ReplicateDocuments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } using (var scope = stats.StartRecording("Attachments")) { switch (ReplicateAttachments(destination, destinationsReplicationInformationForSource, scope)) { case true: replicated = true; break; case false: return false; } } docDb.WorkContext.MetricsCounters.GetReplicationDurationMetric(destination).Mark((long)stats.ElapsedTime.TotalMilliseconds); docDb.WorkContext.MetricsCounters.GetReplicationDurationHistogram(destination).Update((long)stats.ElapsedTime.TotalMilliseconds); return replicated ?? false; } } finally { var holder = activeReplicationTasks.GetOrAdd(destination.ConnectionStringOptions.Url, s => new SemaphoreSlim(0,1)); holder.Release(); } }
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { Tuple<RavenJArray, Etag> tuple; RavenJArray attachments; using (var scope = recorder.StartRecording("Get")) { tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope); attachments = tuple.Item1; if (attachments == null || attachments.Length == 0) { if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag) { SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2); } return null; } } using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments:false); return true; } } scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2, forDocuments: false); return true; }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { JsonDocumentsToReplicate documentsToReplicate; using (var scope = recorder.StartRecording("Get")) { documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, scope); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555 { using (scope.StartRecording("Notify")) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); scope.Record(new RavenJObject { { "LastDocEtag", documentsToReplicate.LastEtag.ToString() } }); } } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } } using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private void SetLastReplicatedEtagForServer(ReplicationStrategy destination, Etag lastDocEtag = null, Etag lastAttachmentEtag = null) { try { var url = destination.ConnectionStringOptions.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl() + "&dbid=" + docDb.TransactionalStorage.Id; if (lastDocEtag != null) url += "&docEtag=" + lastDocEtag; if (lastAttachmentEtag != null) url += "&attachmentEtag=" + lastAttachmentEtag; var request = httpRavenRequestFactory.Create(url, "PUT", destination.ConnectionStringOptions, GetRequestBuffering(destination)); request.Write(new byte[0]); request.ExecuteRequest(); log.Debug("Sent last replicated document Etag {0} to server {1}", lastDocEtag, destination.ConnectionStringOptions.Url); } catch (WebException e) { HandleRequestBufferingErrors(e, destination); var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) log.WarnException("Replication is not enabled on: " + destination, e); else log.WarnException("Failed to contact replication destination: " + destination, e); } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); } }
private void ReplicateSingleSideBySideIndex(ReplicationStrategy destination, IndexDefinition indexDefinition, IndexDefinition sideBySideIndexDefinition) { var url = string.Format("{0}/replication/side-by-side?{1}", destination.ConnectionStringOptions.Url, GetDebugInfomration()); IndexReplaceDocument indexReplaceDocument; try { indexReplaceDocument = docDb.Documents.Get(Constants.IndexReplacePrefix + sideBySideIndexDefinition.Name, null).DataAsJson.JsonDeserialization<IndexReplaceDocument>(); } catch (Exception e) { log.Warn("Cannot get side-by-side index replacement document. Aborting operation. (this exception should not happen and the cause should be investigated)", e); return; } var sideBySideReplicationInfo = new SideBySideReplicationInfo { Index = indexDefinition, SideBySideIndex = sideBySideIndexDefinition, OriginDatabaseId = destination.CurrentDatabaseId, IndexReplaceDocument = indexReplaceDocument }; var replicationRequest = httpRavenRequestFactory.Create(url, "POST", destination.ConnectionStringOptions, GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(sideBySideReplicationInfo)); replicationRequest.ExecuteRequest(); }
private bool GetRequestBuffering(ReplicationStrategy destination) { return destinationForceBuffering.GetOrAdd(destination.ConnectionStringOptions.Url, docDb.Configuration.Replication.ForceReplicationRequestBuffering); }
private static ReplicationStrategy CreateReplicationStrategyFromConnectionString(ReplicationDestination x, ReplicationStrategy replicationStrategy) { var connectionStringParser = ConnectionStringParser <RavenConnectionStringOptions> .FromConnectionStringName(x.ConnectionStringName); connectionStringParser.Parse(); var options = connectionStringParser.ConnectionStringOptions; if (string.IsNullOrEmpty(options.Url)) { throw new InvalidOperationException("Could not figure out what the replication URL is"); } if (string.IsNullOrEmpty(options.DefaultDatabase) == false) { options.Url += "/databases/" + options.DefaultDatabase; } replicationStrategy.ConnectionStringOptions = options; return(replicationStrategy); }
private ReplicatonNodeState CheckDestinationConnectionState(ReplicationStrategy destination) { return CheckConnectionState(destination.ConnectionStringOptions.Url, destination.ConnectionStringOptions); }
private void ReplicateIndexDeletionIfNeeded(List<JsonDocument> indexTombstones, ReplicationStrategy destination, Dictionary<string, int> replicatedIndexTombstones) { if (indexTombstones.Count == 0) return; foreach (var tombstone in indexTombstones) { try { int value; //In case the index was recreated under the same name we will increase the destination count for this tombstone //As if we sent the delete request but without actually sending the request, ending with a NOOP and deleting the index tombstone. if (Database.IndexStorage.HasIndex(tombstone.Key)) { replicatedIndexTombstones.TryGetValue(tombstone.Key, out value); replicatedIndexTombstones[tombstone.Key] = value + 1; continue; } var url = string.Format("{0}/indexes/{1}?{2}", destination.ConnectionStringOptions.Url, Uri.EscapeUriString(tombstone.Key), GetDebugInformation()); var replicationRequest = HttpRavenRequestFactory.Create(url, "DELETE", destination.ConnectionStringOptions, Replication.GetRequestBuffering(destination)); replicationRequest.Write(RavenJObject.FromObject(EmptyRequestBody)); replicationRequest.ExecuteRequest(); Log.Info("Replicated index deletion (index name = {0})", tombstone.Key); replicatedIndexTombstones.TryGetValue(tombstone.Key, out value); replicatedIndexTombstones[tombstone.Key] = value + 1; } catch (Exception e) { Replication.HandleRequestBufferingErrors(e, destination); Log.ErrorException(string.Format("Failed to replicate index deletion (index name = {0})", tombstone.Key), e); } } }