private void OnGet(IHttpContext context, string src) { using (Database.DisableAllTriggersForCurrentThread()) { var document = Database.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null); SourceReplicationInformation sourceReplicationInformation; Guid serverInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote serve if (document == null) { sourceReplicationInformation = new SourceReplicationInformation() { Source = src }; } else { sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>(); sourceReplicationInformation.ServerInstanceId = serverInstanceId; } var currentEtag = context.Request.QueryString["currentEtag"]; log.Debug("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src, sourceReplicationInformation.LastDocumentEtag, currentEtag); context.WriteJson(sourceReplicationInformation); } }
private ReplicationTopologySourceNode HandleSource(SourceReplicationInformation source) { if (from.Contains(source.Source)) { var state = CheckSourceConnectionState(source.Source); switch (state) { case ReplicatonNodeState.Online: return ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag); case ReplicatonNodeState.Offline: return ReplicationTopologySourceNode.Offline(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag); default: throw new NotSupportedException(state.ToString()); } } string error; ReplicationTopologyRootNode rootNode; if (TryGetSchema(source.Source, new RavenConnectionStringOptions(), out rootNode, out error)) { var node = ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag); node.Destinations = rootNode.Destinations; node.Sources = rootNode.Sources; node.Errors = rootNode.Errors; return node; } var offline = ReplicationTopologySourceNode.Online(source.Source, source.ServerInstanceId, source.LastDocumentEtag, source.LastAttachmentEtag); if (string.IsNullOrEmpty(error) == false) offline.Errors.Add(error); return offline; }
public override void Respond(IHttpContext context) { var src = context.Request.QueryString["from"]; var currentEtag = context.Request.QueryString["currentEtag"]; if (string.IsNullOrEmpty(src)) { context.SetStatusToBadRequest(); return; } while (src.EndsWith("/")) src = src.Substring(0, src.Length - 1);// remove last /, because that has special meaning for Raven if (string.IsNullOrEmpty(src)) { context.SetStatusToBadRequest(); return; } using (Database.DisableAllTriggersForCurrentThread()) { var document = Database.Get(ReplicationConstants.RavenReplicationSourcesBasePath + "/" + src, null); SourceReplicationInformation sourceReplicationInformation; if (document == null) { sourceReplicationInformation = new SourceReplicationInformation() { ServerInstanceId = Database.TransactionalStorage.Id }; } else { sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>(); sourceReplicationInformation.ServerInstanceId = Database.TransactionalStorage.Id; } log.Debug("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src, sourceReplicationInformation.LastDocumentEtag, currentEtag); context.WriteJson(sourceReplicationInformation); } }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > 15) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false)// failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError))// success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var tuple = GetAttachments(destinationsReplicationInformationForSource, destination); var attachments = tuple.Item1; if (attachments == null || attachments.Length == 0) { if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag) { SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2); } return null; } string lastError; if (TryReplicationAttachments(destination, attachments, out lastError) == false)// failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationAttachments(destination, attachments, out lastError))// success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2); return true; } } RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2); return true; }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; Guid lastDocumentEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = actions.Documents.GetDocumentsAfter(lastDocumentEtag, 100).ToList(); filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(document, destinationId)).ToList(); docsSinceLastReplEtag += docsToReplicate.Count; if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } JsonDocument jsonDocument = docsToReplicate.Last(); Debug.Assert(jsonDocument.Etag != null); Guid documentEtag = jsonDocument.Etag.Value; log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", docsToReplicate); lastDocumentEtag = documentEtag; } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("Nothing to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag); if(docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag); }); jsonDocuments = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return jsonDocuments; }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source .Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Select(x=> { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return jsonDocuments; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(result.LastEtag) > 0) { log.Debug("Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag); return false; } } return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); result.CountOfFilteredDocumentsWhichOriginFromDestination += docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); scope.Record(new RavenJObject { {"StartEtag", lastEtag.ToString()}, {"EndEtag", result.LastEtag.ToString()}, {"Count", docsSinceLastReplEtag}, {"FilteredCount", filteredDocsToReplicate.Count} }); docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag); docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { scope.RecordError(e); log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024*1024*10) .Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => x.Etag) .ToList(); filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)). ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.AllFilteredDocumentsAreSystemDocuments = docsToReplicate.All(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } JsonDocument jsonDocument = docsToReplicate.Last(); Debug.Assert(jsonDocument.Etag != null); Guid documentEtag = jsonDocument.Etag.Value; log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag); result.LastEtag = documentEtag; } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonAttachments = null; try { string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag) .Where(destination.FilterAttachments) // we don't replicate stuff that was created there .Where(x=>x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId) .Take(100) .Select(x => new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", actions.Attachments.GetAttachment(x.Key).Data().ReadData()} })); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return jsonAttachments; }
private RavenJArray GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonDocuments = null; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonDocuments = new RavenJArray(actions.Documents.GetDocumentsAfter(destinationsReplicationInformationForSource.LastDocumentEtag) .Where(destination.FilterDocuments) .Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationId) // prevent replicating back to source .Select(x=> { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Take(100) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return jsonDocuments; }
private bool? ReplicateAttachments(string destination, SourceReplicationInformation sourceReplicationInformation) { var attachments = GetAttachments(sourceReplicationInformation.LastAttachmentEtag); if (attachments == null || attachments.Count == 0) return null; if (TryReplicationAttachments(destination, attachments) == false)// failed to replicate, start error handling strategy { if (IsFirstFailue(destination)) { log.InfoFormat( "This is the first failure for {0}, assuming transinet failure and trying again", destination); if (TryReplicationAttachments(destination, attachments))// success on second faile return true; } IncrementFailureCount(destination); return false; } return true; }
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray jsonAttachments = null; try { string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag,100) .Where(information => destination.FilterAttachments(information, destinationInstanceId)) .Select(x => new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", actions.Attachments.GetAttachment(x.Key).Data().ReadData()} })); }); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return jsonAttachments; }
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { Tuple<RavenJArray, Etag> tuple; RavenJArray attachments; using (var scope = recorder.StartRecording("Get")) { tuple = GetAttachments(destinationsReplicationInformationForSource, destination, scope); attachments = tuple.Item1; if (attachments == null || attachments.Length == 0) { if (tuple.Item2 != destinationsReplicationInformationForSource.LastAttachmentEtag) { SetLastReplicatedEtagForServer(destination, lastAttachmentEtag: tuple.Item2); } return null; } } using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationAttachments(destination, attachments, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info("This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationAttachments(destination, attachments, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2); return true; } } scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } RecordSuccess(destination.ConnectionStringOptions.Url, lastReplicatedEtag: tuple.Item2); return true; }
private Tuple<RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { RavenJArray attachments = null; Guid lastAttachmentEtag = Guid.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List<AttachmentInformation> attachmentsToReplicate; List<AttachmentInformation> filteredAttachmentsToReplicate; lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; while (true) { attachmentsToReplicate = actions.Attachments.GetAttachmentsAfter(lastAttachmentEtag, 100, 1024 * 1024 * 10) .Concat(actions.Lists.Read(Constants.RavenReplicationAttachmentsTombstones, lastAttachmentEtag, 100) .Select(x => new AttachmentInformation { Key = x.Key, Etag = x.Etag, Metadata = x.Data, Size = 0, })) .OrderBy(x => x.Etag) .ToList(); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Guid attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; } log.Debug(() => { if (attachmentSinceLastEtag == 0) return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag); if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) return string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination); var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag); }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } return new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", data} }; })); }); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return Tuple.Create(attachments, lastAttachmentEtag); }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { JsonDocumentsToReplicate documentsToReplicate; using (var scope = recorder.StartRecording("Get")) { documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, scope); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555 { using (scope.StartRecording("Notify")) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); scope.Record(new RavenJObject { { "LastDocEtag", documentsToReplicate.LastEtag.ToString() } }); } } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } } using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag(); var lastEtag = etagSynchronizer.CalculateSynchronizationEtag( synchronizationEtag, destinationsReplicationInformationForSource.LastDocumentEtag); int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (info.PreTouchEtag.CompareTo(result.LastEtag) >= 0) return false; } return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { RavenJArray attachments = null; Etag lastAttachmentEtag = Etag.Empty; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int attachmentSinceLastEtag = 0; List<AttachmentInformation> attachmentsToReplicate; List<AttachmentInformation> filteredAttachmentsToReplicate; var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag; lastAttachmentEtag = startEtag; while (true) { attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag); filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList(); attachmentSinceLastEtag += attachmentsToReplicate.Count; if (attachmentsToReplicate.Count == 0 || filteredAttachmentsToReplicate.Count != 0) { break; } AttachmentInformation jsonDocument = attachmentsToReplicate.Last(); Etag attachmentEtag = jsonDocument.Etag; log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag); lastAttachmentEtag = attachmentEtag; } log.Debug(() => { if (attachmentSinceLastEtag == 0) return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastAttachmentEtag); if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count) return string.Format("Replicating {0} attachments [>{1}] to {2}.", attachmentSinceLastEtag, destinationsReplicationInformationForSource.LastAttachmentEtag, destination); var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", attachmentSinceLastEtag, filteredAttachmentsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastAttachmentEtag); }); scope.Record(new RavenJObject { {"StartEtag", startEtag.ToString()}, {"EndEtag", lastAttachmentEtag.ToString()}, {"Count", attachmentSinceLastEtag}, {"FilteredCount", filteredAttachmentsToReplicate.Count} }); attachments = new RavenJArray(filteredAttachmentsToReplicate .Select(x => { var data = new byte[0]; if (x.Size > 0) { data = actions.Attachments.GetAttachment(x.Key).Data().ReadData(); } return new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", data} }; })); }); } catch (InvalidDataException e) { RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e)); scope.RecordError(e); log.ErrorException("Data is corrupted, could not proceed with replication", e); } catch (Exception e) { log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return Tuple.Create(attachments, lastAttachmentEtag); }
public HttpResponseMessage ReplicationLastEtagGet() { string src; string dbid; var result = GetValuesForLastEtag(out src, out dbid); if (result != null) return result; using (Database.DisableAllTriggersForCurrentThread()) { var document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null); SourceReplicationInformation sourceReplicationInformation; var serverInstanceId = Database.TransactionalStorage.Id; // this is my id, sent to the remote serve if (document == null) { sourceReplicationInformation = new SourceReplicationInformation() { Source = src, ServerInstanceId = serverInstanceId }; } else { sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>(); sourceReplicationInformation.ServerInstanceId = serverInstanceId; } var currentEtag = GetQueryStringValue("currentEtag"); Log.Debug(() => string.Format("Got replication last etag request from {0}: [Local: {1} Remote: {2}]", src, sourceReplicationInformation.LastDocumentEtag, currentEtag)); return GetMessageWithObject(sourceReplicationInformation); } }
private void OnPut(IHttpContext context, string src) { using (Database.DisableAllTriggersForCurrentThread()) { var document = Database.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null); SourceReplicationInformation sourceReplicationInformation; Etag docEtag = null, attachmentEtag = null; try { docEtag = Etag.Parse(context.Request.QueryString["docEtag"]); } catch { } try { attachmentEtag = Etag.Parse(context.Request.QueryString["attachmentEtag"]); } catch { } Guid serverInstanceId; if (Guid.TryParse(context.Request.QueryString["dbid"], out serverInstanceId) == false) serverInstanceId = Database.TransactionalStorage.Id; if (document == null) { sourceReplicationInformation = new SourceReplicationInformation() { ServerInstanceId = serverInstanceId, LastAttachmentEtag = attachmentEtag ?? Etag.Empty, LastDocumentEtag = docEtag ?? Etag.Empty, Source = src }; } else { sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>(); sourceReplicationInformation.ServerInstanceId = serverInstanceId; sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag; sourceReplicationInformation.LastAttachmentEtag = attachmentEtag ?? sourceReplicationInformation.LastAttachmentEtag; } var etag = document == null ? Etag.Empty : document.Etag; var metadata = document == null ? new RavenJObject() : document.Metadata; var newDoc = RavenJObject.FromObject(sourceReplicationInformation); log.Debug("Updating replication last etags from {0}: [doc: {1} attachment: {2}]", src, sourceReplicationInformation.LastDocumentEtag, sourceReplicationInformation.LastAttachmentEtag); Database.Put(Constants.RavenReplicationSourcesBasePath + "/" + src, etag, newDoc, metadata, null); } }
public HttpResponseMessage ReplicationLastEtagPut() { string src; string dbid; var result = GetValuesForLastEtag(out src, out dbid); if (result != null) return result; using (Database.DisableAllTriggersForCurrentThread()) { var document = Database.Documents.Get(Constants.RavenReplicationSourcesBasePath + "/" + src, null); SourceReplicationInformation sourceReplicationInformation; Etag docEtag = null, attachmentEtag = null; try { docEtag = Etag.Parse(GetQueryStringValue("docEtag")); } catch { } try { attachmentEtag = Etag.Parse(GetQueryStringValue("attachmentEtag")); } catch { } Guid serverInstanceId; if (Guid.TryParse(dbid, out serverInstanceId) == false) serverInstanceId = Database.TransactionalStorage.Id; if (document == null) { sourceReplicationInformation = new SourceReplicationInformation() { ServerInstanceId = serverInstanceId, LastAttachmentEtag = attachmentEtag ?? Etag.Empty, LastDocumentEtag = docEtag ?? Etag.Empty, Source = src }; } else { sourceReplicationInformation = document.DataAsJson.JsonDeserialization<SourceReplicationInformation>(); sourceReplicationInformation.ServerInstanceId = serverInstanceId; sourceReplicationInformation.LastDocumentEtag = docEtag ?? sourceReplicationInformation.LastDocumentEtag; sourceReplicationInformation.LastAttachmentEtag = attachmentEtag ?? sourceReplicationInformation.LastAttachmentEtag; } var etag = document == null ? Etag.Empty : document.Etag; var metadata = document == null ? new RavenJObject() : document.Metadata; var newDoc = RavenJObject.FromObject(sourceReplicationInformation); log.Debug("Updating replication last etags from {0}: [doc: {1} attachment: {2}]", src, sourceReplicationInformation.LastDocumentEtag, sourceReplicationInformation.LastAttachmentEtag); Database.Documents.Put(Constants.RavenReplicationSourcesBasePath + "/" + src, etag, newDoc, metadata, null); } return GetEmptyMessage(); }
private bool? ReplicateDocuments(RavenConnectionStringOptions destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var jsonDocuments = GetJsonDocuments(destinationsReplicationInformationForSource); if (jsonDocuments == null || jsonDocuments.Length == 0) return null; if (TryReplicationDocuments(destination, jsonDocuments) == false)// failed to replicate, start error handling strategy { if (IsFirstFailue(destination)) { log.Info( "This is the first failure for {0}, assuming transinet failure and trying again", destination); if (TryReplicationDocuments(destination, jsonDocuments))// success on second faile return true; } IncrementFailureCount(destination); return false; } return true; }
private bool? ReplicateDocuments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope recorder) { JsonDocumentsToReplicate documentsToReplicate = null; Stopwatch sp = Stopwatch.StartNew(); IDisposable removeBatch = null; var prefetchingBehavior = prefetchingBehaviors.GetOrAdd(destination.ConnectionStringOptions.Url, x => docDb.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Replicator, autoTuner)); try { using (var scope = recorder.StartRecording("Get")) { documentsToReplicate = GetJsonDocuments(destinationsReplicationInformationForSource, destination, prefetchingBehavior, scope); if (documentsToReplicate.Documents == null || documentsToReplicate.Documents.Length == 0) { if (documentsToReplicate.LastEtag != destinationsReplicationInformationForSource.LastDocumentEtag) { // we don't notify remote server about updates to system docs, see: RavenDB-715 if (documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments == 0 || documentsToReplicate.CountOfFilteredDocumentsWhichAreSystemDocuments > SystemDocsLimitForRemoteEtagUpdate || documentsToReplicate.CountOfFilteredDocumentsWhichOriginFromDestination > DestinationDocsLimitForRemoteEtagUpdate) // see RavenDB-1555 { using (scope.StartRecording("Notify")) { SetLastReplicatedEtagForServer(destination, lastDocEtag: documentsToReplicate.LastEtag); scope.Record(new RavenJObject { { "LastDocEtag", documentsToReplicate.LastEtag.ToString() } }); } } } RecordLastEtagChecked(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag); return null; } } // if the db is idling in all respect except sending out replication, let us keep it that way. docDb.WorkContext.UpdateFoundWork(); removeBatch = prefetchingBehavior.UpdateCurrentlyUsedBatches(documentsToReplicate.LoadedDocs); using (var scope = recorder.StartRecording("Send")) { string lastError; if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError) == false) // failed to replicate, start error handling strategy { if (IsFirstFailure(destination.ConnectionStringOptions.Url)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationDocuments(destination, documentsToReplicate.Documents, out lastError)) // success on second fail { RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; } } // if we had an error sending to this endpoint, it might be because we are sending too much data, or because // the request timed out. This will let us know that the next time we try, we'll use just the initial doc counts // and we'll be much more conservative with increasing the sizes prefetchingBehavior.OutOfMemoryExceptionHappened(); scope.RecordError(lastError); RecordFailure(destination.ConnectionStringOptions.Url, lastError); return false; } } } finally { if (documentsToReplicate != null && documentsToReplicate.LoadedDocs != null) prefetchingBehavior.UpdateAutoThrottler(documentsToReplicate.LoadedDocs, sp.Elapsed); if(removeBatch != null) removeBatch.Dispose(); } RecordSuccess(destination.ConnectionStringOptions.Url, documentsToReplicate.LastEtag, documentsToReplicate.LastLastModified); return true; }
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource) { RavenJArray jsonAttachments = null; try { string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag) .Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs .Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) != destinationInstanceId) // prevent replicating back to source .Where(x => x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict .Take(100) .Select(x => new RavenJObject { {"@metadata", x.Metadata}, {"@id", x.Key}, {"@etag", x.Etag.ToByteArray()}, {"data", actions.Attachments.GetAttachment(x.Key).Data} })); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e); } return jsonAttachments; }
private bool? ReplicateAttachments(ReplicationStrategy destination, SourceReplicationInformation destinationsReplicationInformationForSource) { var attachments = GetAttachments(destinationsReplicationInformationForSource, destination); if (attachments == null || attachments.Length == 0) return null; if (TryReplicationAttachments(destination, attachments) == false)// failed to replicate, start error handling strategy { if (IsFirstFailue(destination)) { log.Info( "This is the first failure for {0}, assuming transient failure and trying again", destination); if (TryReplicationAttachments(destination, attachments))// success on second fail return true; } IncrementFailureCount(destination); return false; } ResetFailureCount(destination); return true; }