private List<JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, PrefetchingBehavior prefetchingBehavior, JsonDocumentsToReplicate result, int? maxNumberOfItemsToReceiveInSingleBatch) { var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag, maxNumberOfItemsToReceiveInSingleBatch); Etag lastEtag = null; if (docsToReplicate.Count > 0) lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag; var maxNumberOfTombstones = Math.Max(1024, docsToReplicate.Count); var tombstones = actions .Lists .Read(Constants.RavenReplicationDocsTombstones, result.LastEtag, lastEtag, maxNumberOfTombstones + 1) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() }) .ToList(); var results = docsToReplicate.Concat(tombstones); if (tombstones.Count >= maxNumberOfTombstones + 1) { var lastTombstoneEtag = tombstones[tombstones.Count - 1].Etag; log.Info("Replication batch trimmed. Found more than '{0}' document tombstones. Last etag from prefetcher: '{1}'. Last tombstone etag: '{2}'.", maxNumberOfTombstones, lastEtag, lastTombstoneEtag); results = results.Where(x => EtagUtil.IsGreaterThan(x.Etag, lastTombstoneEtag) == false); } results = results.OrderBy(x => x.Etag); // can't return earlier, because we need to know if there are tombstones that need to be send if (maxNumberOfItemsToReceiveInSingleBatch.HasValue) results = results.Take(maxNumberOfItemsToReceiveInSingleBatch.Value); return results.ToList(); }
private List<JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result) { var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag); Etag lastEtag = null; if (docsToReplicate.Count > 0) { lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag; } return docsToReplicate.Concat(actions.Lists.Read(Constants.RavenReplicationDocsTombstones, result.LastEtag, lastEtag, 1024) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => x.Etag) .ToList(); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope) { var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds); var duration = Stopwatch.StartNew(); var result = new JsonDocumentsToReplicate { LastEtag = Etag.Empty, }; try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch; docDb.TransactionalStorage.Batch(actions => { var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested(); docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.Documents.GetRecentTouchesFor(document.Key); if (info != null) { if (info.TouchedEtag.CompareTo(result.LastEtag) > 0) { log.Debug( "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.TouchedEtag); return false; } } string reason; return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); result.CountOfFilteredDocumentsWhichOriginFromDestination += docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); if (duration.Elapsed > timeout) break; } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); scope.Record(new RavenJObject { {"StartEtag", lastEtag.ToString()}, {"EndEtag", result.LastEtag.ToString()}, {"Count", docsSinceLastReplEtag}, {"FilteredCount", filteredDocsToReplicate.Count} }); result.LoadedDocs = filteredDocsToReplicate; docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag); docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { JsonDocument.EnsureIdInMetadata(x); EnsureReplicationInformationInMetadata(x.Metadata, docDb); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { scope.RecordError(e); log.WarnException( "Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024*1024*10) .Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => x.Etag) .ToList(); filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)). ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.AllFilteredDocumentsAreSystemDocuments = docsToReplicate.All(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } JsonDocument jsonDocument = docsToReplicate.Last(); Debug.Assert(jsonDocument.Etag != null); Guid documentEtag = jsonDocument.Etag.Value; log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag); result.LastEtag = documentEtag; } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag(); var lastEtag = etagSynchronizer.CalculateSynchronizationEtag( synchronizationEtag, destinationsReplicationInformationForSource.LastDocumentEtag); int docsSinceLastReplEtag = 0; List<JsonDocument> docsToReplicate; List<JsonDocument> filteredDocsToReplicate; result.LastEtag = lastEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (info.PreTouchEtag.CompareTo(result.LastEtag) <= 0) { log.Debug("Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.PreTouchEtag); return false; } } return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag; if (lastDoc.LastModified.HasValue) result.LastLastModified = lastDoc.LastModified.Value; } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, lastEtag); if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) return string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, lastEtag, destination); var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), lastEtag); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return x; }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return result; }
private static List <JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result) { var docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 1024, 1024 * 1024 * 25).ToList(); Guid?lastEtag = null; if (docsToReplicate.Count > 0) { lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag; } return(docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => new ComparableByteArray(x.Etag ?? Guid.Empty)) .ToList()); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List <JsonDocument> docsToReplicate; List <JsonDocument> filteredDocsToReplicate; result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = GetDocsToReplicate(actions, result); filteredDocsToReplicate = docsToReplicate .Where(document => { var info = docDb.GetRecentTouchesFor(document.Key); if (info != null) { if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false) { return(false); } } return(destination.FilterDocuments(destinationId, document.Key, document.Metadata)); }) .ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count > 0) { var lastDoc = docsToReplicate.Last(); Debug.Assert(lastDoc.Etag != null); result.LastEtag = lastDoc.Etag.Value; if (lastDoc.LastModified.HasValue) { result.LastLastModified = lastDoc.LastModified.Value; } } if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag); } log.Debug(() => { if (docsSinceLastReplEtag == 0) { return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag)); } if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) { return(string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination)); } var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag)); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(result); }
private static List<JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result) { var docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 1024, 1024*1024*25).ToList(); Guid? lastEtag = null; if (docsToReplicate.Count > 0) { lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag; } return docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => new ComparableByteArray(x.Etag ?? Guid.Empty)) .ToList(); }
private List <JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result) { var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag); Etag lastEtag = null; if (docsToReplicate.Count > 0) { lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag; } return(docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => x.Etag) .ToList()); }
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination) { var result = new JsonDocumentsToReplicate(); try { var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString(); docDb.TransactionalStorage.Batch(actions => { int docsSinceLastReplEtag = 0; List <JsonDocument> docsToReplicate; List <JsonDocument> filteredDocsToReplicate; result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag; while (true) { docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024 * 1024 * 10) .Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100) .Select(x => new JsonDocument { Etag = x.Etag, Key = x.Key, Metadata = x.Data, DataAsJson = new RavenJObject() })) .OrderBy(x => x.Etag) .ToList(); filteredDocsToReplicate = docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)). ToList(); docsSinceLastReplEtag += docsToReplicate.Count; result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key)); if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0) { break; } JsonDocument jsonDocument = docsToReplicate.Last(); Debug.Assert(jsonDocument.Etag != null); Guid documentEtag = jsonDocument.Etag.Value; log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag); result.LastEtag = documentEtag; } log.Debug(() => { if (docsSinceLastReplEtag == 0) { return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination, destinationsReplicationInformationForSource.LastDocumentEtag)); } if (docsSinceLastReplEtag == filteredDocsToReplicate.Count) { return(string.Format("Replicating {0} docs [>{1}] to {2}.", docsSinceLastReplEtag, destinationsReplicationInformationForSource.LastDocumentEtag, destination)); } var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key); return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]", docsSinceLastReplEtag, filteredDocsToReplicate.Count, destination, string.Join(", ", diff), destinationsReplicationInformationForSource.LastDocumentEtag)); }); result.Documents = new RavenJArray(filteredDocsToReplicate .Select(x => { DocumentRetriever.EnsureIdInMetadata(x); return(x); }) .Select(x => x.ToJson())); }); } catch (Exception e) { log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e); } return(result); }