private List<JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, PrefetchingBehavior prefetchingBehavior, JsonDocumentsToReplicate result, int? maxNumberOfItemsToReceiveInSingleBatch)
{
var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag, maxNumberOfItemsToReceiveInSingleBatch);
Etag lastEtag = null;
if (docsToReplicate.Count > 0)
lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag;
var maxNumberOfTombstones = Math.Max(1024, docsToReplicate.Count);
var tombstones = actions
.Lists
.Read(Constants.RavenReplicationDocsTombstones, result.LastEtag, lastEtag, maxNumberOfTombstones + 1)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
})
.ToList();
var results = docsToReplicate.Concat(tombstones);
if (tombstones.Count >= maxNumberOfTombstones + 1)
{
var lastTombstoneEtag = tombstones[tombstones.Count - 1].Etag;
log.Info("Replication batch trimmed. Found more than '{0}' document tombstones. Last etag from prefetcher: '{1}'. Last tombstone etag: '{2}'.", maxNumberOfTombstones, lastEtag, lastTombstoneEtag);
results = results.Where(x => EtagUtil.IsGreaterThan(x.Etag, lastTombstoneEtag) == false);
}
results = results.OrderBy(x => x.Etag);
// can't return earlier, because we need to know if there are tombstones that need to be send
if (maxNumberOfItemsToReceiveInSingleBatch.HasValue)
results = results.Take(maxNumberOfItemsToReceiveInSingleBatch.Value);
return results.ToList();
}
private List<JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result)
{
var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag);
Etag lastEtag = null;
if (docsToReplicate.Count > 0)
{
lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag;
}
return docsToReplicate.Concat(actions.Lists.Read(Constants.RavenReplicationDocsTombstones, result.LastEtag, lastEtag, 1024)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList();
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
var duration = Stopwatch.StartNew();
var result = new JsonDocumentsToReplicate
{
LastEtag = Etag.Empty,
};
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;
docDb.TransactionalStorage.Batch(actions =>
{
var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested();
docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.Documents.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
{
log.Debug(
"Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
document.Key, destinationId, info.TouchedEtag);
return false;
}
}
string reason;
return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) &&
prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments +=
docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
result.CountOfFilteredDocumentsWhichOriginFromDestination +=
docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
if (duration.Elapsed > timeout)
break;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", lastEtag.ToString()},
{"EndEtag", result.LastEtag.ToString()},
{"Count", docsSinceLastReplEtag},
{"FilteredCount", filteredDocsToReplicate.Count}
});
result.LoadedDocs = filteredDocsToReplicate;
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag);
docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag);
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
JsonDocument.EnsureIdInMetadata(x);
EnsureReplicationInformationInMetadata(x.Metadata, docDb);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
scope.RecordError(e);
log.WarnException(
"Could not get documents to replicate after: " +
destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024*1024*10)
.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList();
filteredDocsToReplicate =
docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)).
ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.AllFilteredDocumentsAreSystemDocuments =
docsToReplicate.All(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag);
result.LastEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag);
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var synchronizationEtag = etagSynchronizer.GetSynchronizationEtag();
var lastEtag = etagSynchronizer.CalculateSynchronizationEtag(
synchronizationEtag,
destinationsReplicationInformationForSource.LastDocumentEtag);
int docsSinceLastReplEtag = 0;
List<JsonDocument> docsToReplicate;
List<JsonDocument> filteredDocsToReplicate;
result.LastEtag = lastEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (info.PreTouchEtag.CompareTo(result.LastEtag) <= 0)
{
log.Debug("Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches", document.Key, destinationId, info.PreTouchEtag);
return false;
}
}
return destination.FilterDocuments(destinationId, document.Key, document.Metadata) && prefetchingBehavior.FilterDocuments(document);
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag;
if (lastDoc.LastModified.HasValue)
result.LastLastModified = lastDoc.LastModified.Value;
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
lastEtag);
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
return string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
lastEtag,
destination);
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
lastEtag);
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return x;
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return result;
}
private static List <JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result)
{
var docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 1024, 1024 * 1024 * 25).ToList();
Guid?lastEtag = null;
if (docsToReplicate.Count > 0)
{
lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag;
}
return(docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => new ComparableByteArray(x.Etag ?? Guid.Empty))
.ToList());
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = GetDocsToReplicate(actions, result);
filteredDocsToReplicate =
docsToReplicate
.Where(document =>
{
var info = docDb.GetRecentTouchesFor(document.Key);
if (info != null)
{
if (Etag.IsGreaterThan(info.PreTouchEtag, result.LastEtag) == false)
{
return(false);
}
}
return(destination.FilterDocuments(destinationId, document.Key, document.Metadata));
})
.ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count > 0)
{
var lastDoc = docsToReplicate.Last();
Debug.Assert(lastDoc.Etag != null);
result.LastEtag = lastDoc.Etag.Value;
if (lastDoc.LastModified.HasValue)
{
result.LastLastModified = lastDoc.LastModified.Value;
}
}
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(result);
}
private static List<JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result)
{
var docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 1024, 1024*1024*25).ToList();
Guid? lastEtag = null;
if (docsToReplicate.Count > 0)
{
lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag;
}
return docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => new ComparableByteArray(x.Etag ?? Guid.Empty))
.ToList();
}
private List <JsonDocument> GetDocsToReplicate(IStorageActionsAccessor actions, JsonDocumentsToReplicate result)
{
var docsToReplicate = prefetchingBehavior.GetDocumentsBatchFrom(result.LastEtag);
Etag lastEtag = null;
if (docsToReplicate.Count > 0)
{
lastEtag = docsToReplicate[docsToReplicate.Count - 1].Etag;
}
return(docsToReplicate.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, lastEtag, 1024)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList());
}
private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
var result = new JsonDocumentsToReplicate();
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int docsSinceLastReplEtag = 0;
List <JsonDocument> docsToReplicate;
List <JsonDocument> filteredDocsToReplicate;
result.LastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;
while (true)
{
docsToReplicate = actions.Documents.GetDocumentsAfter(result.LastEtag, 100, 1024 * 1024 * 10)
.Concat(actions.Lists.Read("Raven/Replication/Docs/Tombstones", result.LastEtag, 100)
.Select(x => new JsonDocument
{
Etag = x.Etag,
Key = x.Key,
Metadata = x.Data,
DataAsJson = new RavenJObject()
}))
.OrderBy(x => x.Etag)
.ToList();
filteredDocsToReplicate =
docsToReplicate.Where(document => destination.FilterDocuments(destinationId, document.Key, document.Metadata)).
ToList();
docsSinceLastReplEtag += docsToReplicate.Count;
result.CountOfFilteredDocumentsWhichAreSystemDocuments += docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
{
break;
}
JsonDocument jsonDocument = docsToReplicate.Last();
Debug.Assert(jsonDocument.Etag != null);
Guid documentEtag = jsonDocument.Etag.Value;
log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", documentEtag);
result.LastEtag = documentEtag;
}
log.Debug(() =>
{
if (docsSinceLastReplEtag == 0)
{
return(string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
{
return(string.Format("Replicating {0} docs [>{1}] to {2}.",
docsSinceLastReplEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
docsSinceLastReplEtag,
filteredDocsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
result.Documents = new RavenJArray(filteredDocsToReplicate
.Select(x =>
{
DocumentRetriever.EnsureIdInMetadata(x);
return(x);
})
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.WarnException("Could not get documents to replicate after: " + destinationsReplicationInformationForSource.LastDocumentEtag, e);
}
return(result);
}