| public FilterAttachments ( Raven.Database.Data.AttachmentInformation attachment, string destinationInstanceId ) : bool | ||
| attachment | Raven.Database.Data.AttachmentInformation | |
| destinationInstanceId | string | |
| return | bool | |
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag, 100)
.Where(information => destination.FilterAttachments(information, destinationInstanceId))
.Select(x => new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", actions.Attachments.GetAttachment(x.Key).Data().ReadData() }
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(jsonAttachments);
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
{
var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
var duration = Stopwatch.StartNew();
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
var startEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
lastAttachmentEtag = startEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag, maxNumberOfItemsToReceiveInSingleBatch);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
if (duration.Elapsed > timeout)
break;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastAttachmentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastAttachmentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastAttachmentEtag);
});
scope.Record(new RavenJObject
{
{"StartEtag", startEtag.ToString()},
{"EndEtag", lastAttachmentEtag.ToString()},
{"Count", attachmentSinceLastEtag},
{"FilteredCount", filteredAttachmentsToReplicate.Count}
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
EnsureReplicationInformationInMetadata(x.Metadata, docDb);
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (InvalidDataException e)
{
RecordFailure(String.Empty, string.Format("Data is corrupted, could not proceed with attachment replication. Exception : {0}", e));
scope.RecordError(e);
log.ErrorException("Data is corrupted, could not proceed with replication", e);
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private Tuple<RavenJArray, Etag> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Etag lastAttachmentEtag = Etag.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List<AttachmentInformation> attachmentsToReplicate;
List<AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Etag attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
return string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag);
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
return string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination);
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag);
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", data}
};
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return Tuple.Create(attachments, lastAttachmentEtag);
}
private Tuple <RavenJArray, Guid> GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray attachments = null;
Guid lastAttachmentEtag = Guid.Empty;
try
{
var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
int attachmentSinceLastEtag = 0;
List <AttachmentInformation> attachmentsToReplicate;
List <AttachmentInformation> filteredAttachmentsToReplicate;
lastAttachmentEtag = destinationsReplicationInformationForSource.LastAttachmentEtag;
while (true)
{
attachmentsToReplicate = GetAttachmentsToReplicate(actions, lastAttachmentEtag);
filteredAttachmentsToReplicate = attachmentsToReplicate.Where(attachment => destination.FilterAttachments(attachment, destinationId)).ToList();
attachmentSinceLastEtag += attachmentsToReplicate.Count;
if (attachmentsToReplicate.Count == 0 ||
filteredAttachmentsToReplicate.Count != 0)
{
break;
}
AttachmentInformation jsonDocument = attachmentsToReplicate.Last();
Guid attachmentEtag = jsonDocument.Etag;
log.Debug("All the attachments were filtered, trying another batch from etag [>{0}]", attachmentEtag);
lastAttachmentEtag = attachmentEtag;
}
log.Debug(() =>
{
if (attachmentSinceLastEtag == 0)
{
return(string.Format("No attachments to replicate to {0} - last replicated etag: {1}", destination,
destinationsReplicationInformationForSource.LastDocumentEtag));
}
if (attachmentSinceLastEtag == filteredAttachmentsToReplicate.Count)
{
return(string.Format("Replicating {0} attachments [>{1}] to {2}.",
attachmentSinceLastEtag,
destinationsReplicationInformationForSource.LastDocumentEtag,
destination));
}
var diff = attachmentsToReplicate.Except(filteredAttachmentsToReplicate).Select(x => x.Key);
return(string.Format("Replicating {1} attachments (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
attachmentSinceLastEtag,
filteredAttachmentsToReplicate.Count,
destination,
string.Join(", ", diff),
destinationsReplicationInformationForSource.LastDocumentEtag));
});
attachments = new RavenJArray(filteredAttachmentsToReplicate
.Select(x =>
{
var data = new byte[0];
if (x.Size > 0)
{
data = actions.Attachments.GetAttachment(x.Key).Data().ReadData();
}
return(new RavenJObject
{
{ "@metadata", x.Metadata },
{ "@id", x.Key },
{ "@etag", x.Etag.ToByteArray() },
{ "data", data }
});
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return(Tuple.Create(attachments, lastAttachmentEtag));
}
private RavenJArray GetAttachments(SourceReplicationInformation destinationsReplicationInformationForSource, ReplicationStrategy destination)
{
RavenJArray jsonAttachments = null;
try
{
string destinationInstanceId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
jsonAttachments = new RavenJArray(actions.Attachments.GetAttachmentsAfter(destinationsReplicationInformationForSource.LastAttachmentEtag,100)
.Where(information => destination.FilterAttachments(information, destinationInstanceId))
.Select(x => new RavenJObject
{
{"@metadata", x.Metadata},
{"@id", x.Key},
{"@etag", x.Etag.ToByteArray()},
{"data", actions.Attachments.GetAttachment(x.Key).Data().ReadData()}
}));
});
}
catch (Exception e)
{
log.WarnException("Could not get attachments to replicate after: " + destinationsReplicationInformationForSource.LastAttachmentEtag, e);
}
return jsonAttachments;
}
