protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag)
{
var totalCount = 0;
while (true)
{
var array = GetAttachments(totalCount, lastEtag);
if (array.Length == 0)
{
var databaseStatistics = GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += array.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount);
foreach (var item in array)
{
item.WriteTo(jsonWriter);
}
lastEtag = new Guid(array.Last().Value <string>("Etag"));
}
}
public void CleanupDocuments(Etag lastIndexedEtag)
{
var highest = new ComparableByteArray(lastIndexedEtag);
foreach (var docToRemove in documentsToRemove)
{
if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false)
{
continue;
}
HashSet <Etag> _;
documentsToRemove.TryRemove(docToRemove.Key, out _);
}
foreach (var updatedDocs in updatedDocuments)
{
if (updatedDocs.Value.All(etag => highest.CompareTo(etag) > 0) == false)
{
continue;
}
HashSet <Etag> _;
updatedDocuments.TryRemove(updatedDocs.Key, out _);
}
JsonDocument result;
while (prefetchingQueue.TryPeek(out result) && highest.CompareTo(result.Etag) >= 0)
{
prefetchingQueue.TryDequeue(out result);
}
}
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag)
{
var totalCount = 0;
while (true)
{
var array = GetAttachments(totalCount, lastEtag);
if (array.Length == 0)
{
var databaseStatistics = GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += array.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount);
foreach (var item in array)
{
item.WriteTo(jsonWriter);
}
lastEtag = new Guid(array.Last().Value<string>("Etag"));
}
}
private bool TryGetInMemoryJsonDocuments(Guid nextDocEtag, List <JsonDocument> items)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing)
{
return(false);
}
JsonDocument result;
bool hasDocs = false;
while (inMemoryDocs.TryPeek(out result) &&
ComparableByteArray.CompareTo(nextDocEtag.ToByteArray(), result.Etag.Value.ToByteArray()) >= 0)
{
// safe to do peek then dequeue because we are the only one doing the dequeues
// and here we are single threaded
inMemoryDocs.TryDequeue(out result);
if (result.Etag.Value != nextDocEtag)
{
continue;
}
items.Add(result);
hasDocs = true;
nextDocEtag = Etag.Increment(nextDocEtag, 1);
}
return(hasDocs);
}
protected void HandleReduceForIndex(IndexToWorkOn indexToWorkOn)
{
List <MappedResultInfo> reduceKeyAndEtags = null;
try
{
transactionalStorage.Batch(actions =>
{
reduceKeyAndEtags = actions.MappedResults.GetMappedResultsReduceKeysAfter
(
indexToWorkOn.IndexName,
indexToWorkOn.LastIndexedEtag,
loadData: false,
// for reduce operations, we use the smaller value, rather than tuning stuff on the fly
// the reason for that is that we may have large number of map values to reduce anyway,
// so we don't want to try to load too much all at once.
take: context.Configuration.InitialNumberOfItemsToIndexInSingleBatch
)
.ToList();
if (log.IsDebugEnabled)
{
if (reduceKeyAndEtags.Count > 0)
{
log.Debug(() => string.Format("Found {0} mapped results for keys [{1}] for index {2}", reduceKeyAndEtags.Count, string.Join(", ", reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct()), indexToWorkOn.IndexName));
}
else
{
log.Debug("No reduce keys found for {0}", indexToWorkOn.IndexName);
}
}
new ReduceTask
{
Index = indexToWorkOn.IndexName,
ReduceKeys = reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct().ToArray(),
}.Execute(context);
});
}
finally
{
if (reduceKeyAndEtags != null && reduceKeyAndEtags.Count > 0)
{
var lastByEtag = GetLastByEtag(reduceKeyAndEtags);
var lastEtag = lastByEtag.Etag;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed mapped results
transactionalStorage.Batch(actions =>
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) <= 0)
{
actions.Indexing.UpdateLastReduced(indexToWorkOn.IndexName, lastByEtag.Etag, lastByEtag.Timestamp);
}
});
}
}
}
private void MarkIndexes(IndexToWorkOn indexToWorkOn, ComparableByteArray lastIndexedEtag, IStorageActionsAccessor actions, Guid lastEtag, DateTime lastModified)
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0)
{
return;
}
actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified);
}
protected void HandleReduceForIndex(IndexToWorkOn indexToWorkOn)
{
List <MappedResultInfo> reduceKeyAndEtags = null;
try
{
transactionalStorage.Batch(actions =>
{
reduceKeyAndEtags = actions.MappedResults.GetMappedResultsReduceKeysAfter
(
indexToWorkOn.IndexName,
indexToWorkOn.LastIndexedEtag,
loadData: false,
take: autoTuner.NumberOfItemsToIndexInSingleBatch
)
.ToList();
if (log.IsDebugEnabled)
{
if (reduceKeyAndEtags.Count > 0)
{
log.Debug(() => string.Format("Found {0} mapped results for keys [{1}] for index {2}", reduceKeyAndEtags.Count, string.Join(", ", reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct()), indexToWorkOn.IndexName));
}
else
{
log.Debug("No reduce keys found for {0}", indexToWorkOn.IndexName);
}
}
new ReduceTask
{
Index = indexToWorkOn.IndexName,
ReduceKeys = reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct().ToArray(),
}.Execute(context);
});
}
finally
{
if (reduceKeyAndEtags != null && reduceKeyAndEtags.Count > 0)
{
var lastByEtag = GetLastByEtag(reduceKeyAndEtags);
var lastEtag = lastByEtag.Etag;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed mapped results
transactionalStorage.Batch(actions =>
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) <= 0)
{
actions.Indexing.UpdateLastReduced(indexToWorkOn.IndexName, lastByEtag.Etag, lastByEtag.Timestamp);
}
});
autoTuner.AutoThrottleBatchSize(reduceKeyAndEtags.Count, reduceKeyAndEtags.Sum(x => x.Size));
}
}
}
protected void HandleReduceForIndex(IndexToWorkOn indexToWorkOn)
{
TimeSpan reduceDuration= TimeSpan.Zero;
List<MappedResultInfo> reduceKeyAndEtags = null;
try
{
transactionalStorage.Batch(actions =>
{
reduceKeyAndEtags = actions.MappedResults.GetMappedResultsReduceKeysAfter
(
indexToWorkOn.IndexName,
indexToWorkOn.LastIndexedEtag,
loadData: false,
take: autoTuner.NumberOfItemsToIndexInSingleBatch
)
.ToList();
if(log.IsDebugEnabled)
{
if (reduceKeyAndEtags.Count > 0)
log.Debug(() => string.Format("Found {0} mapped results for keys [{1}] for index {2}", reduceKeyAndEtags.Count, string.Join(", ", reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct()), indexToWorkOn.IndexName));
else
log.Debug("No reduce keys found for {0}", indexToWorkOn.IndexName);
}
var sw = Stopwatch.StartNew();
new ReduceTask
{
Index = indexToWorkOn.IndexName,
ReduceKeys = reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct().ToArray(),
}.Execute(context);
reduceDuration = sw.Elapsed;
});
}
finally
{
if (reduceKeyAndEtags != null && reduceKeyAndEtags.Count > 0)
{
var lastByEtag = GetLastByEtag(reduceKeyAndEtags);
var lastEtag = lastByEtag.Etag;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed mapped results
transactionalStorage.Batch(actions =>
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) <= 0)
{
actions.Indexing.UpdateLastReduced(indexToWorkOn.IndexName, lastByEtag.Etag, lastByEtag.Timestamp);
}
});
autoTuner.AutoThrottleBatchSize(reduceKeyAndEtags.Count, reduceKeyAndEtags.Sum(x => x.Size), reduceDuration);
}
}
}
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
{
throw new ArgumentException("We don't support maxEtag in SmugglerApi", "maxEtag");
}
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
await commands.CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag, "GET")
.ReadResponseJsonAsync()
.ContinueWith(task => attachmentInfo = (RavenJArray)task.Result);
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
await commands.CreateRequest("/static/" + item.Value <string>("Key"), "GET")
.ReadResponseBytesAsync()
.ContinueWith(task => attachmentData = task.Result);
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(attachmentInfo.Last().Value <string>("Etag"));
}
}
private void ExecuteIndexingInternal(IEnumerable <IndexToWorkOn> indexesToWorkOn, Action <JsonDocument[]> indexingOp)
{
var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid();
JsonDocument[] jsonDocs = null;
try
{
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
});
if (jsonDocs.Length > 0)
{
indexingOp(jsonDocs);
}
}
finally
{
if (jsonDocs != null && jsonDocs.Length > 0)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
transactionalStorage.Batch(actions =>
{
foreach (var indexToWorkOn in indexesToWorkOn)
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0)
{
continue;
}
actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified);
}
});
}
}
}
protected void HandleReduceForIndex(IndexToWorkOn indexToWorkOn)
{
List<MappedResultInfo> reduceKeyAndEtags = null;
try
{
transactionalStorage.Batch(actions =>
{
reduceKeyAndEtags = actions.MappedResults.GetMappedResultsReduceKeysAfter
(
indexToWorkOn.IndexName,
indexToWorkOn.LastIndexedEtag,
loadData: false,
// for reduce operations, we use the smaller value, rather than tuning stuff on the fly
// the reason for that is that we may have large number of map values to reduce anyway,
// so we don't want to try to load too much all at once.
take: context.Configuration.InitialNumberOfItemsToIndexInSingleBatch
)
.ToList();
if(log.IsDebugEnabled)
{
if (reduceKeyAndEtags.Count > 0)
log.Debug(() => string.Format("Found {0} mapped results for keys [{1}] for index {2}", reduceKeyAndEtags.Count, string.Join(", ", reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct()), indexToWorkOn.IndexName));
else
log.Debug("No reduce keys found for {0}", indexToWorkOn.IndexName);
}
new ReduceTask
{
Index = indexToWorkOn.IndexName,
ReduceKeys = reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct().ToArray(),
}.Execute(context);
});
}
finally
{
if (reduceKeyAndEtags != null && reduceKeyAndEtags.Count > 0)
{
var lastByEtag = GetLastByEtag(reduceKeyAndEtags);
var lastEtag = lastByEtag.Etag;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed mapped results
transactionalStorage.Batch(actions =>
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) <= 0)
{
actions.Indexing.UpdateLastReduced(indexToWorkOn.IndexName, lastByEtag.Etag, lastByEtag.Timestamp);
}
});
}
}
}
private IEnumerable <Tuple <IndexToWorkOn, IEnumerable <JsonDocument> > > FilterIndexes(IEnumerable <IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
Action <IStorageActionsAccessor> action = null;
foreach (var indexToWorkOn in indexesToWorkOn)
{
var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray());
if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0)
{
continue;
}
var filteredDocs = jsonDocs.Where(doc => indexLastInedexEtag.CompareTo(new ComparableByteArray(doc.Etag.Value.ToByteArray())) < 0);
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
continue; // probably deleted
}
if (viewGenerator.ForEntityNames.Count != 0) // limit for the items that it care for
{
filteredDocs = filteredDocs.Where(x => viewGenerator.ForEntityNames.Contains(x.Metadata.Value <string>(Constants.RavenEntityName)));
}
List <JsonDocument> jsonDocuments = filteredDocs.ToList();
if (jsonDocuments.Count == 0)
{
// we use it this way to batch all the updates together
action += accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
continue;
}
yield return(Tuple.Create <IndexToWorkOn, IEnumerable <JsonDocument> >(indexToWorkOn, jsonDocuments));
}
if (action != null)
{
transactionalStorage.Batch(action);
}
}
protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
await commands.CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag, "GET")
.ReadResponseJsonAsync()
.ContinueWith(task => attachmentInfo = (RavenJArray) task.Result);
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value<string>("Key"));
byte[] attachmentData = null;
await commands.CreateRequest("/static/" + item.Value<string>("Key"), "GET")
.ReadResponseBytesAsync()
.ContinueWith(task => attachmentData = task.Result);
new RavenJObject
{
{"Data", attachmentData},
{"Metadata", item.Value<RavenJObject>("Metadata")},
{"Key", item.Value<string>("Key")}
}
.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(attachmentInfo.Last().Value<string>("Etag"));
}
}
private static Guid GetLowestEtag(JsonDocument[] past)
{
var lowest = new ComparableByteArray(past[0].Etag.Value);
for (int i = 1; i < past.Length; i++)
{
var etag = past[i].Etag.Value;
if (lowest.CompareTo(etag) < 0)
{
continue;
}
lowest = new ComparableByteArray(etag);
}
return(lowest.ToGuid());
}
protected void HandleReduceForIndex(IndexToWorkOn indexToWorkOn)
{
List<MappedResultInfo> reduceKeyAndEtags = null;
try
{
transactionalStorage.Batch(actions =>
{
reduceKeyAndEtags = actions.MappedResults.GetMappedResultsReduceKeysAfter
(
indexToWorkOn.IndexName,
indexToWorkOn.LastIndexedEtag
).ToList();
if(log.IsDebugEnabled)
{
if (reduceKeyAndEtags.Count > 0)
log.Debug(() => string.Format("Found {0} mapped results for keys [{1}] for index {2}", reduceKeyAndEtags.Count, string.Join(", ", reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct()), indexToWorkOn.IndexName));
else
log.Debug("No reduce keys found for {0}", indexToWorkOn.IndexName);
}
new ReduceTask
{
Index = indexToWorkOn.IndexName,
ReduceKeys = reduceKeyAndEtags.Select(x => x.ReduceKey).Distinct().ToArray(),
}.Execute(context);
});
}
finally
{
if (reduceKeyAndEtags != null && reduceKeyAndEtags.Count > 0)
{
var lastByEtag = GetLastByEtag(reduceKeyAndEtags);
var lastEtag = lastByEtag.Etag;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed mapped results
transactionalStorage.Batch(actions =>
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) <= 0)
{
actions.Indexing.UpdateLastReduced(indexToWorkOn.IndexName, lastByEtag.Etag, lastByEtag.Timestamp);
}
});
}
}
}
public void RemoveAllBefore(string name, Etag etag)
{
var comparable = new ComparableByteArray(etag);
var results = storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
{
{"name", name},
{"etag", Guid.Empty.ToByteArray()}
})
.TakeWhile(x => String.Equals(x.Value<string>("name"), name, StringComparison.OrdinalIgnoreCase) &&
comparable.CompareTo(x.Value<byte[]>("etag")) >= 0);
foreach (var result in results)
{
storage.Lists.Remove(result);
}
}
public void RemoveAllBefore(string name, Guid etag)
{
var comparable = new ComparableByteArray(etag);
var results = storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
{
{ "name", name },
{ "etag", Guid.Empty.ToByteArray() }
})
.TakeWhile(x => String.Equals(x.Value <string>("name"), name, StringComparison.InvariantCultureIgnoreCase) &&
comparable.CompareTo(x.Value <byte[]>("etag")) >= 0);
foreach (var result in results)
{
storage.Lists.Remove(result);
}
}
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
var request = CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest("/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(attachmentInfo.Last().Value <string>("Etag"));
}
}
public static JsonDocument GetHighestJsonDocumentByEtag(List <JsonDocument> past)
{
var highest = new ComparableByteArray(Guid.Empty);
JsonDocument highestDoc = null;
for (int i = past.Count - 1; i >= 0; i--)
{
Guid etag = past[i].Etag.Value;
if (highest.CompareTo(etag) > 0)
{
continue;
}
highest = new ComparableByteArray(etag);
highestDoc = past[i];
}
return(highestDoc);
}
private static JsonDocument GetHighestEtag(JsonDocument[] past)
{
var highest = new ComparableByteArray(Guid.Empty);
JsonDocument highestDoc = null;
for (int i = past.Length - 1; i >= 0; i--)
{
var etag = past[i].Etag.Value;
if (highest.CompareTo(etag) > 0)
{
continue;
}
highest = new ComparableByteArray(etag);
highestDoc = past[i];
}
return(highestDoc);
}
private async static Task <Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize)
{
Etag lastEtag = databaseOptions.StartAttachmentsEtag;
int totalCount = 0;
while (true)
{
var attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize);
if (attachments.Length == 0)
{
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachments.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount);
foreach (var attachmentInformation in attachments)
{
if (databaseOptions.StripReplicationInformation)
{
attachmentInformation.Metadata = StripReplicationInformationFromMetadata(attachmentInformation.Metadata);
}
ShowProgress("Downloading attachment: {0}", attachmentInformation.Key);
var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key);
await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata);
}
lastEtag = Etag.Parse(attachments.Last().Etag);
}
}
private Guid ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Guid lastEtag)
{
int totalCount = 0;
while (true)
{
var watch = Stopwatch.StartNew();
var documents = GetDocuments(lastEtag);
watch.Stop();
if (documents.Length == 0)
{
var databaseStatistics = GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
var currentProcessingTime = watch.Elapsed;
ModifyBatchSize(options, currentProcessingTime);
var final = documents.Where(options.MatchFilters).ToList();
if (options.ShouldExcludeExpired)
{
final = documents.Where(options.ExcludeExpired).ToList();
}
final.ForEach(item => item.WriteTo(jsonWriter));
totalCount += final.Count;
ShowProgress("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount);
lastEtag = new Guid(documents.Last().Value <RavenJObject>("@metadata").Value <string>("@etag"));
}
}
public void CleanupDocumentsToRemove(Guid lastIndexedEtag)
{
var highest = new ComparableByteArray(lastIndexedEtag);
foreach (var docToRemove in documentsToRemove)
{
if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false)
{
continue;
}
HashSet <Guid> _;
documentsToRemove.TryRemove(docToRemove.Key, out _);
}
JsonDocument result;
while (inMemoryDocs.TryPeek(out result) && highest.CompareTo(result.Etag.Value) >= 0)
{
inMemoryDocs.TryDequeue(out result);
}
}
public static JsonDocument GetHighestJsonDocumentByEtag(List<JsonDocument> past)
{
var highest = new ComparableByteArray(Etag.Empty);
JsonDocument highestDoc = null;
for (int i = past.Count - 1; i >= 0; i--)
{
Etag etag = past[i].Etag;
if (highest.CompareTo(etag) > 0)
{
continue;
}
highest = new ComparableByteArray(etag);
highestDoc = past[i];
}
return highestDoc;
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private IEnumerable<Tuple<IndexToWorkOn, IndexingBatch>> FilterIndexes(IList<IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc =>
{
var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index);
return filteredDoc == null ? new
{
Doc = doc,
Json = (object)new FilteredDocument(doc)
} : new
{
Doc = filteredDoc,
Json = JsonToExpando.Convert(doc.ToJson())
};
});
Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new Tuple<IndexToWorkOn, IndexingBatch>[indexesToWorkOn.Count];
var actions = new Action<IStorageActionsAccessor>[indexesToWorkOn.Count];
BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) =>
{
var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray());
if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0)
return;
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
return; // probably deleted
var batch = new IndexingBatch();
foreach (var item in filteredDocs)
{
if (FilterDocuments(item.Doc))
continue;
// did we already indexed this document in this index?
var etag = item.Doc.Etag;
if (etag == null)
continue;
if (indexLastInedexEtag.CompareTo(new ComparableByteArray(etag.Value.ToByteArray())) >= 0)
continue;
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value<string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(item.Doc, item.Json);
if (batch.DateTime == null)
batch.DateTime = item.Doc.LastModified;
else
batch.DateTime = batch.DateTime > item.Doc.LastModified
? item.Doc.LastModified
: batch.DateTime;
}
if (batch.Docs.Count == 0)
{
Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName,
lastEtag, lastModified);
// we use it this way to batch all the updates together
actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
return;
}
if (Log.IsDebugEnabled)
{
Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids));
}
results[i] = Tuple.Create(indexToWorkOn, batch);
});
transactionalStorage.Batch(actionsAccessor =>
{
foreach (var action in actions)
{
if (action != null)
action(actionsAccessor);
}
});
return results.Where(x => x != null);
}
private static JsonDocument GetHighestEtag(JsonDocument[] past)
{
var highest = new ComparableByteArray(Guid.Empty);
JsonDocument highestDoc = null;
for (int i = past.Length - 1; i >= 0; i--)
{
var etag = past[i].Etag.Value;
if (highest.CompareTo(etag) > 0)
{
continue;
}
highest = new ComparableByteArray(etag);
highestDoc = past[i];
}
return highestDoc;
}
private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = databaseOptions.Limit - totalCount;
if (maxRecords > 0)
{
var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords);
using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false))
{
while (await documents.MoveNextAsync().ConfigureAwait(false))
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
lastEtag = tempLastEtag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false);
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer);
if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0)
{
lastEtag = databaseStatistics.LastDocEtag;
}
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
var jsonDoc = doc.ToJson();
AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc)));
totalCount++;
}
});
}
});
await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
private void MarkIndexes(IndexToWorkOn indexToWorkOn, ComparableByteArray lastIndexedEtag, IStorageActionsAccessor actions, Guid lastEtag, DateTime lastModified)
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0)
return;
actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified);
}
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
using (var documents = await GetDocuments(lastEtag))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
var document = documents.Current;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (watch.ElapsedMilliseconds > 100)
{
errorcount++;
}
watch.Start();
}
}
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
var lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
var isLastLoop = false;
try
{
while (true)
{
try
{
var beforeCount = totalCount;
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
var res = await TransferStreamedDocuments(exportStore,
databaseOptions,
now,
jintHelper,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var res = await TransferDocumentsWithoutStreaming(exportStore,
databaseOptions,
exportBatchSize,
operationMetadata,
now,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.Empty;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop)
{
if (totalCount == beforeCount)
{
isLastLoop = true;
ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag);
}
else
ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
return;
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
protected async Task<Etag> ExportDocuments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorsCount = 0;
var reachedMaxEtag = false;
Operations.ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = SmugglerOptions.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
using (var documents = await Operations.GetDocuments(src, lastEtag, Math.Min(SmugglerOptions.BatchSize, maxRecords)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value<RavenJObject>("@metadata").Value<string>("@id")));
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
lastEtag = tempLastEtag;
if (!SmugglerOptions.MatchFilters(document))
continue;
if (SmugglerOptions.ShouldExcludeExpired && SmugglerOptions.ExcludeExpired(document, now))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
Operations.ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
errorsCount++;
watch.Start();
}
}
if (hasDocs)
continue;
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await Operations.GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
Operations.ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
Operations.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
Operations.ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
SmugglerOptions.Filters.ForEach(filter =>
{
if (filter.Path == "@metadata.Raven-Entity-Name")
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = Operations.GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
doc.ToJson().WriteTo(jsonWriter);
totalCount++;
}
});
}
});
Operations.ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, maxRecords))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
if (hasDocs)
continue;
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
}
private IEnumerable <IndexingBatchForIndex> FilterIndexes(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc =>
{
var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index);
return(filteredDoc == null ? new
{
Doc = doc,
Json = (object)new FilteredDocument(doc)
} : new
{
Doc = filteredDoc,
Json = JsonToExpando.Convert(doc.ToJson())
});
});
Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new IndexingBatchForIndex[indexesToWorkOn.Count];
var actions = new Action <IStorageActionsAccessor> [indexesToWorkOn.Count];
BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) =>
{
var indexLastIndexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray());
if (indexLastIndexEtag.CompareTo(lastIndexedEtag) >= 0)
{
return;
}
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
{
return; // probably deleted
}
var batch = new IndexingBatch();
foreach (var item in filteredDocs)
{
if (prefetchingBehavior.FilterDocuments(item.Doc))
{
continue;
}
// did we already indexed this document in this index?
var etag = item.Doc.Etag;
if (etag == null)
{
continue;
}
if (indexLastIndexEtag.CompareTo(new ComparableByteArray(etag.Value.ToByteArray())) >= 0)
{
continue;
}
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value <string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(item.Doc, item.Json, prefetchingBehavior.ShouldSkipDeleteFromIndex(item.Doc));
if (batch.DateTime == null)
{
batch.DateTime = item.Doc.LastModified;
}
else
{
batch.DateTime = batch.DateTime > item.Doc.LastModified
? item.Doc.LastModified
: batch.DateTime;
}
}
if (batch.Docs.Count == 0)
{
Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName,
lastEtag, lastModified);
// we use it this way to batch all the updates together
actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
return;
}
if (Log.IsDebugEnabled)
{
Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids));
}
results[i] = new IndexingBatchForIndex
{
Batch = batch,
IndexName = indexToWorkOn.IndexName,
Index = indexToWorkOn.Index,
LastIndexedEtag = indexToWorkOn.LastIndexedEtag
};
});
transactionalStorage.Batch(actionsAccessor =>
{
foreach (var action in actions)
{
if (action != null)
{
action(actionsAccessor);
}
}
});
return(results.Where(x => x != null));
}
protected override async Task<Etag> ExportAttachments(RavenConnectionStringOptions src,JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
{
throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag);
}
var totalCount = 0;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
RavenJArray attachmentInfo = null;
var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length,
totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value<string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest(src, "/static/" + item.Value<string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{"Data", attachmentData},
{"Metadata", item.Value<RavenJObject>("Metadata")},
{"Key", item.Value<string>("Key")}
}
.WriteTo(jsonWriter);
totalCount++;
lastEtag = Etag.Parse(item.Value<string>("Etag"));
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
private Guid ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Guid lastEtag)
{
int totalCount = 0;
while (true)
{
var watch = Stopwatch.StartNew();
var documents = GetDocuments(lastEtag);
watch.Stop();
if (documents.Length == 0)
{
var databaseStatistics = GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}",lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
var currentProcessingTime = watch.Elapsed;
ModifyBatchSize(options, currentProcessingTime);
var final = documents.Where(options.MatchFilters).ToList();
if (options.ShouldExcludeExpired)
final = documents.Where(options.ExcludeExpired).ToList();
final.ForEach(item => item.WriteTo(jsonWriter));
totalCount += final.Count;
ShowProgress("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount);
lastEtag = new Guid(documents.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
}
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
var lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
var isLastLoop = false;
try
{
while (true)
{
try
{
var beforeCount = totalCount;
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
var res = await TransferStreamedDocuments(exportStore,
databaseOptions,
now,
jintHelper,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var res = await TransferDocumentsWithoutStreaming(exportStore,
databaseOptions,
exportBatchSize,
operationMetadata,
now,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop)
{
if (totalCount == beforeCount)
{
isLastLoop = true;
ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag);
}
else
{
ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag);
}
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
{
return;
}
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
public void CleanupDocumentsToRemove(Guid lastIndexedEtag)
{
var highest = new ComparableByteArray(lastIndexedEtag);
foreach (var docToRemove in documentsToRemove)
{
if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false)
continue;
HashSet<Guid> _;
documentsToRemove.TryRemove(docToRemove.Key, out _);
}
JsonDocument result;
while (inMemoryDocs.TryPeek(out result) && highest.CompareTo(result.Etag.Value) >= 0)
{
inMemoryDocs.TryDequeue(out result);
}
}
private static async Task<Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize)
{
Etag lastEtag = databaseOptions.StartAttachmentsEtag;
int totalCount = 0;
var retries = RetriesCount;
while (true)
{
try
{
AttachmentInformation[] attachments;
try
{
attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize);
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.InvalidEtag;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("Failed to get attachments. {0} retries remaining. Message: {1}", retries, e.Message);
continue;
}
if (attachments.Length == 0)
{
DatabaseStatistics databaseStatistics;
try
{
databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.Empty;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("Failed to get database statistics. Message: {0}", e.Message);
continue;
}
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += attachments.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount);
foreach (var attachmentInformation in attachments)
{
ShowProgress("Downloading attachment: {0}", attachmentInformation.Key);
try
{
var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key);
if (attachment == null)
continue;
if (databaseOptions.StripReplicationInformation)
attachment.Metadata = StripReplicationInformationFromMetadata(attachment.Metadata);
await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of an attachment {0} failed. Message: {1}", attachmentInformation.Key, e.Message);
}
}
lastEtag = Etag.Parse(attachments.Last().Etag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerOptions options, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = options.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = options.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
continue;
metadata = document.Value<RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.Empty;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
return;
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
protected override void ExecuteIndexingWork(IList<IndexToWorkOn> indexesToWorkOn)
{
indexesToWorkOn = context.Configuration.IndexingScheduler.FilterMapIndexes(indexesToWorkOn);
var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid();
JsonDocument[] jsonDocs = null;
try
{
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes, autoTuner.NumberOfItemsToIndexInSingleBatch)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return doc;
})
.ToArray();
});
log.Debug("Found a total of {0} documents that requires indexing since etag: {1}",
jsonDocs.Length, lastIndexedGuidForAllIndexes);
if (jsonDocs.Length > 0)
{
var result = FilterIndexes(indexesToWorkOn, jsonDocs).ToList();
indexesToWorkOn = result.Select(x => x.Item1).ToList();
BackgroundTaskExecuter.Instance.ExecuteAll(context.Configuration, scheduler, result, (indexToWorkOn,_) =>
{
var index = indexToWorkOn.Item1;
var docs = indexToWorkOn.Item2;
transactionalStorage.Batch(
actions => IndexDocuments(actions, index.IndexName, docs));
});
}
}
finally
{
if (jsonDocs != null && jsonDocs.Length > 0)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
transactionalStorage.Batch(actions =>
{
foreach (var indexToWorkOn in indexesToWorkOn)
{
MarkIndexes(indexToWorkOn, lastIndexedEtag, actions, lastEtag, lastModified);
}
});
autoTuner.AutoThrottleBatchSize(jsonDocs.Length, jsonDocs.Sum(x => x.SerializedSizeOnDisk));
}
}
}
private void CleanupDocumentsToRemove(Guid lastIndexedEtag)
{
var highest = new ComparableByteArray(lastIndexedEtag);
documentsToRemove.RemoveWhere(x => x.Etag == null || highest.CompareTo(x.Etag.Value) >= 0);
}
private void ExecuteIndexingInternal(IEnumerable<IndexToWorkOn> indexesToWorkOn, Action<JsonDocument[]> indexingOp)
{
var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid();
JsonDocument[] jsonDocs = null;
try
{
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return doc;
})
.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
});
if (jsonDocs.Length > 0)
indexingOp(jsonDocs);
}
finally
{
if (jsonDocs != null && jsonDocs.Length > 0)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
transactionalStorage.Batch(actions =>
{
foreach (var indexToWorkOn in indexesToWorkOn)
{
if (new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()).CompareTo(lastIndexedEtag) > 0)
continue;
actions.Indexing.UpdateLastIndexed(indexToWorkOn.IndexName, lastEtag, lastModified);
}
});
}
}
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
using (var documents = await GetDocuments(lastEtag))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
var document = documents.Current;
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}
private void CleanupDocumentsToRemove(Guid lastIndexedEtag)
{
var highest = new ComparableByteArray(lastIndexedEtag);
documentsToRemove.RemoveWhere(x => x.Etag == null || highest.CompareTo(x.Etag) <= 0);
}
protected override async Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
{
throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag);
}
var totalCount = 0;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
RavenJArray attachmentInfo = null;
var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length,
totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest(src, "/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
totalCount++;
lastEtag = Etag.Parse(item.Value <string>("Etag"));
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
private static Guid GetLowestEtag(JsonDocument[] past)
{
var lowest = new ComparableByteArray(past[0].Etag.Value);
for (int i = 1; i < past.Length; i++)
{
var etag = past[i].Etag.Value;
if (lowest.CompareTo(etag) < 0)
{
continue;
}
lowest = new ComparableByteArray(etag);
}
return lowest.ToGuid();
}
private IEnumerable<Tuple<IndexToWorkOn, IEnumerable<JsonDocument>>> FilterIndexes(IEnumerable<IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
Action<IStorageActionsAccessor> action = null;
foreach (var indexToWorkOn in indexesToWorkOn)
{
var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray());
if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0)
continue;
var filteredDocs = jsonDocs.Where(doc => indexLastInedexEtag.CompareTo(new ComparableByteArray(doc.Etag.Value.ToByteArray())) < 0);
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if(viewGenerator == null)
continue; // probably deleted
if (viewGenerator.ForEntityNames.Count != 0) // limit for the items that it care for
{
filteredDocs = filteredDocs.Where(x => viewGenerator.ForEntityNames.Contains(x.Metadata.Value<string>(Constants.RavenEntityName)));
}
List<JsonDocument> jsonDocuments = filteredDocs.ToList();
if(jsonDocuments.Count == 0)
{
// we use it this way to batch all the updates together
action += accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
continue;
}
yield return Tuple.Create<IndexToWorkOn, IEnumerable<JsonDocument>>(indexToWorkOn, jsonDocuments);
}
if (action != null)
{
transactionalStorage.Batch(action);
}
}
private async static Task<Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize)
{
Etag lastEtag = databaseOptions.StartAttachmentsEtag;
int totalCount = 0;
while (true)
{
var attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize);
if (attachments.Length == 0)
{
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += attachments.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount);
foreach (var attachmentInformation in attachments)
{
ShowProgress("Downloading attachment: {0}", attachmentInformation.Key);
var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key);
await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata);
}
lastEtag = Etag.Parse(attachments.Last().Etag);
}
}
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, Math.Min(maxRecords, options.BatchSize)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorcount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
public void CleanupDocuments(Etag lastIndexedEtag)
{
var highest = new ComparableByteArray(lastIndexedEtag);
foreach (var docToRemove in documentsToRemove)
{
if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false)
continue;
HashSet<Etag> _;
documentsToRemove.TryRemove(docToRemove.Key, out _);
}
foreach (var updatedDocs in updatedDocuments)
{
if (updatedDocs.Value.All(etag => highest.CompareTo(etag) > 0) == false)
continue;
HashSet<Etag> _;
updatedDocuments.TryRemove(updatedDocs.Key, out _);
}
JsonDocument result;
while (prefetchingQueue.TryPeek(out result) && highest.CompareTo(result.Etag) >= 0)
{
prefetchingQueue.TryDequeue(out result);
}
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
{
return;
}
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag)
{
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
var request = CreateRequest("/static/?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value<string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest("/static/" + item.Value<string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{"Data", attachmentData},
{"Metadata", item.Value<RavenJObject>("Metadata")},
{"Key", item.Value<string>("Key")}
}
.WriteTo(jsonWriter);
}
lastEtag = new Guid(attachmentInfo.Last().Value<string>("Etag"));
}
}
private async static Task <Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize)
{
Etag lastEtag = databaseOptions.StartAttachmentsEtag;
int totalCount = 0;
var retries = RetriesCount;
while (true)
{
try
{
AttachmentInformation[] attachments;
try
{
attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize);
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.InvalidEtag);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("Failed to get attachments. {0} retries remaining. Message: {1}", retries, e.Message);
continue;
}
if (attachments.Length == 0)
{
DatabaseStatistics databaseStatistics;
try
{
databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("Failed to get database statistics. Message: {0}", e.Message);
continue;
}
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachments.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount);
foreach (var attachmentInformation in attachments)
{
ShowProgress("Downloading attachment: {0}", attachmentInformation.Key);
try
{
var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key);
if (attachment == null)
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
attachment.Metadata = StripReplicationInformationFromMetadata(attachment.Metadata);
}
await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of an attachment {0} failed. Message: {1}", attachmentInformation.Key, e.Message);
}
}
lastEtag = Etag.Parse(attachments.Last().Etag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorsCount = 0;
var reachedMaxEtag = false;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
using (var documents = await GetDocuments(src, lastEtag, Math.Min(options.BatchSize, maxRecords)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
lastEtag = tempLastEtag;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorsCount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
options.Filters.ForEach(filter =>
{
if (filter.Path == "@metadata.Raven-Entity-Name")
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
doc.ToJson().WriteTo(jsonWriter);
totalCount++;
}
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
{
indexesToWorkOn = context.Configuration.IndexingScheduler.FilterMapIndexes(indexesToWorkOn);
var lastIndexedGuidForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToGuid();
JsonDocument[] jsonDocs = null;
try
{
transactionalStorage.Batch(actions =>
{
jsonDocs = actions.Documents.GetDocumentsAfter(lastIndexedGuidForAllIndexes, autoTuner.NumberOfItemsToIndexInSingleBatch)
.Where(x => x != null)
.Select(doc =>
{
DocumentRetriever.EnsureIdInMetadata(doc);
return(doc);
})
.ToArray();
});
log.Debug("Found a total of {0} documents that requires indexing since etag: {1}",
jsonDocs.Length, lastIndexedGuidForAllIndexes);
if (jsonDocs.Length > 0)
{
var result = FilterIndexes(indexesToWorkOn, jsonDocs).ToList();
indexesToWorkOn = result.Select(x => x.Item1).ToList();
BackgroundTaskExecuter.Instance.ExecuteAll(context.Configuration, scheduler, result, (indexToWorkOn, _) =>
{
var index = indexToWorkOn.Item1;
var docs = indexToWorkOn.Item2;
transactionalStorage.Batch(
actions => IndexDocuments(actions, index.IndexName, docs));
});
}
}
finally
{
if (jsonDocs != null && jsonDocs.Length > 0)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
transactionalStorage.Batch(actions =>
{
foreach (var indexToWorkOn in indexesToWorkOn)
{
MarkIndexes(indexToWorkOn, lastIndexedEtag, actions, lastEtag, lastModified);
}
});
autoTuner.AutoThrottleBatchSize(jsonDocs.Length, jsonDocs.Sum(x => x.SerializedSizeOnDisk));
}
}
}
