private async Task ImportIncrementalData(CounterConnectionStringOptions connectionString, Stream stream)
{
CountingStream sizeStream;
JsonTextReader jsonReader;
if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false)
{
throw new InvalidOperationException("Failed to get reader for the data stream.");
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
ICounterStore store = null;
try
{
if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
store = new CounterStore
{
Url = connectionString.Url,
Name = connectionString.CounterStoreId,
Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials)
};
store.Initialize(true);
ShowProgress($"Initialized connection to counter store (name = {store.Name})");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
if (jsonReader.TokenType != JsonToken.StartObject)
{
continue;
}
var counterDelta = RavenJToken.ReadFrom(jsonReader).ToObject <CounterState>();
ShowProgress($"Importing counter {counterDelta.GroupName} - {counterDelta.CounterName}");
if (counterDelta.Sign == ValueSign.Negative)
{
counterDelta.Value = -counterDelta.Value;
}
store.Batch.ScheduleChange(counterDelta.GroupName, counterDelta.CounterName, counterDelta.Value);
}
ShowProgress("Finished import of the current file.");
await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
store?.Dispose();
}
}
private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = databaseOptions.Limit - totalCount;
if (maxRecords > 0)
{
var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords);
using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false))
{
while (await documents.MoveNextAsync().ConfigureAwait(false))
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
lastEtag = tempLastEtag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false);
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer);
if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0)
{
lastEtag = databaseStatistics.LastDocEtag;
}
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
var jsonDoc = doc.ToJson();
AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc)));
totalCount++;
}
});
}
});
await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
{
return;
}
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
//assumes that the caller has responsibility to handle data stream disposal ("stream" parameter)
private async Task ImportFullData(CounterConnectionStringOptions connectionString, Stream stream)
{
CountingStream sizeStream;
JsonTextReader jsonReader;
if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false)
{
throw new InvalidOperationException("Failed to get reader for the data stream.");
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
ICounterStore store = null;
try
{
if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
store = new CounterStore
{
Url = connectionString.Url,
Name = connectionString.CounterStoreId,
Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials)
};
store.Initialize(true);
ShowProgress($"Initialized connection to counter store (name = {store.Name})");
var existingCounterGroupsAndNames = await store.Admin.GetCounterStorageNameAndGroups(token : CancellationToken)
.WithCancellation(CancellationToken)
.ConfigureAwait(false);
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
if (jsonReader.TokenType != JsonToken.StartObject)
{
continue;
}
var counterInfo = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var delta = Math.Abs(counterInfo.Value <long>("Positive")) - Math.Abs(counterInfo.Value <long>("Negative"));
var groupName = counterInfo.Value <string>("Group");
var counterName = counterInfo.Value <string>("Name");
if (existingCounterGroupsAndNames.Any(x => x.Group == groupName && x.Name == counterName))
{
ShowProgress($"Counter {groupName} - {counterName} is already there. Reset is performed");
await store.ResetAsync(groupName, counterName, CancellationToken).ConfigureAwait(false); //since it is a full import, the values are overwritten
}
ShowProgress($"Importing counter {groupName} - {counterName}");
store.Batch.ScheduleChange(groupName, counterName, delta);
}
ShowProgress("Finished import...");
await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
store?.Dispose();
}
}
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
using (var actions = Destination.DocumentActions())
{
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false)
{
await Source.SkipDocumentsAsync(cancellationToken).ConfigureAwait(false);
return;
}
var afterEtag = state.LastDocsEtag;
var maxEtag = _maxEtags.LastDocsEtag;
var now = SystemTime.UtcNow;
var totalCount = 0;
var reachedMaxEtag = false;
var affectedCollections = new List <string>();
Options.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
affectedCollections.Add(collectionName);
});
}
});
do
{
var hasDocs = false;
try
{
var maxRecords = Options.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
var pageSize = Source.SupportsPaging ? Math.Min(Options.BatchSize, maxRecords) : int.MaxValue;
using (var documents = await Source.ReadDocumentsAfterAsync(afterEtag, pageSize, cancellationToken).ConfigureAwait(false))
{
while (await documents.MoveNextAsync().ConfigureAwait(false))
{
hasDocs = true;
var currentDocument = documents.Current;
var currentEtag = Etag.Parse(currentDocument.Value <RavenJObject>("@metadata").Value <string>("@etag"));
var currentKey = currentDocument["@metadata"].Value <string>("@id");
Notifications.OnDocumentRead(this, currentKey);
if (maxEtag != null && currentEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
afterEtag = currentEtag;
if (Options.MatchFilters(currentDocument) == false)
{
if (affectedCollections.Count <= 0)
{
continue;
}
if (currentDocument.ContainsKey("@metadata") == false)
{
continue;
}
if (currentKey == null)
{
continue;
}
var isHilo = currentKey.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase);
if (isHilo && Source.SupportsReadingHiLoDocuments)
{
continue;
}
if (isHilo == false || affectedCollections.Any(x => currentKey.EndsWith("/" + x, StringComparison.OrdinalIgnoreCase)) == false)
{
continue;
}
}
if (Options.ShouldExcludeExpired && Options.ExcludeExpired(currentDocument, now))
{
continue;
}
if (string.IsNullOrEmpty(Options.TransformScript) == false)
{
currentDocument = await TransformDocumentAsync(currentDocument).ConfigureAwait(false);
}
// If document is null after a transform we skip it.
if (currentDocument == null)
{
continue;
}
var metadata = currentDocument["@metadata"] as RavenJObject;
if (metadata != null)
{
if (Options.SkipConflicted && metadata.ContainsKey(Constants.RavenReplicationConflictDocument))
{
continue;
}
if (Options.StripReplicationInformation)
{
currentDocument["@metadata"] = SmugglerHelper.StripReplicationInformationFromMetadata(metadata); //TODO arek - why there three methods are in SmugglerHelper, they are used just here
}
if (Options.ShouldDisableVersioningBundle)
{
currentDocument["@metadata"] = SmugglerHelper.DisableVersioning(metadata);
}
currentDocument["@metadata"] = SmugglerHelper.HandleConflictDocuments(metadata);
}
try
{
await actions.WriteDocumentAsync(currentDocument, cancellationToken).ConfigureAwait(false);
Notifications.OnDocumentWrite(this, currentKey);
}
catch (Exception e)
{
if (Options.IgnoreErrorsAndContinue == false)
{
throw;
}
Notifications.ShowProgress(DatabaseSmugglerMessages.Documents_Write_Failure, currentKey, e.Message);
}
totalCount++;
if (totalCount % Options.BatchSize == 0)
{
await SaveOperationStateAsync(state, currentEtag, cancellationToken).ConfigureAwait(false);
// Wait for the batch to be indexed before continue.
if (Destination.SupportsWaitingForIndexing)
{
await Destination.WaitForIndexingAsOfLastWriteAsync(cancellationToken).ConfigureAwait(false);
}
}
}
}
if (hasDocs)
{
continue;
}
}
if (Source.SupportsReadingHiLoDocuments)
{
// Load HiLo documents for selected collections
foreach (var filter in Options.Filters)
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase) == false)
{
continue;
}
foreach (var collectionName in filter.Values)
{
var doc = await Source
.ReadDocumentAsync("Raven/HiLo/" + collectionName, cancellationToken)
.ConfigureAwait(false);
if (doc == null)
{
continue;
}
await actions.WriteDocumentAsync(doc, cancellationToken).ConfigureAwait(false);
totalCount++;
}
}
}
await SaveOperationStateAsync(state, afterEtag, cancellationToken).ConfigureAwait(false);
state.LastDocsEtag = afterEtag;
return;
}
catch (Exception e)
{
throw new SmugglerException(e.Message, e)
{
LastEtag = afterEtag
};
}
} while (Source.SupportsPaging);
}
}
private static async Task <Tuple <int, Etag, DateTime> > TransferStreamedDocuments(DocumentStore exportStore,
SmugglerDatabaseOptions databaseOptions,
DateTime now,
SmugglerJintHelper jintHelper,
BulkInsertOperation bulkInsertOperation,
TimeSpan reportInterval,
int totalCount,
string fromEtag,
DateTime lastReport)
{
Etag lastReadEtag = fromEtag;
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastReadEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
return(Tuple.Create(totalCount, lastReadEtag, lastReport));
}
private static async Task <Tuple <int, string, DateTime> > TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
{
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
fromEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
return(Tuple.Create(totalCount, fromEtag, lastReport));
}
