public Etag CalculateSynchronizationEtag(Etag etag, Etag lastProcessedEtag)
{
if (etag == null)
{
if (lastProcessedEtag != null)
{
lock (locker)
{
if (currentEtag == null && lastProcessedEtag.CompareTo(synchronizationEtag) != 0)
{
synchronizationEtag = lastProcessedEtag;
PersistSynchronizationState();
}
}
return lastProcessedEtag;
}
return Etag.Empty;
}
if (lastProcessedEtag == null)
return Etag.Empty;
if (etag.CompareTo(lastProcessedEtag) < 0)
return EtagUtil.Increment(etag, -1);
return lastProcessedEtag;
}
protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
while (true)
{
var array = GetAttachments(totalCount, lastEtag);
if (array.Length == 0)
{
var databaseStatistics = await GetStats();
if (lastEtag == null) lastEtag = Etag.Empty;
if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += array.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount);
foreach (var item in array)
{
item.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(array.Last().Value<string>("Etag"));
}
}
private bool UpdateSynchronizationStateInternal(Etag lowestEtag)
{
if (currentEtag == null || lowestEtag.CompareTo(currentEtag) < 0)
{
currentEtag = lowestEtag;
}
return lowestEtag.CompareTo(synchronizationEtag) < 0;
}
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
{
return storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
{
{ "name", name },
{ "etag", start.ToByteArray() }
})
.TakeWhile(x=>
StringComparer.OrdinalIgnoreCase.Equals(x.Value<string>("name"), name) &&
(end == null || end.CompareTo(Etag.Parse(x.Value<byte[]>("etag"))) > 0)
)
.Take(take)
.Select(result =>
{
var readResult = storage.Lists.Read(result);
return new ListItem
{
Data = readResult.Data().ToJObject(),
Etag = Etag.Parse(readResult.Key.Value<byte[]>("etag")),
Key = readResult.Key.Value<string>("key")
};
});
}
public static bool IsGreaterThanOrEqual(Etag x, Etag y)
{
return(x.CompareTo(y) >= 0);
}
public static bool IsGreaterThan(Etag x, Etag y)
{
return(x.CompareTo(y) > 0);
}
protected virtual async Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var totalCount = 0;
var maxEtagReached = false;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0 || maxEtagReached)
{
Operations.ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
var attachments = await Operations.GetAttachments(totalCount, lastEtag, maxRecords);
if (attachments.Count == 0)
{
var databaseStatistics = await Operations.GetStats();
if (lastEtag == null) lastEtag = Etag.Empty;
if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
Operations.ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}",
lastEtag);
continue;
}
Operations.ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += attachments.Count;
Operations.ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Count, totalCount);
foreach (var attachment in attachments)
{
if (maxEtag != null && attachment.Etag.CompareTo(maxEtag) > 0)
{
maxEtagReached = true;
break;
}
var attachmentData = await Operations.GetAttachmentData(attachment);
if (attachmentData == null)
continue;
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", attachment.Metadata },
{ "Key", attachment.Key },
{ "Etag", new RavenJValue(attachment.Etag.ToString()) }
}.WriteTo(jsonWriter);
lastEtag = attachment.Etag;
}
}
catch (Exception e)
{
Operations.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
Operations.ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
private bool TryGetDocumentsFromQueue(Etag nextDocEtag, List<JsonDocument> items, int? take)
{
JsonDocument result;
bool hasDocs = false;
while (items.Count < GetNumberOfItemsToProcessInSingleBatch() &&
prefetchingQueue.TryPeek(out result) &&
// we compare to current or _smaller_ so we will remove from the queue old versions
// of documents that we have already loaded
nextDocEtag.CompareTo(result.Etag) >= 0)
{
// safe to do peek then dequeue because we are the only one doing the dequeues
// and here we are single threaded, but still, better to check
if (prefetchingQueue.TryDequeue(out result) == false)
continue;
// this shouldn't happen, but...
if (result == null)
continue;
if (result.Etag != nextDocEtag)
continue;
items.Add(result);
hasDocs = true;
if (take.HasValue && items.Count >= take.Value)
break;
nextDocEtag = Abstractions.Util.EtagUtil.Increment(nextDocEtag, 1);
}
return hasDocs;
}
private Etag ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
int totalCount = 0;
while (true)
{
var watch = Stopwatch.StartNew();
var documents = GetDocuments(lastEtag);
watch.Stop();
if (documents.Length == 0)
{
var databaseStatistics = GetStats();
if (lastEtag == null) lastEtag = Etag.Empty;
if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, smugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}",lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
var currentProcessingTime = watch.Elapsed;
ModifyBatchSize(options, currentProcessingTime);
var final = documents.Where(options.MatchFilters).ToList();
if (options.ShouldExcludeExpired)
final = documents.Where(options.ExcludeExpired).ToList();
final.ForEach(item => item.WriteTo(jsonWriter));
totalCount += final.Count;
ShowProgress("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount);
lastEtag = Etag.Parse(documents.Last().Value<RavenJObject>("@metadata").Value<string>("@etag"));
}
}
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
{
Api.JetSetCurrentIndex(session, Lists, "by_name_and_etag");
Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Lists, start.TransformToValueForEsentSorting(), MakeKeyGrbit.None);
if (Api.TrySeek(session, Lists, SeekGrbit.SeekGT) == false)
yield break;
int count = 0;
do
{
var nameFromDb = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["name"], Encoding.Unicode);
if (string.Equals(name, nameFromDb, StringComparison.InvariantCultureIgnoreCase) == false)
yield break;
var etag = Etag.Parse(Api.RetrieveColumn(session, Lists, tableColumnsCache.ListsColumns["etag"]));
if (end != null && end.CompareTo(etag) <= 0)
yield break;
count++;
using (Stream stream = new BufferedStream(new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"])))
{
yield return new ListItem
{
Etag = etag,
Data = stream.ToJObject(),
Key = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["key"], Encoding.Unicode),
CreatedAt = Api.RetrieveColumnAsDateTime(session, Lists, tableColumnsCache.ListsColumns["created_at"]).Value
};
}
} while (Api.TryMoveNext(session, Lists) && count < take);
}
public void CleanupDocuments(Etag lastIndexedEtag)
{
if (lastIndexedEtag == null)
return;
foreach (var docToRemove in documentsToRemove)
{
if (docToRemove.Value == null)
continue;
if (docToRemove.Value.All(etag => lastIndexedEtag.CompareTo(etag) > 0) == false)
continue;
HashSet<Etag> _;
documentsToRemove.TryRemove(docToRemove.Key, out _);
}
HandleCleanupOfUnusedDocumentsInQueue();
}
public static bool IsGreaterThanOrEqual(Etag x, Etag y)
{
return x.CompareTo(y) >= 0;
}
public static bool IsGreaterThan(Etag x, Etag y)
{
return x.CompareTo(y) > 0;
}
public void CleanupDocuments(Etag lastIndexedEtag)
{
foreach (var docToRemove in documentsToRemove)
{
if (docToRemove.Value.All(etag => lastIndexedEtag.CompareTo(etag) > 0) == false)
continue;
HashSet<Etag> _;
documentsToRemove.TryRemove(docToRemove.Key, out _);
}
updatedDocumentsLock.EnterWriteLock();
try
{
updatedDocuments.RemoveSmallerOrEqual(lastIndexedEtag);
}
finally
{
updatedDocumentsLock.ExitWriteLock();
}
JsonDocument result;
while (prefetchingQueue.TryPeek(out result) && lastIndexedEtag.CompareTo(result.Etag) >= 0)
{
prefetchingQueue.TryDequeue(out result);
}
}
private void HandleCollectingDocumentsAfterCommit(Etag requestedEtag)
{
if (ShouldHandleUnusedDocumentsAddedAfterCommit == false)
return;
var current = lowestInMemoryDocumentAddedAfterCommit;
if (DisableCollectingDocumentsAfterCommit)
{
if (current != null && requestedEtag.CompareTo(current.Etag) > 0)
{
lowestInMemoryDocumentAddedAfterCommit = null;
DisableCollectingDocumentsAfterCommit = false;
numberOfTimesIngestRateWasTooHigh = 0;
}
}
else if (current != null)
{
var oldestTimeInQueue = SystemTime.UtcNow - current.AddedAt;
if (
// this can happen if we have a very long indexing time, which takes
// us a long while to process, so we might as might as well stop keeping
// stuff in memory, because we lag
oldestTimeInQueue > TimeSpan.FromMinutes(10)
)
{
DisableCollectingDocumentsAfterCommit = true;
// If we disable in memory collection of data, we need to also remove all the
// items in the prefetching queue that are after the last in memory docs
// immediately, not wait for them to be indexed. They have already been in
// memory for ten minutes
// But not if our time has finally been reached
if (requestedEtag.CompareTo(current.Etag) >= 0)
prefetchingQueue.RemoveAfter(current.Etag);
}
}
}
protected override async Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var totalCount = 0;
var maxEtagReached = false;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0 || maxEtagReached)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
var array = GetAttachments(totalCount, lastEtag, maxRecords);
if (array.Length == 0)
{
var databaseStatistics = await GetStats();
if (lastEtag == null) lastEtag = Etag.Empty;
if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}",
lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += array.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount);
foreach (var item in array)
{
var tempLastEtag = item.Value<string>("Etag");
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
maxEtagReached = true;
break;
}
item.WriteTo(jsonWriter);
lastEtag = tempLastEtag;
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
private void RemoveOutdatedFutureIndexBatches(Etag etag)
{
foreach (FutureIndexBatch source in futureIndexBatches.Values.Where(x => etag.CompareTo(x.StartingEtag) > 0))
{
ObserveDiscardedTask(source);
DisposeCancellationToken(source.CancellationTokenSource);
FutureIndexBatch batch;
futureIndexBatches.TryRemove(source.StartingEtag, out batch);
}
}
private bool TryGetDocumentsFromQueue(Etag nextDocEtag, ref List<JsonDocument> items)
{
JsonDocument result;
nextDocEtag = HandleEtagGapsIfNeeded(nextDocEtag);
bool hasDocs = false;
while (items.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && prefetchingQueue.TryPeek(out result) && nextDocEtag.CompareTo(result.Etag) >= 0)
{
// safe to do peek then dequeue because we are the only one doing the dequeues
// and here we are single threaded
prefetchingQueue.TryDequeue(out result);
if (result.Etag != nextDocEtag)
continue;
items.Add(result);
hasDocs = true;
nextDocEtag = EtagUtil.Increment(nextDocEtag, 1);
nextDocEtag = HandleEtagGapsIfNeeded(nextDocEtag);
}
return hasDocs;
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = options.Limit - totalCount;
var amountToTakeFromServer = Math.Min(maxRecords, options.BatchSize);
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, amountToTakeFromServer))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
if (hasDocs)
continue;
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, amountToTakeFromServer);
if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0)
{
lastEtag = databaseStatistics.LastDocEtag;
}
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
}