public IEnumerable <AttachmentInformation> GetAttachmentsAfter(Etag etag, int take, long maxTotalSize)
{
if (take == 0 || maxTotalSize == 0)
{
return(Enumerable.Empty <AttachmentInformation>());
}
Api.JetSetCurrentIndex(session, Files, "by_etag");
Api.MakeKey(session, Files, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekGT) == false)
{
return(Enumerable.Empty <AttachmentInformation>());
}
var optimizer = new OptimizedIndexReader();
do
{
optimizer.Add(Session, Files);
} while (Api.TryMoveNext(session, Files) && optimizer.Count < take);
long totalSize = 0;
return(optimizer
.Where(_ => totalSize <= maxTotalSize)
.Select(Session, Files, o => ReadCurrentAttachmentInformation())
.Select(x =>
{
totalSize += x.Size;
return x;
}));
}
public IEnumerable <JsonDocument> GetDocumentsByReverseUpdateOrder(int start, int take)
{
Api.JetSetCurrentIndex(session, Documents, "by_etag");
Api.MoveAfterLast(session, Documents);
if (TryMoveTableRecords(Documents, start, backward: true))
{
return(Enumerable.Empty <JsonDocument>());
}
if (take < 1024 * 4)
{
var optimizer = new OptimizedIndexReader();
while (Api.TryMovePrevious(session, Documents) && optimizer.Count < take)
{
optimizer.Add(Session, Documents);
}
return(optimizer.Select(Session, Documents, ReadCurrentDocument));
}
return(GetDocumentsWithoutBuffering(take));
}
public IEnumerable<RavenJObject> GetMappedResults(params GetMappedResultsParams[] getMappedResultsParams)
{
var optimizedIndexReader = new OptimizedIndexReader<GetMappedResultsParams>(Session, MappedResults, getMappedResultsParams.Length);
Api.JetSetCurrentIndex(session, MappedResults, "by_reduce_key_and_view_hashed");
foreach (var item in getMappedResultsParams)
{
Api.MakeKey(session, MappedResults, item.ViewAndReduceKeyHashed, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekEQ) == false)
continue;
Api.MakeKey(session, MappedResults, item.ViewAndReduceKeyHashed, MakeKeyGrbit.NewKey);
Api.JetSetIndexRange(session, MappedResults, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive);
do
{
optimizedIndexReader.Add(item);
} while (Api.TryMoveNext(session, MappedResults));
}
return optimizedIndexReader
.Where(item =>
{
// we need to check that we don't have hash collisions
var currentReduceKey = Api.RetrieveColumnAsString(session, MappedResults,
tableColumnsCache.MappedResultsColumns["reduce_key"]);
if (currentReduceKey != item.ReduceKey)
return false;
var currentView = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"]);
return currentView == item.View;
})
.Select(item =>
{
using (var stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"])))
{
return stream.ToJObject();
}
});
}
public IEnumerable <AttachmentInformation> GetAttachmentsStartingWith(string idPrefix, int start, int pageSize)
{
if (pageSize <= 0)
{
return(Enumerable.Empty <AttachmentInformation>());
}
Api.JetSetCurrentIndex(session, Files, "by_name");
Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekGE) == false)
{
return(Enumerable.Empty <AttachmentInformation>());
}
var optimizer = new OptimizedIndexReader();
do
{
Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey | MakeKeyGrbit.SubStrLimit);
if (Api.TrySetIndexRange(session, Files, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive) == false)
{
return(Enumerable.Empty <AttachmentInformation>());
}
while (start > 0)
{
if (Api.TryMoveNext(session, Files) == false)
{
return(Enumerable.Empty <AttachmentInformation>());
}
start--;
}
optimizer.Add(Session, Files);
} while (Api.TryMoveNext(session, Files) && optimizer.Count < pageSize);
return(optimizer.Select(Session, Files, ReadCurrentAttachmentInformation));
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(string index, string[] reduceKeys, int level, int take, bool loadData, List<object> itemsToDelete)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key");
foreach (var reduceKey in reduceKeys)
{
Api.MakeKey(session, ScheduledReductions, index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekEQ) == false)
yield break;
Api.MakeKey(session, ScheduledReductions, index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.TrySetIndexRange(session, ScheduledReductions,
SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
// this isn't used for optimized reading, but to make it easier to delete records later on
var reader = new OptimizedIndexReader(session, ScheduledReductions, take);
itemsToDelete.Add(reader);
var seen = new HashSet<Tuple<string, int>>();
do
{
var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
tableColumnsCache.ScheduledReductionColumns["reduce_key"]);
if (string.Equals(index, indexFromDb, StringComparison.InvariantCultureIgnoreCase) == false)
continue;
if (levelFromDb != level)
continue;
if (string.IsNullOrEmpty(reduceKey) == false &&
string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
continue;
var bucket =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;
if (seen.Add(Tuple.Create(reduceKeyFromDb, bucket)))
{
foreach (var mappedResultInfo in GetResultsForBucket(index, level, reduceKeyFromDb, bucket, loadData))
{
take--;
yield return mappedResultInfo;
}
}
reader.Add();
} while (Api.TryMoveNext(session, ScheduledReductions) && take > 0);
if (take <= 0)
break;
}
}
public IEnumerable<AttachmentInformation> GetAttachmentsAfter(Guid etag, int take)
{
Api.JetSetCurrentIndex(session, Files, "by_etag");
Api.MakeKey(session, Files, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekGT) == false)
return Enumerable.Empty<AttachmentInformation>();
var optimizer = new OptimizedIndexReader(Session, Files, take);
do
{
optimizer.Add();
} while (Api.TryMoveNext(session, Files) && optimizer.Count < take);
return optimizer.Select(() => new AttachmentInformation
{
Size = Api.RetrieveColumnSize(session, Files, tableColumnsCache.FilesColumns["data"]) ?? 0,
Etag = Api.RetrieveColumn(session, Files, tableColumnsCache.FilesColumns["etag"]).TransfromToGuidWithProperSorting(),
Key = Api.RetrieveColumnAsString(session, Files, tableColumnsCache.FilesColumns["name"], Encoding.Unicode),
Metadata = RavenJObject.Parse(Api.RetrieveColumnAsString(session, Files, tableColumnsCache.FilesColumns["metadata"], Encoding.Unicode))
});
}
public IEnumerable<AttachmentInformation> GetAttachmentsAfter(Guid etag, int take, long maxTotalSize)
{
Api.JetSetCurrentIndex(session, Files, "by_etag");
Api.MakeKey(session, Files, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekGT) == false)
return Enumerable.Empty<AttachmentInformation>();
var optimizer = new OptimizedIndexReader(Session, Files, take);
do
{
optimizer.Add();
} while (Api.TryMoveNext(session, Files) && optimizer.Count < take);
long totalSize = 0;
return optimizer
.Where(_ => totalSize <= maxTotalSize)
.Select(o => ReadCurrentAttachmentInformation())
.Select(x=>
{
totalSize += x.Size;
return x;
});
}
public IEnumerable<AttachmentInformation> GetAttachmentsStartingWith(string idPrefix, int start, int pageSize)
{
Api.JetSetCurrentIndex(session, Files, "by_name");
Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekGE) == false)
return Enumerable.Empty<AttachmentInformation>();
var optimizer = new OptimizedIndexReader(Session, Files, pageSize);
do
{
Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey | MakeKeyGrbit.SubStrLimit);
if (Api.TrySetIndexRange(session, Files, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive) == false)
return Enumerable.Empty<AttachmentInformation>();
while (start > 0)
{
if (Api.TryMoveNext(session, Files) == false)
return Enumerable.Empty<AttachmentInformation>();
start--;
}
optimizer.Add();
} while (Api.TryMoveNext(session, Files) && optimizer.Count < pageSize);
return optimizer.Select(ReadCurrentAttachmentInformation);
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(string index, int level, int take, List<object> itemsToDelete)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_bucket_and_hashed_reduce_key");
Api.MakeKey(session, ScheduledReductions, index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, level, MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
yield break;
// this isn't used for optimized reading, but to make it easier to delete records
// later on
var reader = new OptimizedIndexReader(session, ScheduledReductions, take);
itemsToDelete.Add(reader);
var seen = new HashSet<Tuple<string, int>>();
do
{
var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var compareResult = string.Compare(index, indexFromDb, StringComparison.InvariantCultureIgnoreCase);
if (compareResult < 0) // not yet here
continue;
if (compareResult > 0) // after the record
break;
if (levelFromDb < level)
continue;
if (levelFromDb > level)
break;
var bucket =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var reduceKey = Api.RetrieveColumnAsString(session, ScheduledReductions,
tableColumnsCache.ScheduledReductionColumns["reduce_key"]);
if (seen.Add(Tuple.Create(reduceKey, bucket)))
{
foreach (var mappedResultInfo in GetResultsForBucket(index, level, reduceKey, bucket))
{
take--;
yield return mappedResultInfo;
}
}
reader.Add();
} while (Api.TryMoveNext(session, ScheduledReductions) && take > 0);
}
public IEnumerable<MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData, int take)
{
Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_etag");
Api.MakeKey(session, MappedResults, indexName, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, MappedResults, lastReducedEtag, MakeKeyGrbit.None);
if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekLE) == false)
return Enumerable.Empty<MappedResultInfo>();
var optimizer = new OptimizedIndexReader(Session, MappedResults, take);
while (
optimizer.Count < take &&
Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex) == indexName)
{
optimizer.Add();
// the index is view ascending and etag descending
// that means that we are going backward to go up
if (Api.TryMovePrevious(session, MappedResults) == false)
break;
}
return optimizer.Select(() => new MappedResultInfo
{
ReduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]),
Etag = new Guid(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])),
Timestamp = Api.RetrieveColumnAsDateTime(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).Value,
Data = loadData
? Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]).ToJObject()
: null,
});
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key");
var seenLocally = new HashSet<Tuple<string, int>>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray())
{
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekEQ) == false)
yield break;
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.TrySetIndexRange(session, ScheduledReductions,
SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit);
// this isn't used for optimized reading, but to make it easier to delete records later on
OptimizedIndexReader reader;
if (getItemsToReduceParams.ItemsToDelete.Count == 0)
{
getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedIndexReader());
}
else
{
reader = (OptimizedIndexReader)getItemsToReduceParams.ItemsToDelete[0];
}
do
{
var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
tableColumnsCache.ScheduledReductionColumns["reduce_key"]);
if (string.Equals(getItemsToReduceParams.Index, indexFromDb, StringComparison.InvariantCultureIgnoreCase) == false)
continue;
if (levelFromDb != getItemsToReduceParams.Level)
continue;
if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
continue;
var bucket =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;
var rowKey = Tuple.Create(reduceKeyFromDb, bucket);
var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions);
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
break;
} while (Api.TryMoveNext(session, ScheduledReductions));
getItemsToReduceParams.ReduceKeys.Remove(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
}
