public IEnumerable <AttachmentInformation> GetAttachmentsAfter(Etag etag, int take, long maxTotalSize) { if (take == 0 || maxTotalSize == 0) { return(Enumerable.Empty <AttachmentInformation>()); } Api.JetSetCurrentIndex(session, Files, "by_etag"); Api.MakeKey(session, Files, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekGT) == false) { return(Enumerable.Empty <AttachmentInformation>()); } var optimizer = new OptimizedIndexReader(); do { optimizer.Add(Session, Files); } while (Api.TryMoveNext(session, Files) && optimizer.Count < take); long totalSize = 0; return(optimizer .Where(_ => totalSize <= maxTotalSize) .Select(Session, Files, o => ReadCurrentAttachmentInformation()) .Select(x => { totalSize += x.Size; return x; })); }
public IEnumerable <JsonDocument> GetDocumentsByReverseUpdateOrder(int start, int take) { Api.JetSetCurrentIndex(session, Documents, "by_etag"); Api.MoveAfterLast(session, Documents); if (TryMoveTableRecords(Documents, start, backward: true)) { return(Enumerable.Empty <JsonDocument>()); } if (take < 1024 * 4) { var optimizer = new OptimizedIndexReader(); while (Api.TryMovePrevious(session, Documents) && optimizer.Count < take) { optimizer.Add(Session, Documents); } return(optimizer.Select(Session, Documents, ReadCurrentDocument)); } return(GetDocumentsWithoutBuffering(take)); }
public IEnumerable<RavenJObject> GetMappedResults(params GetMappedResultsParams[] getMappedResultsParams) { var optimizedIndexReader = new OptimizedIndexReader<GetMappedResultsParams>(Session, MappedResults, getMappedResultsParams.Length); Api.JetSetCurrentIndex(session, MappedResults, "by_reduce_key_and_view_hashed"); foreach (var item in getMappedResultsParams) { Api.MakeKey(session, MappedResults, item.ViewAndReduceKeyHashed, MakeKeyGrbit.NewKey); if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekEQ) == false) continue; Api.MakeKey(session, MappedResults, item.ViewAndReduceKeyHashed, MakeKeyGrbit.NewKey); Api.JetSetIndexRange(session, MappedResults, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive); do { optimizedIndexReader.Add(item); } while (Api.TryMoveNext(session, MappedResults)); } return optimizedIndexReader .Where(item => { // we need to check that we don't have hash collisions var currentReduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]); if (currentReduceKey != item.ReduceKey) return false; var currentView = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"]); return currentView == item.View; }) .Select(item => { using (var stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]))) { return stream.ToJObject(); } }); }
public IEnumerable <AttachmentInformation> GetAttachmentsStartingWith(string idPrefix, int start, int pageSize) { if (pageSize <= 0) { return(Enumerable.Empty <AttachmentInformation>()); } Api.JetSetCurrentIndex(session, Files, "by_name"); Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekGE) == false) { return(Enumerable.Empty <AttachmentInformation>()); } var optimizer = new OptimizedIndexReader(); do { Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey | MakeKeyGrbit.SubStrLimit); if (Api.TrySetIndexRange(session, Files, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive) == false) { return(Enumerable.Empty <AttachmentInformation>()); } while (start > 0) { if (Api.TryMoveNext(session, Files) == false) { return(Enumerable.Empty <AttachmentInformation>()); } start--; } optimizer.Add(Session, Files); } while (Api.TryMoveNext(session, Files) && optimizer.Count < pageSize); return(optimizer.Select(Session, Files, ReadCurrentAttachmentInformation)); }
public IEnumerable<MappedResultInfo> GetItemsToReduce(string index, string[] reduceKeys, int level, int take, bool loadData, List<object> itemsToDelete) { Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key"); foreach (var reduceKey in reduceKeys) { Api.MakeKey(session, ScheduledReductions, index, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, ScheduledReductions, level, MakeKeyGrbit.None); Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None); if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekEQ) == false) yield break; Api.MakeKey(session, ScheduledReductions, index, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, ScheduledReductions, level, MakeKeyGrbit.None); Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None); Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit); // this isn't used for optimized reading, but to make it easier to delete records later on var reader = new OptimizedIndexReader(session, ScheduledReductions, take); itemsToDelete.Add(reader); var seen = new HashSet<Tuple<string, int>>(); do { var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex); var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex). Value; var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"]); if (string.Equals(index, indexFromDb, StringComparison.InvariantCultureIgnoreCase) == false) continue; if (levelFromDb != level) continue; if (string.IsNullOrEmpty(reduceKey) == false && string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false) continue; var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value; if (seen.Add(Tuple.Create(reduceKeyFromDb, bucket))) { foreach (var mappedResultInfo in GetResultsForBucket(index, level, reduceKeyFromDb, bucket, loadData)) { take--; yield return mappedResultInfo; } } reader.Add(); } while (Api.TryMoveNext(session, ScheduledReductions) && take > 0); if (take <= 0) break; } }
public IEnumerable<AttachmentInformation> GetAttachmentsAfter(Guid etag, int take) { Api.JetSetCurrentIndex(session, Files, "by_etag"); Api.MakeKey(session, Files, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekGT) == false) return Enumerable.Empty<AttachmentInformation>(); var optimizer = new OptimizedIndexReader(Session, Files, take); do { optimizer.Add(); } while (Api.TryMoveNext(session, Files) && optimizer.Count < take); return optimizer.Select(() => new AttachmentInformation { Size = Api.RetrieveColumnSize(session, Files, tableColumnsCache.FilesColumns["data"]) ?? 0, Etag = Api.RetrieveColumn(session, Files, tableColumnsCache.FilesColumns["etag"]).TransfromToGuidWithProperSorting(), Key = Api.RetrieveColumnAsString(session, Files, tableColumnsCache.FilesColumns["name"], Encoding.Unicode), Metadata = RavenJObject.Parse(Api.RetrieveColumnAsString(session, Files, tableColumnsCache.FilesColumns["metadata"], Encoding.Unicode)) }); }
public IEnumerable<AttachmentInformation> GetAttachmentsAfter(Guid etag, int take, long maxTotalSize) { Api.JetSetCurrentIndex(session, Files, "by_etag"); Api.MakeKey(session, Files, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekGT) == false) return Enumerable.Empty<AttachmentInformation>(); var optimizer = new OptimizedIndexReader(Session, Files, take); do { optimizer.Add(); } while (Api.TryMoveNext(session, Files) && optimizer.Count < take); long totalSize = 0; return optimizer .Where(_ => totalSize <= maxTotalSize) .Select(o => ReadCurrentAttachmentInformation()) .Select(x=> { totalSize += x.Size; return x; }); }
public IEnumerable<AttachmentInformation> GetAttachmentsStartingWith(string idPrefix, int start, int pageSize) { Api.JetSetCurrentIndex(session, Files, "by_name"); Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekGE) == false) return Enumerable.Empty<AttachmentInformation>(); var optimizer = new OptimizedIndexReader(Session, Files, pageSize); do { Api.MakeKey(session, Files, idPrefix, Encoding.Unicode, MakeKeyGrbit.NewKey | MakeKeyGrbit.SubStrLimit); if (Api.TrySetIndexRange(session, Files, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive) == false) return Enumerable.Empty<AttachmentInformation>(); while (start > 0) { if (Api.TryMoveNext(session, Files) == false) return Enumerable.Empty<AttachmentInformation>(); start--; } optimizer.Add(); } while (Api.TryMoveNext(session, Files) && optimizer.Count < pageSize); return optimizer.Select(ReadCurrentAttachmentInformation); }
public IEnumerable<MappedResultInfo> GetItemsToReduce(string index, int level, int take, List<object> itemsToDelete) { Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_bucket_and_hashed_reduce_key"); Api.MakeKey(session, ScheduledReductions, index, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, ScheduledReductions, level, MakeKeyGrbit.None); if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false) yield break; // this isn't used for optimized reading, but to make it easier to delete records // later on var reader = new OptimizedIndexReader(session, ScheduledReductions, take); itemsToDelete.Add(reader); var seen = new HashSet<Tuple<string, int>>(); do { var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex); var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex). Value; var compareResult = string.Compare(index, indexFromDb, StringComparison.InvariantCultureIgnoreCase); if (compareResult < 0) // not yet here continue; if (compareResult > 0) // after the record break; if (levelFromDb < level) continue; if (levelFromDb > level) break; var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"], RetrieveColumnGrbit.RetrieveFromIndex). Value; var reduceKey = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"]); if (seen.Add(Tuple.Create(reduceKey, bucket))) { foreach (var mappedResultInfo in GetResultsForBucket(index, level, reduceKey, bucket)) { take--; yield return mappedResultInfo; } } reader.Add(); } while (Api.TryMoveNext(session, ScheduledReductions) && take > 0); }
public IEnumerable<MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData, int take) { Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_etag"); Api.MakeKey(session, MappedResults, indexName, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, MappedResults, lastReducedEtag, MakeKeyGrbit.None); if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekLE) == false) return Enumerable.Empty<MappedResultInfo>(); var optimizer = new OptimizedIndexReader(Session, MappedResults, take); while ( optimizer.Count < take && Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex) == indexName) { optimizer.Add(); // the index is view ascending and etag descending // that means that we are going backward to go up if (Api.TryMovePrevious(session, MappedResults) == false) break; } return optimizer.Select(() => new MappedResultInfo { ReduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]), Etag = new Guid(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])), Timestamp = Api.RetrieveColumnAsDateTime(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).Value, Data = loadData ? Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]).ToJObject() : null, }); }
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams) { Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key"); var seenLocally = new HashSet<Tuple<string, int>>(); foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray()) { Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None); Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None); if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekEQ) == false) yield break; Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None); Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None); Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit); // this isn't used for optimized reading, but to make it easier to delete records later on OptimizedIndexReader reader; if (getItemsToReduceParams.ItemsToDelete.Count == 0) { getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedIndexReader()); } else { reader = (OptimizedIndexReader)getItemsToReduceParams.ItemsToDelete[0]; } do { var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex); var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex). Value; var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"]); if (string.Equals(getItemsToReduceParams.Index, indexFromDb, StringComparison.InvariantCultureIgnoreCase) == false) continue; if (levelFromDb != getItemsToReduceParams.Level) continue; if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false) continue; var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value; var rowKey = Tuple.Create(reduceKeyFromDb, bucket); var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions); var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey); if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket) { if (seenLocally.Add(rowKey)) { foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData)) { getItemsToReduceParams.Take--; yield return mappedResultInfo; } } } if (getItemsToReduceParams.Take <= 0) break; } while (Api.TryMoveNext(session, ScheduledReductions)); getItemsToReduceParams.ReduceKeys.Remove(reduceKey); if (getItemsToReduceParams.Take <= 0) break; } }