public IEnumerable <MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData, int take) { return(storage.MappedResults["ByViewAndEtagDesc"] // the index is sorted view ascending and then etag descending // we index before this index, then backward toward the last one. .SkipBefore(new RavenJObject { { "view", indexName }, { "etag", lastReducedEtag.ToByteArray() } }) .TakeWhile(x => StringComparer.InvariantCultureIgnoreCase.Equals(x.Value <string>("view"), indexName)) .Select(key => { var mappedResultInfo = new MappedResultInfo { ReduceKey = key.Value <string>("reduceKey"), Etag = new Guid(key.Value <byte[]>("etag")), Timestamp = key.Value <DateTime>("timestamp"), }; if (loadData) { var readResult = storage.MappedResults.Read(key); if (readResult != null) { mappedResultInfo.Data = readResult.Data().ToJObject(); } } return mappedResultInfo; }).Take(take)); }
public IEnumerable <MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData) { Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_etag"); Api.MakeKey(session, MappedResults, indexName, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, MappedResults, lastReducedEtag, MakeKeyGrbit.None); if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekLE) == false) { yield break; } while (Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"]) == indexName) { var result = new MappedResultInfo { ReduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]), Etag = new Guid(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])), Timestamp = Api.RetrieveColumnAsDateTime(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).Value, }; if (loadData) { result.Data = Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]).ToJObject(); } yield return(result); // the index is view ascending and etag descending // that means that we are going backward to go up if (Api.TryMovePrevious(session, MappedResults) == false) { yield break; } } }
private IEnumerable <MappedResultInfo> GetReducedResultsForBucket(string index, string reduceKey, int level, int bucket, bool loadData) { var results = storage.ReduceResults["ByViewReduceKeyLevelAndBucket"] .SkipTo(new RavenJObject { { "view", index }, { "reduceKey", reduceKey }, { "level", level }, { "bucket", bucket } }) .TakeWhile(x => string.Equals(index, x.Value <string>("view"), StringComparison.InvariantCultureIgnoreCase) && string.Equals(reduceKey, x.Value <string>("reduceKey"), StringComparison.InvariantCultureIgnoreCase) && level == x.Value <int>("level") && bucket == x.Value <int>("bucket")); bool hasResults = false; foreach (var result in results) { hasResults = true; var readResult = storage.ReduceResults.Read(result); var mappedResultInfo = new MappedResultInfo { ReduceKey = readResult.Key.Value <string>("reduceKey"), Etag = new Guid(readResult.Key.Value <byte[]>("etag")), Timestamp = readResult.Key.Value <DateTime>("timestamp"), Bucket = readResult.Key.Value <int>("bucket"), Source = readResult.Key.Value <int>("sourceBucket").ToString(), Size = readResult.Size, Data = loadData ? LoadMappedResult(readResult) : null }; yield return(mappedResultInfo); } if (hasResults) { yield break; } yield return(new MappedResultInfo { Bucket = bucket, ReduceKey = reduceKey }); }
public IEnumerable <MappedResultInfo> GetMappedResultsReduceKeysAfter(string indexName, Guid lastReducedEtag, bool loadData, int take) { Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_etag"); Api.MakeKey(session, MappedResults, indexName, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, MappedResults, lastReducedEtag, MakeKeyGrbit.None); if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekLE) == false) { return(Enumerable.Empty <MappedResultInfo>()); } var results = new Dictionary <string, MappedResultInfo>(); while ( results.Count < take && Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex) == indexName) { var key = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]); var mappedResultInfo = new MappedResultInfo { ReduceKey = key, Etag = new Guid(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])), Timestamp = Api.RetrieveColumnAsDateTime(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]). Value, Data = loadData ? LoadMappedResults(key) : null, Size = Api.RetrieveColumnSize(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]) ?? 0 }; results[mappedResultInfo.ReduceKey] = mappedResultInfo; // the index is view ascending and etag descending // that means that we are going backward to go up if (Api.TryMovePrevious(session, MappedResults) == false) { break; } } return(results.Values); }
public List<MappedResultInfo> GetMappedResults(int view, HashSet<string> keysLeftToReduce, bool loadData, int take, HashSet<string> keysReturned, CancellationToken cancellationToken, List<MappedResultInfo> outputCollection = null) { if (outputCollection == null) outputCollection = new List<MappedResultInfo>(); Api.JetSetCurrentIndex(session, MappedResults, "by_view_hashed_reduce_key_and_bucket"); var keysToReduce = new HashSet<string>(keysLeftToReduce); foreach (var reduceKey in keysToReduce) { cancellationToken.ThrowIfCancellationRequested(); keysLeftToReduce.Remove(reduceKey); Api.MakeKey(session, MappedResults, view, MakeKeyGrbit.NewKey); var hashReduceKey = HashReduceKey(reduceKey); keysReturned.Add(reduceKey); Api.MakeKey(session, MappedResults, hashReduceKey, MakeKeyGrbit.None); if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekGE) == false) continue; do { cancellationToken.ThrowIfCancellationRequested(); var indexFromDb = Api.RetrieveColumnAsInt32(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"]); var hashKeyFromDb = Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["hashed_reduce_key"]); if (indexFromDb != view || hashReduceKey.SequenceEqual(hashKeyFromDb) == false) break; var timestamp = Api.RetrieveColumnAsInt64(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"]).Value; var keyFromDb = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"]); take--; // We have worked with this reduce key, so we consider it an output even if we don't add it. RavenJObject data = null; if ( loadData ) { data = LoadMappedResults(keyFromDb); if ( data == null ) continue; } var result = new MappedResultInfo { Bucket = Api.RetrieveColumnAsInt32(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"]).Value, ReduceKey = keyFromDb, Etag = Etag.Parse(Api.RetrieveColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"])), Timestamp = DateTime.FromBinary(timestamp), Data = data, Size = Api.RetrieveColumnSize(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]) ?? 0 }; outputCollection.Add(result); } while (Api.TryMoveNext(session, MappedResults)); if (take < 0) return outputCollection; } return outputCollection; }
public List<MappedResultInfo> GetMappedResults(int view, HashSet<string> keysLeftToReduce, bool loadData, int take, HashSet<string> keysReturned, CancellationToken cancellationToken, List<MappedResultInfo> outputCollection = null) { if (outputCollection == null) outputCollection = new List<MappedResultInfo>(); var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey); var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data); var keysToReduce = new HashSet<string>(keysLeftToReduce); foreach (var reduceKey in keysToReduce) { cancellationToken.ThrowIfCancellationRequested(); keysLeftToReduce.Remove(reduceKey); var reduceKeyHash = HashKey(reduceKey); var viewAndReduceKey = (Slice)CreateKey(view, ReduceKeySizeLimited(reduceKey), reduceKeyHash); using (var iterator = mappedResultsByViewAndReduceKey.MultiRead(Snapshot, viewAndReduceKey)) { keysReturned.Add(reduceKey); if (!iterator.Seek(Slice.BeforeAllKeys)) continue; do { cancellationToken.ThrowIfCancellationRequested(); ushort version; var value = LoadStruct(tableStorage.MappedResults, iterator.CurrentKey, writeBatch.Value, out version); if (value == null) continue; var size = tableStorage.MappedResults.GetDataSize(Snapshot, iterator.CurrentKey); var readReduceKey = value.ReadString(MappedResultFields.ReduceKey); take--; // We have worked with this reduce key, so we consider it an output even if we don't add it. RavenJObject data = null; if ( loadData ) { data = LoadMappedResult(iterator.CurrentKey, readReduceKey, mappedResultsData); if (data == null) continue; // If we request to load data and it is not there, we ignore it } var mappedResult = new MappedResultInfo { Bucket = value.ReadInt(MappedResultFields.Bucket), ReduceKey = readReduceKey, Etag = Etag.Parse(value.ReadBytes(MappedResultFields.Etag)), Timestamp = DateTime.FromBinary(value.ReadLong(MappedResultFields.Timestamp)), Data = data, Size = size }; outputCollection.Add(mappedResult); } while (iterator.MoveNext()); } if (take < 0) return outputCollection; } return outputCollection; }