public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams) { var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey); var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o => { var etag = o as Etag; if (etag == null) return null; return (Slice)etag.ToString(); }); var seenLocally = new HashSet<Tuple<string, int>>(); foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray()) { var reduceKeyHash = HashKey(reduceKey); var viewAndLevelAndReduceKey = (Slice) CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, ReduceKeySizeLimited(reduceKey), reduceKeyHash); using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey)) { if (!iterator.Seek(Slice.BeforeAllKeys)) continue; do { if (getItemsToReduceParams.Take <= 0) break; ushort version; var value = LoadStruct(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version); var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey); var bucket = value.ReadInt(ScheduledReductionFields.Bucket); var rowKey = Tuple.Create(reduceKeyFromDb, bucket); var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag))); var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey); if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket) { if (seenLocally.Add(rowKey)) { foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData)) { getItemsToReduceParams.Take--; yield return mappedResultInfo; } } } if (getItemsToReduceParams.Take <= 0) yield break; } while (iterator.MoveNext()); } getItemsToReduceParams.ReduceKeys.Remove(reduceKey); if (getItemsToReduceParams.Take <= 0) yield break; } }
public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken) { var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey); var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o => { var etag = o as Etag; if (etag == null) return null; return (Slice)etag.ToString(); }); var keysToRemove = new List<string>(); try { var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance); var mappedResults = new List<MappedResultInfo>(); var first = true; foreach (var reduceKey in getItemsToReduceParams.ReduceKeys) { cancellationToken.ThrowIfCancellationRequested(); Slice start = Slice.BeforeAllKeys; bool needToMoveNext = false; if (first) { first = false; if (getItemsToReduceParams.LastReduceKeyAndBucket != null) { if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey) { throw new InvalidOperationException("Mismatches last reduce key with the remaining reduce keys in the params"); } needToMoveNext = true; start = CreateBucketAndEtagKey(getItemsToReduceParams.LastReduceKeyAndBucket.Bucket, Etag.Empty); } } var viewAndLevelAndReduceKey = CreateScheduleReductionKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey); using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey)) { if (!iterator.Seek(start)) continue; if(needToMoveNext && iterator.MoveNext() ==false) continue; do { cancellationToken.ThrowIfCancellationRequested(); if (getItemsToReduceParams.Take <= 0) break; var idValueReader = iterator.CurrentKey.CreateReader(); idValueReader.ReadBigEndianInt32(); // bucket int _; var id = new Slice(Etag.Parse(idValueReader.ReadBytes(16, out _))); ushort version; var value = LoadStruct(tableStorage.ScheduledReductions, id, writeBatch.Value, out version); if (value == null) // TODO: Check if this is correct. continue; var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey); var bucket = value.ReadInt(ScheduledReductionFields.Bucket); var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb); var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag))); if (thisIsNewScheduledReductionRow) { if (seenLocally.Add(rowKey)) { getItemsToReduceParams.LastReduceKeyAndBucket = rowKey; foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken)) { getItemsToReduceParams.Take--; mappedResults.Add(mappedResultInfo); } } } if (getItemsToReduceParams.Take <= 0) return mappedResults; } while (iterator.MoveNext()); } keysToRemove.Add(reduceKey); if (getItemsToReduceParams.Take <= 0) break; } return mappedResults; } finally { foreach (var keyToRemove in keysToRemove) getItemsToReduceParams.ReduceKeys.Remove(keyToRemove); } }
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken) { var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey); var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o => { var etag = o as Etag; if (etag == null) return null; return (Slice)etag.ToString(); }); var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance); var keysToRemove = new List<string>(); foreach (var reduceKey in getItemsToReduceParams.ReduceKeys) { cancellationToken.ThrowIfCancellationRequested(); var reduceKeyHash = HashKey(reduceKey); var viewAndLevelAndReduceKey = (Slice) CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, ReduceKeySizeLimited(reduceKey), reduceKeyHash); using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey)) { if (!iterator.Seek(Slice.BeforeAllKeys)) continue; do { cancellationToken.ThrowIfCancellationRequested(); if (getItemsToReduceParams.Take <= 0) break; ushort version; var value = LoadStruct(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version); if (value == null) // TODO: Check if this is correct. continue; var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey); var bucket = value.ReadInt(ScheduledReductionFields.Bucket); var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb); var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag))); var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey); if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket) { if (seenLocally.Add(rowKey)) { foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken)) { getItemsToReduceParams.Take--; yield return mappedResultInfo; } } } if (getItemsToReduceParams.Take <= 0) yield break; } while (iterator.MoveNext()); } keysToRemove.Add(reduceKey); if (getItemsToReduceParams.Take <= 0) yield break; } foreach (var keyToRemove in keysToRemove) getItemsToReduceParams.ReduceKeys.Remove(keyToRemove); }
public IEnumerable <MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams) { var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey); var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o => { var json = o as RavenJObject; if (json == null) { return(null); } var etag = Etag.Parse(json.Value <byte[]>("etag")); return(etag.ToString()); }); var seenLocally = new HashSet <Tuple <string, int> >(); foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray()) { var reduceKeyHash = HashKey(reduceKey); var viewAndLevelAndReduceKey = CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey, reduceKeyHash); using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey)) { if (!iterator.Seek(Slice.BeforeAllKeys)) { continue; } do { if (getItemsToReduceParams.Take <= 0) { break; } ushort version; var value = LoadJson(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version); var reduceKeyFromDb = value.Value <string>("reduceKey"); var bucket = value.Value <int>("bucket"); var rowKey = Tuple.Create(reduceKeyFromDb, bucket); var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, value); var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey); if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket) { if (seenLocally.Add(rowKey)) { foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData)) { getItemsToReduceParams.Take--; yield return(mappedResultInfo); } } } if (getItemsToReduceParams.Take <= 0) { yield break; } }while (iterator.MoveNext()); } getItemsToReduceParams.ReduceKeys.Remove(reduceKey); if (getItemsToReduceParams.Take <= 0) { yield break; } } }