public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var etag = o as Etag;
if (etag == null)
return null;
return (Slice)etag.ToString();
});
var seenLocally = new HashSet<Tuple<string, int>>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray())
{
var reduceKeyHash = HashKey(reduceKey);
var viewAndLevelAndReduceKey = (Slice) CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
continue;
do
{
if (getItemsToReduceParams.Take <= 0)
break;
ushort version;
var value = LoadStruct(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version);
var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);
var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
var rowKey = Tuple.Create(reduceKeyFromDb, bucket);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
yield break;
}
while (iterator.MoveNext());
}
getItemsToReduceParams.ReduceKeys.Remove(reduceKey);
if (getItemsToReduceParams.Take <= 0)
yield break;
}
}
public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var etag = o as Etag;
if (etag == null)
return null;
return (Slice)etag.ToString();
});
var keysToRemove = new List<string>();
try
{
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
var mappedResults = new List<MappedResultInfo>();
var first = true;
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
Slice start = Slice.BeforeAllKeys;
bool needToMoveNext = false;
if (first)
{
first = false;
if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
{
if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
{
throw new InvalidOperationException("Mismatches last reduce key with the remaining reduce keys in the params");
}
needToMoveNext = true;
start = CreateBucketAndEtagKey(getItemsToReduceParams.LastReduceKeyAndBucket.Bucket, Etag.Empty);
}
}
var viewAndLevelAndReduceKey = CreateScheduleReductionKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(start))
continue;
if(needToMoveNext && iterator.MoveNext() ==false)
continue;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
break;
var idValueReader = iterator.CurrentKey.CreateReader();
idValueReader.ReadBigEndianInt32(); // bucket
int _;
var id = new Slice(Etag.Parse(idValueReader.ReadBytes(16, out _)));
ushort version;
var value = LoadStruct(tableStorage.ScheduledReductions, id, writeBatch.Value, out version);
if (value == null) // TODO: Check if this is correct.
continue;
var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);
var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));
if (thisIsNewScheduledReductionRow)
{
if (seenLocally.Add(rowKey))
{
getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
mappedResults.Add(mappedResultInfo);
}
}
}
if (getItemsToReduceParams.Take <= 0)
return mappedResults;
}
while (iterator.MoveNext());
}
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
return mappedResults;
}
finally
{
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var etag = o as Etag;
if (etag == null)
return null;
return (Slice)etag.ToString();
});
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
var keysToRemove = new List<string>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
var reduceKeyHash = HashKey(reduceKey);
var viewAndLevelAndReduceKey = (Slice) CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
continue;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
break;
ushort version;
var value = LoadStruct(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null) // TODO: Check if this is correct.
continue;
var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);
var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
yield break;
}
while (iterator.MoveNext());
}
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
yield break;
}
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}
public IEnumerable <MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var json = o as RavenJObject;
if (json == null)
{
return(null);
}
var etag = Etag.Parse(json.Value <byte[]>("etag"));
return(etag.ToString());
});
var seenLocally = new HashSet <Tuple <string, int> >();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray())
{
var reduceKeyHash = HashKey(reduceKey);
var viewAndLevelAndReduceKey = CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey, reduceKeyHash);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
continue;
}
do
{
if (getItemsToReduceParams.Take <= 0)
{
break;
}
ushort version;
var value = LoadJson(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version);
var reduceKeyFromDb = value.Value <string>("reduceKey");
var bucket = value.Value <int>("bucket");
var rowKey = Tuple.Create(reduceKeyFromDb, bucket);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, value);
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData))
{
getItemsToReduceParams.Take--;
yield return(mappedResultInfo);
}
}
}
if (getItemsToReduceParams.Take <= 0)
{
yield break;
}
}while (iterator.MoveNext());
}
getItemsToReduceParams.ReduceKeys.Remove(reduceKey);
if (getItemsToReduceParams.Take <= 0)
{
yield break;
}
}
}
