public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");
var viewReductionColumn = tableColumnsCache.ScheduledReductionColumns["view"];
var levelReductionColumn = tableColumnsCache.ScheduledReductionColumns["level"];
var reduceReductionColumn = tableColumnsCache.ScheduledReductionColumns["reduce_key"];
var bucketReductionColumn = tableColumnsCache.ScheduledReductionColumns["bucket"];
var keysToRemove = new List<string>();
var output = new List<MappedResultInfo>();
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
try
{
var first = true;
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
int initialBucket = 0;
bool needToMoveNext = false;
if (first)
{
first = false;
if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
{
if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
{
needToMoveNext = true;
initialBucket = getItemsToReduceParams.LastReduceKeyAndBucket.Bucket;
}
}
}
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, initialBucket, MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false ||
(needToMoveNext && Api.TryMoveNext(session, ScheduledReductions) == false) )
{
keysToRemove.Add(reduceKey);
continue;
}
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);
if (Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
{
keysToRemove.Add(reduceKey);
continue;
}
// this isn't used for optimized reading, but to make it easier to delete records later on
OptimizedDeleter reader;
if (getItemsToReduceParams.ItemsToDelete.Count == 0)
{
getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
}
else
{
reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
}
reader.IndexId = getItemsToReduceParams.Index;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
return output;
var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, viewReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, levelReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex).Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, reduceReductionColumn);
if (getItemsToReduceParams.Index != indexFromDb)
break;
if (levelFromDb != getItemsToReduceParams.Level)
break;
if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
break;
var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, bucketReductionColumn).Value;
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);
if (thisIsNewScheduledReductionRow)
{
if (seenLocally.Add(rowKey))
{
getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
output.Add(mappedResultInfo);
}
}
}
}
while (Api.TryMoveNext(session, ScheduledReductions));
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
return output;
}
finally
{
// In whatever condition we would have to return, we must signal the removal of the reduce keys.
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");
var seenLocally = new HashSet<Tuple<string, int>>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray())
{
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, 0, MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
continue;
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);
if(Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
continue;
// this isn't used for optimized reading, but to make it easier to delete records later on
OptimizedDeleter reader;
if (getItemsToReduceParams.ItemsToDelete.Count == 0)
{
getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
}
else
{
reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete[0];
}
do
{
if (getItemsToReduceParams.Take <= 0)
break;
var indexFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], Encoding.Unicode, RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
tableColumnsCache.ScheduledReductionColumns["reduce_key"]);
if (string.Equals(getItemsToReduceParams.Index, indexFromDb, StringComparison.OrdinalIgnoreCase) == false)
continue;
if (levelFromDb != getItemsToReduceParams.Level)
continue;
if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
continue;
var bucket =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;
var rowKey = Tuple.Create(reduceKeyFromDb, bucket);
var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions);
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
yield break;
} while (Api.TryMoveNext(session, ScheduledReductions));
getItemsToReduceParams.ReduceKeys.Remove(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
var keysToRemove = new List<string>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, 0, MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
continue;
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);
if(Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
continue;
// this isn't used for optimized reading, but to make it easier to delete records later on
OptimizedDeleter reader;
if (getItemsToReduceParams.ItemsToDelete.Count == 0)
{
getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
}
else
{
reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
}
reader.IndexId = getItemsToReduceParams.Index;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
break;
var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
tableColumnsCache.ScheduledReductionColumns["reduce_key"]);
if (getItemsToReduceParams.Index != indexFromDb)
continue;
if (levelFromDb != getItemsToReduceParams.Level)
continue;
if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
continue;
var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
yield break;
}
while (Api.TryMoveNext(session, ScheduledReductions));
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}