public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary <ReduceKeyAndBucket, int> removed)
{
var viewAndDocumentId = CreateKey(view, documentId);
var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
using (var iterator = mappedResultsByViewAndDocumentId.MultiRead(Snapshot, viewAndDocumentId))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return;
}
do
{
var id = iterator.CurrentKey.Clone();
ushort version;
var value = LoadJson(tableStorage.MappedResults, id, writeBatch.Value, out version);
var reduceKey = value.Value <string>("reduceKey");
var bucket = value.Value <int>("bucket");
DeleteMappedResult(id, view, documentId, reduceKey, bucket.ToString(CultureInfo.InvariantCulture));
var reduceKeyAndBucket = new ReduceKeyAndBucket(bucket, reduceKey);
removed[reduceKeyAndBucket] = removed.GetOrDefault(reduceKeyAndBucket) + 1;
}while (iterator.MoveNext());
}
}
public void ScheduleReductions(string view, int level, ReduceKeyAndBucket reduceKeysAndBucket)
{
var etag = generator.CreateSequentialUuid(UuidType.ScheduledReductions).ToByteArray();
storage.ScheduleReductions.UpdateKey(new RavenJObject
{
{ "view", view },
{ "reduceKey", reduceKeysAndBucket.ReduceKey },
{ "bucket", reduceKeysAndBucket.Bucket },
{ "level", level },
{ "etag", etag },
{ "timestamp", SystemTime.UtcNow }
});
}
public void DeleteMappedResultsForDocumentId(string documentId, string view, Dictionary <ReduceKeyAndBucket, int> removed)
{
foreach (var key in storage.MappedResults["ByViewAndDocumentId"].SkipTo(new RavenJObject
{
{ "view", view },
{ "docId", documentId }
}).TakeWhile(x => StringComparer.OrdinalIgnoreCase.Equals(x.Value <string>("view"), view) &&
StringComparer.OrdinalIgnoreCase.Equals(x.Value <string>("docId"), documentId)))
{
storage.MappedResults.Remove(key);
var reduceKey = key.Value <string>("reduceKey");
var bucket = new ReduceKeyAndBucket(key.Value <int>("bucket"), reduceKey);
removed[bucket] = removed.GetOrDefault(bucket) + 1;
}
}
public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
{
var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
var idAsString = id.ToString();
var reduceHashKey = HashKey(reduceKeysAndBuckets.ReduceKey);
tableStorage.ScheduledReductions.Add(writeBatch.Value, idAsString, new RavenJObject
{
{ "view", view },
{ "reduceKey", reduceKeysAndBuckets.ReduceKey },
{ "bucket", reduceKeysAndBuckets.Bucket },
{ "level", level },
{ "etag", id.ToByteArray() },
{ "timestamp", SystemTime.UtcNow }
});
scheduledReductionsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString);
scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, level, reduceKeysAndBuckets.ReduceKey, reduceHashKey), idAsString);
}
public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary<ReduceKeyAndBucket, int> removed)
{
Api.JetSetCurrentIndex(session, MappedResults, "by_view_and_doc_key");
Api.MakeKey(session, MappedResults, view, MakeKeyGrbit.NewKey);
Api.MakeKey(session, MappedResults, documentId, Encoding.Unicode, MakeKeyGrbit.None);
if (Api.TrySeek(session, MappedResults, SeekGrbit.SeekEQ) == false)
return;
Api.MakeKey(session, MappedResults, view, MakeKeyGrbit.NewKey);
Api.MakeKey(session, MappedResults, documentId, Encoding.Unicode, MakeKeyGrbit.None);
Api.JetSetIndexRange(session, MappedResults, SetIndexRangeGrbit.RangeUpperLimit | SetIndexRangeGrbit.RangeInclusive);
do
{
// esent index ranges are approximate, and we need to check them ourselves as well
var documentIdFromDb = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"]);
if (StringComparer.OrdinalIgnoreCase.Equals(documentIdFromDb, documentId) == false)
continue;
var reduceKey = Api.RetrieveColumnAsString(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"],
Encoding.Unicode);
var bucket = Api.RetrieveColumnAsInt32(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"]).Value;
var key = new ReduceKeyAndBucket(bucket, reduceKey);
removed[key] = removed.GetOrDefault(key) + 1;
Api.JetDelete(session, MappedResults);
} while (Api.TryMoveNext(session, MappedResults));
}
public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");
var viewReductionColumn = tableColumnsCache.ScheduledReductionColumns["view"];
var levelReductionColumn = tableColumnsCache.ScheduledReductionColumns["level"];
var reduceReductionColumn = tableColumnsCache.ScheduledReductionColumns["reduce_key"];
var bucketReductionColumn = tableColumnsCache.ScheduledReductionColumns["bucket"];
var keysToRemove = new List<string>();
var output = new List<MappedResultInfo>();
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
try
{
var first = true;
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
int initialBucket = 0;
bool needToMoveNext = false;
if (first)
{
first = false;
if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
{
if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
{
needToMoveNext = true;
initialBucket = getItemsToReduceParams.LastReduceKeyAndBucket.Bucket;
}
}
}
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, initialBucket, MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false ||
(needToMoveNext && Api.TryMoveNext(session, ScheduledReductions) == false) )
{
keysToRemove.Add(reduceKey);
continue;
}
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);
if (Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
{
keysToRemove.Add(reduceKey);
continue;
}
// this isn't used for optimized reading, but to make it easier to delete records later on
OptimizedDeleter reader;
if (getItemsToReduceParams.ItemsToDelete.Count == 0)
{
getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
}
else
{
reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
}
reader.IndexId = getItemsToReduceParams.Index;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
return output;
var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, viewReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, levelReductionColumn, RetrieveColumnGrbit.RetrieveFromIndex).Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions, reduceReductionColumn);
if (getItemsToReduceParams.Index != indexFromDb)
break;
if (levelFromDb != getItemsToReduceParams.Level)
break;
if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
break;
var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, bucketReductionColumn).Value;
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);
if (thisIsNewScheduledReductionRow)
{
if (seenLocally.Add(rowKey))
{
getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
output.Add(mappedResultInfo);
}
}
}
}
while (Api.TryMoveNext(session, ScheduledReductions));
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
return output;
}
finally
{
// In whatever condition we would have to return, we must signal the removal of the reduce keys.
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}
}
public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBucket)
{
var bucket = reduceKeysAndBucket.Bucket;
using (var map = new Update(session, ScheduledReductions, JET_prep.Insert))
{
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], view);
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"],
reduceKeysAndBucket.ReduceKey, Encoding.Unicode);
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["hashed_reduce_key"],
HashReduceKey(reduceKeysAndBucket.ReduceKey));
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["etag"], uuidGenerator.CreateSequentialUuid(UuidType.ScheduledReductions).ToByteArray());
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["timestamp"],
SystemTime.UtcNow.ToBinary());
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"],
bucket);
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], level);
map.Save();
}
if (scheduledReductionsPerViewAndLevel != null)
scheduledReductionsPerViewAndLevel.AddOrUpdate(view, new RemainingReductionPerLevel(level), (key, oldvalue) => oldvalue.IncrementPerLevelCounters(level));
}
public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
{
var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
var idSlice = (Slice)id.ToString();
var reduceHashKey = HashKey(reduceKeysAndBuckets.ReduceKey);
var scheduledReduction = new Structure<ScheduledReductionFields>(tableStorage.ScheduledReductions.Schema);
scheduledReduction.Set(ScheduledReductionFields.IndexId, view)
.Set(ScheduledReductionFields.ReduceKey, reduceKeysAndBuckets.ReduceKey)
.Set(ScheduledReductionFields.Bucket, reduceKeysAndBuckets.Bucket)
.Set(ScheduledReductionFields.Level, level)
.Set(ScheduledReductionFields.Etag, id.ToByteArray())
.Set(ScheduledReductionFields.Timestamp, SystemTime.UtcNow.ToBinary());
tableStorage.ScheduledReductions.AddStruct(writeBatch.Value, idSlice, scheduledReduction);
var viewKey = CreateKey(view);
var viewKeySlice = (Slice)viewKey;
scheduledReductionsByView.MultiAdd(writeBatch.Value, viewKeySlice, idSlice);
scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, (Slice)AppendToKey(viewKey, level, ReduceKeySizeLimited(reduceKeysAndBuckets.ReduceKey), reduceHashKey), idSlice);
}
public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary<ReduceKeyAndBucket, int> removed)
{
var viewKey = CreateKey(view);
var viewKeySlice = new Slice(viewKey);
var viewAndDocumentId = new Slice(AppendToKey(viewKey, documentId));
var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView);
var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey);
var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket);
var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data);
using (var iterator = mappedResultsByViewAndDocumentId.MultiRead(Snapshot, viewAndDocumentId))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
return;
do
{
// TODO: Check if we can relax the clone.
var id = iterator.CurrentKey.Clone();
ushort version;
var value = LoadStruct(tableStorage.MappedResults, id, writeBatch.Value, out version);
var reduceKey = value.ReadString(MappedResultFields.ReduceKey);
var bucket = value.ReadInt(MappedResultFields.Bucket);
var reduceKeyHash = HashKey(reduceKey);
var viewAndReduceKey = AppendToKey(viewKey, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
var viewAndReduceKeyAndSourceBucket = AppendToKey(viewAndReduceKey, bucket);
tableStorage.MappedResults.Delete(writeBatch.Value, id);
mappedResultsByViewAndDocumentId.MultiDelete(writeBatch.Value, viewAndDocumentId, id);
mappedResultsByView.MultiDelete(writeBatch.Value, viewKeySlice, id);
mappedResultsByViewAndReduceKey.MultiDelete(writeBatch.Value, (Slice)viewAndReduceKey, id);
mappedResultsByViewAndReduceKeyAndSourceBucket.MultiDelete(writeBatch.Value, (Slice)viewAndReduceKeyAndSourceBucket, id);
mappedResultsData.Delete(writeBatch.Value, id);
var reduceKeyAndBucket = new ReduceKeyAndBucket(bucket, reduceKey);
removed[reduceKeyAndBucket] = removed.GetOrDefault(reduceKeyAndBucket) + 1;
}
while (iterator.MoveNext());
}
}
public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBucket)
{
var bucket = reduceKeysAndBucket.Bucket;
using (var map = new Update(session, ScheduledReductions, JET_prep.Insert))
{
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], view);
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["reduce_key"],
reduceKeysAndBucket.ReduceKey, Encoding.Unicode);
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["hashed_reduce_key"],
HashReduceKey(reduceKeysAndBucket.ReduceKey));
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["etag"], uuidGenerator.CreateSequentialUuid(UuidType.ScheduledReductions).ToByteArray());
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["timestamp"],
SystemTime.UtcNow.ToBinary());
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"],
bucket);
Api.SetColumn(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], level);
map.Save();
}
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var etag = o as Etag;
if (etag == null)
return null;
return (Slice)etag.ToString();
});
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
var keysToRemove = new List<string>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
var reduceKeyHash = HashKey(reduceKey);
var viewAndLevelAndReduceKey = (Slice) CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, ReduceKeySizeLimited(reduceKey), reduceKeyHash);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
continue;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
break;
ushort version;
var value = LoadStruct(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null) // TODO: Check if this is correct.
continue;
var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);
var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
yield break;
}
while (iterator.MoveNext());
}
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
yield break;
}
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}
public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
{
var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
var idAsString = id.ToString();
var reduceHashKey = HashKey(reduceKeysAndBuckets.ReduceKey);
tableStorage.ScheduledReductions.Add(writeBatch.Value, idAsString, new RavenJObject
{
{"view", view},
{"reduceKey", reduceKeysAndBuckets.ReduceKey},
{"bucket", reduceKeysAndBuckets.Bucket},
{"level", level},
{"etag", id.ToByteArray()},
{"timestamp", SystemTime.UtcNow}
});
scheduledReductionsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString);
scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, level, reduceKeysAndBuckets.ReduceKey, reduceHashKey), idAsString);
}
public void DeleteMappedResultsForDocumentId(string documentId, int view, Dictionary<ReduceKeyAndBucket, int> removed)
{
var viewAndDocumentId = CreateKey(view, documentId);
var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
using (var iterator = mappedResultsByViewAndDocumentId.MultiRead(Snapshot, viewAndDocumentId))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
return;
do
{
var id = iterator.CurrentKey.Clone();
ushort version;
var value = LoadJson(tableStorage.MappedResults, id, writeBatch.Value, out version);
var reduceKey = value.Value<string>("reduceKey");
var bucket = value.Value<int>("bucket");
DeleteMappedResult(id, view, documentId, reduceKey, bucket.ToString(CultureInfo.InvariantCulture));
var reduceKeyAndBucket = new ReduceKeyAndBucket(bucket, reduceKey);
removed[reduceKeyAndBucket] = removed.GetOrDefault(reduceKeyAndBucket) + 1;
}
while (iterator.MoveNext());
}
}
public IList<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var etag = o as Etag;
if (etag == null)
return null;
return (Slice)etag.ToString();
});
var keysToRemove = new List<string>();
try
{
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
var mappedResults = new List<MappedResultInfo>();
var first = true;
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
Slice start = Slice.BeforeAllKeys;
bool needToMoveNext = false;
if (first)
{
first = false;
if (getItemsToReduceParams.LastReduceKeyAndBucket != null)
{
if (getItemsToReduceParams.LastReduceKeyAndBucket.ReduceKey != reduceKey)
{
throw new InvalidOperationException("Mismatches last reduce key with the remaining reduce keys in the params");
}
needToMoveNext = true;
start = CreateBucketAndEtagKey(getItemsToReduceParams.LastReduceKeyAndBucket.Bucket, Etag.Empty);
}
}
var viewAndLevelAndReduceKey = CreateScheduleReductionKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(start))
continue;
if(needToMoveNext && iterator.MoveNext() ==false)
continue;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
break;
var idValueReader = iterator.CurrentKey.CreateReader();
idValueReader.ReadBigEndianInt32(); // bucket
int _;
var id = new Slice(Etag.Parse(idValueReader.ReadBytes(16, out _)));
ushort version;
var value = LoadStruct(tableStorage.ScheduledReductions, id, writeBatch.Value, out version);
if (value == null) // TODO: Check if this is correct.
continue;
var reduceKeyFromDb = value.ReadString(ScheduledReductionFields.ReduceKey);
var bucket = value.ReadInt(ScheduledReductionFields.Bucket);
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, Etag.Parse(value.ReadBytes(ScheduledReductionFields.Etag)));
if (thisIsNewScheduledReductionRow)
{
if (seenLocally.Add(rowKey))
{
getItemsToReduceParams.LastReduceKeyAndBucket = rowKey;
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
mappedResults.Add(mappedResultInfo);
}
}
}
if (getItemsToReduceParams.Take <= 0)
return mappedResults;
}
while (iterator.MoveNext());
}
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
return mappedResults;
}
finally
{
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}
}
public void ScheduleReductions(int view, int level, ReduceKeyAndBucket reduceKeysAndBuckets)
{
var scheduledReductionsByView = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByView);
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var id = generator.CreateSequentialUuid(UuidType.ScheduledReductions);
var idSlice = (Slice)id.ToString();
var scheduledReduction = new Structure<ScheduledReductionFields>(tableStorage.ScheduledReductions.Schema);
scheduledReduction.Set(ScheduledReductionFields.IndexId, view)
.Set(ScheduledReductionFields.ReduceKey, reduceKeysAndBuckets.ReduceKey)
.Set(ScheduledReductionFields.Bucket, reduceKeysAndBuckets.Bucket)
.Set(ScheduledReductionFields.Level, level)
.Set(ScheduledReductionFields.Etag, id.ToByteArray())
.Set(ScheduledReductionFields.Timestamp, SystemTime.UtcNow.ToBinary());
tableStorage.ScheduledReductions.AddStruct(writeBatch.Value, idSlice, scheduledReduction);
var viewKey = CreateViewKey(view);
scheduledReductionsByView.MultiAdd(writeBatch.Value, viewKey, idSlice);
var scheduleReductionKey = CreateScheduleReductionKey(view, level, reduceKeysAndBuckets.ReduceKey);
scheduledReductionsByViewAndLevelAndReduceKey.MultiAdd(writeBatch.Value, scheduleReductionKey, CreateBucketAndEtagKey(reduceKeysAndBuckets.Bucket, id));
if (scheduledReductionsPerViewAndLevel != null)
scheduledReductionsPerViewAndLevel.AddOrUpdate(view, new RemainingReductionPerLevel(level), (key, oldvalue) => oldvalue.IncrementPerLevelCounters(level));
}
public IEnumerable<MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams, CancellationToken cancellationToken)
{
Api.JetSetCurrentIndex(session, ScheduledReductions, "by_view_level_and_hashed_reduce_key_and_bucket");
var seenLocally = new HashSet<ReduceKeyAndBucket>(ReduceKeyAndBucketEqualityComparer.Instance);
var keysToRemove = new List<string>();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys)
{
cancellationToken.ThrowIfCancellationRequested();
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, 0, MakeKeyGrbit.None);
if (Api.TrySeek(session, ScheduledReductions, SeekGrbit.SeekGE) == false)
continue;
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Index, MakeKeyGrbit.NewKey);
Api.MakeKey(session, ScheduledReductions, getItemsToReduceParams.Level, MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, HashReduceKey(reduceKey), MakeKeyGrbit.None);
Api.MakeKey(session, ScheduledReductions, int.MaxValue, MakeKeyGrbit.None);
if(Api.TrySetIndexRange(session, ScheduledReductions, SetIndexRangeGrbit.RangeInclusive | SetIndexRangeGrbit.RangeUpperLimit) == false)
continue;
// this isn't used for optimized reading, but to make it easier to delete records later on
OptimizedDeleter reader;
if (getItemsToReduceParams.ItemsToDelete.Count == 0)
{
getItemsToReduceParams.ItemsToDelete.Add(reader = new OptimizedDeleter());
}
else
{
reader = (OptimizedDeleter)getItemsToReduceParams.ItemsToDelete.First();
}
reader.IndexId = getItemsToReduceParams.Index;
do
{
cancellationToken.ThrowIfCancellationRequested();
if (getItemsToReduceParams.Take <= 0)
break;
var indexFromDb = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["view"], RetrieveColumnGrbit.RetrieveFromIndex);
var levelFromDb =
Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["level"], RetrieveColumnGrbit.RetrieveFromIndex).
Value;
var reduceKeyFromDb = Api.RetrieveColumnAsString(session, ScheduledReductions,
tableColumnsCache.ScheduledReductionColumns["reduce_key"]);
if (getItemsToReduceParams.Index != indexFromDb)
continue;
if (levelFromDb != getItemsToReduceParams.Level)
continue;
if (string.Equals(reduceKeyFromDb, reduceKey, StringComparison.Ordinal) == false)
continue;
var bucket = Api.RetrieveColumnAsInt32(session, ScheduledReductions, tableColumnsCache.ScheduledReductionColumns["bucket"]).Value;
var rowKey = new ReduceKeyAndBucket(bucket, reduceKeyFromDb);
var thisIsNewScheduledReductionRow = reader.Add(session, ScheduledReductions, getItemsToReduceParams.Level);
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData, cancellationToken))
{
getItemsToReduceParams.Take--;
yield return mappedResultInfo;
}
}
}
if (getItemsToReduceParams.Take <= 0)
yield break;
}
while (Api.TryMoveNext(session, ScheduledReductions));
keysToRemove.Add(reduceKey);
if (getItemsToReduceParams.Take <= 0)
break;
}
foreach (var keyToRemove in keysToRemove)
getItemsToReduceParams.ReduceKeys.Remove(keyToRemove);
}