public void WillExpand()
{
using (var context = new TransactionOperationContext(Env, 1024, 1024))
{
var key1 = context.GetLazyString("orders/1");
var key2 = context.GetLazyString("orders/2");
using (var tx = context.OpenWriteTransaction())
{
var collection = CollectionOfBloomFilters.Load(2, context);
Assert.Equal(1, collection.Count);
Assert.True(collection.Add(key1));
Assert.Equal(1, collection.Count);
Assert.True(collection.Add(key2));
Assert.Equal(2, collection.Count);
tx.Commit();
}
using (context.OpenWriteTransaction())
{
var collection = CollectionOfBloomFilters.Load(2, context);
Assert.Equal(2, collection.Count);
}
}
}
private unsafe void UpdateInternal(TransactionOperationContext context, string guid, string type, long index, long term, HistoryStatus status, object result, Exception exception)
{
var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice);
TableValueReader reader;
using (Slice.From(context.Allocator, guid, out var guidSlice))
{
if (table.ReadByKey(guidSlice, out reader) == false)
{
return;
}
}
if (TypeConverter.IsSupportedType(result) == false)
{
throw new RachisApplyException("We don't support type " + result.GetType().FullName + ".");
}
using (Slice.From(context.Allocator, guid, out var guidSlice))
using (Slice.From(context.Allocator, type, out var typeSlice))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(guidSlice);
tvb.Add(Bits.SwapBytes(index));
tvb.Add(Bits.SwapBytes(GetUniqueTicks(context.Transaction.InnerTransaction)));
tvb.Add(*(long *)reader.Read((int)(LogHistoryColumn.Term), out _));
tvb.Add(term);
tvb.Add(typeSlice);
tvb.Add((byte)status);
if (result == null)
{
tvb.Add(Slices.Empty);
}
else
{
var blittableResult = context.ReadObject(new DynamicJsonValue {
["Result"] = result
}, "set-history-result");
tvb.Add(blittableResult.BasePointer, blittableResult.Size);
}
if (exception == null)
{
tvb.Add(Slices.Empty);
tvb.Add(Slices.Empty);
}
else
{
var exceptionType = context.GetLazyString(exception.GetType().AssemblyQualifiedName);
var exceptionMsg = context.GetLazyString(exception.ToString());
tvb.Add(exceptionType.Buffer, exceptionType.Size);
tvb.Add(exceptionMsg.Buffer, exceptionMsg.Size);
}
table.Set(tvb);
}
}
public void SuccessRate_BloomFilters_2(int count, double expectedSuccessRate, long?version)
{
using (var context = new TransactionOperationContext(Env, 1024, 1024, 32 * 1024, SharedMultipleUseFlag.None))
{
var added = 0;
SetUpCollectionOfBloomFilters(context, version);
using (var tx = context.OpenWriteTransaction())
{
using (var filter = CollectionOfBloomFilters.Load(CollectionOfBloomFilters.Mode.X64, context))
{
for (var i = 0; i < count; i++)
{
var key = context.GetLazyString($"orders/{i:D19}-A");
if (filter.Add(key))
{
added++;
}
}
}
}
var successRate = added / (double)count;
Assert.True(successRate >= expectedSuccessRate, $"{successRate} >= {expectedSuccessRate}");
}
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation,
IndexingStatsScope stats, CancellationToken token)
{
if (_mapReduceContext.StoreByReduceKeyHash.Count == 0)
{
WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop
return(false);
}
ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32);
var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName);
var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction;
var writer = writeOperation.Value;
var treeScopeStats = stats.For(IndexingOperation.Reduce.TreeScope, start: false);
var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false);
foreach (var store in _mapReduceContext.StoreByReduceKeyHash)
{
token.ThrowIfCancellationRequested();
using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture)))
using (store.Value)
using (_aggregationBatch)
{
var modifiedStore = store.Value;
switch (modifiedStore.Type)
{
case MapResultsStorageType.Tree:
using (treeScopeStats.Start())
{
HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
break;
case MapResultsStorageType.Nested:
using (nestedValuesScopeStats.Start())
{
HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token);
}
break;
default:
throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString());
}
}
}
using (MemoryUsageGuard.GetProcessMemoryUsage(out var memoryUsage, out _))
{
stats.RecordReduceMemoryStats(memoryUsage.WorkingSet, memoryUsage.PrivateMemory);
}
WriteLastEtags(indexContext);
return(false);
}
public void CanPersist()
{
using (var context = new TransactionOperationContext(Env, 1024, 1024, SharedMultipleUseFlag.None))
{
var key1 = context.GetLazyString("orders/1");
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
using (var filter = new CollectionOfBloomFilters.BloomFilter64(0, tree, writable: true, allocator: context.Allocator))
{
Assert.True(filter.Add(key1));
Assert.Equal(1, filter.Count);
}
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
using (var filter = new CollectionOfBloomFilters.BloomFilter64(0, tree, writable: false, allocator: context.Allocator))
{
Assert.False(filter.Add(key1));
Assert.Equal(1, filter.Count);
}
}
context.ReturnMemory(key1.AllocatedMemoryData);
}
}
public void WillExpand()
{
using (var context = new TransactionOperationContext(Env, 1024, 1024, SharedMultipleUseFlag.None))
{
using (var tx = context.OpenWriteTransaction())
{
using (var collection = CollectionOfBloomFilters.Load(CollectionOfBloomFilters.Mode.X86, context))
{
Assert.Equal(1, collection.Count);
for (var i = 0; i < CollectionOfBloomFilters.BloomFilter32.MaxCapacity * 1.2; i++)
{
var key = context.GetLazyString("orders/" + i);
collection.Add(key);
}
Assert.Equal(2, collection.Count);
}
tx.Commit();
}
using (context.OpenWriteTransaction())
{
using (var collection = CollectionOfBloomFilters.Load(CollectionOfBloomFilters.Mode.X86, context))
{
Assert.Equal(2, collection.Count);
}
}
}
}
public void WillGetConsumedFromStorage()
{
using (var context = new TransactionOperationContext(Env, 1024, 1024, 32 * 1024, SharedMultipleUseFlag.None))
{
using (var tx = context.OpenWriteTransaction())
{
using (var collection = CollectionOfBloomFilters.Load(CollectionOfBloomFilters.Mode.X86, context))
{
var i = 0;
while (collection.Consumed == false)
{
var key = context.GetLazyString($"orders/{i++}");
collection.Add(key);
}
Assert.Equal(20, collection.Count);
collection.Flush();
Assert.Equal(0, collection.Count);
}
tx.Commit();
}
using (context.OpenWriteTransaction())
{
using (var collection = CollectionOfBloomFilters.Load(CollectionOfBloomFilters.Mode.X86, context))
{
Assert.Equal(0, collection.Count);
Assert.True(collection.Consumed);
}
}
}
}
public void CanPersist()
{
Slice key;
using (var context = new TransactionOperationContext(Env, 1024, 1024))
using (Slice.From(context.Allocator, "f1", out key))
{
var key1 = context.GetLazyString("orders/1");
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
var ptr = tree.DirectAdd(key, CollectionOfBloomFilters.BloomFilter.PtrSize);
var filter = new CollectionOfBloomFilters.BloomFilter(key, ptr, tree, writeable: true);
Assert.True(filter.Add(key1));
Assert.Equal(1, filter.Count);
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
var read = tree.Read("f1");
var filter = new CollectionOfBloomFilters.BloomFilter(key, read.Reader.Base, tree, writeable: false);
Assert.False(filter.Add(key1));
Assert.Equal(1, filter.Count);
}
}
}
public void CheckWritability()
{
using (var context = new TransactionOperationContext(Env, 1024, 1024, 32 * 1024, SharedMultipleUseFlag.None))
{
var key1 = context.GetLazyString("orders/1");
var key2 = context.GetLazyString("orders/2");
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
using (var filter = new CollectionOfBloomFilters.BloomFilter64(0, tree, writable: true, allocator: context.Allocator))
{
Assert.True(filter.Add(key1));
Assert.Equal(1, filter.Count);
Assert.True(filter.Writable);
Assert.False(filter.ReadOnly);
filter.Flush();
}
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
using (var filter = new CollectionOfBloomFilters.BloomFilter64(0, tree, writable: false, allocator: context.Allocator))
{
Assert.False(filter.Writable);
Assert.False(filter.Add(key1));
Assert.Equal(1, filter.Count);
Assert.False(filter.Writable);
Assert.True(filter.Add(key2));
Assert.Equal(2, filter.Count);
Assert.True(filter.Writable);
}
}
}
}
private ReplicationBatchItem ConvertForTest(TransactionOperationContext context, ReplicationBatchItem item)
{
if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment)
{
item.Stream.Position = 0;
var stream = new MemoryStream();
item.Stream.CopyTo(stream);
stream.Position = 0;
item.Stream.Position = 0;
return(new ReplicationBatchItem
{
Base64Hash = item.Base64Hash.Clone(context.Allocator),
ContentType = context.GetLazyString(item.ContentType),
Name = context.GetLazyString(item.Name),
Id = context.GetLazyString(item.Id),
Stream = stream,
Type = item.Type,
ChangeVector = item.ChangeVector,
LastModifiedTicks = item.LastModifiedTicks,
TransactionMarker = item.TransactionMarker,
Etag = item.Etag
});
}
if (item.Type == ReplicationBatchItem.ReplicationItemType.Document)
{
return(new ReplicationBatchItem
{
Id = context.GetLazyString(item.Id),
Data = item.Data?.Clone(context),
Collection = context.GetLazyString(item.Collection),
Flags = item.Flags,
Type = item.Type,
ChangeVector = item.ChangeVector,
LastModifiedTicks = item.LastModifiedTicks,
TransactionMarker = item.TransactionMarker,
Etag = item.Etag
});
}
throw new InvalidOperationException($"This test should receive only the following types: ReplicationItemType.Document, ReplicationItemType.Attachment.");
}
public void Basic()
{
using (var context = new TransactionOperationContext(Env, 1024, 1024, 32 * 1024, SharedMultipleUseFlag.None))
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("Filters");
var key1 = context.GetLazyString("orders/1");
var key2 = context.GetLazyString("orders/2");
using (var filter = new CollectionOfBloomFilters.BloomFilter64(0, BloomFilterVersion.CurrentVersion, tree, writable: true, allocator: context.Allocator))
{
Assert.False(filter.Contains(key1));
Assert.False(filter.Contains(key2));
Assert.Equal(0, filter.Count);
Assert.True(filter.Add(key1));
Assert.True(filter.Contains(key1));
Assert.False(filter.Contains(key2));
Assert.Equal(1, filter.Count);
Assert.False(filter.Add(key1));
Assert.True(filter.Contains(key1));
Assert.False(filter.Contains(key2));
Assert.Equal(1, filter.Count);
Assert.True(filter.Add(key2));
Assert.True(filter.Contains(key1));
Assert.True(filter.Contains(key2));
Assert.Equal(2, filter.Count);
Assert.False(filter.Add(key1));
Assert.True(filter.Contains(key1));
Assert.True(filter.Contains(key2));
Assert.Equal(2, filter.Count);
}
tx.Commit();
}
}
}
public unsafe void AddAlert(Alert alert, TransactionOperationContext context, RavenTransaction tx)
{
_store?.TrackChangeAfterTransactionCommit(context, "AlertRaised", alert.Key);
var table = tx.InnerTransaction.OpenTable(_alertsSchema, AlertsSchema.AlertsTree);
var alertId = alert.Id;
var alertAsJson = alert.ToJson();
// if previous alert has dismissed until value pass this value to newly saved alert
Slice slice;
using (Slice.From(tx.InnerTransaction.Allocator, alertId, out slice))
{
var existingTvr = table.ReadByKey(slice);
if (existingTvr != null)
{
var existingAlert = Read(context, existingTvr);
object dismissedUntilValue;
existingAlert.TryGetMember(nameof(alert.DismissedUntil), out dismissedUntilValue);
if (dismissedUntilValue != null)
{
var dismissedUntil = (LazyStringValue)dismissedUntilValue;
alertAsJson[nameof(alert.DismissedUntil)] = dismissedUntil;
}
}
}
using (var id = context.GetLazyString(alertId))
using (var json = context.ReadObject(alertAsJson, "Alert", BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
var tvb = new TableValueBuilder
{
{ id.Buffer, id.Size },
{ json.BasePointer, json.Size }
};
table.Set(tvb);
}
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation,
IndexingStatsScope stats, CancellationToken token)
{
if (_mapReduceContext.StoreByReduceKeyHash.Count == 0)
{
WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop
return(false);
}
ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32);
var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName);
var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction;
var writer = writeOperation.Value;
var treeScopeStats = stats.For(IndexingOperation.Reduce.TreeScope, start: false);
var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false);
foreach (var store in _mapReduceContext.StoreByReduceKeyHash)
{
token.ThrowIfCancellationRequested();
using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture)))
using (store.Value)
using (_aggregationBatch)
{
var modifiedStore = store.Value;
switch (modifiedStore.Type)
{
case MapResultsStorageType.Tree:
using (treeScopeStats.Start())
{
HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
break;
case MapResultsStorageType.Nested:
using (nestedValuesScopeStats.Start())
{
HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token);
}
break;
default:
throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString());
}
}
if (_mapReduceContext.FreedPages.Count > 0)
{
long tmp = 0;
using (treeScopeStats.Start())
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in _mapReduceContext.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
}
}
if (stats.Duration >= MinReduceDurationToCalculateProcessMemoryUsage)
{
var workingSet = MemoryInformation.GetWorkingSetInBytes();
var privateMemory = MemoryInformation.GetManagedMemoryInBytes() + MemoryInformation.GetUnManagedAllocationsInBytes();
stats.RecordReduceMemoryStats(workingSet, privateMemory);
}
WriteLastEtags(indexContext);
_mapReduceContext.StoreNextMapResultId();
return(false);
}