public void CouldOpenBucketsAndGetBuffers()
{
var path = TestUtils.GetPath();
var buckets = new SharedMemoryBuckets(path);
var available = buckets.GetAvailableFreeSpace();
Console.WriteLine("AVAILABLE: " + available);
for (int i = 0; i <= BufferRef.MaxBucketIdx; i++)
{
var br = BufferRef.Create(i, 1);
var buffer = buckets[br];
var bufferSize = SharedMemoryPool.BucketIndexToBufferSize(i, 4096);
buffer.Span.Fill(0);
Assert.AreEqual(bufferSize, buffer.Length);
}
var sndSmall = BufferRef.Create(0, 40000);
var buffer1 = buckets[sndSmall];
buffer1.Span.Fill(0);
Assert.AreEqual(4096, buffer1.Length);
buckets.Dispose();
}
public StreamLogStateStorage(string directoryPath)
: base(directoryPath, StartupConfig.StreamStateEnvFlags, default, ProcessConfig.ProcessConfigRecord.BufferSize,
1024L * 1024 * 1024) // 1GB is way too much but it takes no resources above actually used
{
// BRA dbs and _env are set in base class
// InstanceId -> BufferRef
_stateDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
_buckets = new SharedMemoryBuckets(directoryPath, StreamLogState.StreamLogStateRecord.BufferSize, 0);
using (var txn = Environment.BeginTransaction())
{
try
{
long sharedLogId = 0;
if (_stateDb.TryGet(txn, ref sharedLogId, out BufferRef sharedBufferRef))
{
txn.Abort();
}
else
{
sharedBufferRef = Allocate(txn, 0, out var fromFreeList, null);
_stateDb.Put(txn, sharedLogId, sharedBufferRef, TransactionPutOptions.NoOverwrite);
txn.Commit();
if (!fromFreeList)
{
Environment.Sync(true);
}
}
SharedState = new StreamLogState(default, (IntPtr)_buckets[sharedBufferRef].Data);
public ProcessConfigStorage(string directoryPath)
: base(directoryPath, StartupConfig.ProcessConfigEnvFlags, default, ProcessConfigRecord.BufferSize,
1024L * 1024 * 1024) // 1GB is way too much but it takes no resources above actually used
{
// BRA dbs and _env are set in base class
// InstanceId -> BufferRef
_processDb = Environment.OpenDatabase("_processConfig",
new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
_buckets = new SharedMemoryBuckets(directoryPath, ProcessConfigRecord.BufferSize, 0);
using (var txn = Environment.BeginTransaction())
{
try
{
uint sharedInstanceId = 0;
if (_processDb.TryGet(txn, ref sharedInstanceId, out BufferRef sharedBufferRef))
{
txn.Abort();
}
else
{
sharedBufferRef = Allocate(txn, 0, out var fromFreeList, null);
_processDb.Put(txn, sharedInstanceId, sharedBufferRef, TransactionPutOptions.NoOverwrite);
txn.Commit();
if (!fromFreeList)
{
Environment.Sync(true);
}
}
SharedRecord = new ProcessConfigRecord(_buckets[sharedBufferRef]);
}
catch (Exception ex)
{
txn.Abort();
Trace.TraceError(ex.ToString());
throw;
}
}
}
public void CouldUseSmallPages()
{
var path = TestUtils.GetPath();
var buckets = new SharedMemoryBuckets(path, pageSize: 32, 0);
var tooBigBucket = BufferRef.Create(1, 1);
Assert.Throws <ArgumentException>(() =>
{
var _ = buckets[tooBigBucket];
});
var br = BufferRef.Create(0, 40000);
var buffer1 = buckets[br];
buffer1.Span.Fill(0);
Assert.AreEqual(32, buffer1.Length);
buckets.Dispose();
}
public StreamBlockManager(string path,
LMDBEnvironmentFlags envFlags,
Wpid wpid,
IStreamBlockStorage blockStorage = null,
long maxTotalSize = 1024L * 1024 * 1024 * 1024 * 1024,
uint envSizeMb = 1024) // 1Gb takes 2 MB of TLB, do not make it too large TODO calculate realistic required size that is derived from maxTotalSize when using smallest buffers
: base(path, envFlags, wpid, PageSize, maxTotalSize, envSizeMb)
{
_blocksDb = Environment.OpenDatabase("_streamBlocks",
new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesSort | DbFlags.IntegerDuplicates | DbFlags.DuplicatesFixed)
{
DupSortPrefix = 64 * 64
}
);
_pidDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
_buckets = new SharedMemoryBuckets(Path.Combine(path, "buckets"), pageSize: PageSize, maxBucketIndex: BufferRef.MaxBucketIdx);
_blockStorage = blockStorage;
}
public unsafe void BufferAccessBench()
{
Settings.DoAdditionalCorrectnessChecks = false;
var path = TestUtils.GetPath();
var buckets = new SharedMemoryBuckets(path);
var bufferCount = 100_000; // c.2800 = 12 MB (L3 cache on i7-8700)
using (Benchmark.Run("Init (K)", bufferCount * 1000))
{
for (int i = 0; i < bufferCount; i++)
{
var bi = i + 1;
var br = BufferRef.Create(0, bi);
var buffer = buckets[br];
buffer.WriteInt32(0, bi);
}
}
var count = 10_000_000;
for (int r = 0; r < 10; r++)
{
using (Benchmark.Run("Access Unsafe", count))
{
for (int i = 0; i < count; i++)
{
var bi = 1 + (i % bufferCount);
var br = BufferRef.Create(0, bi);
var buffer = buckets.DangerousGet(br);
if (buffer.Length != 4096)
{
Assert.Fail();
}
if (bi != *(int *)buffer.Data)
{
// Assert.Fail($"bi [{bi}] != buffer.ReadInt32(0) [{buffer.ReadInt32(0)}]");
}
}
}
using (Benchmark.Run("Access Safe", count))
{
for (int i = 0; i < count; i++)
{
var bi = 1 + (i % bufferCount);
var br = BufferRef.Create(0, bi);
var buffer = buckets[br];
if (buffer.Length != 4096)
{
Assert.Fail();
}
if (bi != *(int *)buffer.Data)
{
// Assert.Fail($"bi [{bi}] != buffer.ReadInt32(0) [{buffer.ReadInt32(0)}]");
}
}
}
}
Benchmark.Dump();
buckets.Dispose();
}
