public unsafe void UncontentedBench()
{
var ptr = (long *)Marshal.AllocHGlobal(8);
* ptr = 0;
var wpid = Wpid.Create();
var sl = new SharedSpinLock(ptr);
var count = 10_000_000;
for (int _ = 0; _ < 10; _++)
{
using (Benchmark.Run("Uncontented", count))
{
for (int i = 0; i < count; i++)
{
sl.TryAcquireLock(wpid);
sl.TryReleaseLock(wpid);
}
}
}
Benchmark.Dump();
}
public Wpid TryAcquireLock(Wpid wpid, IWpidHelper wpidHelper, int spinLimit = 0)
{
var content = *(long *)(_statePointer);
*(long *)(_statePointer) = 0;
return(SharedSpinLock.TryAcquireLock(ref *(long *)(_statePointer + StreamLogStateRecord.LockerOffset), wpid, spinLimit, wpidHelper));
}
// Note: base RetainableMemoryPool works with Pow2 sizes, this pool adds additional space due to headers
/// <summary>
/// SharedMemoryPool constructor.
/// </summary>
/// <param name="path">Directory path where shared memory files are stored.</param>
/// <param name="maxLogSizeMb"></param>
/// <param name="ownerId"></param>
/// <param name="rmMaxBufferLength"></param>
/// <param name="rmMaxBuffersPerBucket"></param>
/// <param name="rentAlwaysClean"></param>
public SharedMemoryPool(string path, uint maxLogSizeMb,
Wpid ownerId,
int rmMaxBufferLength = RmDefaultMaxBufferLength,
int rmMaxBuffersPerBucket = RmDefaultMaxBuffersPerBucket, bool rentAlwaysClean = false)
: this(path, maxLogSizeMb, LMDBEnvironmentFlags.WriteMap, ownerId, rmMaxBufferLength, rmMaxBuffersPerBucket, rentAlwaysClean)
{
}
public unsafe void CouldAcquireReleaseExclusiveLock()
{
var ptr = (long *)Marshal.AllocHGlobal(8);
* ptr = 0;
var wpid = Wpid.Create();
var wpid2 = Wpid.Create();
var sl = new SharedSpinLock(ptr);
Assert.AreEqual(Wpid.Empty, sl.TryAcquireExclusiveLock(wpid, out _, spinLimit: 0)); // fast path
Assert.AreEqual(Wpid.Empty, sl.TryReleaseLock(wpid));
Assert.AreEqual(Wpid.Empty, sl.TryAcquireExclusiveLock(wpid, out _));
var sw = new Stopwatch();
sw.Start();
Assert.AreEqual(wpid, sl.TryAcquireExclusiveLock(wpid2, out _, spinLimit: 1000));
sw.Stop();
Console.WriteLine($"Elapsed: {sw.ElapsedMilliseconds}");
Assert.AreEqual(Wpid.Empty, sl.TryReleaseLock(wpid));
}
internal BlockMemoryPool(string path, uint maxLogSizeMb,
LMDBEnvironmentFlags envFlags,
Wpid ownerId,
int maxBufferLength = RmDefaultMaxBufferLength,
int maxBuffersPerBucket = RmDefaultMaxBuffersPerBucket)
: base(path, maxLogSizeMb, envFlags, ownerId, maxBufferLength, maxBuffersPerBucket, rentAlwaysClean: true)
{
}
// TODO this is all internal with very limited usage, but refactor
public BufferRefAllocator(string directoryPath, LMDBEnvironmentFlags envFlags, Wpid wpid, int pageSize = 4096, long maxTotalSize = 64L * 1024 * 1024 * 1024, uint envSizeMb = 128)
{
_wpid = wpid;
_pageSize = pageSize;
_maxTotalSize = maxTotalSize;
// TODO Log size to settings. Also handle MapFull later and reduce size.
SetupEnv(directoryPath, envSizeMb, envFlags);
InitDbs();
}
internal unsafe SharedMemoryPool(string path, uint maxLogSizeMb,
LMDBEnvironmentFlags envFlags,
Wpid ownerId,
int rmMaxBufferLength = RmDefaultMaxBufferLength,
int rmMaxBuffersPerBucket = RmDefaultMaxBuffersPerBucket, bool rentAlwaysClean = false)
: base(
(pool, bucketSize) =>
{
if (bucketSize > pool.MaxBufferSize)
{
BuffersThrowHelper.ThrowBadLength();
}
var smbp = (SharedMemoryPool)pool;
#pragma warning disable 618
var sm = smbp.RentNative(bucketSize);
Debug.Assert(!sm.IsDisposed);
Debug.Assert(sm.ReferenceCount == 0);
Debug.Assert(Unsafe.ReadUnaligned <uint>(sm.HeaderPointer) == HeaderFlags.IsOwned);
#pragma warning restore 618
// TODO review if Releasing -> IsOwned should keep disposed state?
// RMP calls CreateNew outside lock, so if we call RentNative only here
// then it could return IsOwned without IsDisposed. But we a technically
// inside the pool until this factory returns. Buffers from RentNative
// should be unusable without explicit un-dispose action.
// Set counter to zero
sm.CounterRef &= ~AtomicCounter.CountMask;
return(sm);
},
RmMinPoolBufferLength,
Math.Max(RmDefaultMaxBufferLength, Math.Min(RmMaxPoolBufferLength, BitUtil.FindNextPositivePowerOfTwo(rmMaxBufferLength))),
// from ProcCount to DefaultMaxNumberOfBuffersPerBucket x 2
Math.Max(Environment.ProcessorCount, Math.Min(RmDefaultMaxBuffersPerBucket * 2, rmMaxBuffersPerBucket)),
MaxBucketsToTry, rentAlwaysClean: rentAlwaysClean)
{
if (ownerId <= 0)
{
ThrowHelper.ThrowArgumentOutOfRangeException("ownerId <= 0");
}
Directory.CreateDirectory(path);
_bra = new BufferRefAllocator(path, envFlags, ownerId, PageSize, maxLogSizeMb * 1024 * 1024L);
_buckets = new SharedMemoryBuckets(Path.Combine(path, "buckets"), pageSize: PageSize, maxBucketIndex: BufferRef.MaxBucketIdx);
StartMonitoringTask();
}
/// <summary>
/// Increment shared instance id and return a process buffer with new Wpid
/// </summary>
/// <returns></returns>
public ProcessConfigRecord CreateNew()
{
using (var txn = Environment.BeginTransaction())
{
while (true)
{
// ReSharper disable once ImpureMethodCallOnReadonlyValueField
var newInsatnceId = unchecked ((uint)SharedRecord._processBuffer.InterlockedIncrementInt32(ProcessConfigRecord.SharedInstanceIdCounterOffset));
try
{
// We try to increment instance id instead of reuse existing
// At some point after running for very very long time it might overflow
// and start from zero again - this is fine.
if (_processDb.TryGet(txn, ref newInsatnceId, out BufferRef bufferRef))
{
continue;
}
bufferRef = Allocate(txn, 0, out var fromFreeList, null);
_processDb.Put(txn, newInsatnceId, bufferRef, TransactionPutOptions.NoOverwrite);
txn.Commit();
if (!fromFreeList)
{
Environment.Sync(true);
}
var result = _buckets[bufferRef];
if (fromFreeList)
{
// in Delete we clear the buffer after commit, there is small chance a process died after commit but before cleaning
result.Clear(0, result.Length);
}
else
{
Debug.Assert(result.IsFilledWithValue(0), "a new ProcessConfig buffer must be clean.");
}
result.WriteInt64(ProcessConfigRecord.WpidOffset, Wpid.Create(newInsatnceId));
var record = new ProcessConfigRecord(result);
return(record);
}
catch (Exception ex)
{
txn.Abort();
Trace.TraceError(ex.ToString());
throw;
}
}
}
}
public unsafe void CouldAcquireEnterExitReleaseExclusiveLock()
{
var ptr = (long *)Marshal.AllocHGlobal(8);
* ptr = 0;
var wpid = Wpid.Create();
var wpid2 = Wpid.Create();
var sl = new SharedSpinLock(ptr);
Assert.AreEqual(Wpid.Empty, sl.TryAcquireExclusiveLock(wpid, out var tt, spinLimit: 0));
Assert.AreEqual(Wpid.Empty, sl.TryReEnterExclusiveLock(wpid, tt, spinLimit: 0));
Assert.Throws <InvalidOperationException>(() => { sl.TryReleaseLock(wpid); });
Assert.AreEqual(Wpid.Empty, sl.TryExitExclusiveLock(wpid, tt));
Assert.AreEqual(Wpid.Empty, sl.TryReleaseLock(wpid));
}
// ReSharper disable once MemberHidesStaticFromOuterClass
public ProcessConfigRecord GetRecord(Wpid wpid)
{
using (var txn = Environment.BeginReadOnlyTransaction())
{
var insatnceId = wpid.InstanceId;
try
{
if (_processDb.TryGet(txn, ref insatnceId, out BufferRef bufferRef))
{
return(new ProcessConfigRecord(_buckets[bufferRef]));
}
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
throw;
}
}
return(default);
public StreamBlockManager(string path,
LMDBEnvironmentFlags envFlags,
Wpid wpid,
IStreamBlockStorage blockStorage = null,
long maxTotalSize = 1024L * 1024 * 1024 * 1024 * 1024,
uint envSizeMb = 1024) // 1Gb takes 2 MB of TLB, do not make it too large TODO calculate realistic required size that is derived from maxTotalSize when using smallest buffers
: base(path, envFlags, wpid, PageSize, maxTotalSize, envSizeMb)
{
_blocksDb = Environment.OpenDatabase("_streamBlocks",
new DatabaseConfig(DbFlags.Create | DbFlags.DuplicatesSort | DbFlags.IntegerDuplicates | DbFlags.DuplicatesFixed)
{
DupSortPrefix = 64 * 64
}
);
_pidDb = Environment.OpenDatabase("_streamLogState", new DatabaseConfig(DbFlags.Create | DbFlags.IntegerKey));
_buckets = new SharedMemoryBuckets(Path.Combine(path, "buckets"), pageSize: PageSize, maxBucketIndex: BufferRef.MaxBucketIdx);
_blockStorage = blockStorage;
}
public unsafe void ReleasingOthersLockReturnsOthersWpid()
{
var ptr = (long *)Marshal.AllocHGlobal(8);
* ptr = 0;
var wpid = Wpid.Create();
var wpid2 = Wpid.Create();
var sl = new SharedSpinLock(ptr);
Assert.AreEqual(Wpid.Empty, sl.TryAcquireLock(wpid, spinLimit: 0)); // fast path
Assert.AreEqual(Wpid.Empty, sl.TryReleaseLock(wpid));
Assert.AreEqual(Wpid.Empty, sl.TryAcquireLock(wpid));
// wpid holding the lock
var holder = sl.TryReleaseLock(wpid2);
Assert.AreEqual(wpid, holder);
}
public Wpid TryReEnterExclusiveLock(Wpid wpid, IWpidHelper wpidHelper, byte threadToken, int spinLimit = 0)
{
var holder = LockHolder;
return(SharedSpinLock.TryReEnterExclusiveLock(ref *(long *)(_statePointer + StreamLogStateRecord.LockerOffset), wpid, threadToken, spinLimit, wpidHelper));
}
public Wpid TryReleaseLock(Wpid wpid)
{
return(SharedSpinLock.TryReleaseLock(ref *(long *)(_statePointer + StreamLogStateRecord.LockerOffset), wpid));
}
public Wpid TryExitExclusiveLock(Wpid wpid, byte threadToken)
{
return(SharedSpinLock.TryExitExclusiveLock(ref *(long *)(_statePointer + StreamLogStateRecord.LockerOffset), wpid, threadToken));
}
public StreamLogManager(ProcessConfig processConfig,
string dataStoreName,
string dataStorePath = null,
uint maxLogSizeMb = 1024,
bool disableNotificationLog = false,
bool disablePacker = false,
IStreamBlockStorage blockStorage = null)
{
if (LeaksDetection.Enabled)
{
Spreads.Buffers.BufferPool.PinnedArrayMemoryPool.AddStackTraceOnRent = true;
}
DataStorePath = dataStorePath ?? Path.Combine(processConfig.DataRootPath, dataStoreName);
ProcessConfig = processConfig;
_wpidValue = ProcessConfig.Wpid;
DisableNotificationLog = disableNotificationLog;
DisablePacker = disablePacker;
var bufferPoolPath = Path.Combine(DataStorePath, "log", "logbuffer");
var bufferPoolFlags = StartupConfig.StreamLogBufferPoolFlags;
BufferPool = new BlockMemoryPool(bufferPoolPath, maxLogSizeMb, bufferPoolFlags, processConfig.Wpid,
maxBufferLength: MaxBufferSize,
maxBuffersPerBucket: Environment.ProcessorCount * 4);
if (blockStorage == null)
{
var dataStoragePath = Path.Combine(DataStorePath, "storage");
Directory.CreateDirectory(dataStoragePath);
var path = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db"));
var uri = new Uri(path);
var absoluteUri = uri.AbsoluteUri;
blockStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared");
}
var blockIndexPath = Path.Combine(DataStorePath, "log", "blockindex");
var blockIndexFlags = StartupConfig.StreamBlockIndexFlags;
var blockIndexSizeMb = Math.Max(StartupConfig.StreamBlockTableMaxSizeMb, 128);
BlockIndex = new StreamBlockIndex(blockIndexPath, blockIndexSizeMb, blockIndexFlags, BufferPool, blockStorage);
var logStateStoragePath = Path.Combine(DataStorePath, "log", "logstate");
StateStorage = new StreamLogStateStorage(logStateStoragePath);
// For Log0 tests we need state but we removed it from Log0 ctor, so always init it.
// In real code _disableNotificationLog is always false.
Log0State = StateStorage.GetState(StreamLogId.Log0Id);
Log0State.CheckInit(StreamLogId.Log0Id, StreamLogNotification.Size, StreamLogFlags.IsBinary | StreamLogFlags.NoTimestamp | StreamLogFlags.DropPacked | StreamLogFlags.Pow2Payload);
Log0State.HintRatePerMinute(MaxBufferSize);
if (!DisableNotificationLog)
{
Log0 = new NotificationLog(this);
OpenStreams.TryAdd((long)StreamLogId.Log0Id, Log0);
Log0Reader = new NotificationLog.Reader(Log0, Cts.Token);
var lastVersion = (ulong)Log0.CurrentVersion;
if (lastVersion > 1)
{
Log0Reader.MoveGT(lastVersion - 1);
}
}
Packer = new Packer(this, StateStorage, disablePacker);
StartLog0(dataStoreName);
// TODO we need more general unlocking locking that detects missed updates
// and works for all waiters, not only for ack. See Log0.Reader comments.
//_unlockTimer = new Timer(o =>
//{
// TryCompleteAckRequests();
//}, null, 0, 1000);
}
public Wpid TryAcquireExclusiveLock(Wpid wpid, IWpidHelper wpidHelper, out byte threadToken, int spinLimit = 0)
{
var res = SharedSpinLock.TryAcquireExclusiveLock(ref *(long *)(_statePointer + StreamLogStateRecord.LockerOffset), wpid, out threadToken, spinLimit, wpidHelper);
return(res);
}
