internal async ValueTask <bool> CreateStreamAsync <T>(DataPath dataPath,
Metadata metadata, DataStreamCreateFlags flags)
{
var slf = (StreamLogFlags)flags;
short fixedSize = 0;
if (slf.IsBinary() && TypeEnumHelper <T> .FixedSize > 0)
{
if (slf.NoTimestamp())
{
fixedSize = TypeEnumHelper <T> .FixedSize;
}
else
{
fixedSize = checked ((short)(Timestamp.Size + TypeEnumHelper <T> .FixedSize));
}
}
// TODO this is only first take to move further and start writing stuff,
// needs review and proper ContainerSchema generation from types T.
// Also there are so many different flags that they need review.
var schema = new ContainerSchema()
{
ContainerType = (byte)ContainerLayout.None,
ContainerFlags = (byte)slf,
ValuesSchema = new TypeSchema()
{
DataTypeHeader = TypeEnumHelper <T> .DataTypeHeader,
TypeName = typeof(T).Name,
TypeFullName = typeof(T).FullName,
FixedSize = fixedSize,
}
};
var mdr = await MetadataManager.CreateStreamAsync(dataPath, metadata, schema).ConfigureAwait(false);
if (mdr.PK != null)
{
var containerId = mdr.ContainerId;
var streamId = StreamLogId.ContainerIdToStreamId(containerId);
var slid = new StreamLogId(mdr.RepoId, streamId);
var state = StreamLogManager.StateStorage.GetState(slid);
var lockHolder = state.LockHolder;
state.CheckInit(slid, fixedSize, slf);
return(true);
}
mdr = await MetadataManager.GetMetadataRecordByPathAsync(dataPath).ConfigureAwait(false);
if (mdr.PK != null)
{
return(false);
}
throw new ArgumentException($"Cannot create a new data stream {dataPath}.");
}
public void CouldOpenCloseSlm()
{
var path = TestUtils.GetPath();
var pc = new ProcessConfig(path);
var slm = new StreamLogManager(pc, "test");
var slid = new StreamLogId(-1, 1);
var inited = slm.InitStreamLog(slid, 0, StreamLogFlags.IsBinary);
Assert.IsTrue(inited);
var sl = slm.OpenStreamLog(slid);
var mem = sl.ClaimRestOfBlockMemory(1);
var freeSpace = mem.Length;
mem.Span[0] = 123;
sl.Commit(1);
var mem1 = sl.ClaimRestOfBlockMemory(1);
Assert.AreEqual(mem1.Length, freeSpace - 1 - 4);
sl.Commit(1);
var db = sl.DangerousGetUnpacked(1);
Assert.AreEqual(123, db.ReadByte(0));
sl.Dispose();
slm.Dispose();
}
public StreamBlockRecordTimestampCursor(StreamBlockIndex blockIndex, StreamLogId slid)
{
_blockIndex = blockIndex;
_slid = slid;
_chunkCursorState = CursorState.Initialized;
_currentKey = default;
_currentValue = default;
}
internal async ValueTask <DataStream <T> > OpenStreamAsync <T>(DataPath dataPath)
{
var mdr = await MetadataManager.GetMetadataRecordByPathAsync(dataPath).ConfigureAwait(false);
if (mdr.PK != null)
{
var containerId = mdr.ContainerId;
var streamId = StreamLogId.ContainerIdToStreamId(containerId);
var slid = new StreamLogId(mdr.RepoId, streamId);
var sl = StreamLogManager.OpenStreamLog(slid, textId: dataPath.ToString());
var ds = new DataStream <T>(new DataStreamCursor <T>(sl.GetCursor()), mdr.Metadata);
return(ds);
}
throw new KeyNotFoundException($"Data stream {dataPath} does not exist.");
}
internal async ValueTask <DataStreamWriter <T> > OpenStreamWriterAsync <T>(DataPath dataPath,
WriteMode writeMode = WriteMode.LocalSync,
int rateHint = 0)
{
var mdr = await MetadataManager.GetMetadataRecordByPathAsync(dataPath).ConfigureAwait(false);
if (mdr.PK != null)
{
var containerId = mdr.ContainerId;
var streamId = StreamLogId.ContainerIdToStreamId(containerId);
var slid = new StreamLogId(mdr.RepoId, streamId);
var sl = StreamLogManager.OpenStreamLog(slid, rateHint, dataPath.ToString());
// var state = StreamLogManager.StateStorage.GetState(slid);
var writer = new DataStreamWriter <T>(sl, KeySorting.NotEnforced, writeMode);
return(writer);
}
throw new KeyNotFoundException($"Data stream {dataPath} does not exist.");
}
internal bool TryPackBlocks(StreamLogId slid)
{
var state = _stateStorage.GetState(slid);
long packerLock;
if (state.IsValid && (packerLock = state.AcquirePackerLock()) > 0)
{
try
{
var isPackable = !_disablePacking && !state.StreamLogFlags.NoPacking();
var couldHaveMore = isPackable;
while (couldHaveMore)
{
var packAction =
state.StreamLogFlags.DropPacked()
? PackAction.Delete
: PackAction.Pack;
couldHaveMore = _streamLogManager.BlockIndex.PackBlocks(state, packAction);
if (couldHaveMore)
{
// extend lock to avoid timeout
packerLock = state.AcquirePackerLock(packerLock);
}
}
}
finally
{
if (!state.ReleasePackerLock(packerLock))
{
ThrowHelper.FailFast("Cannot release owned packer lock");
}
}
return(true);
}
return(false);
}
public void Add(StreamLogId id)
{
_jobs.Set(id, 0);
Pool.UnsafeQueueCompletableItem(this, false);
}