public void Initialize()
{
// recovery on edge crash
if (File.Exists(TmpStorePath) && !File.Exists(_storePath))
{
File.Move(TmpStorePath, _storePath);
_log.Log(LogSeverity.Warning, $"File recovered: {TmpStorePath}");
}
// Number of sectors is inferred from the file size.
_firstNotAllocated = SectorIndex.First;
if (File.Exists(_storePath))
{
var info = new FileInfo(_storePath);
_firstNotAllocated = SectorIndex.OffsetInBytes(info.Length, _sectorSizeInBytes);
}
// hot stream does not apply to volatile store
if (!File.Exists(_storePath))
{
_storeStream = new FileStream(_storePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None);
_log?.Log(LogSeverity.Info, $"Empty file created: {_storePath}.");
_firstNotAllocated = SectorIndex.First;
_memoryStream = new MemoryStream();
}
else
{
_storeStream = new FileStream(_storePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None);
_memoryStream = new MemoryStream((int)_storeStream.Length);
_storeStream.CopyTo(_memoryStream);
}
}
public void StoreAlternateWriteRead()
{
SectorIndex index = new SectorIndex(1);
_store.Initialize();
// manually make a sector
var sector = new Sector(_sectorSize);
var contentBytes = new byte[sector.Contents.Length];
_rnd.NextBytes(contentBytes);
contentBytes.AsSpan().CopyTo(sector.Contents);
_store.AllocateSectors(4);
_store.Write(index, sector);
var storeProvided = _store.Read(index);
// second write
var contentBytes2 = new byte[sector.Contents.Length];
_rnd.NextBytes(contentBytes2);
contentBytes2.AsSpan().CopyTo(sector.Contents);
SectorIndex otherIndex = new SectorIndex(2);
_store.Write(otherIndex, sector);
// read first sector
storeProvided = _store.Read(index);
Assert.True(storeProvided.Contents.SequenceEqual(contentBytes.AsSpan()));
}
public void EnsurePersistenceTest()
{
var numberOfSectorTesting = 5;
_store.Initialize();
// manually make a sector
var sector = new Sector(_sectorSize);
var contentBytes = new byte[sector.Contents.Length];
for (var i = 0; i < numberOfSectorTesting; i++)
{
_rnd.NextBytes(contentBytes);
contentBytes.AsSpan().CopyTo(sector.Contents);
var index = _store.AllocateSectors(4);
_store.Write(index, sector);
}
_store.EnsurePersistence();
_store.Dispose();
// reinit store
_store = _storeFixiture.Create();
_store.Initialize();
var sectorIndex = new SectorIndex((numberOfSectorTesting - 1) * 4);
var storeProvided = _store.Read(sectorIndex);
Assert.True(storeProvided.Contents.SequenceEqual(contentBytes.AsSpan()));
}
/// <summary>
/// The returned 'Sector' is invalidated at the next call.
/// </summary>
public Sector Read(SectorIndex index)
{
_memoryStream.Seek(index.ByteIndex(_sectorSizeInBytes), SeekOrigin.Begin);
_sector.Read(_memoryStream, index.ByteIndex(_sectorSizeInBytes));
return(_sector);
}
public SectorIndex AllocateSectors(int count)
{
if (count <= 0)
{
throw new ArgumentOutOfRangeException(nameof(count));
}
var firstNewlyAllocated = _firstNotAllocated;
_firstNotAllocated = _firstNotAllocated.GetNext(count);
_storeStream.SetLength(_storeStream.Length + _sectorSizeInBytes * count);
_sector.Clear();
for (var idx = firstNewlyAllocated;
idx.IsWithinRange(_firstNotAllocated);
idx = idx.GetNext())
{
_sector.Write(_storeStream, idx.ByteIndex(_sectorSizeInBytes));
}
Debug.Assert(_storeStream.Length == _firstNotAllocated.ByteIndex(_sectorSizeInBytes));
return(firstNewlyAllocated);
}
public void Initialize()
{
if (!File.Exists(_storePath))
{
_storeStream = new FileStream(_storePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None);
_firstNotAllocated = SectorIndex.First;
_log?.Log(LogSeverity.Info, $"Empty file created: {_storePath}.");
}
else
{
_storeStream = new FileStream(_storePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None);
}
// Number of sectors is inferred from the file size.
var info = new FileInfo(_storePath);
_firstNotAllocated = SectorIndex.OffsetInBytes(info.Length, _sectorSizeInBytes);
if (!File.Exists(_hotPath))
{
_hotStream = new FileStream(_hotPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None);
_log?.Log(LogSeverity.Info, $"Empty file created: {_hotPath}.");
}
else
{
_hotStream = new FileStream(_hotPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None);
TryRecovery();
}
}
public VolatileStore(int sectorSizeInBytes, string storePath, ILog log = null)
{
_sectorSizeInBytes = sectorSizeInBytes;
_storePath = storePath;
_firstNotAllocated = SectorIndex.Unknown; // not initialized
_log = log;
_sector = new Sector(_sectorSizeInBytes);
}
public TwoPhaseStore(int sectorSizeInBytes, string storePath, ILog log = null)
{
_sectorSizeInBytes = sectorSizeInBytes;
_hotPath = Path.Combine(Path.GetDirectoryName(storePath), @"hot-" + Path.GetFileName(storePath));
_storePath = storePath;
_firstNotAllocated = SectorIndex.Unknown; // dummy initialization
_log = log;
_sector = new Sector(_sectorSizeInBytes);
}
/// <summary>
/// Deserialize a TxoPack from Sector.
/// </summary>
/// <param name="sector">Sector to deserialize</param>
/// <param name="pack">TxoPack to fill</param>
/// <param name="childSectorIndex">sectorIndex of child</param>
/// <remarks>
/// Life-time of the response lasts until the next call to 'Read'.
/// Size of TxoPack should be large enough to not overflow.
/// </remarks>
public static void Deserialize(this Sector sector, TxoPack pack, out SectorIndex childSectorIndex)
{
var sectorContent = sector.Contents;
childSectorIndex =
new SectorIndex(BitConverter.ToInt32(sectorContent.Slice(ChildSectorIndexStart, ChildSectorIndexSizeInBytes)));
var numberOfTxo = BitConverter.ToInt32(sectorContent.Slice(TxoCountStart, TxoCountSizeInBytes));
var payloadLengthAccumulator = 0;
ref var writer = ref pack.GetResetWriter();
/// <remarks>
/// The returned 'Sector' is invalidated at the next call.
/// </remarks>
public Sector Read(SectorIndex index)
{
if (!index.IsWithinRange(_firstNotAllocated))
{
throw new ArgumentOutOfRangeException(
$"Try to read from index {index} out of total {_firstNotAllocated.Value} sectors.");
}
_storeStream.Seek(index.ByteIndex(_sectorSizeInBytes), SeekOrigin.Begin);
_sector.Read(_storeStream, index.ByteIndex(_sectorSizeInBytes));
return(_sector);
}
/// <summary>
/// Write to hot storage first, then regular storage secondly.
/// </summary>
public void Write(SectorIndex index, Sector source)
{
if (!index.IsWithinRange(_firstNotAllocated))
{
throw new ArgumentOutOfRangeException(
$"Try to write to index {index.Value} out of total {_firstNotAllocated.Value} sectors.");
}
_hotStream.Seek(0, SeekOrigin.Begin);
source.Write(_hotStream, index.ByteIndex(_sectorSizeInBytes));
_storeStream.Seek(index.ByteIndex(_sectorSizeInBytes), SeekOrigin.Begin);
source.Write(_storeStream, index.ByteIndex(_sectorSizeInBytes));
}
public void StoreWriteRead()
{
SectorIndex index = new SectorIndex(3);
_store.Initialize();
// manually make a sector
var sector = new Sector(_sectorSize);
var rnd = new Random(42);
var contentBytes = new byte[sector.Contents.Length];
rnd.NextBytes(contentBytes);
contentBytes.AsSpan().CopyTo(sector.Contents);
_store.AllocateSectors(4);
_store.Write(index, sector);
var storeProvided = _store.Read(index);
Assert.True(storeProvided.Contents.SequenceEqual(contentBytes.AsSpan()));
}
public bool TryRead(SectorIndex index, out Sector sector)
{
sector = null;
// upper layer saved -1 as sectorIndex if child sector doesn't exist.
if (index.IsUnknown())
{
return(false);
}
var position = _storeStream.Seek(index.ByteIndex(_sectorSizeInBytes), SeekOrigin.Begin);
// seek a position beyond file length
if (position == _storeStream.Length)
{
return(false);
}
_sector.Read(_storeStream, index.ByteIndex(_sectorSizeInBytes));
sector = _sector;
return(true);
}
/// <summary>
/// Recover a halfway written sector.
/// </summary>
/// <remarks>
/// TODO: verify this method does what it is supposed to do, after replacing TryRead with a Read that throws exceptions,
/// instead of returning a bool.
/// </remarks>
private void TryRecovery()
{
try
{
var offset = _sector.Read(_hotStream); // hotstream is valid
var deducedIndex = SectorIndex.OffsetInBytes(offset, _sectorSizeInBytes);
try
{
//try read last recorded sector
Read(deducedIndex);
// reach here means that both streams are ok
}
catch (SectorOperationException)
{
// something went wrong in storeStream
_hotStream.Seek(0, SeekOrigin.Begin);
_sector.Read(_hotStream, offset);
// reach here means that hostream is ok
// copying '_hotstream' back to '_storeStream'
_storeStream.Seek(deducedIndex.ByteIndex(_sectorSizeInBytes), SeekOrigin.Begin);
_sector.Write(_storeStream, deducedIndex.ByteIndex(_sectorSizeInBytes));
// TODO: add log to indicate successful recovery.
}
}
catch (SectorOperationException)
{
// hotstream invalid
// normally thanks to double writing, main store stream is valid
_log?.Log(LogSeverity.Warning, "Hotstream error when trying to recover sector store.");
throw;
}
}
public void Write(SectorIndex index, Sector source)
{
_memoryStream.Seek(index.ByteIndex(_sectorSizeInBytes), SeekOrigin.Begin);
source.Write(_memoryStream, index.ByteIndex(_sectorSizeInBytes));
}
