/// <summary>
/// Write a block and return the offset after the block
/// </summary>
/// <param name="block"></param>
/// <param name="offset"></param>
/// <returns></returns>
private long WriteBlock(PersistentBlock block, long offset)
{
StorageStream.Seek(offset, SeekOrigin.Begin);
var writer = new BinaryWriter(StorageStream);
block.Write(writer);
writer.Flush();
return(StorageStream.Position);
}
/// <summary>
/// Add a new bloc or update an existing one (depending if the primary key is already stored)
/// </summary>
/// <param name="data"></param>
/// <param name="primaryKey"></param>
/// <param name="transactionId"></param>
public void StoreBlock(byte[] data, string primaryKey, int transactionId)
{
var block = new PersistentBlock
{
RawData = data,
PrimaryKey = primaryKey,
BlockStatus = BlockStatus.Active,
LastTransactionId = transactionId,
UsedDataSize = data.Length
};
if (BlockInfoByPrimaryKey.TryGetValue(primaryKey, out var blockInfo))
{
// load the old version of the block
StorageStream.Seek(blockInfo.Offset, SeekOrigin.Begin);
var reader = new BinaryReader(StorageStream);
var oldBlock = new PersistentBlock();
oldBlock.Read(reader);
// if enough space is available do in-place update
if (oldBlock.ReservedDataSize > block.UsedDataSize)
{
block.ReservedDataSize = oldBlock.ReservedDataSize;
WriteBlock(block, blockInfo.Offset);
}
else // the old block is marked as deleted and the new version is added at the end of the stream
{
oldBlock.BlockStatus = BlockStatus.Dirty;
InactiveBlockCount++;
WriteBlock(oldBlock, blockInfo.Offset);
block.ReservedDataSize = (int)(block.UsedDataSize * 1.5);
StorageSize = WriteBlock(block, StorageSize);
BlockInfoByPrimaryKey[primaryKey] = new BlockInfo(StorageSize, block.LastTransactionId);
}
}
else // a new object not already in the persistent storage
{
// reserve some more space to allow for in-place updating
block.ReservedDataSize = (int)(block.UsedDataSize * 1.5);
var offset = StorageSize;
StorageSize = WriteBlock(block, StorageSize);
BlockInfoByPrimaryKey[primaryKey] = new BlockInfo(offset, block.LastTransactionId);
}
_backupStorage?.StoreBlock(data, primaryKey, transactionId);
}
/// <summary>
/// Remove dirty blocks thus compacting the storage
/// </summary>
public void CleanStorage()
{
if (File.Exists(TempFileName))
{
File.Delete(TempFileName);
}
var fullPath = Path.Combine(DataPath, StorageFileName);
var tempPath = Path.Combine(DataPath, TempFileName);
using (var tempStream = new FileStream(tempPath, FileMode.Create))
{
var writer = new BinaryWriter(tempStream);
StorageStream.Dispose();
StorageStream = new FileStream(fullPath, FileMode.OpenOrCreate);
var reader = new BinaryReader(StorageStream);
var block = new PersistentBlock();
long offset = 0;
// read all the blocks
while (block.Read(reader))
{
if (block.BlockStatus == BlockStatus.Active)
{
offset = (int)StorageStream.Position;
block.Write(writer);
}
}
StorageSize = offset;
writer.Flush();
StorageStream.Dispose();
}
File.Delete(fullPath);
File.Move(tempPath, fullPath);
StorageStream = new FileStream(fullPath, FileMode.OpenOrCreate);
InactiveBlockCount = 0;
}
/// <summary>
/// For recovery tests only
/// </summary>
/// <param name="primaryKey"></param>
public void MakeCorruptedBlock(string primaryKey)
{
if (BlockInfoByPrimaryKey.TryGetValue(primaryKey, out var info))
{
var block = new PersistentBlock();
StorageStream.Seek(info.Offset, SeekOrigin.Begin);
var reader = new BinaryReader(StorageStream);
block.Read(reader);
block.Hash = block.Hash + 1; // the hash will not match so the block is corrupted
StorageStream.Seek(info.Offset, SeekOrigin.Begin);
var writer = new BinaryWriter(StorageStream);
block.Write(writer);
}
}
public PersistentBlock ReadBlock(string primaryKey)
{
if (BlockInfoByPrimaryKey.TryGetValue(primaryKey, out var info))
{
var block = new PersistentBlock();
StorageStream.Seek(info.Offset, SeekOrigin.Begin);
var reader = new BinaryReader(StorageStream);
block.Read(reader);
return(block);
}
throw new NotSupportedException("primary key not found in backup storage");
}
private void MarkInvalidBlocksAsDirty(InvalidBlockException e)
{
var nextOffset = FindNextBeginMarker(e.Offset + PersistentBlock.MinSize);
var size = nextOffset - e.Offset;
var dirtyBlock = PersistentBlock.MakeDirtyBlock(size);
StorageStream.Seek(e.Offset, SeekOrigin.Begin);
var writer = new BinaryWriter(StorageStream);
dirtyBlock.Write(writer);
InactiveBlockCount++;
CorruptedBlocks++;
}
public void DeleteBlock(string primaryKey, int transactionId)
{
if (BlockInfoByPrimaryKey.TryGetValue(primaryKey, out var blockInfo))
{
StorageStream.Seek(blockInfo.Offset, SeekOrigin.Begin);
var reader = new BinaryReader(StorageStream);
var block = new PersistentBlock();
block.Read(reader);
if (block.BlockStatus != BlockStatus.Active)
{
// if the same id ok. We are re executing a transaction that was not marked as Processed.
// It happens if the server crashes during the update of persistence blocks. The transaction is simply played
// again when the server is restarted
if (block.LastTransactionId != transactionId)
{
throw new NotSupportedException(
$"Trying to delete an inactive block for primary key {primaryKey}");
}
}
block.BlockStatus = BlockStatus.Deleted;
block.LastTransactionId = transactionId;
StorageStream.Seek(blockInfo.Offset, SeekOrigin.Begin);
var writer = new BinaryWriter(StorageStream);
block.Write(writer);
writer.Flush();
BlockInfoByPrimaryKey.Remove(primaryKey);
InactiveBlockCount++;
_backupStorage?.DeleteBlock(primaryKey, transactionId);
}
else
{
throw new NotSupportedException($"Active block not found for primary key {primaryKey}");
}
}
private void LoadAll(bool useObjectProcessor)
{
var reader = new BinaryReader(StorageStream);
var block = new PersistentBlock();
long offset = 0;
// read all the blocks
while (block.Read(reader))
{
// get information for deleted blocks too as there may be an uncomplete delete transaction to
// reprocess
if (block.BlockStatus == BlockStatus.Active || block.BlockStatus == BlockStatus.Deleted)
{
BlockInfoByPrimaryKey[block.PrimaryKey] =
new BlockInfo(offset, block.LastTransactionId);
// only active blocks contain objects
if (useObjectProcessor && block.BlockStatus == BlockStatus.Active)
{
ObjectProcessor.Process(block.RawData);
}
}
if (block.BlockStatus != BlockStatus.Active)
{
InactiveBlockCount++;
}
offset = (int)StorageStream.Position;
}
if (useObjectProcessor)
{
ObjectProcessor.EndProcess(DataPath);
}
StorageSize = offset;
}
