/// <summary>
/// Write a group of Blocks into Disk. NOTE: it will be more optimal if Blocks
/// are sorted by its Data Address so this function can write contiguous blocks
/// in one async write.
/// </summary>
public int WriteBlocksToDisk(Algorithm.Collection.ICollectionOnDisk parent,
IDictionary <long, Sop.DataBlock> blocks, bool clear)
{
if (!parent.IsOpen)
{
return(0);
}
var blockSize = (int)parent.DataBlockSize;
int r = blocks.Count;
if (BinaryWriter == null)
{
if (BufferStream == null)
{
BufferStream = new MemoryStream();
}
BinaryWriter = new BinaryWriter(BufferStream, parent.File.Server.Encoding);
}
else
{
BinaryWriter.Seek(0, SeekOrigin.Begin);
}
const int sizeOfNumerics = Sop.DataBlock.OverheadSize;
int chunkSize = (int)DataBlockSize.FiveTwentyFourTwoEightyEight * 4;
if (chunkSize > blocks.Count * blockSize)
{
chunkSize = blocks.Count * blockSize;
}
if (_writeBuffer == null || _writeBuffer.Length < chunkSize)
{
_writeBuffer = new byte[chunkSize];
}
int bufferIndex = 0, startIndex = 0, currentTargetBufferIndex = 0;
long runningAddress = -1, startBlockAddress = -1;
var bulkWriter = new BulkWriter();
var dataChunks = new List <BulkWriter.DataChunk>(4);
foreach (Sop.DataBlock block in blocks.Values)
{
SetIsDirty(block, false);
if (block.DataAddress >= 0)
{
#region Process special states, e.g. - buffer is full
if (startBlockAddress == -1)
{
startBlockAddress = runningAddress = block.DataAddress;
}
else
{
bool bufferIsFull = (bufferIndex - startIndex) + sizeOfNumerics + block.Data.Length +
currentTargetBufferIndex > _writeBuffer.Length - block.Length;
if (block.DataAddress != runningAddress || bufferIsFull)
{
dataChunks.Add(new BulkWriter.DataChunk
{
TargetDataAddress =
startBlockAddress == -1
? block.DataAddress
: startBlockAddress,
Index = currentTargetBufferIndex + startIndex,
Size = bufferIndex - startIndex
});
if (bufferIsFull)
{
//** write to disk
bulkWriter.Write(parent, _writeBuffer, dataChunks);
//** reset buffer
dataChunks.Clear();
currentTargetBufferIndex = 0;
}
else
{
currentTargetBufferIndex += (bufferIndex - startIndex);
}
startIndex = bufferIndex = 0;
runningAddress = startBlockAddress = block.DataAddress;
}
}
#endregion
}
else
{
throw new InvalidOperationException("Invalid (-) Block.DataAddress detected.");
}
//**** write Block Header and Data to disk
BinaryWriter.Seek(0, SeekOrigin.Begin);
// Byte 0 to 7: Next Item Address (64 bit long int) = 0 (no next item)
BinaryWriter.Write(block.NextItemAddress);
// Byte 8 to 11: Size Occupied
BinaryWriter.Write(block.SizeOccupied);
// Byte 12 to 19: Low-level next datablock address
BinaryWriter.Write(block.InternalNextBlockAddress);
// Byte 20: count of member blocks, max is 255.
byte memberCount = 0;
if (block.IsHead)
{
int cm = block.CountMembers(true);
memberCount = cm > byte.MaxValue ? byte.MaxValue : (byte)cm;
}
BinaryWriter.Write(memberCount);
byte[] b2 = BufferStream.GetBuffer();
Array.Copy(b2, 0, _writeBuffer, currentTargetBufferIndex + bufferIndex, sizeOfNumerics);
bufferIndex += sizeOfNumerics;
//** Byte 20 to 20 + Data Length: USER DATA
int cs = block.Data.Length;
if (currentTargetBufferIndex + cs + bufferIndex > _writeBuffer.Length - block.Length)
{
cs = _writeBuffer.Length - (currentTargetBufferIndex + bufferIndex);
}
Array.Copy(block.Data, 0, _writeBuffer, currentTargetBufferIndex + bufferIndex, cs);
bufferIndex += block.Data.Length;
runningAddress += block.Length;
}
if (startBlockAddress != -1)
{
//** write to disk
dataChunks.Add(new BulkWriter.DataChunk
{
TargetDataAddress = startBlockAddress,
Index = currentTargetBufferIndex + startIndex,
Size = bufferIndex - startIndex
});
}
if (dataChunks.Count > 0)
{
bulkWriter.Write(parent, _writeBuffer, dataChunks);
}
return(r);
}
private void WriteBlocksToDisk(ConcurrentIOPoolManager readPool, ConcurrentIOPoolManager writePool,
Algorithm.Collection.ICollectionOnDisk parent, IDictionary <long, Sop.DataBlock> blocks)
{
// todo: do this Async way, when time permits. :)
if (BinaryWriter == null)
{
if (BufferStream == null)
{
BufferStream = new MemoryStream();
}
BinaryWriter = new BinaryWriter(BufferStream, parent.File.Server.Encoding);
}
else
{
BinaryWriter.Seek(0, SeekOrigin.Begin);
}
const int sizeOfNumerics = Sop.DataBlock.OverheadSize;
var writeBuffer = _writeBuffer;
int bufferIndex = 0, currentTargetBufferIndex = 0;
long runningAddress = -1, startBlockAddress = -1;
var bulkWriter = new BulkWriter();
var dataChunks = new List <BulkWriter.DataChunk>(4);
#region resize data file before appending to it...
if (readPool == null || writePool == null)
{
long currentAddress = -1;
if (((Collections.Generic.ISortedDictionary <long, Sop.DataBlock>)blocks).MoveLast())
{
currentAddress = ((Collections.Generic.ISortedDictionary <long, Sop.DataBlock>)blocks).CurrentKey;
}
if (currentAddress > -1)
{
// thread safe increase File Size to accomodate data to be appended...
var FileSize = currentAddress + (int)parent.DataBlockSize;
if (parent.FileStream.Length < FileSize)
{
if (parent.Transaction != null)
{
lock (parent.Transaction)
{
if (parent.FileStream.Length < FileSize)
{
parent.FileStream.SetLength(FileSize);
}
}
}
else
{
parent.FileStream.SetLength(FileSize);
}
}
}
}
#endregion
Sop.DataBlock[] blocksCopy = new Sop.DataBlock[blocks.Count];
blocks.Values.CopyTo(blocksCopy, 0);
foreach (Sop.DataBlock block in blocksCopy)
{
SetIsDirty(block, false);
if (block.DataAddress >= 0)
{
#region Process special states, e.g. - buffer is full, current block Address is fragmented from previous block's
if (startBlockAddress == -1)
{
startBlockAddress = runningAddress = block.DataAddress;
}
else
{
bool bufferIsFull = bufferIndex + sizeOfNumerics + block.Data.Length +
currentTargetBufferIndex > writeBuffer.Length - block.Length;
if (block.DataAddress != runningAddress || bufferIsFull)
{
dataChunks.Add(new BulkWriter.DataChunk
{
TargetDataAddress = startBlockAddress,
Index = currentTargetBufferIndex, // Index in the buffer of 1st byte of this segment
Size = bufferIndex // size of the segment
});
if (bufferIsFull)
{
//** write to disk
if (readPool != null && writePool != null)
{
bulkWriter.Backup(readPool, writePool, parent, writeBuffer, dataChunks);
if (writePool.AsyncThreadException != null)
{
throw writePool.AsyncThreadException;
}
else if (readPool.AsyncThreadException != null)
{
throw readPool.AsyncThreadException;
}
}
else if (writePool != null)
{
bulkWriter.Write(writePool, parent, writeBuffer, dataChunks);
if (writePool.AsyncThreadException != null)
{
throw writePool.AsyncThreadException;
}
}
else
{
throw new SopException("WriteBlocksToDisk has a bug!");
}
// create new buffer for succeeding chunks...
dataChunks = new List <BulkWriter.DataChunk>(4);
writeBuffer = new byte[writeBuffer.Length];
currentTargetBufferIndex = 0;
}
else
{
currentTargetBufferIndex += bufferIndex;
}
bufferIndex = 0;
runningAddress = startBlockAddress = block.DataAddress;
}
}
#endregion
}
else
{
throw new InvalidOperationException("Invalid (-) Block.DataAddress detected.");
}
//**** write Block Header and Data to disk
BinaryWriter.Seek(0, SeekOrigin.Begin);
// Byte 0 to 7: Next Item Address (64 bit long int) = 0 (no next item)
BinaryWriter.Write(block.NextItemAddress);
// Byte 8 to 11: Size Occupied
BinaryWriter.Write(block.SizeOccupied);
// Byte 12 to 19: Low-level next datablock address
BinaryWriter.Write(block.InternalNextBlockAddress);
// Byte 20: count of member blocks, max is 65535.
ushort memberCount = 0;
if (block.IsHead)
{
int cm = block.CountMembers(true);
memberCount = cm > Sop.DataBlock.MaxChainMemberCount ? Sop.DataBlock.MaxChainMemberCount : (ushort)cm;
}
BinaryWriter.Write(memberCount);
byte[] b2 = BufferStream.GetBuffer();
Array.Copy(b2, 0, writeBuffer, currentTargetBufferIndex + bufferIndex, sizeOfNumerics);
bufferIndex += sizeOfNumerics;
//** Byte 20 to 20 + Data Length: USER DATA
int cs = block.Data.Length;
//if (currentTargetBufferIndex + cs + bufferIndex > writeBuffer.Length - block.Length)
// cs = writeBuffer.Length - (currentTargetBufferIndex + bufferIndex);
Array.Copy(block.Data, 0, writeBuffer, currentTargetBufferIndex + bufferIndex, cs);
bufferIndex += block.Data.Length;
runningAddress += block.Length;
}
// write the last chunk set to disk...
if (startBlockAddress != -1)
{
//** write to disk
dataChunks.Add(new BulkWriter.DataChunk
{
TargetDataAddress = startBlockAddress,
Index = currentTargetBufferIndex,
Size = bufferIndex
});
}
if (dataChunks.Count > 0)
{
if (readPool != null && writePool != null)
{
bulkWriter.Backup(readPool, writePool, parent, writeBuffer, dataChunks);
}
else if (writePool != null)
{
bulkWriter.Write(writePool, parent, writeBuffer, dataChunks);
}
else
{
throw new SopException("WriteBlocksToDisk has a bug!");
}
}
}