/// <summary>
/// Release taken memory stream (set it back to <see cref="_cachedMemoryReadStream"/>)
/// </summary>
/// <param name="bufferingReadStream">Buffered stream to release</param>
private void ReleaseMemoryReadStream(RegionBinaryReader bufferingReadStream)
{
Debug.Assert(bufferingReadStream != null);
Debug.Assert(Monitor.IsEntered(_readLock));
Debug.Assert(_cachedMemoryReadStream == null);
if (bufferingReadStream.BaseStream.InnerStream.Length <= _maxCachedMemoryReadStreamSize)
{
if (bufferingReadStream.BaseStream.InnerStream.Capacity > _maxCachedMemoryReadStreamSize)
{
bufferingReadStream.BaseStream.SetOriginLength(0, -1);
bufferingReadStream.BaseStream.InnerStream.SetLength(0);
bufferingReadStream.BaseStream.InnerStream.Capacity = _maxCachedMemoryReadStreamSize;
}
_cachedMemoryReadStream = bufferingReadStream;
}
}
/// <summary>
/// Release taken memory stream (set it back to <see cref="_cachedMemoryReadStream"/>)
/// </summary>
/// <param name="bufferingReadStream">Buffered stream to release</param>
private void ReleaseMemoryReadStream(RegionBinaryReader bufferingReadStream)
{
TurboContract.Requires(bufferingReadStream != null, conditionString: "bufferingReadStream != null");
TurboContract.Assert(Monitor.IsEntered(_readLock), conditionString: "Monitor.IsEntered(_readLock)");
TurboContract.Assert(_cachedMemoryReadStream == null, conditionString: "_cachedMemoryReadStream == null");
if (bufferingReadStream.BaseStream.InnerStream.Length <= _maxCachedMemoryReadStreamSize)
{
if (bufferingReadStream.BaseStream.InnerStream.Capacity > _maxCachedMemoryReadStreamSize)
{
bufferingReadStream.BaseStream.SetOriginLength(0, -1);
bufferingReadStream.BaseStream.InnerStream.SetLength(0);
bufferingReadStream.BaseStream.InnerStream.Capacity = _maxCachedMemoryReadStreamSize;
}
_cachedMemoryReadStream = bufferingReadStream;
}
}
/// <summary>
/// Take or peek item inside readLock
/// Steps:
/// - Reads item from disk
/// - Reads item from disk in exclusive mode (lock on _writeLock)
/// - Reads item from writeBuffer (with lock on _writeLock)
/// </summary>
private bool TryTakeOrPeek(out T item, bool take)
{
lock (_readLock)
{
if (_isDisposed)
{
throw new ObjectDisposedException(this.GetType().Name);
}
RegionBinaryReader memoryBuffer = GetMemoryReadStream();
Debug.Assert(memoryBuffer.BaseStream.Length == 0);
Debug.Assert(memoryBuffer.BaseStream.InnerStream.Length == 0);
try
{
if (TryTakeOrPeekItemFromDisk(out item, memoryBuffer, take))
{
return(true);
}
// Should enter write lock to observe fully saved items
lock (_writeLock)
{
// Retry read from disk
if (TryTakeOrPeekItemFromDisk(out item, memoryBuffer, take))
{
return(true);
}
// Now attempt to read from write buffer
if (_writeBuffer != null && TryTakeOrPeekFromWriteBuffer(out item, take))
{
return(true);
}
}
}
finally
{
ReleaseMemoryReadStream(memoryBuffer);
}
return(false);
}
}
/// <summary>
/// Read signle item from disk and deserialize it
/// </summary>
/// <param name="item">Taken item</param>
/// <param name="buffer">Buffer</param>
/// <param name="take">True = take, False = peek</param>
/// <returns>Success or not</returns>
private bool TryTakeOrPeekItemFromDisk(out T item, RegionBinaryReader buffer, bool take)
{
Debug.Assert(buffer != null);
Debug.Assert(Monitor.IsEntered(_readLock));
buffer.BaseStream.SetOriginLength(0, -1);
int itemSize = 0;
if (!TryTakeOrPeekSingleItemBytesFromDisk(buffer.BaseStream.InnerStream, out itemSize, take)) // Read from disk
{
item = default(T);
return(false);
}
buffer.BaseStream.SetOriginLength(ItemHeaderSize, itemSize);
Debug.Assert(buffer.BaseStream.Length == itemSize);
item = _serializer.Deserialize(buffer); // Deserialize
return(true);
}
// ============= MemoryReadStream =================
/// <summary>
/// Gets cached memory stream to read
/// </summary>
/// <returns>BinaryReader with stream for in-memory reading</returns>
private RegionBinaryReader GetMemoryReadStream(int itemCount = 1)
{
Debug.Assert(Monitor.IsEntered(_readLock));
Debug.Assert(itemCount > 0);
RegionBinaryReader result = _cachedMemoryReadStream;
_cachedMemoryReadStream = null;
if (result == null)
{
int expectedItemSize = _serializer.ExpectedSizeInBytes;
if (expectedItemSize <= 0)
{
expectedItemSize = InitialCacheSizePerItem;
}
result = new RegionBinaryReader(Math.Min(MaxCachedMemoryStreamSize, Math.Max(0, itemCount * (expectedItemSize + ItemHeaderSize)))); // Max used to protect from overflow
}
else
{
result.BaseStream.SetOriginLength(0, -1);
result.BaseStream.SetLength(0);
}
return(result);
}
/// <summary>
/// Take or peek item inside readLock through read buffer. Also populate readBuffer with items
/// Steps:
/// - Reads item from readBuffer;
/// - Reads items from disk
/// - Reads items from disk in exclusive mode (lock on _writeLock)
/// - Reads items from writeBuffer (with lock on _writeLock)
/// </summary>
private bool TryTakeOrPeekThroughReadBuffer(out T item, bool take)
{
Debug.Assert(_readBuffer != null);
Debug.Assert(_maxReadBufferSize > 0);
lock (_readLock)
{
if (_isDisposed)
{
throw new ObjectDisposedException(this.GetType().Name);
}
// retry read from buffer
if (TryTakeOrPeekFromReadBuffer(out item, take))
{
return(true);
}
// Read buffer is empty => should read from disk
RegionBinaryReader memoryBuffer = GetMemoryReadStream();
Debug.Assert(memoryBuffer.BaseStream.Length == 0);
Debug.Assert(memoryBuffer.BaseStream.InnerStream.Length == 0);
try
{
int itemTransfered = 0;
T tmpItem = default(T);
while (itemTransfered < _maxReadBufferSize && TryTakeOrPeekItemFromDisk(out tmpItem, memoryBuffer, take: true)) // take = true as we transfer items to buffer
{
if (itemTransfered == 0)
{
item = tmpItem;
}
if (itemTransfered > 0 || !take) // First item should always be ours
{
_readBuffer.Enqueue(tmpItem);
}
itemTransfered++;
}
if (itemTransfered < _maxReadBufferSize)
{
// Should enter write lock to observe fully saved items
lock (_writeLock)
{
// Retry read from disk
while (itemTransfered < _maxReadBufferSize && TryTakeOrPeekItemFromDisk(out tmpItem, memoryBuffer, take: true)) // take = true as we transfer items to buffer
{
if (itemTransfered == 0)
{
item = tmpItem;
}
if (itemTransfered > 0 || !take) // First item should always be ours
{
_readBuffer.Enqueue(tmpItem);
}
itemTransfered++;
}
// attempt to read from write buffer
if (itemTransfered < _maxReadBufferSize && _writeBuffer != null)
{
while (itemTransfered < _maxReadBufferSize && TryTakeOrPeekFromWriteBuffer(out tmpItem, take: true)) // take = true as we transfer items to buffer
{
if (itemTransfered == 0)
{
item = tmpItem;
}
if (itemTransfered > 0 || !take) // First item should always be ours
{
_readBuffer.Enqueue(tmpItem);
}
itemTransfered++;
}
}
}
}
return(itemTransfered > 0);
}
finally
{
ReleaseMemoryReadStream(memoryBuffer);
}
}
}
/// <summary>
/// NonPersistentDiskQueueSegment constructor
/// </summary>
/// <param name="segmentNumber">Segment number</param>
/// <param name="fileName">Full file name for the segment</param>
/// <param name="serializer">Items serializing/deserializing logic</param>
/// <param name="capacity">Maximum number of stored items inside the segement (overall capacity)</param>
/// <param name="writeBufferSize">Determines the number of items, that are stored in memory before save them to disk (-1 - set to default value, 0 - disable write buffer)</param>
/// <param name="cachedMemoryWriteStreamSize">Maximum size of the cached byte stream that used to serialize items in memory (-1 - set to default value, 0 - disable byte stream caching)</param>
/// <param name="readBufferSize">Determines the number of items, that are stored in memory for read purposes (-1 - set to default value, 0 - disable read buffer)</param>
/// <param name="cachedMemoryReadStreamSize">Maximum size of the cached byte stream that used to deserialize items in memory (-1 - set to default value, 0 - disable byte stream caching)</param>
public NonPersistentDiskQueueSegment(long segmentNumber, string fileName, IDiskQueueItemSerializer <T> serializer, int capacity,
int writeBufferSize, int cachedMemoryWriteStreamSize, int readBufferSize, int cachedMemoryReadStreamSize)
: base(segmentNumber, capacity, 0, 0)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException(nameof(fileName));
}
if (serializer == null)
{
throw new ArgumentNullException(nameof(serializer));
}
if (File.Exists(fileName))
{
throw new ArgumentException($"Can't create NonPersistentDiskQueueSegment on existing file '{fileName}'", nameof(fileName));
}
_fileName = fileName;
_serializer = serializer;
try
{
_writeStream = new FileStream(_fileName, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
_readStream = new FileStream(_fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
_writeLock = new object();
_readLock = new object();
_writeBufferSize = 0;
_maxWriteBufferSize = writeBufferSize >= 0 ? writeBufferSize : DefaultWriteBufferSize;
if (writeBufferSize < 0 && serializer.ExpectedSizeInBytes > 0)
{
_maxWriteBufferSize = checked (Math.Max(DefaultWriteBufferSize, 4096 / Math.Min(serializer.ExpectedSizeInBytes + ItemHeaderSize, MaxChacheSizePerItem)));
}
if (_maxWriteBufferSize > 0)
{
_writeBuffer = new ConcurrentQueue <T>();
}
_cachedMemoryWriteStream = null;
_maxCachedMemoryWriteStreamSize = cachedMemoryWriteStreamSize;
if (cachedMemoryWriteStreamSize < 0)
{
checked
{
int expectedItemSize = MaxChacheSizePerItem;
if (serializer.ExpectedSizeInBytes > 0)
{
expectedItemSize = Math.Min(serializer.ExpectedSizeInBytes + ItemHeaderSize, MaxChacheSizePerItem);
}
_maxCachedMemoryWriteStreamSize = Math.Min(MaxCachedMemoryStreamSize, (_maxWriteBufferSize + 1) * expectedItemSize);
}
}
_maxReadBufferSize = readBufferSize >= 0 ? readBufferSize : Math.Max(_writeBufferSize / 2, DefaultReadBufferSize);
if (_maxReadBufferSize > 0)
{
_readBuffer = new ConcurrentQueue <T>();
}
_cachedMemoryReadStream = null;
_maxCachedMemoryReadStreamSize = cachedMemoryReadStreamSize;
if (cachedMemoryReadStreamSize < 0)
{
checked
{
int expectedItemSize = MaxChacheSizePerItem;
if (serializer.ExpectedSizeInBytes > 0)
{
expectedItemSize = Math.Min(serializer.ExpectedSizeInBytes + ItemHeaderSize, MaxChacheSizePerItem);
}
_maxCachedMemoryReadStreamSize = Math.Min(MaxCachedMemoryStreamSize, expectedItemSize);
}
}
// Prepare file header
WriteSegmentHeader(_writeStream, Number, Capacity);
_readStream.Seek(0, SeekOrigin.End); // Offset readStream after the header
_isDisposed = false;
Debug.Assert(_maxWriteBufferSize >= 0);
Debug.Assert(_maxCachedMemoryWriteStreamSize >= 0);
Debug.Assert(_maxReadBufferSize >= 0);
Debug.Assert(_maxCachedMemoryReadStreamSize >= 0);
Debug.Assert(_writeStream.Position == _readStream.Position);
}
catch
{
// Should close streams to make created files available to delete
if (_writeStream != null)
{
_writeStream.Dispose();
}
if (_readStream != null)
{
_readStream.Dispose();
}
throw;
}
}