// =================================
/// <summary>
/// Writes items from <see cref="_writeBuffer"/> to disk
/// </summary>
private void SaveWriteBufferToDisk()
{
Debug.Assert(_writeBuffer != null);
Debug.Assert(_maxWriteBufferSize > 0);
lock (_writeLock)
{
if (_isDisposed)
{
throw new ObjectDisposedException(this.GetType().Name);
}
// Nothing to do check
if (_writeBuffer.IsEmpty || _writeBufferSize < _maxWriteBufferSize)
{
return;
}
RegionBinaryWriter writer = GetMemoryWriteStream(_maxWriteBufferSize);
Debug.Assert(writer.BaseStream.Length == 0);
Debug.Assert(writer.BaseStream.InnerStream.Length == 0);
int itemCount = 0;
T item = default(T);
while (itemCount < _maxWriteBufferSize && _writeBuffer.TryDequeue(out item))
{
// write item to mem stream
SerializeItemToStream(item, writer);
itemCount++;
}
if (itemCount > 0)
{
// Reduce counter safely
int curWriteBufferSize = Interlocked.Add(ref _writeBufferSize, -itemCount);
Debug.Assert(curWriteBufferSize >= 0);
Debug.Assert(curWriteBufferSize <= _writeBuffer.Count);
// Write all data to disk
writer.BaseStream.InnerStream.WriteTo(_writeStream);
_writeStream.Flush(flushToDisk: false);
}
ReleaseMemoryWriteStream(writer);
}
}
/// <summary>
/// Release taken memory stream (set it back to <see cref="_cachedMemoryWriteStream"/>)
/// </summary>
/// <param name="bufferingWriteStream">Buffered stream to release</param>
private void ReleaseMemoryWriteStream(RegionBinaryWriter bufferingWriteStream)
{
Debug.Assert(bufferingWriteStream != null);
Debug.Assert(Monitor.IsEntered(_writeLock));
Debug.Assert(_cachedMemoryWriteStream == null);
if (bufferingWriteStream.BaseStream.InnerStream.Length <= _maxCachedMemoryWriteStreamSize)
{
if (bufferingWriteStream.BaseStream.InnerStream.Capacity > _maxCachedMemoryWriteStreamSize)
{
bufferingWriteStream.BaseStream.SetOriginLength(0, -1);
bufferingWriteStream.BaseStream.InnerStream.SetLength(0);
bufferingWriteStream.BaseStream.InnerStream.Capacity = _maxCachedMemoryWriteStreamSize;
}
_cachedMemoryWriteStream = bufferingWriteStream;
}
}
/// <summary>
/// Release taken memory stream (set it back to <see cref="_cachedMemoryWriteStream"/>)
/// </summary>
/// <param name="bufferingWriteStream">Buffered stream to release</param>
private void ReleaseMemoryWriteStream(RegionBinaryWriter bufferingWriteStream)
{
TurboContract.Requires(bufferingWriteStream != null, conditionString: "bufferingWriteStream != null");
TurboContract.Assert(Monitor.IsEntered(_writeLock), conditionString: "Monitor.IsEntered(_writeLock)");
TurboContract.Assert(_cachedMemoryWriteStream == null, conditionString: "_cachedMemoryWriteStream == null");
if (bufferingWriteStream.BaseStream.InnerStream.Length <= _maxCachedMemoryWriteStreamSize)
{
if (bufferingWriteStream.BaseStream.InnerStream.Capacity > _maxCachedMemoryWriteStreamSize)
{
bufferingWriteStream.BaseStream.SetOriginLength(0, -1);
bufferingWriteStream.BaseStream.InnerStream.SetLength(0);
bufferingWriteStream.BaseStream.InnerStream.Capacity = _maxCachedMemoryWriteStreamSize;
}
_cachedMemoryWriteStream = bufferingWriteStream;
}
}
/// <summary>
/// Writes single item to disk
/// </summary>
/// <param name="item">Item</param>
private void SaveSingleItemToDisk(T item)
{
lock (_writeLock)
{
if (_isDisposed)
{
throw new ObjectDisposedException(this.GetType().Name);
}
RegionBinaryWriter writer = GetMemoryWriteStream(1);
Debug.Assert(writer.BaseStream.Length == 0);
Debug.Assert(writer.BaseStream.InnerStream.Length == 0);
SerializeItemToStream(item, writer);
// Write all data to disk
writer.BaseStream.InnerStream.WriteTo(_writeStream);
_writeStream.Flush(flushToDisk: false);
ReleaseMemoryWriteStream(writer);
}
}
/// <summary>
/// Build segment record with specified item in memory stream.
/// (writes length + item bytes)
/// </summary>
/// <param name="item">Item</param>
/// <param name="writer">Writer that wraps memory stream</param>
private void SerializeItemToStream(T item, RegionBinaryWriter writer)
{
Debug.Assert(writer != null);
Debug.Assert(Monitor.IsEntered(_writeLock));
checked
{
var stream = writer.BaseStream;
int origin = stream.InnerStreamPosition;
stream.SetOriginLength(origin + 4, -1); // Offset 4 to store length later
Debug.Assert(stream.Length == 0);
_serializer.Serialize(writer, item);
writer.Flush();
Debug.Assert(stream.Length >= 0);
int length = (int)stream.Length;
stream.SetOrigin(origin); // Offset back to the beggining
writer.Write((int)length); // Write length
stream.Seek(0, SeekOrigin.End); // Seek to the end of the stream
}
}
// ================= MemoryWriteStream ==================
/// <summary>
/// Gets cached memory stream to write
/// </summary>
/// <returns>BinaryWriter with stream for in-memory writing</returns>
private RegionBinaryWriter GetMemoryWriteStream(int itemCount)
{
Debug.Assert(Monitor.IsEntered(_writeLock));
Debug.Assert(itemCount > 0);
RegionBinaryWriter result = _cachedMemoryWriteStream;
_cachedMemoryWriteStream = null;
if (result == null)
{
int expectedItemSize = _serializer.ExpectedSizeInBytes;
if (expectedItemSize <= 0)
{
expectedItemSize = InitialCacheSizePerItem;
}
result = new RegionBinaryWriter(Math.Min(MaxCachedMemoryStreamSize, Math.Max(0, itemCount * (expectedItemSize + ItemHeaderSize)))); // Max used to protect from overflow
}
else
{
result.BaseStream.SetOriginLength(0, -1);
result.BaseStream.SetLength(0);
}
return(result);
}
/// <summary>
/// Build segment record with specified item in memory stream.
/// (writes length + item bytes)
/// </summary>
/// <param name="item">Item</param>
/// <param name="writer">Writer that wraps memory stream</param>
private void SerializeItemToStream(T item, RegionBinaryWriter writer)
{
TurboContract.Requires(writer != null, conditionString: "writer != null");
TurboContract.Assert(Monitor.IsEntered(_writeLock), conditionString: "Monitor.IsEntered(_writeLock)");
checked
{
var stream = writer.BaseStream;
int origin = stream.InnerStreamPosition;
stream.SetOriginLength(origin + 4, -1); // Offset 4 to store length later
TurboContract.Assert(stream.Length == 0, conditionString: "stream.Length == 0");
_serializer.Serialize(writer, item);
writer.Flush();
TurboContract.Assert(stream.Length >= 0, conditionString: "stream.Length >= 0");
int length = (int)stream.Length;
stream.SetOrigin(origin); // Offset back to the beggining
writer.Write((int)length); // Write length
stream.Seek(0, SeekOrigin.End); // Seek to the end of the stream
}
}
/// <summary>
/// NonPersistentDiskQueueSegment constructor
/// </summary>
/// <param name="segmentNumber">Segment number</param>
/// <param name="fileName">Full file name for the segment</param>
/// <param name="serializer">Items serializing/deserializing logic</param>
/// <param name="capacity">Maximum number of stored items inside the segement (overall capacity)</param>
/// <param name="writeBufferSize">Determines the number of items, that are stored in memory before save them to disk (-1 - set to default value, 0 - disable write buffer)</param>
/// <param name="cachedMemoryWriteStreamSize">Maximum size of the cached byte stream that used to serialize items in memory (-1 - set to default value, 0 - disable byte stream caching)</param>
/// <param name="readBufferSize">Determines the number of items, that are stored in memory for read purposes (-1 - set to default value, 0 - disable read buffer)</param>
/// <param name="cachedMemoryReadStreamSize">Maximum size of the cached byte stream that used to deserialize items in memory (-1 - set to default value, 0 - disable byte stream caching)</param>
public NonPersistentDiskQueueSegment(long segmentNumber, string fileName, IDiskQueueItemSerializer <T> serializer, int capacity,
int writeBufferSize, int cachedMemoryWriteStreamSize, int readBufferSize, int cachedMemoryReadStreamSize)
: base(segmentNumber, capacity, 0, 0)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException(nameof(fileName));
}
if (serializer == null)
{
throw new ArgumentNullException(nameof(serializer));
}
if (File.Exists(fileName))
{
throw new ArgumentException($"Can't create NonPersistentDiskQueueSegment on existing file '{fileName}'", nameof(fileName));
}
_fileName = fileName;
_serializer = serializer;
try
{
_writeStream = new FileStream(_fileName, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
_readStream = new FileStream(_fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
_writeLock = new object();
_readLock = new object();
_writeBufferSize = 0;
_maxWriteBufferSize = writeBufferSize >= 0 ? writeBufferSize : DefaultWriteBufferSize;
if (writeBufferSize < 0 && serializer.ExpectedSizeInBytes > 0)
{
_maxWriteBufferSize = checked (Math.Max(DefaultWriteBufferSize, 4096 / Math.Min(serializer.ExpectedSizeInBytes + ItemHeaderSize, MaxChacheSizePerItem)));
}
if (_maxWriteBufferSize > 0)
{
_writeBuffer = new ConcurrentQueue <T>();
}
_cachedMemoryWriteStream = null;
_maxCachedMemoryWriteStreamSize = cachedMemoryWriteStreamSize;
if (cachedMemoryWriteStreamSize < 0)
{
checked
{
int expectedItemSize = MaxChacheSizePerItem;
if (serializer.ExpectedSizeInBytes > 0)
{
expectedItemSize = Math.Min(serializer.ExpectedSizeInBytes + ItemHeaderSize, MaxChacheSizePerItem);
}
_maxCachedMemoryWriteStreamSize = Math.Min(MaxCachedMemoryStreamSize, (_maxWriteBufferSize + 1) * expectedItemSize);
}
}
_maxReadBufferSize = readBufferSize >= 0 ? readBufferSize : Math.Max(_writeBufferSize / 2, DefaultReadBufferSize);
if (_maxReadBufferSize > 0)
{
_readBuffer = new ConcurrentQueue <T>();
}
_cachedMemoryReadStream = null;
_maxCachedMemoryReadStreamSize = cachedMemoryReadStreamSize;
if (cachedMemoryReadStreamSize < 0)
{
checked
{
int expectedItemSize = MaxChacheSizePerItem;
if (serializer.ExpectedSizeInBytes > 0)
{
expectedItemSize = Math.Min(serializer.ExpectedSizeInBytes + ItemHeaderSize, MaxChacheSizePerItem);
}
_maxCachedMemoryReadStreamSize = Math.Min(MaxCachedMemoryStreamSize, expectedItemSize);
}
}
// Prepare file header
WriteSegmentHeader(_writeStream, Number, Capacity);
_readStream.Seek(0, SeekOrigin.End); // Offset readStream after the header
_isDisposed = false;
Debug.Assert(_maxWriteBufferSize >= 0);
Debug.Assert(_maxCachedMemoryWriteStreamSize >= 0);
Debug.Assert(_maxReadBufferSize >= 0);
Debug.Assert(_maxCachedMemoryReadStreamSize >= 0);
Debug.Assert(_writeStream.Position == _readStream.Position);
}
catch
{
// Should close streams to make created files available to delete
if (_writeStream != null)
{
_writeStream.Dispose();
}
if (_readStream != null)
{
_readStream.Dispose();
}
throw;
}
}
