/// <summary>
/// Creates a <see cref="LargeArray{T}"/> with the specified jagged array depth.
/// </summary>
/// <param name="jaggedArrayDepth">the number of elements per jagged array. Rounds up to the nearest power of 2.</param>
public LargeArray(int jaggedArrayDepth)
{
m_size = (int)BitMath.RoundUpToNearestPowerOfTwo((uint)jaggedArrayDepth);
m_mask = m_size - 1;
m_bitShift = BitMath.CountBitsSet((uint)m_mask);
m_array = new T[0][];
m_capacity = 0;
}
/// <summary>
/// Create a new <see cref="UnmanagedMemoryStreamCore"/> that allocates its own unmanaged memory.
/// </summary>
protected UnmanagedMemoryStreamCore(int allocationSize = 4096)
{
if (!BitMath.IsPowerOfTwo(allocationSize) || allocationSize < 4096 || allocationSize > 1024 * 1024)
{
throw new ArgumentOutOfRangeException("allocationSize", "Must be a power of 2 between 4KB and 1MB");
}
m_shiftLength = BitMath.CountBitsSet((uint)(allocationSize - 1));
m_pageSize = allocationSize;
m_invertMask = ~(allocationSize - 1);
m_settings = new Settings();
m_syncRoot = new object();
m_memoryBlocks = new List <Memory>();
}
/// <summary>
/// Create a thread safe list of MemoryPool pages.
/// </summary>
/// <param name="pageSize">The desired page size. Must be between 4KB and 256KB</param>
/// <param name="maximumBufferSize">The desired maximum size of the allocation. Note: could be less if there is not enough system memory.
/// A value of -1 will default based on available system memory</param>
public MemoryPoolPageList(int pageSize, long maximumBufferSize)
{
m_syncRoot = new object();
PageSize = pageSize;
#if SQLCLR
long totalMemory = int.MaxValue;
long availableMemory = int.MaxValue;
#else
long totalMemory = (long)Common.GetTotalPhysicalMemory();
long availableMemory = (long)Common.GetAvailablePhysicalMemory();
#endif
if (!Environment.Is64BitProcess)
{
Log.Publish(MessageLevel.Info, "Process running in 32-bit mode. Memory Pool is Limited in size.");
totalMemory = Math.Min(int.MaxValue, totalMemory); //Clip at 2GB
availableMemory = Math.Min(int.MaxValue - GC.GetTotalMemory(false), availableMemory); //Clip at 2GB
}
m_memoryBlockSize = CalculateMemoryBlockSize(PageSize, totalMemory);
MemoryPoolCeiling = CalculateMemoryPoolCeiling(m_memoryBlockSize, totalMemory);
if (maximumBufferSize < 0)
{
//Maximum size defaults to the larger of:
//50% of the free ram
//25% of the total system memory.
MaximumPoolSize = Math.Max(MemoryPool.MinimumTestedSupportedMemoryFloor, availableMemory / 2);
MaximumPoolSize = Math.Max(MaximumPoolSize, totalMemory / 4);
}
else
{
MaximumPoolSize = Math.Max(Math.Min(MemoryPoolCeiling, maximumBufferSize), MemoryPool.MinimumTestedSupportedMemoryFloor);
}
Log.Publish(MessageLevel.Info, "Memory Pool Maximum Defaulted To: " + MaximumPoolSize / 1024 / 1024 + "MB of Ram");
m_memoryBlockAllocations = 0;
m_usedPageCount = 0;
m_pagesPerMemoryBlock = (m_memoryBlockSize / PageSize);
m_pagesPerMemoryBlockMask = m_pagesPerMemoryBlock - 1;
m_pagesPerMemoryBlockShiftBits = BitMath.CountBitsSet((uint)m_pagesPerMemoryBlockMask);
m_isPageFree = new BitArray(false);
m_memoryBlocks = new List <Memory>();
}
public void CountBitsSet()
{
Assert.AreEqual(0, BitMath.CountBitsSet(0));
Assert.AreEqual(32, BitMath.CountBitsSet(uint.MaxValue));
Assert.AreEqual(31, BitMath.CountBitsSet(int.MaxValue));
Assert.AreEqual(1, BitMath.CountBitsSet(short.MaxValue + 1));
Assert.AreEqual(10, BitMath.CountBitsSet(0xF030207));
Assert.AreEqual(0, BitMath.CountBitsSet((ulong)0));
Assert.AreEqual(32, BitMath.CountBitsSet((ulong)uint.MaxValue));
Assert.AreEqual(31, BitMath.CountBitsSet((ulong)int.MaxValue));
Assert.AreEqual(1, BitMath.CountBitsSet((ulong)short.MaxValue + 1));
Assert.AreEqual(10, BitMath.CountBitsSet((ulong)0xF030207));
int x = -1;
Assert.AreEqual(64, BitMath.CountBitsSet(ulong.MaxValue));
Assert.AreEqual(64, BitMath.CountBitsSet((ulong)x));
}
/// <summary>
/// Creates a new instance of <see cref="PageReplacementAlgorithm"/>.
/// </summary>
/// <param name="pool">The memory pool that blocks will be allocated from.</param>
public PageReplacementAlgorithm(MemoryPool pool)
{
if (pool.PageSize < 4096)
{
throw new ArgumentOutOfRangeException("PageSize Must be greater than 4096", "pool");
}
if (!BitMath.IsPowerOfTwo(pool.PageSize))
{
throw new ArgumentException("PageSize Must be a power of 2", "pool");
}
m_maxValidPosition = (int.MaxValue - 1) * (long)pool.PageSize; //Max position
m_syncRoot = new object();
m_memoryPageSizeMask = pool.PageSize - 1;
m_memoryPageSizeShiftBits = BitMath.CountBitsSet((uint)m_memoryPageSizeMask);
m_pageList = new PageList(pool);
m_arrayIndexLocks = new WeakList <PageLock>();
}
/// <summary>
/// Creates a new <see cref="MemoryPool"/>.
/// </summary>
/// <param name="pageSize">The desired page size. Must be between 4KB and 256KB</param>
/// <param name="maximumBufferSize">The desired maximum size of the allocation. Note: could be less if there is not enough system memory.</param>
/// <param name="utilizationLevel">Specifies the desired utilization level of the allocated space.</param>
public MemoryPool(int pageSize = 64 * 1024, long maximumBufferSize = -1, TargetUtilizationLevels utilizationLevel = TargetUtilizationLevels.Low)
{
if (pageSize < 4096 || pageSize > 256 * 1024)
{
throw new ArgumentOutOfRangeException("pageSize", "Page size must be between 4KB and 256KB and a power of 2");
}
if (!BitMath.IsPowerOfTwo((uint)pageSize))
{
throw new ArgumentOutOfRangeException("pageSize", "Page size must be between 4KB and 256KB and a power of 2");
}
m_syncRoot = new object();
m_syncAllocate = new object();
PageSize = pageSize;
PageMask = PageSize - 1;
PageShiftBits = BitMath.CountBitsSet((uint)PageMask);
m_pageList = new MemoryPoolPageList(PageSize, maximumBufferSize);
m_requestCollectionEvent = new ThreadSafeList <WeakEventHandler <CollectionEventArgs> >();
SetTargetUtilizationLevel(utilizationLevel);
}
/// <summary>
/// This will return a byte array of data that can be written to an archive file.
/// </summary>
public byte[] GetBytes()
{
if (!IsFileAllocationTableValid())
{
throw new InvalidOperationException("File Allocation Table is invalid");
}
byte[] dataBytes = new byte[m_blockSize];
MemoryStream stream = new MemoryStream(dataBytes);
BinaryWriter dataWriter = new BinaryWriter(stream);
dataWriter.Write(FileAllocationTableHeaderBytes);
if (BitConverter.IsLittleEndian)
{
dataWriter.Write('L');
}
else
{
dataWriter.Write('B');
}
dataWriter.Write((byte)(BitMath.CountBitsSet((uint)(m_blockSize - 1))));
dataWriter.Write(FileAllocationReadTableVersion);
dataWriter.Write(FileAllocationWriteTableVersion);
dataWriter.Write(FileAllocationHeaderVersion);
dataWriter.Write((byte)(m_isSimplifiedFileFormat ? 2 : 1));
dataWriter.Write(m_headerBlockCount);
dataWriter.Write(m_lastAllocatedBlock);
dataWriter.Write(m_snapshotSequenceNumber);
dataWriter.Write(m_nextFileId);
dataWriter.Write(m_archiveId.ToByteArray());
dataWriter.Write(m_archiveType.ToByteArray());
dataWriter.Write((short)m_files.Count);
foreach (SubFileHeader node in m_files)
{
node.Save(dataWriter);
}
//Metadata Flags
if (m_flags.Count > 0)
{
Encoding7Bit.WriteInt15(dataWriter.Write, (short)FileHeaderAttributes.FileFlags);
Encoding7Bit.WriteInt15(dataWriter.Write, (short)(m_flags.Count * 16));
foreach (var flag in m_flags)
{
dataWriter.Write(GuidExtensions.ToLittleEndianBytes(flag));
}
}
if (m_unknownAttributes != null)
{
foreach (var md in m_unknownAttributes)
{
Encoding7Bit.WriteInt15(dataWriter.Write, md.Key);
Encoding7Bit.WriteInt15(dataWriter.Write, (short)md.Value.Length);
dataWriter.Write(md.Value);
}
}
if (m_userAttributes != null)
{
foreach (var md in m_userAttributes)
{
Encoding7Bit.WriteInt15(dataWriter.Write, (short)FileHeaderAttributes.UserAttributes);
dataWriter.Write(GuidExtensions.ToLittleEndianBytes(md.Key));
Encoding7Bit.WriteInt15(dataWriter.Write, (short)md.Value.Length);
dataWriter.Write(md.Value);
}
}
Encoding7Bit.WriteInt15(dataWriter.Write, (short)FileHeaderAttributes.EndOfAttributes);
Encoding7Bit.WriteInt15(dataWriter.Write, 0);
if (stream.Position + 32 > dataBytes.Length)
{
throw new Exception("the file size exceedes the allowable size.");
}
WriteFooterData(dataBytes);
return(dataBytes);
}