// Mark a range of physical memory as being used. THIS IS ONLY
// USED DURING INITIALIZATION, and so is not particularly
// efficient.
//
// Addresses do not have to be page-aligned, but whole pages will
// be marked.
//
// To allow SMAP overlaps with HAL regions make this ignore
// region overlaps.
private static unsafe void MarkRangeUsed(ulong startAddr, ulong length)
{
ulong firstPage = startAddr & ~MemoryManager.PageMask;
// lastPage ends up being the address of the page immediately *after*
// the data ends.
ulong lastPage = MemoryManager.PagePad(startAddr + length);
for (ulong pageAddr = firstPage; pageAddr < lastPage; pageAddr += MemoryManager.PageSize)
{
ulong pageNum = MemoryManager.PagesFromBytes(pageAddr);
uint blockIdx = (uint)(pageNum / (ulong)PageBlock.PagesPerBlock);
uint pageInBlock = (uint)(pageNum % (ulong)PageBlock.PagesPerBlock);
PageBlock *pBlock = GetBlock(blockIdx);
if (!pBlock->PageInUse(pageInBlock))
{
pBlock->MarkAsUsed(pageInBlock);
if (pBlock->Full)
{
pBlock->Unlink(ref freeList);
}
}
}
}
/////////////////////////////////////
// PUBLIC METHODS
/////////////////////////////////////
public unsafe PhysicalHeap(UIntPtr start, UIntPtr limit)
{
DebugStub.Assert(MemoryManager.IsPageAligned(start));
DebugStub.Assert(MemoryManager.IsPageAligned(limit));
// Note that this wastes a little bit of memory by allocating
// table space to describe page-table memory!
UIntPtr numPages = MemoryManager.PagesFromBytes(limit - start);
UIntPtr bytesForTable = numPages * BytesPerTableEntry;
bytesForTable = MemoryManager.PagePad(bytesForTable);
UIntPtr pagesForTable = MemoryManager.PagesFromBytes(bytesForTable);
pageCount = numPages - pagesForTable;
startAddr = start + bytesForTable;
heapLimit = limit;
pageTable = (ushort *)start;
heapLock = new SpinLock(SpinLock.Types.PhysicalHeap);
// The entire heap is free to start out with
freeList = new FreeList();
// Initialize the page table
SetPages(startAddr, pageCount, FreePage);
fixed(PhysicalHeap *thisPtr = &this)
{
freeList.CreateAndInsert(thisPtr, startAddr, pageCount);
}
CheckConsistency();
}
public unsafe void Free(UIntPtr addr, UIntPtr bytes, Process process)
{
if (addr == UIntPtr.Zero)
{
// Silently accept freeing null
return;
}
// We always hand out memory in page-size chunks, so round up what
// the caller thinks their block size is
bytes = MemoryManager.PagePad(bytes);
// Our blocks always start on page boundaries
DebugStub.Assert(MemoryManager.IsPageAligned(addr));
ushort tag = process != null ? (ushort)process.ProcessId : KernelPage;
bool iflag = Lock();
try {
CheckConsistency();
UIntPtr numPages = MemoryManager.PagesFromBytes(bytes);
VerifyOwner(addr, numPages, tag);
FreeNode *nextBlock = null;
FreeNode *prevBlock = null;
if ((addr + bytes) < heapLimit)
{
fixed(PhysicalHeap *thisPtr = &this)
{
nextBlock = FreeNode.GetNodeAt(thisPtr, addr + bytes);
}
}
if (addr > startAddr)
{
fixed(PhysicalHeap *thisPtr = &this)
{
prevBlock = LastNode.GetNodeFromLast(thisPtr, addr - MemoryManager.PageSize);
}
}
// Don't mark pages as free until we're done discovering the
// previous and next blocks, or the attempt to discover
// the previous and next blocks gets confused to find itself
// adjacent to a free block.
SetPages(addr, numPages, FreePage);
// Coalesce with the preceding region
if (prevBlock != null)
{
addr = (UIntPtr)prevBlock;
bytes += prevBlock->bytes;
freeList.Remove(prevBlock);
}
// Coalesce with the following region
if (nextBlock != null)
{
bytes += nextBlock->bytes;
freeList.Remove(nextBlock);
}
// Blocks should always be integral numbers of pages
DebugStub.Assert(MemoryManager.IsPageAligned(bytes));
// Create the free node.
fixed(PhysicalHeap *thisPtr = &this)
{
freeList.CreateAndInsert(thisPtr, addr, bytes / MemoryManager.PageSize);
}
CheckConsistency();
}
finally {
Unlock(iflag);
}
}
public unsafe UIntPtr Allocate(UIntPtr limitAddr,
UIntPtr bytes,
UIntPtr alignment,
Process process)
{
ushort tag = process != null ? (ushort)process.ProcessId : KernelPage;
UIntPtr blockPtr;
bool iflag = Lock();
if (alignment < MemoryManager.PageSize)
{
alignment = MemoryManager.PageSize;
}
try {
CheckConsistency();
// Find an appropriately-sized block
FreeNode *foundNode = freeList.FindGoodFit(bytes, alignment);
if (foundNode == null)
{
return(UIntPtr.Zero);
}
DebugStub.Assert(MemoryManager.IsPageAligned((UIntPtr)foundNode));
// Respect alignment within the node
blockPtr = MemoryManager.Pad((UIntPtr)foundNode, alignment);
UIntPtr alignedSize = bytes + SpaceToAlign((UIntPtr)foundNode, alignment);
DebugStub.Assert(alignedSize == (blockPtr + bytes) - (UIntPtr)foundNode);
DebugStub.Assert(foundNode->bytes >= alignedSize);
// Give back any extra pages
UIntPtr numPages = MemoryManager.PagesFromBytes(MemoryManager.PagePad(alignedSize));
UIntPtr chunkPages = MemoryManager.PagesFromBytes(foundNode->bytes);
DebugStub.Assert(chunkPages >= numPages);
UIntPtr extraPages = chunkPages - numPages;
if (extraPages > 0)
{
// Give back the extra memory
UIntPtr remainderPtr = (UIntPtr)foundNode + (numPages * MemoryManager.PageSize);
fixed(PhysicalHeap *thisPtr = &this)
{
freeList.CreateAndInsert(thisPtr, remainderPtr, extraPages);
}
}
SetPages((UIntPtr)foundNode, numPages, tag);
CheckConsistency();
}
finally {
Unlock(iflag);
}
// TODO: Flexible limit specification not yet implemented
if (limitAddr > UIntPtr.Zero)
{
DebugStub.Assert(blockPtr < limitAddr);
}
return(blockPtr);
}
/////////////////////////////////////
// PUBLIC METHODS
/////////////////////////////////////
//
// The range of memory turned over to a VirtualMemoryRange structure
// must not have *any* mapped pages in it to start out with
//
// A VirtualMemoryRange can build a pagetable that *describes*
// more memory than it *manages* (this supports some kernel GC
// oddities). In that case, pages out-of-range are marked
// as PageType.Unknown. Obviously, allocation requests are never
// satisfied with out-of-bounds data.
//
internal unsafe VirtualMemoryRange_struct(
UIntPtr baseAddr, UIntPtr limitAddr,
UIntPtr descrBaseAddr, UIntPtr descrLimitAddr,
ProtectionDomain inDomain)
{
DebugStub.Assert(MemoryManager.IsPageAligned(baseAddr));
DebugStub.Assert(MemoryManager.IsPageAligned(limitAddr));
DebugStub.Assert(MemoryManager.IsPageAligned(descrBaseAddr));
DebugStub.Assert(MemoryManager.IsPageAligned(descrLimitAddr));
// The descriptive range can't be smaller than the managed range
DebugStub.Assert(descrLimitAddr >= limitAddr);
DebugStub.Assert(descrBaseAddr <= baseAddr);
descrBase = descrBaseAddr;
descrLimit = descrLimitAddr;
rangeBase = baseAddr;
rangeLimit = limitAddr;
rangeLock = new SpinLock(SpinLock.Types.VirtualMemoryRange);
describedPages = MemoryManager.PagesFromBytes(descrLimit - descrBase);
// Figure out how many pages we need for a page description table
UIntPtr pageTableSize = MemoryManager.PagePad(describedPages * sizeof(uint));
dataStart = baseAddr + pageTableSize;
nextAlloc = dataStart;
// Commit and prepare the page table
pageTable = (uint *)baseAddr;
bool success = MemoryManager.CommitAndMapRange(
baseAddr, baseAddr + pageTableSize, inDomain);
if (!success)
{
Kernel.Panic("Couldn't get pages to create a new VirtualMemoryRange page table");
}
allocatedBytes = 0;
allocatedCount = 0;
freedBytes = 0;
freedCount = 0;
// Describe the pages outside our range as Unknown
if (descrBase < rangeBase)
{
SetRange(descrBase, rangeBase, MemoryManager.PageUnknown);
}
if (descrLimit > rangeLimit)
{
SetRange(rangeLimit, descrLimit, MemoryManager.PageUnknown);
}
// The page-table pages themselves are in use by the System
SetRange((UIntPtr)pageTable,
(UIntPtr)pageTable + pageTableSize,
MemoryManager.KernelPageNonGC);
// Describe pages in-range as Free
SetRange(dataStart, rangeLimit, MemoryManager.PageFree);
#if DEBUG
// Check that our data range is pristine
for (UIntPtr stepAddr = dataStart;
stepAddr < rangeLimit;
stepAddr += MemoryManager.PageSize)
{
DebugStub.Assert(!VMManager.IsPageMapped(stepAddr));
}
#endif
}
/////////////////////////////////////
// PUBLIC METHODS
/////////////////////////////////////
//
// Initialize() must be called during OS boot *BEFORE* paging
// is enabled, so that physical memory can be accessed
// directly. We consult the system's memory map to pick a
// range of memory where we can situate the physical page
// table, and initialize it.
//
// the reserveSize argument specifies how much physical
// memory to set aside and return a PhysicalAddress to.
// This is used by the MemoryManager to reserve an area
// of contiguous physical memory for use in I/O operations.
//
// After Initialize()ation, the PhysicalPages module
// assumes that its page table will be identity-mapped
// into the kernel's virtual memory space.
//
internal static unsafe PhysicalAddress Initialize(ulong reserveSize)
{
Platform p = Platform.ThePlatform;
UIntPtr lowerAddress = UIntPtr.Zero;
UIntPtr upperAddress = UIntPtr.Zero;
// NB: our initialization is currently naive; we
// set up a table to describe all of physical memory,
// even though there are holes in it for reserved
// memory that we will never be able to use.
// This makes indexing much easier, though.
InitializeLock();
// Retrieve the highest RAM address
MemoryManager.PhysicalMemoryLimits(out lowerAddress, out upperAddress);
addressLimit = (ulong)upperAddress;
DebugStub.WriteLine("Highest usable RAM address is 0x{0:x8}",
__arglist(addressLimit));
// How much room do we need for the pageblock table?
numBlocks = GetNumBlocks(addressLimit);
ulong requiredSpace = (ulong)numBlocks * (ulong)sizeof(PageBlock);
// Now we need to find a location in physical memory
// where we can stick the physical pageblock table.
ulong startAddr = LowerRAMBlackout;
if (p.BootAllocatedMemorySize != 0) {
if (startAddr > p.BootAllocatedMemory &&
startAddr < p.BootAllocatedMemory + p.BootAllocatedMemorySize) {
startAddr = (ulong)p.BootAllocatedMemory + (ulong)p.BootAllocatedMemorySize;
}
}
ulong physLocation = FindFreeBlockInSMAP(startAddr, requiredSpace);
if (physLocation == 0) {
// Failed to find a spot to park the physical page
// table. This is fatal!
Kernel.Panic("Couldn't find enough room to initialize hardware page table");
}
// Initialize all the descriptor blocks as "free" (zeroed)
physTable = (PageBlock*)physLocation;
tableSize = requiredSpace;
freeList = 0; // First descriptor
for (uint i = 0; i < numBlocks; i++) {
PageBlock* thisBlock = GetBlock(i);
thisBlock->Initialize();
thisBlock->prevIdx = (i == 0) ? PageBlock.Sentinel : i - 1;
thisBlock->nextIdx = (i == numBlocks - 1) ? PageBlock.Sentinel : i + 1;
}
// Mark the blackout area of low physical memory as used
MarkRangeUsed(0, startAddr - 1);
// Now mark the range of physical pages occupied by the
// hardware table itself as being used!
MarkRangeUsed((ulong)physTable, requiredSpace);
// Mark any non-Free areas (according to SMAP) as in use
SMAPINFO* smap = p.Smap;
for (uint i = 0; i < p.SmapCount; i++) {
if ((smap[i].type != (ulong)SMAPINFO.AddressType.Free) &&
((smap[i].addr + smap[i].size) > startAddr) &&
(smap[i].addr < addressLimit)) {
ulong unaccountedStart, unaccountedLength;
if (smap[i].addr >= startAddr) {
unaccountedStart = smap[i].addr;
unaccountedLength = smap[i].size;
}
else {
// Part of this memory window is already accounted for
unaccountedStart = startAddr;
unaccountedLength = smap[i].size - (startAddr - smap[i].addr);
}
MarkRangeUsed(unaccountedStart, unaccountedLength);
}
}
// Mark out all the Platform special regions as busy
if (p.MiniDumpLimit != 0) {
ulong miniDumpSize = (ulong)p.MiniDumpLimit - (ulong)p.MiniDumpBase;
MarkRangeUsed((ulong)p.MiniDumpBase,
MemoryManager.PagePad(miniDumpSize));
}
if (p.KernelDllSize != 0) {
MarkRangeUsed((ulong)p.KernelDllBase,
MemoryManager.PagePad((ulong)p.KernelDllSize));
}
if (p.BootAllocatedMemorySize != 0) {
MarkRangeUsed((ulong)p.BootAllocatedMemory,
MemoryManager.PagePad((ulong)p.BootAllocatedMemorySize));
}
// Last step: find an available area to situate the caller's
// requested reserve-block, if any.
PhysicalAddress reservedPtr = PhysicalAddress.Null;
if (reserveSize != 0) {
reservedPtr = new PhysicalAddress(
FindFreeBlockInSMAP(
MemoryManager.PagePad((ulong)physTable + tableSize),
reserveSize));
if (reservedPtr == PhysicalAddress.Null) {
Kernel.Panic("Couldn't find enough physically contiguous memory to reserve");
}
if (reservedPtr.Value + reserveSize > 0xFFFFFF) {
Kernel.Panic("Couldn't find enough physically contiguous memory below 0xFFFFFF for I/O memory");
}
}
// Mark the reserved block as used. It's up to the caller
// to administer its use.
MarkRangeUsed(reservedPtr.Value, reserveSize);
CheckConsistency();
DebugStub.WriteLine("PhysicalPages initialized at 0x{0:x8} - 0x{1:x8} with {2} physical pages available.",
__arglist(reservedPtr.Value,
reservedPtr.Value + reserveSize,
GetFreePageCount()));
return reservedPtr;
}
internal static unsafe StackHead *GetStackSegmentRaw(UIntPtr size,
ref ThreadContext context,
bool kernelAllocation,
bool initialStack)
{
// Allocate a new chunk, making room for StackHead at the top.
// If you change these constants to add more data, see the
// comment about InitialStackSize at the top of this file!
#if DO_TRACE_STACKS
Kernel.Waypoint(667);
#endif
if (size == UIntPtr.Zero)
{
size = InitialStackSize;
}
size = MemoryManager.PagePad(size + sizeof(StackHead) + SafetyBufferSize);
UIntPtr chunk;
Process owner = Process.GetProcessByID(context.processId);
//
//// NOTE: here's where we should be clever about
//// whether to allocate a stack chunk in the user range
//// or the kernel range. Except, if we switch contexts
//// during an ABI call while using a user-range stack
//// segment on a paging machine, we die. Gloss over
//// this hackily by always getting stack segments
//// from the kernel range.
//if (kernelAllocation || (owner == Process.kernelProcess)) {
// chunk = MemoryManager.KernelAllocate(
// MemoryManager.PagesFromBytes(size), owner, 0, PageType.Stack);
//}
//else {
// chunk = MemoryManager.UserAllocate(
// MemoryManager.PagesFromBytes(size), owner, 0, PageType.Stack);
//}
//
UIntPtr pageCount = MemoryManager.PagesFromBytes(size);
#if DEBUG_STACK_VERBOSE
fixed(ThreadContext *ptr = &context)
{
Tracing.Log(Tracing.Debug,
"GetStackSegmentRaw(ctx={0:x8},size={1:d}) pages={2} [{3:x8}..{4:x8}]",
(UIntPtr)ptr, size, pageCount,
context.stackLimit, context.stackBegin);
}
#endif
chunk = MemoryManager.StackAllocate(pageCount, owner, 0, kernelAllocation, initialStack);
if (chunk != UIntPtr.Zero)
{
// NB: We do _not_ zero out stack memory!
// We assume that Bartok prevents access to prev contents.
StackHead *head = (StackHead *)(chunk + size - sizeof(StackHead));
context.stackBegin = chunk + size;
context.stackLimit = chunk + SafetyBufferSize;
#if DEBUG_STACK_VERBOSE
Tracing.Log(Tracing.Debug,
"GetStackSegmentRaw(size={0:d}) -> [{1:x8}..{2:x8}]",
size, context.stackLimit, context.stackBegin);
#endif
return(head);
}
else
{
// Stack allocation failed. In the future, we should
// trigger a kernel exception; for now, we break to the
// debugger.
#if DEBUG_STACK_VERBOSE
Tracing.Log(Tracing.Debug,
"GetStackSegmentRaw: KernelAllocate failed!(siz={0:d})",
size);
#endif
//DebugStub.Break();
return(null);
}
}