public unsafe Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
if (pages > _FreePages)
{
Panic.Error("Out of Memory");
return(null);
}
Page *page;
if (AddressSpaceKind == AddressSpaceKind.Virtual || options.Continuous)
{
page = AllocatePagesContinuous(pages, options);
}
else
{
page = AllocatePagesNormal(pages, options);
}
if (page == null)
{
KernelMessage.WriteLine("DebugName: {0}", DebugName);
KernelMessage.WriteLine("Free pages: {0:X8}, Requested: {1:X8}", FreePages, pages);
Panic.Error("Out of Memory");
}
KernelMessage.Path(DebugName, "SimpleAlloc: Request {0} Pages, Addr {1:X8}", pages, GetAddress(page));
return(page);
}
public static MemoryRegion AllocateRegion(USize size, AllocatePageOptions options = default)
{
var pages = KMath.DivCeil(size, 4096);
var start = AllocatePages(pages, options);
return(new MemoryRegion(start, pages * 4096));
}
public static Page *AllocatePage(AllocatePageOptions options = default)
{
var p = Default.AllocatePage(options);
//if (p->PhysicalAddress == 0x01CA4000)
// Panic.Error("DEBUG-MARKER");
return(p);
}
private unsafe Page *AllocatePagesContinuous(uint pages, AllocatePageOptions options = default)
{
for (var i = 0; i < _TotalPages; i++)
{
if (i + pages >= _TotalPages)
{
return(null);
}
var head = &Pages[i];
if (!InUse(head))
{
var foundContinuous = true;
for (var n = 1; n < pages; n++)
{
if (InUse(&Pages[++i]))
{
foundContinuous = false;
break;
}
}
if (!foundContinuous)
{
continue;
}
var p = head;
for (var n = 0; n < pages; n++)
{
SetInUse(p);
if (n == pages - 1)
{
SetNext(p, null);
}
else
{
SetNext(p, p + 1);
}
p++;
}
_FreePages -= pages;
return(head);
}
}
return(null);
}
public Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
var configuredMaxPagesPerAllocation = 64;
var maxAllocation = Math.Min(configuredMaxPagesPerAllocation, Allocators[0].MaxPagesPerAllocation);
if (pages <= maxAllocation && Allocators[0].FreePages > Allocators[0].CriticalLowPages)
{
return(Allocators[0].AllocatePages(pages, options));
}
else
{
return(Allocators[1].AllocatePages(pages, options));
}
}
private static unsafe Addr AllocateGlobalPages(uint pages, AllocatePageOptions options = default)
{
if (AddProtectedRegions)
{
pages += 2;
}
//lock (lockObj)
//{
// LockCount++;
var physHead = PhysicalPageManager.AllocatePages(pages, options);
if (physHead == null)
{
return(Addr.Zero);
}
var virtHead = GlobalAllocator.AllocatePages(pages, options);
var p = physHead;
var v = virtHead;
for (var i = 0; i < pages; i++)
{
var map = true;
if (AddProtectedRegions && (i == 0 || i == pages - 1))
{
map = false;
}
if (map)
{
PageTable.KernelTable.Map(GlobalAllocator.GetAddress(v), PhysicalPageManager.GetAddress(p));
}
p = PhysicalPageManager.NextCompoundPage(p);
v = GlobalAllocator.NextCompoundPage(v);
}
PageTable.KernelTable.Flush();
if (AddProtectedRegions)
{
virtHead = GlobalAllocator.NextCompoundPage(virtHead);
}
//LockCount--;
return(GlobalAllocator.GetAddress(virtHead));
//}
}
public Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
Page *page = AllocateInternal(pages, options);
if (page == null)
{
//KernelMessage.WriteLine("DebugName: {0}", DebugName);
KernelMessage.WriteLine("Free pages: {0:X8}, Requested: {1:X8}", FreePages, pages);
Panic.Error("Out of Memory");
}
if (FreePages < 1000)
{
KernelMessage.Path(DebugName, "WARNING: Low pages. Available: {0}", FreePages);
}
return(page);
}
public static unsafe Addr AllocatePages(uint pages, AllocatePageOptions options = default)
{
switch (options.Pool)
{
case PageAllocationPool.Normal:
return(AllocatePagesNormal(pages, options));
case PageAllocationPool.Identity:
return(AllocateIdentityMappedPages(pages, options));
case PageAllocationPool.Global:
return(AllocateGlobalPages(pages, options));
default:
Panic.Error("invalid pool");
break;
}
return(Addr.Zero);
}
/// <summary>
/// Returns pages, where virtAddr equals physAddr.
/// </summary>
private static unsafe Addr AllocateIdentityMappedPages(uint pages, AllocatePageOptions options = default)
{
if (AddProtectedRegions)
{
pages += 2;
}
var virtHead = IdentityAllocator.AllocatePages(pages, options);
var v = virtHead;
for (var i = 0; i < pages; i++)
{
var addr = IdentityAllocator.GetAddress(v);
var map = true;
if (AddProtectedRegions && (i == 0 || i == pages - 1))
{
map = false;
}
if (map)
{
PageTable.KernelTable.Map(addr, addr);
}
v = IdentityAllocator.NextCompoundPage(v);
}
PageTable.KernelTable.Flush();
if (AddProtectedRegions)
{
virtHead = IdentityAllocator.NextCompoundPage(virtHead);
}
return(IdentityAllocator.GetAddress(virtHead));
}
private unsafe Page *AllocatePagesNormal(uint pages, AllocatePageOptions options = default)
{
Page *prevHead = null;
Page *firstHead = null;
var pagesFound = 0;
for (var i = 0; i < _TotalPages; i++)
{
var page = &Pages[i];
if (!InUse(page))
{
if (firstHead == null)
{
firstHead = page;
}
if (prevHead != null)
{
SetNext(prevHead, page);
}
SetInUse(page);
pagesFound++;
_FreePages--;
if (pagesFound >= pages)
{
return(firstHead);
}
prevHead = page;
}
}
return(null);
}
private Page *AllocateInternal(uint pages, AllocatePageOptions options = default)
{
if (KConfig.Log.PageAllocation && TraceOptions.Enabled && pages >= TraceOptions.MinPages)
{
KernelMessage.Path(DebugName, "Requesting Pages: {1}. Available: {2} DebugName={0}", options.DebugName, pages, _FreePages);
}
if (pages == 256)
{
Debug.Nop();
}
UninterruptableMonitor.Enter(this);
try
{
SelfCheck("SC1");
if (pages > 1 && (AddressSpaceKind == AddressSpaceKind.Virtual || options.Continuous))
{
if (!MoveToFreeContinuous(pages))
{
// Compact
//KernelMessage.Path(DebugName, "Compacting Linked List");
//this.DumpPages();
BuildLinkedLists();
if (!MoveToFreeContinuous(pages))
{
this.DumpPages();
KernelMessage.WriteLine("Requesting {0} pages failed", pages);
Panic.Error("Requesting pages failed: out of memory");
}
}
}
// ---
var head = FreeList;
var headPage = (Page *)head;
FreeList = head->next;
list_head.list_del_init(head);
headPage->Status = PageStatus.Used;
if (KConfig.Log.PageAllocation)
{
if (options.DebugName != null)
{
headPage->DebugTag = (uint)Intrinsic.GetObjectAddress(options.DebugName);
}
else
{
headPage->DebugTag = null;
}
}
_FreePages--;
// ---
for (var i = 1; i < pages; i++)
{
var tmpNextFree = FreeList->next;
list_head.list_move_tail(FreeList, head);
var p = (Page *)FreeList;
if (p->Status == PageStatus.Used)
{
this.DumpPages();
this.DumpPage(p);
KernelMessage.Path(DebugName, "Double Alloc pages={0} allocs={1} free={2} ptr={3:X8}", pages, (uint)_Requests, _FreePages, (uint)p);
Panic.Error("Double Alloc");
}
p->Status = PageStatus.Used;
FreeList = tmpNextFree;
_FreePages--;
}
if (KConfig.Log.PageAllocation && TraceOptions.Enabled && pages >= TraceOptions.MinPages)
{
KernelMessage.Path(DebugName, "Allocation done. Addr: {0:X8} Available: {1}", GetAddress(headPage), _FreePages);
}
_Requests++;
CheckAllocation(headPage, pages);
SelfCheck("SC2");
return(headPage);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public static MemoryRegion AllocateRegion(this IPageFrameAllocator allocator, USize size, AllocatePageOptions options = default)
{
var pages = KMath.DivCeil(size, 4096);
var p = allocator.AllocatePages(pages, options);
return(new MemoryRegion(allocator.GetAddress(p), pages * 4096));
}
public static MemoryRegion AllocateRegion(USize size, AllocatePageOptions options = default)
{
return(Default.AllocateRegion(size, options));
}
public static Addr AllocatePageAddr(this IPageFrameAllocator allocator, AllocatePageOptions options = default)
{
return(allocator.GetAddress(allocator.AllocatePage(options)));
}
public Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
return(Allocate(pages));
}
public Page *AllocatePage(AllocatePageOptions options = default)
{
return(AllocatePages(1, options));
}
public static Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
return(Default.AllocatePages(pages, options));
}
public Page *AllocatePage(AllocatePageOptions options = default)
{
return(Allocators[0].AllocatePage(options));
}
public Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
lock (this)
{
if (pages == 0)
{
KernelMessage.WriteLine("Requesting zero pages");
return(null);
}
else if (pages > 1 && KConfig.Log.PageAllocation)
{
KernelMessage.WriteLine("Requesting {0} pages", pages);
}
//KernelMessage.WriteLine("Request {0} pages...", num);
uint statBlocks = 0;
uint statFreeBlocks = 0;
int statMaxBlockPages = 0;
uint statRangeChecks = 0;
uint cnt = 0;
if (NextTryPage == null)
{
NextTryPage = PageArray;
}
Page *p = NextTryPage;
while (true)
{
statBlocks++;
if (p == null)
{
p = PageArray;
}
if (p->Status == PageStatus.Free)
{
statFreeBlocks++;
var head = p;
// Found free Page. Check now free range.
for (var i = 0; i < pages; i++)
{
statRangeChecks++;
statMaxBlockPages = Math.Max(statMaxBlockPages, i);
if (p == null)
{
break; // Reached end. Our Range is incomplete
}
if (p->Status != PageStatus.Free) // Used -> so we can abort the search
{
break;
}
if (i == pages - 1)
{ // all loops successful. So we found our range.
if (p == null)
{
Panic.Error("Tail is null");
}
head->Tail = p;
head->PagesUsed = pages;
p = head;
for (var n = 0; n < pages; n++)
{
if (p->Status != PageStatus.Free)
{
Panic.Error("Page is not Free. PageFrame Array corrupted?");
}
p->Status = PageStatus.Used;
p->Head = head;
p->Tail = head->Tail;
p = NextPage(p);
_FreePages--;
}
// correct version:
NextTryPage = p;
// TODO: HACK! Currently, we have somewhere a buffer overrun? Fix that!
//NextTryPage = p + 1;
//var t = head->Tail;
//var a = t->Address;
//var anum = (uint)a;
////(uint)head->Tail->Address + 4096 - 1
//KernelMessage.Write("<");
//KernelMessage.WriteLine("Allocated from {0:X8} to {1:X8}, Status={2}", (uint)head->Address, anum, (uint)head->Status);
//KernelMessage.Write(">");
//if (head->PhysicalAddress == 0x01CA4000)
//{
// KernelMessage.WriteLine("DEBUG-MARKER 2");
// DumpPage(head);
//}
return(head);
}
p = NextPage(p);
}
}
if (p->Tail != null)
{
p = p->Tail;
}
p = NextPage(p);
if (++cnt > _TotalPages)
{
break;
}
}
KernelMessage.WriteLine("Blocks={0} FreeBlocks={1} MaxBlockPages={2} RangeChecks={3} cnt={4}", statBlocks, statFreeBlocks, (uint)statMaxBlockPages, statRangeChecks, cnt);
this.DumpPages();
Panic.Error("PageFrameAllocator: Could not allocate " + pages + " Pages.");
return(null);
}
}
public static Addr AllocatePageAddr(uint pages, AllocatePageOptions options = default)
{
return(Default.AllocatePagesAddr(pages, options));
}
public Page *AllocatePage(AllocatePageOptions options = default)
{
var p = Allocate(1);
return(p);
}
public static Addr AllocatePageAddr(AllocatePageOptions options = default)
{
return(Default.AllocatePageAddr(options));
}