private void InitializeMemoryManager(AddressSpaceType addrSpaceType, MemoryRegion memRegion)
{
int addrSpaceBits;
switch (addrSpaceType)
{
case AddressSpaceType.Addr32Bits: addrSpaceBits = 32; break;
case AddressSpaceType.Addr36Bits: addrSpaceBits = 36; break;
case AddressSpaceType.Addr32BitsNoMap: addrSpaceBits = 32; break;
case AddressSpaceType.Addr39Bits: addrSpaceBits = 39; break;
default: throw new ArgumentException(nameof(addrSpaceType));
}
bool useFlatPageTable = memRegion == MemoryRegion.Application;
CpuMemory = new MemoryManager(_system.Device.Memory.RamPointer, addrSpaceBits, useFlatPageTable);
MemoryManager = new KMemoryManager(_system, CpuMemory);
Translator = new Translator(CpuMemory);
Translator.CpuTrace += CpuTraceHandler;
}
private void InitializeMemoryManager(AddressSpaceType addrSpaceType, MemoryRegion memRegion)
{
int addrSpaceBits;
switch (addrSpaceType)
{
case AddressSpaceType.Addr32Bits: addrSpaceBits = 32; break;
case AddressSpaceType.Addr36Bits: addrSpaceBits = 36; break;
case AddressSpaceType.Addr32BitsNoMap: addrSpaceBits = 32; break;
case AddressSpaceType.Addr39Bits: addrSpaceBits = 39; break;
default: throw new ArgumentException(nameof(addrSpaceType));
}
bool useFlatPageTable = memRegion == MemoryRegion.Application;
if (_system.UseLegacyJit)
{
CpuMemory = new ChocolArm64.Memory.MemoryManager(_system.Device.Memory.RamPointer, addrSpaceBits, useFlatPageTable);
Translator = new ChocolArm64.Translation.Translator((ChocolArm64.Memory.MemoryManager)CpuMemory);
}
else
{
CpuMemory = new MemoryManager(_system.Device.Memory.RamPointer, addrSpaceBits, useFlatPageTable);
Translator = new Translator((MemoryManager)CpuMemory);
}
MemoryManager = new KMemoryManager(_system, CpuMemory);
}
/// <summary>
/// Creates a set of instructions to realize a generic lea operation.
/// </summary>
/// <param name="sourceType">The source address type (pointer or view).</param>
/// <param name="elementIndex">The current element index (the offset).</param>
/// <param name="targetAddressRegister">The allocated target pointer register to write to.</param>
/// <param name="address">The source address.</param>
private void MakeLoadElementAddress(
AddressSpaceType sourceType,
PrimitiveRegister elementIndex,
PrimitiveRegister targetAddressRegister,
PrimitiveRegister address)
{
var elementSize = ABI.GetSizeOf(sourceType.ElementType);
var offsetRegister = AllocatePlatformRegister(out RegisterDescription _);
using (var command = BeginCommand(
PTXInstructions.GetLEAMulOperation(ABI.PointerArithmeticType)))
{
command.AppendArgument(offsetRegister);
command.AppendArgument(elementIndex);
command.AppendConstant(elementSize);
}
using (var command = BeginCommand(
PTXInstructions.GetArithmeticOperation(
BinaryArithmeticKind.Add,
ABI.PointerArithmeticType,
false)))
{
command.AppendArgument(targetAddressRegister);
command.AppendArgument(address);
command.AppendArgument(offsetRegister);
}
FreeRegister(offsetRegister);
}
/// <summary>
/// Constructs a new cast value.
/// </summary>
/// <param name="basicBlock">The parent basic block.</param>
/// <param name="value">The value to convert.</param>
/// <param name="targetType">The target type to convert the value to.</param>
internal BaseAddressSpaceCast(
BasicBlock basicBlock,
ValueReference value,
AddressSpaceType targetType)
: base(basicBlock, value, targetType)
{
}
private void InitializeMemoryManager(AddressSpaceType addrSpaceType, MemoryRegion memRegion)
{
int addrSpaceBits;
switch (addrSpaceType)
{
case AddressSpaceType.Addr32Bits: addrSpaceBits = 32; break;
case AddressSpaceType.Addr36Bits: addrSpaceBits = 36; break;
case AddressSpaceType.Addr32BitsNoMap: addrSpaceBits = 32; break;
case AddressSpaceType.Addr39Bits: addrSpaceBits = 39; break;
default: throw new ArgumentException(nameof(addrSpaceType));
}
bool useFlatPageTable = memRegion == MemoryRegion.Application;
CpuMemory = new MemoryManager(_system.Device.Memory.RamPointer, addrSpaceBits, useFlatPageTable);
Translator = new Translator(CpuMemory);
// TODO: This should eventually be removed.
// The GPU shouldn't depend on the CPU memory manager at all.
_system.Device.Gpu.SetVmm(CpuMemory);
MemoryManager = new KMemoryManager(_system, CpuMemory);
}
private void InitializeMemoryManager(AddressSpaceType addrSpaceType, MemoryRegion memRegion)
{
int addrSpaceBits = addrSpaceType switch
{
AddressSpaceType.Addr32Bits => 32,
AddressSpaceType.Addr36Bits => 36,
AddressSpaceType.Addr32BitsNoMap => 32,
AddressSpaceType.Addr39Bits => 39,
_ => throw new ArgumentException(nameof(addrSpaceType))
};
CpuMemory = new MemoryManager(KernelContext.Memory, 1UL << addrSpaceBits, InvalidAccessHandler);
CpuContext = new CpuContext(CpuMemory);
// TODO: This should eventually be removed.
// The GPU shouldn't depend on the CPU memory manager at all.
KernelContext.Device.Gpu.SetVmm(CpuMemory);
MemoryManager = new KMemoryManager(KernelContext, CpuMemory);
}
/// <summary>
/// Specializes the address space of the given <see cref="AddressSpaceType"/>.
/// </summary>
/// <param name="addressSpaceType">The source type.</param>
/// <param name="addressSpace">The new address space.</param>
/// <returns>The created specialzized <see cref="AddressSpaceType"/>.</returns>
public AddressSpaceType SpecializeAddressSpaceType(
AddressSpaceType addressSpaceType,
MemoryAddressSpace addressSpace) =>
TypeContext.SpecializeAddressSpaceType(addressSpaceType, addressSpace);
public KernelResult Initialize(
ProcessCreationInfo creationInfo,
int[] caps,
KResourceLimit resourceLimit,
MemoryRegion memRegion)
{
ResourceLimit = resourceLimit;
_memRegion = memRegion;
ulong personalMmHeapSize = GetPersonalMmHeapSize((ulong)creationInfo.PersonalMmHeapPagesCount, memRegion);
ulong codePagesCount = (ulong)creationInfo.CodePagesCount;
ulong neededSizeForProcess = personalMmHeapSize + codePagesCount * KMemoryManager.PageSize;
if (neededSizeForProcess != 0 && resourceLimit != null)
{
if (!resourceLimit.Reserve(LimitableResource.Memory, neededSizeForProcess))
{
return(KernelResult.ResLimitExceeded);
}
}
void CleanUpForError()
{
if (neededSizeForProcess != 0 && resourceLimit != null)
{
resourceLimit.Release(LimitableResource.Memory, neededSizeForProcess);
}
}
PersonalMmHeapPagesCount = (ulong)creationInfo.PersonalMmHeapPagesCount;
KMemoryBlockAllocator memoryBlockAllocator;
if (PersonalMmHeapPagesCount != 0)
{
memoryBlockAllocator = new KMemoryBlockAllocator(PersonalMmHeapPagesCount * KMemoryManager.PageSize);
}
else
{
memoryBlockAllocator = (MmuFlags & 0x40) != 0
? System.LargeMemoryBlockAllocator
: System.SmallMemoryBlockAllocator;
}
AddressSpaceType addrSpaceType = (AddressSpaceType)((creationInfo.MmuFlags >> 1) & 7);
InitializeMemoryManager(addrSpaceType, memRegion);
bool aslrEnabled = ((creationInfo.MmuFlags >> 5) & 1) != 0;
ulong codeAddress = creationInfo.CodeAddress;
ulong codeSize = codePagesCount * KMemoryManager.PageSize;
KernelResult result = MemoryManager.InitializeForProcess(
addrSpaceType,
aslrEnabled,
!aslrEnabled,
memRegion,
codeAddress,
codeSize,
memoryBlockAllocator);
if (result != KernelResult.Success)
{
CleanUpForError();
return(result);
}
if (!ValidateCodeAddressAndSize(codeAddress, codeSize))
{
CleanUpForError();
return(KernelResult.InvalidMemRange);
}
result = MemoryManager.MapNewProcessCode(
codeAddress,
codePagesCount,
MemoryState.CodeStatic,
MemoryPermission.None);
if (result != KernelResult.Success)
{
CleanUpForError();
return(result);
}
result = Capabilities.InitializeForUser(caps, MemoryManager);
if (result != KernelResult.Success)
{
CleanUpForError();
return(result);
}
Pid = System.GetProcessId();
if (Pid == -1 || (ulong)Pid < Horizon.InitialProcessId)
{
throw new InvalidOperationException($"Invalid Process Id {Pid}.");
}
result = ParseProcessInfo(creationInfo);
if (result != KernelResult.Success)
{
CleanUpForError();
}
return(result);
}
public KernelResult InitializeKip(
ProcessCreationInfo creationInfo,
int[] caps,
KPageList pageList,
KResourceLimit resourceLimit,
MemoryRegion memRegion)
{
ResourceLimit = resourceLimit;
_memRegion = memRegion;
AddressSpaceType addrSpaceType = (AddressSpaceType)((creationInfo.MmuFlags >> 1) & 7);
InitializeMemoryManager(addrSpaceType, memRegion);
bool aslrEnabled = ((creationInfo.MmuFlags >> 5) & 1) != 0;
ulong codeAddress = creationInfo.CodeAddress;
ulong codeSize = (ulong)creationInfo.CodePagesCount * KMemoryManager.PageSize;
KMemoryBlockAllocator memoryBlockAllocator = (MmuFlags & 0x40) != 0
? System.LargeMemoryBlockAllocator
: System.SmallMemoryBlockAllocator;
KernelResult result = MemoryManager.InitializeForProcess(
addrSpaceType,
aslrEnabled,
!aslrEnabled,
memRegion,
codeAddress,
codeSize,
memoryBlockAllocator);
if (result != KernelResult.Success)
{
return(result);
}
if (!ValidateCodeAddressAndSize(codeAddress, codeSize))
{
return(KernelResult.InvalidMemRange);
}
result = MemoryManager.MapPages(
codeAddress,
pageList,
MemoryState.CodeStatic,
MemoryPermission.None);
if (result != KernelResult.Success)
{
return(result);
}
result = Capabilities.InitializeForKernel(caps, MemoryManager);
if (result != KernelResult.Success)
{
return(result);
}
Pid = System.GetKipId();
if (Pid == 0 || (ulong)Pid >= Horizon.InitialProcessId)
{
throw new InvalidOperationException($"Invalid KIP Id {Pid}.");
}
result = ParseProcessInfo(creationInfo);
return(result);
}
public KernelResult Initialize(
ProcessCreationInfo creationInfo,
ReadOnlySpan <int> capabilities,
KResourceLimit resourceLimit,
MemoryRegion memRegion,
IProcessContextFactory contextFactory,
ThreadStart customThreadStart = null)
{
ResourceLimit = resourceLimit;
_memRegion = memRegion;
_contextFactory = contextFactory ?? new ProcessContextFactory();
_customThreadStart = customThreadStart;
ulong personalMmHeapSize = GetPersonalMmHeapSize((ulong)creationInfo.SystemResourcePagesCount, memRegion);
ulong codePagesCount = (ulong)creationInfo.CodePagesCount;
ulong neededSizeForProcess = personalMmHeapSize + codePagesCount * KMemoryManager.PageSize;
if (neededSizeForProcess != 0 && resourceLimit != null)
{
if (!resourceLimit.Reserve(LimitableResource.Memory, neededSizeForProcess))
{
return(KernelResult.ResLimitExceeded);
}
}
void CleanUpForError()
{
if (neededSizeForProcess != 0 && resourceLimit != null)
{
resourceLimit.Release(LimitableResource.Memory, neededSizeForProcess);
}
}
PersonalMmHeapPagesCount = (ulong)creationInfo.SystemResourcePagesCount;
KMemoryBlockAllocator memoryBlockAllocator;
if (PersonalMmHeapPagesCount != 0)
{
memoryBlockAllocator = new KMemoryBlockAllocator(PersonalMmHeapPagesCount * KMemoryManager.PageSize);
}
else
{
memoryBlockAllocator = creationInfo.Flags.HasFlag(ProcessCreationFlags.IsApplication)
? KernelContext.LargeMemoryBlockAllocator
: KernelContext.SmallMemoryBlockAllocator;
}
AddressSpaceType addrSpaceType = (AddressSpaceType)((int)(creationInfo.Flags & ProcessCreationFlags.AddressSpaceMask) >> (int)ProcessCreationFlags.AddressSpaceShift);
InitializeMemoryManager(creationInfo.Flags);
bool aslrEnabled = creationInfo.Flags.HasFlag(ProcessCreationFlags.EnableAslr);
ulong codeAddress = creationInfo.CodeAddress;
ulong codeSize = codePagesCount * KMemoryManager.PageSize;
KernelResult result = MemoryManager.InitializeForProcess(
addrSpaceType,
aslrEnabled,
!aslrEnabled,
memRegion,
codeAddress,
codeSize,
memoryBlockAllocator);
if (result != KernelResult.Success)
{
CleanUpForError();
return(result);
}
if (!MemoryManager.CanContain(codeAddress, codeSize, MemoryState.CodeStatic))
{
CleanUpForError();
return(KernelResult.InvalidMemRange);
}
result = MemoryManager.MapNewProcessCode(
codeAddress,
codePagesCount,
MemoryState.CodeStatic,
KMemoryPermission.None);
if (result != KernelResult.Success)
{
CleanUpForError();
return(result);
}
result = Capabilities.InitializeForUser(capabilities, MemoryManager);
if (result != KernelResult.Success)
{
CleanUpForError();
return(result);
}
Pid = KernelContext.NewProcessId();
if (Pid == -1 || (ulong)Pid < KernelConstants.InitialProcessId)
{
throw new InvalidOperationException($"Invalid Process Id {Pid}.");
}
result = ParseProcessInfo(creationInfo);
if (result != KernelResult.Success)
{
CleanUpForError();
}
return(result);
}
public KernelResult InitializeKip(
ProcessCreationInfo creationInfo,
ReadOnlySpan <int> capabilities,
KPageList pageList,
KResourceLimit resourceLimit,
MemoryRegion memRegion,
IProcessContextFactory contextFactory,
ThreadStart customThreadStart = null)
{
ResourceLimit = resourceLimit;
_memRegion = memRegion;
_contextFactory = contextFactory ?? new ProcessContextFactory();
_customThreadStart = customThreadStart;
AddressSpaceType addrSpaceType = (AddressSpaceType)((int)(creationInfo.Flags & ProcessCreationFlags.AddressSpaceMask) >> (int)ProcessCreationFlags.AddressSpaceShift);
InitializeMemoryManager(creationInfo.Flags);
bool aslrEnabled = creationInfo.Flags.HasFlag(ProcessCreationFlags.EnableAslr);
ulong codeAddress = creationInfo.CodeAddress;
ulong codeSize = (ulong)creationInfo.CodePagesCount * KMemoryManager.PageSize;
KMemoryBlockAllocator memoryBlockAllocator = creationInfo.Flags.HasFlag(ProcessCreationFlags.IsApplication)
? KernelContext.LargeMemoryBlockAllocator
: KernelContext.SmallMemoryBlockAllocator;
KernelResult result = MemoryManager.InitializeForProcess(
addrSpaceType,
aslrEnabled,
!aslrEnabled,
memRegion,
codeAddress,
codeSize,
memoryBlockAllocator);
if (result != KernelResult.Success)
{
return(result);
}
if (!MemoryManager.CanContain(codeAddress, codeSize, MemoryState.CodeStatic))
{
return(KernelResult.InvalidMemRange);
}
result = MemoryManager.MapPages(
codeAddress,
pageList,
MemoryState.CodeStatic,
KMemoryPermission.None);
if (result != KernelResult.Success)
{
return(result);
}
result = Capabilities.InitializeForKernel(capabilities, MemoryManager);
if (result != KernelResult.Success)
{
return(result);
}
Pid = KernelContext.NewKipId();
if (Pid == 0 || (ulong)Pid >= KernelConstants.InitialProcessId)
{
throw new InvalidOperationException($"Invalid KIP Id {Pid}.");
}
return(ParseProcessInfo(creationInfo));
}
public KMemoryManager(Process Process)
{
CpuMemory = Process.Memory;
Allocator = Process.Device.Memory.Allocator;
long CodeRegionSize;
long MapRegionSize;
long HeapRegionSize;
long NewMapRegionSize;
long TlsIoRegionSize;
int AddrSpaceWidth;
AddressSpaceType AddrType = AddressSpaceType.Addr39Bits;
if (Process.MetaData != null)
{
AddrType = (AddressSpaceType)Process.MetaData.AddressSpaceWidth;
}
switch (AddrType)
{
case AddressSpaceType.Addr32Bits:
CodeRegionStart = 0x200000;
CodeRegionSize = 0x3fe00000;
MapRegionSize = 0x40000000;
HeapRegionSize = 0x40000000;
NewMapRegionSize = 0;
TlsIoRegionSize = 0;
AddrSpaceWidth = 32;
break;
case AddressSpaceType.Addr36Bits:
CodeRegionStart = 0x8000000;
CodeRegionSize = 0x78000000;
MapRegionSize = 0x180000000;
HeapRegionSize = 0x180000000;
NewMapRegionSize = 0;
TlsIoRegionSize = 0;
AddrSpaceWidth = 36;
break;
case AddressSpaceType.Addr36BitsNoMap:
CodeRegionStart = 0x200000;
CodeRegionSize = 0x3fe00000;
MapRegionSize = 0;
HeapRegionSize = 0x80000000;
NewMapRegionSize = 0;
TlsIoRegionSize = 0;
AddrSpaceWidth = 36;
break;
case AddressSpaceType.Addr39Bits:
CodeRegionStart = 0x8000000;
CodeRegionSize = 0x80000000;
MapRegionSize = 0x1000000000;
HeapRegionSize = 0x180000000;
NewMapRegionSize = 0x80000000;
TlsIoRegionSize = 0x1000000000;
AddrSpaceWidth = 39;
break;
default: throw new InvalidOperationException();
}
AddrSpaceStart = 0;
AddrSpaceEnd = 1L << AddrSpaceWidth;
CodeRegionEnd = CodeRegionStart + CodeRegionSize;
MapRegionStart = CodeRegionEnd;
MapRegionEnd = CodeRegionEnd + MapRegionSize;
HeapRegionStart = MapRegionEnd;
HeapRegionEnd = MapRegionEnd + HeapRegionSize;
NewMapRegionStart = HeapRegionEnd;
NewMapRegionEnd = HeapRegionEnd + NewMapRegionSize;
TlsIoRegionStart = NewMapRegionEnd;
TlsIoRegionEnd = NewMapRegionEnd + TlsIoRegionSize;
CurrentHeapAddr = HeapRegionStart;
if (NewMapRegionSize == 0)
{
NewMapRegionStart = AddrSpaceStart;
NewMapRegionEnd = AddrSpaceEnd;
}
Blocks = new LinkedList <KMemoryBlock>();
long AddrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize;
InsertBlock(AddrSpaceStart, AddrSpacePagesCount, MemoryState.Unmapped);
}
public KernelResult Initialize(
ProcessCreationInfo CreationInfo,
int[] Caps,
KResourceLimit ResourceLimit,
MemoryRegion MemRegion)
{
this.ResourceLimit = ResourceLimit;
this.MemRegion = MemRegion;
ulong PersonalMmHeapSize = GetPersonalMmHeapSize((ulong)CreationInfo.PersonalMmHeapPagesCount, MemRegion);
ulong CodePagesCount = (ulong)CreationInfo.CodePagesCount;
ulong NeededSizeForProcess = PersonalMmHeapSize + CodePagesCount * KMemoryManager.PageSize;
if (NeededSizeForProcess != 0 && ResourceLimit != null)
{
if (!ResourceLimit.Reserve(LimitableResource.Memory, NeededSizeForProcess))
{
return(KernelResult.ResLimitExceeded);
}
}
void CleanUpForError()
{
if (NeededSizeForProcess != 0 && ResourceLimit != null)
{
ResourceLimit.Release(LimitableResource.Memory, NeededSizeForProcess);
}
}
PersonalMmHeapPagesCount = (ulong)CreationInfo.PersonalMmHeapPagesCount;
KMemoryBlockAllocator MemoryBlockAllocator;
if (PersonalMmHeapPagesCount != 0)
{
MemoryBlockAllocator = new KMemoryBlockAllocator(PersonalMmHeapPagesCount * KMemoryManager.PageSize);
}
else
{
MemoryBlockAllocator = (MmuFlags & 0x40) != 0
? System.LargeMemoryBlockAllocator
: System.SmallMemoryBlockAllocator;
}
AddressSpaceType AddrSpaceType = (AddressSpaceType)((CreationInfo.MmuFlags >> 1) & 7);
bool AslrEnabled = ((CreationInfo.MmuFlags >> 5) & 1) != 0;
ulong CodeAddress = CreationInfo.CodeAddress;
ulong CodeSize = CodePagesCount * KMemoryManager.PageSize;
KernelResult Result = MemoryManager.InitializeForProcess(
AddrSpaceType,
AslrEnabled,
!AslrEnabled,
MemRegion,
CodeAddress,
CodeSize,
MemoryBlockAllocator);
if (Result != KernelResult.Success)
{
CleanUpForError();
return(Result);
}
if (!ValidateCodeAddressAndSize(CodeAddress, CodeSize))
{
CleanUpForError();
return(KernelResult.InvalidMemRange);
}
Result = MemoryManager.MapNewProcessCode(
CodeAddress,
CodePagesCount,
MemoryState.CodeStatic,
MemoryPermission.None);
if (Result != KernelResult.Success)
{
CleanUpForError();
return(Result);
}
Result = Capabilities.InitializeForUser(Caps, MemoryManager);
if (Result != KernelResult.Success)
{
CleanUpForError();
return(Result);
}
Pid = System.GetProcessId();
if (Pid == -1 || (ulong)Pid < Horizon.InitialProcessId)
{
throw new InvalidOperationException($"Invalid Process Id {Pid}.");
}
Result = ParseProcessInfo(CreationInfo);
if (Result != KernelResult.Success)
{
CleanUpForError();
}
return(Result);
}
public KernelResult InitializeKip(
ProcessCreationInfo CreationInfo,
int[] Caps,
KPageList PageList,
KResourceLimit ResourceLimit,
MemoryRegion MemRegion)
{
this.ResourceLimit = ResourceLimit;
this.MemRegion = MemRegion;
AddressSpaceType AddrSpaceType = (AddressSpaceType)((CreationInfo.MmuFlags >> 1) & 7);
bool AslrEnabled = ((CreationInfo.MmuFlags >> 5) & 1) != 0;
ulong CodeAddress = CreationInfo.CodeAddress;
ulong CodeSize = (ulong)CreationInfo.CodePagesCount * KMemoryManager.PageSize;
KMemoryBlockAllocator MemoryBlockAllocator = (MmuFlags & 0x40) != 0
? System.LargeMemoryBlockAllocator
: System.SmallMemoryBlockAllocator;
KernelResult Result = MemoryManager.InitializeForProcess(
AddrSpaceType,
AslrEnabled,
!AslrEnabled,
MemRegion,
CodeAddress,
CodeSize,
MemoryBlockAllocator);
if (Result != KernelResult.Success)
{
return(Result);
}
if (!ValidateCodeAddressAndSize(CodeAddress, CodeSize))
{
return(KernelResult.InvalidMemRange);
}
Result = MemoryManager.MapPages(
CodeAddress,
PageList,
MemoryState.CodeStatic,
MemoryPermission.None);
if (Result != KernelResult.Success)
{
return(Result);
}
Result = Capabilities.InitializeForKernel(Caps, MemoryManager);
if (Result != KernelResult.Success)
{
return(Result);
}
Pid = System.GetKipId();
if (Pid == 0 || (ulong)Pid >= Horizon.InitialProcessId)
{
throw new InvalidOperationException($"Invalid KIP Id {Pid}.");
}
Result = ParseProcessInfo(CreationInfo);
return(Result);
}
