private NvInternalResult Remap(Span <RemapArguments> arguments)
{
for (int index = 0; index < arguments.Length; index++)
{
MemoryManager gmm = GetAddressSpaceContext(Context).Gmm;
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, arguments[index].NvMapHandle, true);
if (map == null)
{
Logger.PrintWarning(LogClass.ServiceNv, $"Invalid NvMap handle 0x{arguments[index].NvMapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
long result = (long)gmm.Map(
((ulong)arguments[index].MapOffset << 16) + (ulong)map.Address,
(ulong)arguments[index].GpuOffset << 16,
(ulong)arguments[index].Pages << 16);
if (result < 0)
{
Logger.PrintWarning(LogClass.ServiceNv,
$"Page 0x{arguments[index].GpuOffset:x16} size 0x{arguments[index].Pages:x16} not allocated!");
return(NvInternalResult.InvalidInput);
}
}
return(NvInternalResult.Success);
}
private NvInternalResult Remap(Span <RemapArguments> arguments)
{
AddressSpaceContext addressSpaceContext = GetAddressSpaceContext(Context);
for (int index = 0; index < arguments.Length; index++)
{
MemoryManager gmm = GetAddressSpaceContext(Context).Gmm;
ulong mapOffs = (ulong)arguments[index].MapOffset << 16;
ulong gpuVa = (ulong)arguments[index].GpuOffset << 16;
ulong size = (ulong)arguments[index].Pages << 16;
if (arguments[index].NvMapHandle == 0)
{
gmm.Unmap(gpuVa, size);
}
else
{
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, arguments[index].NvMapHandle);
if (map == null)
{
Logger.Warning?.Print(LogClass.ServiceNv, $"Invalid NvMap handle 0x{arguments[index].NvMapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
gmm.Map(mapOffs + map.Address, gpuVa, size);
}
}
return(NvInternalResult.Success);
}
private void PostFrameBuffer(Layer layer, BufferItem item)
{
int frameBufferWidth = item.GraphicBuffer.Object.Width;
int frameBufferHeight = item.GraphicBuffer.Object.Height;
int nvMapHandle = item.GraphicBuffer.Object.Buffer.Surfaces[0].NvMapHandle;
if (nvMapHandle == 0)
{
nvMapHandle = item.GraphicBuffer.Object.Buffer.NvMapId;
}
int bufferOffset = item.GraphicBuffer.Object.Buffer.Surfaces[0].Offset;
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(layer.Owner, nvMapHandle);
ulong frameBufferAddress = (ulong)(map.Address + bufferOffset);
Format format = ConvertColorFormat(item.GraphicBuffer.Object.Buffer.Surfaces[0].ColorFormat);
int bytesPerPixel =
format == Format.B5G6R5Unorm ||
format == Format.R4G4B4A4Unorm ? 2 : 4;
int gobBlocksInY = 1 << item.GraphicBuffer.Object.Buffer.Surfaces[0].BlockHeightLog2;
// Note: Rotation is being ignored.
Rect cropRect = item.Crop;
bool flipX = item.Transform.HasFlag(NativeWindowTransform.FlipX);
bool flipY = item.Transform.HasFlag(NativeWindowTransform.FlipY);
ImageCrop crop = new ImageCrop(
cropRect.Left,
cropRect.Right,
cropRect.Top,
cropRect.Bottom,
flipX,
flipY);
TextureCallbackInformation textureCallbackInformation = new TextureCallbackInformation
{
Layer = layer,
Item = item,
};
_device.Gpu.Window.EnqueueFrameThreadSafe(
frameBufferAddress,
frameBufferWidth,
frameBufferHeight,
0,
false,
gobBlocksInY,
format,
bytesPerPixel,
crop,
AcquireBuffer,
ReleaseBuffer,
textureCallbackInformation);
}
private NvInternalResult UnmapCommandBuffer(Span <byte> arguments)
{
int headerSize = Unsafe.SizeOf <MapCommandBufferArguments>();
MapCommandBufferArguments commandBufferHeader = MemoryMarshal.Cast <byte, MapCommandBufferArguments>(arguments)[0];
Span <CommandBufferHandle> commandBufferEntries = MemoryMarshal.Cast <byte, CommandBufferHandle>(arguments.Slice(headerSize)).Slice(0, commandBufferHeader.NumEntries);
MemoryManager gmm = NvHostAsGpuDeviceFile.GetAddressSpaceContext(Context).Gmm;
foreach (ref CommandBufferHandle commandBufferEntry in commandBufferEntries)
{
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBufferEntry.MapHandle);
if (map == null)
{
Logger.PrintWarning(LogClass.ServiceNv, $"Invalid handle 0x{commandBufferEntry.MapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
lock (map)
{
if (map.DmaMapAddress != 0)
{
gmm.Free((ulong)map.DmaMapAddress, (uint)map.Size);
map.DmaMapAddress = 0;
}
}
}
return(NvInternalResult.Success);
}
private NvInternalResult UnmapCommandBuffer(Span <byte> arguments)
{
int headerSize = Unsafe.SizeOf <MapCommandBufferArguments>();
MapCommandBufferArguments commandBufferHeader = MemoryMarshal.Cast <byte, MapCommandBufferArguments>(arguments)[0];
Span <CommandBufferHandle> commandBufferEntries = MemoryMarshal.Cast <byte, CommandBufferHandle>(arguments.Slice(headerSize)).Slice(0, commandBufferHeader.NumEntries);
MemoryManager gmm = NvHostAsGpuDeviceFile.GetAddressSpaceContext(Context).Gmm;
foreach (ref CommandBufferHandle commandBufferEntry in commandBufferEntries)
{
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBufferEntry.MapHandle);
if (map == null)
{
Logger.Warning?.Print(LogClass.ServiceNv, $"Invalid handle 0x{commandBufferEntry.MapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
lock (map)
{
if (map.DmaMapAddress != 0)
{
// FIXME:
// To make unmapping work, we need separate address space per channel.
// Right now NVDEC and VIC share the GPU address space which is not correct at all.
// gmm.Free((ulong)map.DmaMapAddress, (uint)map.Size);
// map.DmaMapAddress = 0;
}
}
}
return(NvInternalResult.Success);
}
private NvInternalResult MapCommandBuffer(Span <byte> arguments)
{
int headerSize = Unsafe.SizeOf <MapCommandBufferArguments>();
MapCommandBufferArguments commandBufferHeader = MemoryMarshal.Cast <byte, MapCommandBufferArguments>(arguments)[0];
Span <CommandBufferHandle> commandBufferEntries = MemoryMarshal.Cast <byte, CommandBufferHandle>(arguments.Slice(headerSize)).Slice(0, commandBufferHeader.NumEntries);
MemoryManager gmm = NvHostAsGpuDeviceFile.GetAddressSpaceContext(Context).Gmm;
foreach (ref CommandBufferHandle commandBufferEntry in commandBufferEntries)
{
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBufferEntry.MapHandle);
if (map == null)
{
Logger.Warning?.Print(LogClass.ServiceNv, $"Invalid handle 0x{commandBufferEntry.MapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
lock (map)
{
if (map.DmaMapAddress == 0)
{
ulong va = _memoryAllocator.GetFreeAddress((ulong)map.Size, out ulong freeAddressStartPosition, 1, MemoryManager.PageSize);
if (va != NvMemoryAllocator.PteUnmapped && va <= uint.MaxValue && (va + (uint)map.Size) <= uint.MaxValue)
{
_memoryAllocator.AllocateRange(va, (uint)map.Size, freeAddressStartPosition);
gmm.Map(map.Address, va, (uint)map.Size);
map.DmaMapAddress = va;
}
else
{
map.DmaMapAddress = NvMemoryAllocator.PteUnmapped;
}
}
commandBufferEntry.MapAddress = (int)map.DmaMapAddress;
}
}
return(NvInternalResult.Success);
}
private NvInternalResult Submit(Span <byte> arguments)
{
SubmitArguments submitHeader = GetSpanAndSkip <SubmitArguments>(ref arguments, 1)[0];
Span <CommandBuffer> commandBuffers = GetSpanAndSkip <CommandBuffer>(ref arguments, submitHeader.CmdBufsCount);
Span <Reloc> relocs = GetSpanAndSkip <Reloc>(ref arguments, submitHeader.RelocsCount);
Span <uint> relocShifts = GetSpanAndSkip <uint>(ref arguments, submitHeader.RelocsCount);
Span <SyncptIncr> syncptIncrs = GetSpanAndSkip <SyncptIncr>(ref arguments, submitHeader.SyncptIncrsCount);
Span <SyncptIncr> waitChecks = GetSpanAndSkip <SyncptIncr>(ref arguments, submitHeader.SyncptIncrsCount); // ?
Span <Fence> fences = GetSpanAndSkip <Fence>(ref arguments, submitHeader.FencesCount);
lock (_device)
{
for (int i = 0; i < syncptIncrs.Length; i++)
{
SyncptIncr syncptIncr = syncptIncrs[i];
uint id = syncptIncr.Id;
fences[i].Id = id;
fences[i].Thresh = Context.Device.System.HostSyncpoint.IncrementSyncpointMax(id, syncptIncr.Incrs);
}
foreach (CommandBuffer commandBuffer in commandBuffers)
{
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBuffer.Mem);
var data = _memory.GetSpan(map.Address + commandBuffer.Offset, commandBuffer.WordsCount * 4);
_device.Host1x.Submit(MemoryMarshal.Cast <byte, int>(data));
}
}
fences[0].Thresh = Context.Device.System.HostSyncpoint.IncrementSyncpointMax(fences[0].Id, 1);
Span <int> tmpCmdBuff = stackalloc int[1];
tmpCmdBuff[0] = (4 << 28) | (int)fences[0].Id;
_device.Host1x.Submit(tmpCmdBuff);
return(NvInternalResult.Success);
}
private NvInternalResult Submit(Span <byte> arguments)
{
int headerSize = Unsafe.SizeOf <SubmitArguments>();
SubmitArguments submitHeader = MemoryMarshal.Cast <byte, SubmitArguments>(arguments)[0];
Span <CommandBuffer> commandBufferEntries = MemoryMarshal.Cast <byte, CommandBuffer>(arguments.Slice(headerSize)).Slice(0, submitHeader.CmdBufsCount);
MemoryManager gmm = NvHostAsGpuDeviceFile.GetAddressSpaceContext(Context).Gmm;
foreach (CommandBuffer commandBufferEntry in commandBufferEntries)
{
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBufferEntry.MemoryId);
int[] commandBufferData = new int[commandBufferEntry.WordsCount];
for (int offset = 0; offset < commandBufferData.Length; offset++)
{
commandBufferData[offset] = _memory.ReadInt32(map.Address + commandBufferEntry.Offset + offset * 4);
}
// TODO: Submit command to engines.
}
return(NvInternalResult.Success);
}
private NvInternalResult MapBufferEx(ref MapBufferExArguments arguments)
{
const string mapErrorMsg = "Failed to map fixed buffer with offset 0x{0:x16}, size 0x{1:x16} and alignment 0x{2:x16}!";
AddressSpaceContext addressSpaceContext = GetAddressSpaceContext(Context);
ulong physicalAddress;
if ((arguments.Flags & AddressSpaceFlags.RemapSubRange) != 0)
{
lock (addressSpaceContext)
{
if (addressSpaceContext.TryGetMapPhysicalAddress(arguments.Offset, out physicalAddress))
{
ulong virtualAddress = arguments.Offset + arguments.BufferOffset;
physicalAddress += arguments.BufferOffset;
addressSpaceContext.Gmm.Map(physicalAddress, virtualAddress, arguments.MappingSize);
return(NvInternalResult.Success);
}
else
{
Logger.Warning?.Print(LogClass.ServiceNv, $"Address 0x{arguments.Offset:x16} not mapped!");
return(NvInternalResult.InvalidInput);
}
}
}
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, arguments.NvMapHandle);
if (map == null)
{
Logger.Warning?.Print(LogClass.ServiceNv, $"Invalid NvMap handle 0x{arguments.NvMapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
ulong pageSize = (ulong)arguments.PageSize;
if (pageSize == 0)
{
pageSize = (ulong)map.Align;
}
physicalAddress = map.Address + arguments.BufferOffset;
ulong size = arguments.MappingSize;
if (size == 0)
{
size = (uint)map.Size;
}
NvInternalResult result = NvInternalResult.Success;
lock (addressSpaceContext)
{
// Note: When the fixed offset flag is not set,
// the Offset field holds the alignment size instead.
bool virtualAddressAllocated = (arguments.Flags & AddressSpaceFlags.FixedOffset) == 0;
if (!virtualAddressAllocated)
{
if (addressSpaceContext.ValidateFixedBuffer(arguments.Offset, size, pageSize))
{
addressSpaceContext.Gmm.Map(physicalAddress, arguments.Offset, size);
}
else
{
string message = string.Format(mapErrorMsg, arguments.Offset, size, pageSize);
Logger.Warning?.Print(LogClass.ServiceNv, message);
result = NvInternalResult.InvalidInput;
}
}
else
{
ulong va = _memoryAllocator.GetFreeAddress(size, out ulong freeAddressStartPosition, pageSize);
if (va != NvMemoryAllocator.PteUnmapped)
{
_memoryAllocator.AllocateRange(va, size, freeAddressStartPosition);
}
addressSpaceContext.Gmm.Map(physicalAddress, va, size);
arguments.Offset = va;
}
if (arguments.Offset == NvMemoryAllocator.PteUnmapped)
{
arguments.Offset = 0;
Logger.Warning?.Print(LogClass.ServiceNv, $"Failed to map size 0x{size:x16}!");
result = NvInternalResult.InvalidInput;
}
else
{
addressSpaceContext.AddMap(arguments.Offset, size, physicalAddress, virtualAddressAllocated);
}
}
return(result);
}
private void PostFrameBuffer(Layer layer, BufferItem item)
{
int frameBufferWidth = item.GraphicBuffer.Object.Width;
int frameBufferHeight = item.GraphicBuffer.Object.Height;
int nvMapHandle = item.GraphicBuffer.Object.Buffer.Surfaces[0].NvMapHandle;
if (nvMapHandle == 0)
{
nvMapHandle = item.GraphicBuffer.Object.Buffer.NvMapId;
}
ulong bufferOffset = (ulong)item.GraphicBuffer.Object.Buffer.Surfaces[0].Offset;
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(layer.Owner, nvMapHandle);
ulong frameBufferAddress = map.Address + bufferOffset;
Format format = ConvertColorFormat(item.GraphicBuffer.Object.Buffer.Surfaces[0].ColorFormat);
int bytesPerPixel =
format == Format.B5G6R5Unorm ||
format == Format.R4G4B4A4Unorm ? 2 : 4;
int gobBlocksInY = 1 << item.GraphicBuffer.Object.Buffer.Surfaces[0].BlockHeightLog2;
// Note: Rotation is being ignored.
Rect cropRect = item.Crop;
bool flipX = item.Transform.HasFlag(NativeWindowTransform.FlipX);
bool flipY = item.Transform.HasFlag(NativeWindowTransform.FlipY);
AspectRatio aspectRatio = _device.Configuration.AspectRatio;
bool isStretched = aspectRatio == AspectRatio.Stretched;
ImageCrop crop = new ImageCrop(
cropRect.Left,
cropRect.Right,
cropRect.Top,
cropRect.Bottom,
flipX,
flipY,
isStretched,
aspectRatio.ToFloatX(),
aspectRatio.ToFloatY());
TextureCallbackInformation textureCallbackInformation = new TextureCallbackInformation
{
Layer = layer,
Item = item
};
_device.Gpu.Window.EnqueueFrameThreadSafe(
layer.Owner,
frameBufferAddress,
frameBufferWidth,
frameBufferHeight,
0,
false,
gobBlocksInY,
format,
bytesPerPixel,
crop,
AcquireBuffer,
ReleaseBuffer,
textureCallbackInformation);
if (item.Fence.FenceCount == 0)
{
_device.Gpu.Window.SignalFrameReady();
_device.Gpu.GPFifo.Interrupt();
}
else
{
item.Fence.RegisterCallback(_device.Gpu, (x) =>
{
_device.Gpu.Window.SignalFrameReady();
_device.Gpu.GPFifo.Interrupt();
});
}
}
private NvInternalResult MapBufferEx(ref MapBufferExArguments arguments)
{
const string mapErrorMsg = "Failed to map fixed buffer with offset 0x{0:x16} and size 0x{1:x16}!";
AddressSpaceContext addressSpaceContext = GetAddressSpaceContext(Owner);
NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, arguments.NvMapHandle, true);
if (map == null)
{
Logger.PrintWarning(LogClass.ServiceNv, $"Invalid NvMap handle 0x{arguments.NvMapHandle:x8}!");
return(NvInternalResult.InvalidInput);
}
long physicalAddress;
if ((arguments.Flags & AddressSpaceFlags.RemapSubRange) != 0)
{
lock (addressSpaceContext)
{
if (addressSpaceContext.TryGetMapPhysicalAddress(arguments.Offset, out physicalAddress))
{
long virtualAddress = arguments.Offset + arguments.BufferOffset;
physicalAddress += arguments.BufferOffset;
if (addressSpaceContext.Vmm.Map(physicalAddress, virtualAddress, arguments.MappingSize) < 0)
{
string message = string.Format(mapErrorMsg, virtualAddress, arguments.MappingSize);
Logger.PrintWarning(LogClass.ServiceNv, message);
return(NvInternalResult.InvalidInput);
}
return(NvInternalResult.Success);
}
else
{
Logger.PrintWarning(LogClass.ServiceNv, $"Address 0x{arguments.Offset:x16} not mapped!");
return(NvInternalResult.InvalidInput);
}
}
}
physicalAddress = map.Address + arguments.BufferOffset;
long size = arguments.MappingSize;
if (size == 0)
{
size = (uint)map.Size;
}
NvInternalResult result = NvInternalResult.Success;
lock (addressSpaceContext)
{
// Note: When the fixed offset flag is not set,
// the Offset field holds the alignment size instead.
bool virtualAddressAllocated = (arguments.Flags & AddressSpaceFlags.FixedOffset) == 0;
if (!virtualAddressAllocated)
{
if (addressSpaceContext.ValidateFixedBuffer(arguments.Offset, size))
{
arguments.Offset = addressSpaceContext.Vmm.Map(physicalAddress, arguments.Offset, size);
}
else
{
string message = string.Format(mapErrorMsg, arguments.Offset, size);
Logger.PrintWarning(LogClass.ServiceNv, message);
result = NvInternalResult.InvalidInput;
}
}
else
{
arguments.Offset = addressSpaceContext.Vmm.Map(physicalAddress, size);
}
if (arguments.Offset < 0)
{
arguments.Offset = 0;
Logger.PrintWarning(LogClass.ServiceNv, $"Failed to map size 0x{size:x16}!");
result = NvInternalResult.InvalidInput;
}
else
{
addressSpaceContext.AddMap(arguments.Offset, size, physicalAddress, virtualAddressAllocated);
}
}
return(result);
}