public static Size GetBlockLinearAlignedSize(
int width,
int height,
int depth,
int blockWidth,
int blockHeight,
int bytesPerPixel,
int gobBlocksInY,
int gobBlocksInZ,
int gobBlocksInTileX)
{
width = BitUtils.DivRoundUp(width, blockWidth);
height = BitUtils.DivRoundUp(height, blockHeight);
int gobWidth = gobBlocksInTileX * (GobStride / bytesPerPixel);
int blockOfGobsHeight = gobBlocksInY * GobHeight;
int blockOfGobsDepth = gobBlocksInZ;
width = BitUtils.AlignUp(width, gobWidth);
height = BitUtils.AlignUp(height, blockOfGobsHeight);
depth = BitUtils.AlignUp(depth, blockOfGobsDepth);
return(new Size(width, height, depth));
}
private static int AlignLayerSize(
int size,
int height,
int depth,
int blockHeight,
int gobBlocksInY,
int gobBlocksInZ)
{
height = BitUtils.DivRoundUp(height, blockHeight);
while (height <= (gobBlocksInY >> 1) * GobHeight && gobBlocksInY != 1)
{
gobBlocksInY >>= 1;
}
while (depth <= (gobBlocksInZ >> 1) && gobBlocksInZ != 1)
{
gobBlocksInZ >>= 1;
}
int blockOfGobsSize = gobBlocksInY * gobBlocksInZ * GobSize;
int sizeInBlockOfGobs = size / blockOfGobsSize;
if (size != sizeInBlockOfGobs * blockOfGobsSize)
{
size = (sizeInBlockOfGobs + 1) * blockOfGobsSize;
}
return(size);
}
/// <summary>
/// Recreates the texture storage (or view, in the case of child textures) of this texture.
/// This allows recreating the texture with a new size.
/// A copy is automatically performed from the old to the new texture.
/// </summary>
/// <param name="width">The new texture width</param>
/// <param name="height">The new texture height</param>
/// <param name="width">The block width related to the given width</param>
/// <param name="height">The block height related to the given height</param>
/// <param name="depthOrLayers">The new texture depth (for 3D textures) or layers (for layered textures)</param>
private void RecreateStorageOrView(int width, int height, int blockWidth, int blockHeight, int depthOrLayers)
{
RecreateStorageOrView(
BitUtils.DivRoundUp(width * Info.FormatInfo.BlockWidth, blockWidth),
BitUtils.DivRoundUp(height * Info.FormatInfo.BlockHeight, blockHeight),
depthOrLayers);
}
public KernelResult UnmapFromProcess(
KPageTableBase memoryManager,
ulong address,
ulong size,
KProcess process)
{
if (_storage == null)
{
throw new NotImplementedException();
}
ulong pagesCountRounded = BitUtils.DivRoundUp(size, KPageTableBase.PageSize);
var pageList = _storage.GetPageList();
ulong pagesCount = pageList.GetPagesCount();
if (pagesCount != pagesCountRounded)
{
return(KernelResult.InvalidSize);
}
var ranges = _storage.GetRanges();
MemoryState state = Permission == KMemoryPermission.None ? MemoryState.TransferMemoryIsolated : MemoryState.TransferMemory;
KernelResult result = memoryManager.UnmapPages(address, pagesCount, ranges, state);
if (result == KernelResult.Success)
{
_isMapped = false;
}
return(result);
}
public KernelResult Map(ulong address, ulong size, KMemoryPermission perm)
{
if (_pageCount != BitUtils.DivRoundUp(size, KPageTableBase.PageSize))
{
return(KernelResult.InvalidSize);
}
lock (_lock)
{
if (_isMapped)
{
return(KernelResult.InvalidState);
}
KProcess proc = KernelStatic.GetCurrentProcess();
// TODO: Mark pages as MemoryState.CodeWritable
KernelResult resultCode = proc.MemoryManager.MapPages(address, _hostPagelist, KMemoryPermission.ReadAndWrite);
if (resultCode != KernelResult.Success)
{
return(KernelResult.InvalidState);
}
_isMapped = true;
}
return(KernelResult.Success);
}
public KernelResult MapIntoProcess(
KMemoryManager MemoryManager,
ulong Address,
ulong Size,
KProcess Process,
MemoryPermission Permission)
{
ulong PagesCountRounded = BitUtils.DivRoundUp(Size, KMemoryManager.PageSize);
if (PageList.GetPagesCount() != PagesCountRounded)
{
return(KernelResult.InvalidSize);
}
MemoryPermission ExpectedPermission = Process.Pid == OwnerPid
? OwnerPermission
: UserPermission;
if (Permission != ExpectedPermission)
{
return(KernelResult.InvalidPermission);
}
return(MemoryManager.MapPages(Address, PageList, MemoryState.SharedMemory, Permission));
}
public KernelResult MapIntoProcess(
KPageTableBase memoryManager,
ulong address,
ulong size,
KProcess process,
KMemoryPermission permission)
{
if (_pageList.GetPagesCount() != BitUtils.DivRoundUp(size, KPageTableBase.PageSize))
{
return(KernelResult.InvalidSize);
}
if (permission != Permission || _isMapped)
{
return(KernelResult.InvalidState);
}
MemoryState state = Permission == KMemoryPermission.None ? MemoryState.TransferMemoryIsolated : MemoryState.TransferMemory;
KernelResult result = memoryManager.MapPages(address, _pageList, state, KMemoryPermission.ReadAndWrite);
if (result == KernelResult.Success)
{
_isMapped = true;
}
return(result);
}
/// <summary>
/// Launches Inline-to-Memory engine DMA copy.
/// </summary>
/// <param name="state">Current class state</param>
/// <param name="argument">Method call argument</param>
public void LaunchDma(ref InlineToMemoryClassState state, int argument)
{
_isLinear = (argument & 1) != 0;
_offset = 0;
_size = (int)(state.LineLengthIn * state.LineCount);
int count = BitUtils.DivRoundUp(_size, 4);
if (_buffer == null || _buffer.Length < count)
{
_buffer = new int[count];
}
ulong dstGpuVa = ((ulong)state.OffsetOutUpperValue << 32) | state.OffsetOut;
ulong dstBaseAddress = _channel.MemoryManager.Translate(dstGpuVa);
// Trigger read tracking, to flush any managed resources in the destination region.
_channel.MemoryManager.Physical.GetSpan(dstBaseAddress, _size, true);
_dstGpuVa = dstGpuVa;
_dstX = state.SetDstOriginBytesXV;
_dstY = state.SetDstOriginSamplesYV;
_dstWidth = (int)state.SetDstWidth;
_dstHeight = (int)state.SetDstHeight;
_dstStride = (int)state.PitchOut;
_dstGobBlocksInY = 1 << (int)state.SetDstBlockSizeHeight;
_lineLengthIn = (int)state.LineLengthIn;
_lineCount = (int)state.LineCount;
_finished = false;
}
public KernelResult MapToOwner(ulong address, ulong size, KMemoryPermission permission)
{
if (_pageList.GetPagesCount() != BitUtils.DivRoundUp(size, KPageTableBase.PageSize))
{
return(KernelResult.InvalidSize);
}
lock (_lock)
{
if (_isOwnerMapped)
{
return(KernelResult.InvalidState);
}
Debug.Assert(permission == KMemoryPermission.Read || permission == KMemoryPermission.ReadAndExecute);
KernelResult result = Owner.MemoryManager.MapPages(address, _pageList, MemoryState.CodeReadOnly, permission);
if (result != KernelResult.Success)
{
return(result);
}
_isOwnerMapped = true;
}
return(KernelResult.Success);
}
public KernelResult Map(ulong address, ulong size, KMemoryPermission perm)
{
if (_pageList.GetPagesCount() != BitUtils.DivRoundUp(size, KPageTableBase.PageSize))
{
return(KernelResult.InvalidSize);
}
lock (_lock)
{
if (_isMapped)
{
return(KernelResult.InvalidState);
}
KProcess process = KernelStatic.GetCurrentProcess();
KernelResult result = process.MemoryManager.MapPages(address, _pageList, MemoryState.CodeWritable, KMemoryPermission.ReadAndWrite);
if (result != KernelResult.Success)
{
return(result);
}
_isMapped = true;
}
return(KernelResult.Success);
}
public static Span <byte> ConvertLinearToLinearStrided(
int width,
int height,
int blockWidth,
int blockHeight,
int stride,
int bytesPerPixel,
ReadOnlySpan <byte> data)
{
int w = BitUtils.DivRoundUp(width, blockWidth);
int h = BitUtils.DivRoundUp(height, blockHeight);
int inStride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
Span <byte> output = new byte[h * stride];
int inOffs = 0;
for (int y = 0; y < h; y++)
{
for (int x = 0; x < w; x++)
{
int offset = y * stride + x * bytesPerPixel;
Span <byte> dest = output.Slice(offset, bytesPerPixel);
data.Slice(inOffs + x * bytesPerPixel, bytesPerPixel).CopyTo(dest);
}
inOffs += inStride;
}
return(output);
}
public KernelResult MapIntoProcess(
KPageTableBase memoryManager,
ulong address,
ulong size,
KProcess process,
KMemoryPermission permission)
{
ulong pagesCountRounded = BitUtils.DivRoundUp(size, KPageTableBase.PageSize);
var pageList = _storage.GetPageList();
if (pageList.GetPagesCount() != pagesCountRounded)
{
return(KernelResult.InvalidSize);
}
KMemoryPermission expectedPermission = process.Pid == _ownerPid
? _ownerPermission
: _userPermission;
if (permission != expectedPermission)
{
return(KernelResult.InvalidPermission);
}
KernelResult result = memoryManager.MapPages(address, pageList, MemoryState.SharedMemory, permission);
if (result == KernelResult.Success && !memoryManager.SupportsMemoryAliasing)
{
_storage.Borrow(process, address);
}
return(result);
}
private static int GetTextureSize(
int width,
int height,
int depth,
int levels,
int layers,
int blockWidth,
int blockHeight,
int bytesPerPixel)
{
int layerSize = 0;
for (int level = 0; level < levels; level++)
{
int w = Math.Max(1, width >> level);
int h = Math.Max(1, height >> level);
int d = Math.Max(1, depth >> level);
w = BitUtils.DivRoundUp(w, blockWidth);
h = BitUtils.DivRoundUp(h, blockHeight);
int stride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
layerSize += stride * h * d;
}
return(layerSize * layers);
}
public static Span <byte> ConvertLinearToLinearStrided(
int width,
int height,
int blockWidth,
int blockHeight,
int stride,
int bytesPerPixel,
ReadOnlySpan <byte> data)
{
int w = BitUtils.DivRoundUp(width, blockWidth);
int h = BitUtils.DivRoundUp(height, blockHeight);
int inStride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
int lineSize = width * bytesPerPixel;
Span <byte> output = new byte[h * stride];
int inOffs = 0;
int outOffs = 0;
for (int y = 0; y < h; y++)
{
data.Slice(inOffs, lineSize).CopyTo(output.Slice(outOffs, lineSize));
inOffs += inStride;
outOffs += stride;
}
return(output);
}
public static ISwizzle GetSwizzle(GalImage image)
{
int blockWidth = ImageUtils.GetBlockWidth(image.Format);
int blockHeight = ImageUtils.GetBlockHeight(image.Format);
int blockDepth = ImageUtils.GetBlockDepth(image.Format);
int bytesPerPixel = ImageUtils.GetBytesPerPixel(image.Format);
int width = BitUtils.DivRoundUp(image.Width, blockWidth);
int height = BitUtils.DivRoundUp(image.Height, blockHeight);
int depth = BitUtils.DivRoundUp(image.Depth, blockDepth);
if (image.Layout == GalMemoryLayout.BlockLinear)
{
int alignMask = image.TileWidth * (64 / bytesPerPixel) - 1;
width = (width + alignMask) & ~alignMask;
return(new BlockLinearSwizzle(
width,
height,
depth,
image.GobBlockHeight,
image.GobBlockDepth,
bytesPerPixel));
}
else
{
return(new LinearSwizzle(image.Pitch, bytesPerPixel, width, height));
}
}
public KernelResult MapIntoProcess(
KMemoryManager memoryManager,
ulong address,
ulong size,
KProcess process,
MemoryPermission permission)
{
ulong pagesCountRounded = BitUtils.DivRoundUp(size, KMemoryManager.PageSize);
if (_pageList.GetPagesCount() != pagesCountRounded)
{
return(KernelResult.InvalidSize);
}
MemoryPermission expectedPermission = process.Pid == _ownerPid
? _ownerPermission
: _userPermission;
if (permission != expectedPermission)
{
return(KernelResult.InvalidPermission);
}
return(memoryManager.MapPages(address, _pageList, MemoryState.SharedMemory, permission));
}
public static void DeclareAll(CodeGenContext context, StructuredProgramInfo info)
{
if (context.Config.Stage == ShaderStage.Compute)
{
int localMemorySize = BitUtils.DivRoundUp(context.Config.GpuAccessor.QueryComputeLocalMemorySize(), 4);
if (localMemorySize != 0)
{
DeclareLocalMemory(context, localMemorySize);
}
int sharedMemorySize = BitUtils.DivRoundUp(context.Config.GpuAccessor.QueryComputeSharedMemorySize(), 4);
if (sharedMemorySize != 0)
{
DeclareSharedMemory(context, sharedMemorySize);
}
}
else if (context.Config.LocalMemorySize != 0)
{
int localMemorySize = BitUtils.DivRoundUp(context.Config.LocalMemorySize, 4);
DeclareLocalMemory(context, localMemorySize);
}
DeclareUniformBuffers(context, context.Config.GetConstantBufferDescriptors());
DeclareStorageBuffers(context, context.Config.GetStorageBufferDescriptors());
DeclareSamplers(context, context.Config.GetTextureDescriptors());
DeclareImages(context, context.Config.GetImageDescriptors());
DeclareInputAttributes(context, info);
DeclareOutputAttributes(context, info);
}
private static ReadOnlySpan <Vp9MvRef> GetMvsInput(MemoryManager gmm, FrameSize size, uint offset)
{
int miCols = BitUtils.DivRoundUp(size.Width, 8);
int miRows = BitUtils.DivRoundUp(size.Height, 8);
return(MemoryMarshal.Cast <byte, Vp9MvRef>(gmm.DeviceGetSpan(offset, miRows * miCols * 16)));
}
public static Size GetBlockLinearAlignedSize(
int width,
int height,
int depth,
int blockWidth,
int blockHeight,
int bytesPerPixel,
int gobBlocksInY,
int gobBlocksInZ,
int gobBlocksInTileX)
{
width = BitUtils.DivRoundUp(width, blockWidth);
height = BitUtils.DivRoundUp(height, blockHeight);
int gobWidth = (GobStride / bytesPerPixel) * gobBlocksInTileX;
int gobHeight = gobBlocksInY * GobHeight;
int alignment = gobWidth;
if (depth < gobBlocksInZ || width <= gobWidth || height <= gobHeight)
{
alignment = GobStride / bytesPerPixel;
}
(gobBlocksInY, gobBlocksInZ) = GetMipGobBlockSizes(height, depth, blockHeight, gobBlocksInY, gobBlocksInZ);
int blockOfGobsHeight = gobBlocksInY * GobHeight;
int blockOfGobsDepth = gobBlocksInZ;
width = BitUtils.AlignUp(width, alignment);
height = BitUtils.AlignUp(height, blockOfGobsHeight);
depth = BitUtils.AlignUp(depth, blockOfGobsDepth);
return(new Size(width, height, depth));
}
public KernelResult Unmap(ulong address, ulong size)
{
if (_pageList.GetPagesCount() != BitUtils.DivRoundUp(size, KPageTableBase.PageSize))
{
return(KernelResult.InvalidSize);
}
lock (_lock)
{
KProcess process = KernelStatic.GetCurrentProcess();
KernelResult result = process.MemoryManager.UnmapPages(address, _pageList, MemoryState.CodeWritable);
if (result != KernelResult.Success)
{
return(result);
}
Debug.Assert(_isMapped);
_isMapped = false;
}
return(KernelResult.Success);
}
/// <summary>
/// Launches Inline-to-Memory engine DMA copy.
/// </summary>
/// <param name="state">Current class state</param>
/// <param name="argument">Method call argument</param>
public void LaunchDma(ref InlineToMemoryClassState state, int argument)
{
_isLinear = (argument & 1) != 0;
_offset = 0;
_size = (int)(state.LineLengthIn * state.LineCount);
int count = BitUtils.DivRoundUp(_size, 4);
if (_buffer == null || _buffer.Length < count)
{
_buffer = new int[count];
}
ulong dstGpuVa = ((ulong)state.OffsetOutUpperValue << 32) | state.OffsetOut;
_dstGpuVa = dstGpuVa;
_dstX = state.SetDstOriginBytesXV;
_dstY = state.SetDstOriginSamplesYV;
_dstWidth = (int)state.SetDstWidth;
_dstHeight = (int)state.SetDstHeight;
_dstStride = (int)state.PitchOut;
_dstGobBlocksInY = 1 << (int)state.SetDstBlockSizeHeight;
_lineLengthIn = (int)state.LineLengthIn;
_lineCount = (int)state.LineCount;
_finished = false;
}
public static SizeInfo GetLinearTextureSize(int stride, int height, int blockHeight)
{
// Non-2D or mipmapped linear textures are not supported by the Switch GPU,
// so we only need to handle a single case (2D textures without mipmaps).
int totalSize = stride * BitUtils.DivRoundUp(height, blockHeight);
return(new SizeInfo(new int[] { 0 }, new int[] { 0 }, 1, totalSize, totalSize));
}
public KernelResult UnmapFromProcess(KPageTableBase memoryManager, ulong address, ulong size, KProcess process)
{
if (_pageList.GetPagesCount() != BitUtils.DivRoundUp(size, KPageTableBase.PageSize))
{
return(KernelResult.InvalidSize);
}
return(memoryManager.UnmapPages(address, _pageList, MemoryState.SharedMemory));
}
public unsafe static void Decode(NvdecDevice device, ResourceManager rm, ref NvdecRegisters state)
{
PictureInfo pictureInfo = rm.Gmm.DeviceRead <PictureInfo>(state.SetPictureInfoOffset);
EntropyProbs entropy = rm.Gmm.DeviceRead <EntropyProbs>(state.SetVp9EntropyProbsOffset);
ISurface Rent(uint lumaOffset, uint chromaOffset, FrameSize size)
{
return(rm.Cache.Get(_decoder, CodecId.Vp9, lumaOffset, chromaOffset, size.Width, size.Height));
}
ISurface lastSurface = Rent(state.SetSurfaceLumaOffset[0], state.SetSurfaceChromaOffset[0], pictureInfo.LastFrameSize);
ISurface goldenSurface = Rent(state.SetSurfaceLumaOffset[1], state.SetSurfaceChromaOffset[1], pictureInfo.GoldenFrameSize);
ISurface altSurface = Rent(state.SetSurfaceLumaOffset[2], state.SetSurfaceChromaOffset[2], pictureInfo.AltFrameSize);
ISurface currentSurface = Rent(state.SetSurfaceLumaOffset[3], state.SetSurfaceChromaOffset[3], pictureInfo.CurrentFrameSize);
Vp9PictureInfo info = pictureInfo.Convert();
info.LastReference = lastSurface;
info.GoldenReference = goldenSurface;
info.AltReference = altSurface;
entropy.Convert(ref info.Entropy);
ReadOnlySpan <byte> bitstream = rm.Gmm.DeviceGetSpan(state.SetBitstreamOffset, (int)pictureInfo.BitstreamSize);
ReadOnlySpan <Vp9MvRef> mvsIn = ReadOnlySpan <Vp9MvRef> .Empty;
if (info.UsePrevInFindMvRefs)
{
mvsIn = GetMvsInput(rm.Gmm, pictureInfo.CurrentFrameSize, state.SetVp9LastFrameMvsOffset);
}
int miCols = BitUtils.DivRoundUp(pictureInfo.CurrentFrameSize.Width, 8);
int miRows = BitUtils.DivRoundUp(pictureInfo.CurrentFrameSize.Height, 8);
using var mvsRegion = rm.Gmm.GetWritableRegion(ExtendOffset(state.SetVp9CurrFrameMvsOffset), miRows * miCols * 16);
Span <Vp9MvRef> mvsOut = MemoryMarshal.Cast <byte, Vp9MvRef>(mvsRegion.Memory.Span);
uint lumaOffset = state.SetSurfaceLumaOffset[3];
uint chromaOffset = state.SetSurfaceChromaOffset[3];
if (_decoder.Decode(ref info, currentSurface, bitstream, mvsIn, mvsOut))
{
SurfaceWriter.Write(rm.Gmm, currentSurface, lumaOffset, chromaOffset);
device.OnFrameDecoded(CodecId.Vp9, lumaOffset, chromaOffset);
}
WriteBackwardUpdates(rm.Gmm, state.SetVp9BackwardUpdatesOffset, ref info.BackwardUpdateCounts);
rm.Cache.Put(lastSurface);
rm.Cache.Put(goldenSurface);
rm.Cache.Put(altSurface);
rm.Cache.Put(currentSurface);
}
private RobAndSliceSizes GetRobAndSliceSizes(int width, int height, GobBlockSizes gbSizes)
{
int widthInGobs = BitUtils.DivRoundUp(width * _texBpp, GobWidth);
int robSize = GobSize * gbSizes.Height * gbSizes.Depth * widthInGobs;
int sliceSize = BitUtils.DivRoundUp(height, gbSizes.Height * GobHeight) * robSize;
return(new RobAndSliceSizes(robSize, sliceSize));
}
private RobAndSliceSizes GetRobAndSliceSizes(int width, int height, int gobBlocksInY, int gobBlocksInZ)
{
int widthInGobs = BitUtils.DivRoundUp(width * _texBpp, GobStride);
int robSize = GobSize * gobBlocksInY * gobBlocksInZ * widthInGobs;
int sliceSize = BitUtils.DivRoundUp(height, gobBlocksInY * GobHeight) * robSize;
return(new RobAndSliceSizes(robSize, sliceSize));
}
public Extents2D Reduce(int level)
{
int div = 1 << level;
return(new Extents2D(
X1 >> level,
Y1 >> level,
BitUtils.DivRoundUp(X2, div),
BitUtils.DivRoundUp(Y2, div)));
}
public KernelResult Initialize(ulong address, ulong size)
{
_owner = KernelStatic.GetCurrentProcess();
_hostPagelist = _owner.MemoryManager.GetPhysicalRegions(address, size);
_pageCount = BitUtils.DivRoundUp(size, KPageTableBase.PageSize);
_isOwnerMapped = false;
_isMapped = false;
return(KernelResult.Success);
}
public static int CalculateManagementOverheadSize(ulong regionSize)
{
int overheadBits = 0;
for (int depth = GetRequiredDepth(regionSize) - 1; depth >= 0; depth--)
{
regionSize = BitUtils.DivRoundUp(regionSize, UInt64BitSize);
overheadBits += (int)regionSize;
}
return(overheadBits * sizeof(ulong));
}
private ulong GetA8B8G8R8LayerSize(int width, int height, out int pitch, out int alignment)
{
const int defaultAlignment = 0x1000;
const ulong defaultSize = 0x20000;
alignment = defaultAlignment;
pitch = BitUtils.AlignUp(BitUtils.DivRoundUp(width * 32, 8), 64);
int memorySize = pitch * BitUtils.AlignUp(height, 64);
ulong requiredMemorySize = (ulong)BitUtils.AlignUp(memorySize, alignment);
return((requiredMemorySize + defaultSize - 1) / defaultSize * defaultSize);
}
