using Ryujinx.Common; using Ryujinx.Graphics.Device; using Ryujinx.Graphics.Gpu.Engine.Threed; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Texture; using System; using System.Buffers; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; namespace Ryujinx.Graphics.Gpu.Engine.Dma { /// /// Represents a DMA copy engine class. /// class DmaClass : IDeviceState { private readonly GpuContext _context; private readonly GpuChannel _channel; private readonly ThreedClass _3dEngine; private readonly DeviceState _state; /// /// Copy flags passed on DMA launch. /// [Flags] private enum CopyFlags { SrcLinear = 1 << 7, DstLinear = 1 << 8, MultiLineEnable = 1 << 9, RemapEnable = 1 << 10, } /// /// Texture parameters for copy. /// private readonly struct TextureParams { /// /// Copy region X coordinate. /// public readonly int RegionX; /// /// Copy region Y coordinate. /// public readonly int RegionY; /// /// Offset from the base pointer of the data in memory. /// public readonly int BaseOffset; /// /// Bytes per pixel. /// public readonly int Bpp; /// /// Whether the texture is linear. If false, the texture is block linear. /// public readonly bool Linear; /// /// Pixel offset from XYZ coordinates calculator. /// public readonly OffsetCalculator Calculator; /// /// Creates texture parameters. /// /// Copy region X coordinate /// Copy region Y coordinate /// Offset from the base pointer of the data in memory /// Bytes per pixel /// Whether the texture is linear. If false, the texture is block linear /// Pixel offset from XYZ coordinates calculator public TextureParams(int regionX, int regionY, int baseOffset, int bpp, bool linear, OffsetCalculator calculator) { RegionX = regionX; RegionY = regionY; BaseOffset = baseOffset; Bpp = bpp; Linear = linear; Calculator = calculator; } } [StructLayout(LayoutKind.Sequential, Size = 3, Pack = 1)] private struct UInt24 { public byte Byte0; public byte Byte1; public byte Byte2; } /// /// Creates a new instance of the DMA copy engine class. /// /// GPU context /// GPU channel /// 3D engine public DmaClass(GpuContext context, GpuChannel channel, ThreedClass threedEngine) { _context = context; _channel = channel; _3dEngine = threedEngine; _state = new DeviceState(new Dictionary { { nameof(DmaClassState.LaunchDma), new RwCallback(LaunchDma, null) }, }); } /// /// Reads data from the class registers. /// /// Register byte offset /// Data at the specified offset public int Read(int offset) => _state.Read(offset); /// /// Writes data to the class registers. /// /// Register byte offset /// Data to be written public void Write(int offset, int data) => _state.Write(offset, data); /// /// Determine if a buffer-to-texture region covers the entirety of a texture. /// /// Texture to compare /// True if the texture is linear, false if block linear /// Texture bytes per pixel /// Texture stride /// Number of pixels to be copied /// Number of lines to be copied /// private static bool IsTextureCopyComplete(DmaTexture tex, bool linear, int bpp, int stride, int xCount, int yCount) { if (linear) { // If the stride is negative, the texture has to be flipped, so // the fast copy is not trivial, use the slow path. if (stride <= 0) { return false; } int alignWidth = Constants.StrideAlignment / bpp; return stride / bpp == BitUtils.AlignUp(xCount, alignWidth); } else { int alignWidth = Constants.GobAlignment / bpp; return tex.RegionX == 0 && tex.RegionY == 0 && tex.Width == BitUtils.AlignUp(xCount, alignWidth) && tex.Height == yCount; } } /// /// Releases a semaphore for a given LaunchDma method call. /// /// The LaunchDma call argument private void ReleaseSemaphore(int argument) { LaunchDmaSemaphoreType type = (LaunchDmaSemaphoreType)((argument >> 3) & 0x3); if (type != LaunchDmaSemaphoreType.None) { ulong address = ((ulong)_state.State.SetSemaphoreA << 32) | _state.State.SetSemaphoreB; if (type == LaunchDmaSemaphoreType.ReleaseOneWordSemaphore) { _channel.MemoryManager.Write(address, _state.State.SetSemaphorePayload); } else /* if (type == LaunchDmaSemaphoreType.ReleaseFourWordSemaphore) */ { _channel.MemoryManager.Write(address + 8, _context.GetTimestamp()); _channel.MemoryManager.Write(address, (ulong)_state.State.SetSemaphorePayload); } } } /// /// Performs a buffer to buffer, or buffer to texture copy. /// /// The LaunchDma call argument private void DmaCopy(int argument) { var memoryManager = _channel.MemoryManager; CopyFlags copyFlags = (CopyFlags)argument; bool srcLinear = copyFlags.HasFlag(CopyFlags.SrcLinear); bool dstLinear = copyFlags.HasFlag(CopyFlags.DstLinear); bool copy2D = copyFlags.HasFlag(CopyFlags.MultiLineEnable); bool remap = copyFlags.HasFlag(CopyFlags.RemapEnable); uint size = _state.State.LineLengthIn; if (size == 0) { return; } ulong srcGpuVa = ((ulong)_state.State.OffsetInUpperUpper << 32) | _state.State.OffsetInLower; ulong dstGpuVa = ((ulong)_state.State.OffsetOutUpperUpper << 32) | _state.State.OffsetOutLower; int xCount = (int)_state.State.LineLengthIn; int yCount = (int)_state.State.LineCount; _channel.TextureManager.RefreshModifiedTextures(); _3dEngine.CreatePendingSyncs(); _3dEngine.FlushUboDirty(); if (copy2D) { // Buffer to texture copy. int componentSize = (int)_state.State.SetRemapComponentsComponentSize + 1; int srcComponents = (int)_state.State.SetRemapComponentsNumSrcComponents + 1; int dstComponents = (int)_state.State.SetRemapComponentsNumDstComponents + 1; int srcBpp = remap ? srcComponents * componentSize : 1; int dstBpp = remap ? dstComponents * componentSize : 1; var dst = Unsafe.As(ref _state.State.SetDstBlockSize); var src = Unsafe.As(ref _state.State.SetSrcBlockSize); int srcRegionX = 0, srcRegionY = 0, dstRegionX = 0, dstRegionY = 0; if (!srcLinear) { srcRegionX = src.RegionX; srcRegionY = src.RegionY; } if (!dstLinear) { dstRegionX = dst.RegionX; dstRegionY = dst.RegionY; } int srcStride = (int)_state.State.PitchIn; int dstStride = (int)_state.State.PitchOut; var srcCalculator = new OffsetCalculator( src.Width, src.Height, srcStride, srcLinear, src.MemoryLayout.UnpackGobBlocksInY(), src.MemoryLayout.UnpackGobBlocksInZ(), srcBpp); var dstCalculator = new OffsetCalculator( dst.Width, dst.Height, dstStride, dstLinear, dst.MemoryLayout.UnpackGobBlocksInY(), dst.MemoryLayout.UnpackGobBlocksInZ(), dstBpp); (int srcBaseOffset, int srcSize) = srcCalculator.GetRectangleRange(srcRegionX, srcRegionY, xCount, yCount); (int dstBaseOffset, int dstSize) = dstCalculator.GetRectangleRange(dstRegionX, dstRegionY, xCount, yCount); if (srcLinear && srcStride < 0) { srcBaseOffset += srcStride * (yCount - 1); } if (dstLinear && dstStride < 0) { dstBaseOffset += dstStride * (yCount - 1); } // If remapping is disabled, we always copy the components directly, in order. // If it's enabled, but the mapping is just XYZW, we also copy them in order. bool isIdentityRemap = !remap || (_state.State.SetRemapComponentsDstX == SetRemapComponentsDst.SrcX && (dstComponents < 2 || _state.State.SetRemapComponentsDstY == SetRemapComponentsDst.SrcY) && (dstComponents < 3 || _state.State.SetRemapComponentsDstZ == SetRemapComponentsDst.SrcZ) && (dstComponents < 4 || _state.State.SetRemapComponentsDstW == SetRemapComponentsDst.SrcW)); bool completeSource = IsTextureCopyComplete(src, srcLinear, srcBpp, srcStride, xCount, yCount); bool completeDest = IsTextureCopyComplete(dst, dstLinear, dstBpp, dstStride, xCount, yCount); // Check if the source texture exists on the GPU, if it does, do a GPU side copy. // Otherwise, we would need to flush the source texture which is costly. // We don't expect the source to be linear in such cases, as linear source usually indicates buffer or CPU written data. if (completeSource && completeDest && !srcLinear && isIdentityRemap) { var source = memoryManager.Physical.TextureCache.FindTexture( memoryManager, srcGpuVa, srcBpp, srcStride, src.Height, xCount, yCount, srcLinear, src.MemoryLayout.UnpackGobBlocksInY(), src.MemoryLayout.UnpackGobBlocksInZ()); if (source != null && source.Height == yCount) { source.SynchronizeMemory(); var target = memoryManager.Physical.TextureCache.FindOrCreateTexture( memoryManager, source.Info.FormatInfo, dstGpuVa, xCount, yCount, dstStride, dstLinear, dst.MemoryLayout.UnpackGobBlocksInY(), dst.MemoryLayout.UnpackGobBlocksInZ()); if (source.ScaleFactor != target.ScaleFactor) { target.PropagateScale(source); } source.HostTexture.CopyTo(target.HostTexture, 0, 0); target.SignalModified(); return; } } ReadOnlySpan srcSpan = memoryManager.GetSpan(srcGpuVa + (ulong)srcBaseOffset, srcSize, true); // Try to set the texture data directly, // but only if we are doing a complete copy, // and not for block linear to linear copies, since those are typically accessed from the CPU. if (completeSource && completeDest && !(dstLinear && !srcLinear) && isIdentityRemap) { var target = memoryManager.Physical.TextureCache.FindTexture( memoryManager, dstGpuVa, dstBpp, dstStride, dst.Height, xCount, yCount, dstLinear, dst.MemoryLayout.UnpackGobBlocksInY(), dst.MemoryLayout.UnpackGobBlocksInZ()); if (target != null) { IMemoryOwner data; if (srcLinear) { data = LayoutConverter.ConvertLinearStridedToLinear( target.Info.Width, target.Info.Height, 1, 1, xCount * srcBpp, srcStride, target.Info.FormatInfo.BytesPerPixel, srcSpan); } else { data = LayoutConverter.ConvertBlockLinearToLinear( src.Width, src.Height, src.Depth, 1, 1, 1, 1, 1, srcBpp, src.MemoryLayout.UnpackGobBlocksInY(), src.MemoryLayout.UnpackGobBlocksInZ(), 1, new SizeInfo((int)target.Size), srcSpan); } target.SynchronizeMemory(); target.SetData(data); target.SignalModified(); return; } else if (srcCalculator.LayoutMatches(dstCalculator)) { // No layout conversion has to be performed, just copy the data entirely. memoryManager.Write(dstGpuVa + (ulong)dstBaseOffset, srcSpan); return; } } // OPT: This allocates a (potentially) huge temporary array and then copies an existing // region of memory into it, data that might get overwritten entirely anyways. Ideally this should // all be rewritten to use pooled arrays, but that gets complicated with packed data and strides Span dstSpan = memoryManager.GetSpan(dstGpuVa + (ulong)dstBaseOffset, dstSize).ToArray(); TextureParams srcParams = new(srcRegionX, srcRegionY, srcBaseOffset, srcBpp, srcLinear, srcCalculator); TextureParams dstParams = new(dstRegionX, dstRegionY, dstBaseOffset, dstBpp, dstLinear, dstCalculator); if (isIdentityRemap) { // The order of the components doesn't change, so we can just copy directly // (with layout conversion if necessary). switch (srcBpp) { case 1: Copy(dstSpan, srcSpan, dstParams, srcParams); break; case 2: Copy(dstSpan, srcSpan, dstParams, srcParams); break; case 4: Copy(dstSpan, srcSpan, dstParams, srcParams); break; case 8: Copy(dstSpan, srcSpan, dstParams, srcParams); break; case 12: Copy(dstSpan, srcSpan, dstParams, srcParams); break; case 16: Copy>(dstSpan, srcSpan, dstParams, srcParams); break; default: throw new NotSupportedException($"Unable to copy ${srcBpp} bpp pixel format."); } } else { // The order or value of the components might change. switch (componentSize) { case 1: CopyShuffle(dstSpan, srcSpan, dstParams, srcParams); break; case 2: CopyShuffle(dstSpan, srcSpan, dstParams, srcParams); break; case 3: CopyShuffle(dstSpan, srcSpan, dstParams, srcParams); break; case 4: CopyShuffle(dstSpan, srcSpan, dstParams, srcParams); break; default: throw new NotSupportedException($"Unable to copy ${componentSize} component size."); } } memoryManager.Write(dstGpuVa + (ulong)dstBaseOffset, dstSpan); } else { if (remap && _state.State.SetRemapComponentsDstX == SetRemapComponentsDst.ConstA && _state.State.SetRemapComponentsDstY == SetRemapComponentsDst.ConstA && _state.State.SetRemapComponentsDstZ == SetRemapComponentsDst.ConstA && _state.State.SetRemapComponentsDstW == SetRemapComponentsDst.ConstA && _state.State.SetRemapComponentsNumSrcComponents == SetRemapComponentsNumComponents.One && _state.State.SetRemapComponentsNumDstComponents == SetRemapComponentsNumComponents.One && _state.State.SetRemapComponentsComponentSize == SetRemapComponentsComponentSize.Four) { // Fast path for clears when remap is enabled. memoryManager.Physical.BufferCache.ClearBuffer(memoryManager, dstGpuVa, size * 4, _state.State.SetRemapConstA); } else { // TODO: Implement remap functionality. // Buffer to buffer copy. bool srcIsPitchKind = memoryManager.GetKind(srcGpuVa).IsPitch(); bool dstIsPitchKind = memoryManager.GetKind(dstGpuVa).IsPitch(); if (!srcIsPitchKind && dstIsPitchKind) { CopyGobBlockLinearToLinear(memoryManager, srcGpuVa, dstGpuVa, size); } else if (srcIsPitchKind && !dstIsPitchKind) { CopyGobLinearToBlockLinear(memoryManager, srcGpuVa, dstGpuVa, size); } else { memoryManager.Physical.BufferCache.CopyBuffer(memoryManager, srcGpuVa, dstGpuVa, size); } } } } /// /// Copies data from one texture to another, while performing layout conversion if necessary. /// /// Pixel type /// Destination texture memory region /// Source texture memory region /// Destination texture parameters /// Source texture parameters private unsafe void Copy(Span dstSpan, ReadOnlySpan srcSpan, TextureParams dst, TextureParams src) where T : unmanaged { int xCount = (int)_state.State.LineLengthIn; int yCount = (int)_state.State.LineCount; if (src.Linear && dst.Linear && src.Bpp == dst.Bpp) { // Optimized path for purely linear copies - we don't need to calculate every single byte offset, // and we can make use of Span.CopyTo which is very very fast (even compared to pointers) for (int y = 0; y < yCount; y++) { src.Calculator.SetY(src.RegionY + y); dst.Calculator.SetY(dst.RegionY + y); int srcOffset = src.Calculator.GetOffset(src.RegionX); int dstOffset = dst.Calculator.GetOffset(dst.RegionX); srcSpan.Slice(srcOffset - src.BaseOffset, xCount * src.Bpp) .CopyTo(dstSpan.Slice(dstOffset - dst.BaseOffset, xCount * dst.Bpp)); } } else { fixed (byte* dstPtr = dstSpan, srcPtr = srcSpan) { byte* dstBase = dstPtr - dst.BaseOffset; // Layout offset is relative to the base, so we need to subtract the span's offset. byte* srcBase = srcPtr - src.BaseOffset; for (int y = 0; y < yCount; y++) { src.Calculator.SetY(src.RegionY + y); dst.Calculator.SetY(dst.RegionY + y); for (int x = 0; x < xCount; x++) { int srcOffset = src.Calculator.GetOffset(src.RegionX + x); int dstOffset = dst.Calculator.GetOffset(dst.RegionX + x); *(T*)(dstBase + dstOffset) = *(T*)(srcBase + srcOffset); } } } } } /// /// Sets texture pixel data to a constant value, while performing layout conversion if necessary. /// /// Pixel type /// Destination texture memory region /// Destination texture parameters /// Constant pixel value to be set private unsafe void Fill(Span dstSpan, TextureParams dst, T fillValue) where T : unmanaged { int xCount = (int)_state.State.LineLengthIn; int yCount = (int)_state.State.LineCount; fixed (byte* dstPtr = dstSpan) { byte* dstBase = dstPtr - dst.BaseOffset; // Layout offset is relative to the base, so we need to subtract the span's offset. for (int y = 0; y < yCount; y++) { dst.Calculator.SetY(dst.RegionY + y); for (int x = 0; x < xCount; x++) { int dstOffset = dst.Calculator.GetOffset(dst.RegionX + x); *(T*)(dstBase + dstOffset) = fillValue; } } } } /// /// Copies data from one texture to another, while performing layout conversion and component shuffling if necessary. /// /// Pixel type /// Destination texture memory region /// Source texture memory region /// Destination texture parameters /// Source texture parameters private void CopyShuffle(Span dstSpan, ReadOnlySpan srcSpan, TextureParams dst, TextureParams src) where T : unmanaged { int dstComponents = (int)_state.State.SetRemapComponentsNumDstComponents + 1; for (int i = 0; i < dstComponents; i++) { SetRemapComponentsDst componentsDst = i switch { 0 => _state.State.SetRemapComponentsDstX, 1 => _state.State.SetRemapComponentsDstY, 2 => _state.State.SetRemapComponentsDstZ, _ => _state.State.SetRemapComponentsDstW, }; switch (componentsDst) { case SetRemapComponentsDst.SrcX: Copy(dstSpan[(Unsafe.SizeOf() * i)..], srcSpan, dst, src); break; case SetRemapComponentsDst.SrcY: Copy(dstSpan[(Unsafe.SizeOf() * i)..], srcSpan[Unsafe.SizeOf()..], dst, src); break; case SetRemapComponentsDst.SrcZ: Copy(dstSpan[(Unsafe.SizeOf() * i)..], srcSpan[(Unsafe.SizeOf() * 2)..], dst, src); break; case SetRemapComponentsDst.SrcW: Copy(dstSpan[(Unsafe.SizeOf() * i)..], srcSpan[(Unsafe.SizeOf() * 3)..], dst, src); break; case SetRemapComponentsDst.ConstA: Fill(dstSpan[(Unsafe.SizeOf() * i)..], dst, Unsafe.As(ref _state.State.SetRemapConstA)); break; case SetRemapComponentsDst.ConstB: Fill(dstSpan[(Unsafe.SizeOf() * i)..], dst, Unsafe.As(ref _state.State.SetRemapConstB)); break; } } } /// /// Copies block linear data with block linear GOBs to a block linear destination with linear GOBs. /// /// GPU memory manager /// Source GPU virtual address /// Destination GPU virtual address /// Size in bytes of the copy private static void CopyGobBlockLinearToLinear(MemoryManager memoryManager, ulong srcGpuVa, ulong dstGpuVa, ulong size) { if (((srcGpuVa | dstGpuVa | size) & 0xf) == 0) { for (ulong offset = 0; offset < size; offset += 16) { Vector128 data = memoryManager.Read>(ConvertGobLinearToBlockLinearAddress(srcGpuVa + offset), true); memoryManager.Write(dstGpuVa + offset, data); } } else { for (ulong offset = 0; offset < size; offset++) { byte data = memoryManager.Read(ConvertGobLinearToBlockLinearAddress(srcGpuVa + offset), true); memoryManager.Write(dstGpuVa + offset, data); } } } /// /// Copies block linear data with linear GOBs to a block linear destination with block linear GOBs. /// /// GPU memory manager /// Source GPU virtual address /// Destination GPU virtual address /// Size in bytes of the copy private static void CopyGobLinearToBlockLinear(MemoryManager memoryManager, ulong srcGpuVa, ulong dstGpuVa, ulong size) { if (((srcGpuVa | dstGpuVa | size) & 0xf) == 0) { for (ulong offset = 0; offset < size; offset += 16) { Vector128 data = memoryManager.Read>(srcGpuVa + offset, true); memoryManager.Write(ConvertGobLinearToBlockLinearAddress(dstGpuVa + offset), data); } } else { for (ulong offset = 0; offset < size; offset++) { byte data = memoryManager.Read(srcGpuVa + offset, true); memoryManager.Write(ConvertGobLinearToBlockLinearAddress(dstGpuVa + offset), data); } } } /// /// Calculates the GOB block linear address from a linear address. /// /// Linear address /// Block linear address private static ulong ConvertGobLinearToBlockLinearAddress(ulong address) { // y2 y1 y0 x5 x4 x3 x2 x1 x0 -> x5 y2 y1 x4 y0 x3 x2 x1 x0 return (address & ~0x1f0UL) | ((address & 0x40) >> 2) | ((address & 0x10) << 1) | ((address & 0x180) >> 1) | ((address & 0x20) << 3); } /// /// Performs a buffer to buffer, or buffer to texture copy, then optionally releases a semaphore. /// /// Method call argument private void LaunchDma(int argument) { DmaCopy(argument); ReleaseSemaphore(argument); } } }