using Ryujinx.Common.Logging; using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Texture; using Ryujinx.Memory.Range; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Numerics; using System.Threading; namespace Ryujinx.Graphics.Gpu.Image { /// /// Texture pool. /// class TexturePool : Pool, IPool { /// /// A request to dereference a texture from a pool. /// private readonly struct DereferenceRequest { /// /// Whether the dereference is due to a mapping change or not. /// public readonly bool IsRemapped; /// /// The texture being dereferenced. /// public readonly Texture Texture; /// /// The ID of the pool entry this reference belonged to. /// public readonly int ID; /// /// Create a dereference request for a texture with a specific pool ID, and remapped flag. /// /// Whether the dereference is due to a mapping change or not /// The texture being dereferenced /// The ID of the pool entry, used to restore remapped textures private DereferenceRequest(bool isRemapped, Texture texture, int id) { IsRemapped = isRemapped; Texture = texture; ID = id; } /// /// Create a dereference request for a texture removal. /// /// The texture being removed /// A texture removal dereference request public static DereferenceRequest Remove(Texture texture) { return new DereferenceRequest(false, texture, 0); } /// /// Create a dereference request for a texture remapping with a specific pool ID. /// /// The texture being remapped /// The ID of the pool entry, used to restore remapped textures /// A remap dereference request public static DereferenceRequest Remap(Texture texture, int id) { return new DereferenceRequest(true, texture, id); } } private readonly GpuChannel _channel; private readonly ConcurrentQueue _dereferenceQueue = new(); private TextureDescriptor _defaultDescriptor; /// /// List of textures that shares the same memory region, but have different formats. /// private class TextureAliasList { /// /// Alias texture. /// /// Texture format /// Texture private readonly record struct Alias(Format Format, Texture Texture); /// /// List of texture aliases. /// private readonly List _aliases; /// /// Creates a new instance of the texture alias list. /// public TextureAliasList() { _aliases = new List(); } /// /// Adds a new texture alias. /// /// Alias format /// Alias texture public void Add(Format format, Texture texture) { _aliases.Add(new Alias(format, texture)); texture.IncrementReferenceCount(); } /// /// Finds a texture with the requested format, or returns null if not found. /// /// Format to find /// Texture with the requested format, or null if not found public Texture Find(Format format) { foreach (var alias in _aliases) { if (alias.Format == format) { return alias.Texture; } } return null; } /// /// Removes all alias textures. /// public void Destroy() { foreach (var entry in _aliases) { entry.Texture.DecrementReferenceCount(); } _aliases.Clear(); } } private readonly Dictionary _aliasLists; /// /// Linked list node used on the texture pool cache. /// public LinkedListNode CacheNode { get; set; } /// /// Timestamp used by the texture pool cache, updated on every use of this texture pool. /// public ulong CacheTimestamp { get; set; } /// /// Creates a new instance of the texture pool. /// /// GPU context that the texture pool belongs to /// GPU channel that the texture pool belongs to /// Address of the texture pool in guest memory /// Maximum texture ID of the texture pool (equal to maximum textures minus one) public TexturePool(GpuContext context, GpuChannel channel, ulong address, int maximumId) : base(context, channel.MemoryManager.Physical, address, maximumId) { _channel = channel; _aliasLists = new Dictionary(); } /// /// Gets the texture descripor and texture with the given ID with no bounds check or synchronization. /// /// ID of the texture. This is effectively a zero-based index /// The texture with the given ID /// The texture descriptor with the given ID private ref readonly TextureDescriptor GetInternal(int id, out Texture texture) { texture = Items[id]; ref readonly TextureDescriptor descriptor = ref GetDescriptorRef(id); if (texture == null) { texture = PhysicalMemory.TextureCache.FindShortCache(descriptor); if (texture == null) { // The dereference queue can put our texture back on the cache. if ((texture = ProcessDereferenceQueue(id)) != null) { return ref descriptor; } TextureInfo info = GetInfo(descriptor, out int layerSize); texture = PhysicalMemory.TextureCache.FindOrCreateTexture(_channel.MemoryManager, TextureSearchFlags.ForSampler, info, layerSize); // If this happens, then the texture address is invalid, we can't add it to the cache. if (texture == null) { return ref descriptor; } } else { texture.SynchronizeMemory(); } Items[id] = texture; texture.IncrementReferenceCount(this, id, descriptor.UnpackAddress()); DescriptorCache[id] = descriptor; } else { // On the path above (texture not yet in the pool), memory is automatically synchronized on texture creation. texture.SynchronizeMemory(); } return ref descriptor; } /// /// Gets the texture with the given ID. /// /// ID of the texture. This is effectively a zero-based index /// The texture with the given ID public override Texture Get(int id) { return Get(id, srgbSampler: true); } /// /// Gets the texture with the given ID. /// /// ID of the texture. This is effectively a zero-based index /// Whether the texture is being accessed with a sampler that has sRGB conversion enabled /// The texture with the given ID public Texture Get(int id, bool srgbSampler) { if ((uint)id >= Items.Length) { return null; } if (SequenceNumber != Context.SequenceNumber) { SequenceNumber = Context.SequenceNumber; SynchronizeMemory(); } GetForBinding(id, srgbSampler, out Texture texture); return texture; } /// /// Gets the texture descriptor and texture with the given ID. /// /// /// This method assumes that the pool has been manually synchronized before doing binding. /// /// ID of the texture. This is effectively a zero-based index /// Whether the texture is being accessed with a sampler that has sRGB conversion enabled /// The texture with the given ID /// The texture descriptor with the given ID public ref readonly TextureDescriptor GetForBinding(int id, bool srgbSampler, out Texture texture) { if ((uint)id >= Items.Length) { texture = null; return ref _defaultDescriptor; } // When getting for binding, assume the pool has already been synchronized. if (!srgbSampler) { // If the sampler does not have the sRGB bit enabled, then the texture can't use a sRGB format. ref readonly TextureDescriptor tempDescriptor = ref GetDescriptorRef(id); if (tempDescriptor.UnpackSrgb() && FormatTable.TryGetTextureFormat(tempDescriptor.UnpackFormat(), isSrgb: false, out FormatInfo formatInfo)) { // Get a view of the texture with the right format. return ref GetForBinding(id, formatInfo, out texture); } } return ref GetInternal(id, out texture); } /// /// Gets the texture descriptor and texture with the given ID. /// /// /// This method assumes that the pool has been manually synchronized before doing binding. /// /// ID of the texture. This is effectively a zero-based index /// Texture format information /// The texture with the given ID /// The texture descriptor with the given ID public ref readonly TextureDescriptor GetForBinding(int id, FormatInfo formatInfo, out Texture texture) { if ((uint)id >= Items.Length) { texture = null; return ref _defaultDescriptor; } ref readonly TextureDescriptor descriptor = ref GetInternal(id, out texture); if (texture != null && formatInfo.Format != 0 && texture.Format != formatInfo.Format) { if (!_aliasLists.TryGetValue(texture, out TextureAliasList aliasList)) { _aliasLists.Add(texture, aliasList = new TextureAliasList()); } texture = aliasList.Find(formatInfo.Format); if (texture == null) { TextureInfo info = GetInfo(descriptor, out int layerSize); info = ChangeFormat(info, formatInfo); texture = PhysicalMemory.TextureCache.FindOrCreateTexture(_channel.MemoryManager, TextureSearchFlags.ForSampler, info, layerSize); if (texture != null) { aliasList.Add(formatInfo.Format, texture); } } } return ref descriptor; } /// /// Checks if the pool was modified, and returns the last sequence number where a modification was detected. /// /// A number that increments each time a modification is detected public int CheckModified() { if (SequenceNumber != Context.SequenceNumber) { SequenceNumber = Context.SequenceNumber; SynchronizeMemory(); } return ModifiedSequenceNumber; } /// /// Forcibly remove a texture from this pool's items. /// If deferred, the dereference will be queued to occur on the render thread. /// /// The texture being removed /// The ID of the texture in this pool /// If true, queue the dereference to happen on the render thread, otherwise dereference immediately public void ForceRemove(Texture texture, int id, bool deferred) { var previous = Interlocked.Exchange(ref Items[id], null); if (deferred) { if (previous != null) { _dereferenceQueue.Enqueue(DereferenceRequest.Remove(texture)); } } else { texture.DecrementReferenceCount(); RemoveAliasList(texture); } } /// /// Queues a request to update a texture's mapping. /// Mapping is updated later to avoid deleting the texture if it is still sparsely mapped. /// /// Texture with potential mapping change /// ID in cache of texture with potential mapping change public void QueueUpdateMapping(Texture texture, int id) { if (Interlocked.Exchange(ref Items[id], null) == texture) { _dereferenceQueue.Enqueue(DereferenceRequest.Remap(texture, id)); } } /// /// Process the dereference queue, decrementing the reference count for each texture in it. /// This is used to ensure that texture disposal happens on the render thread. /// /// The ID of the entry that triggered this method /// Texture that matches the entry ID if it has been readded to the cache. private Texture ProcessDereferenceQueue(int id = -1) { while (_dereferenceQueue.TryDequeue(out DereferenceRequest request)) { Texture texture = request.Texture; // Unmapped storage textures can swap their ranges. The texture must be storage with no views or dependencies. // TODO: Would need to update ranges on views, or guarantee that ones where the range changes can be instantly deleted. if (request.IsRemapped && texture.Group.Storage == texture && !texture.HasViews && !texture.Group.HasCopyDependencies) { // Has the mapping for this texture changed? ref readonly TextureDescriptor descriptor = ref GetDescriptorRef(request.ID); ulong address = descriptor.UnpackAddress(); if (!descriptor.Equals(ref DescriptorCache[request.ID])) { // If the pool entry has already been replaced, just remove the texture. texture.DecrementReferenceCount(); continue; } MultiRange range = _channel.MemoryManager.Physical.TextureCache.UpdatePartiallyMapped(_channel.MemoryManager, address, texture); // If the texture is not mapped at all, delete its reference. if (range.Count == 1 && range.GetSubRange(0).Address == MemoryManager.PteUnmapped) { texture.DecrementReferenceCount(); continue; } Items[request.ID] = texture; // Create a new pool reference, as the last one was removed on unmap. texture.IncrementReferenceCount(this, request.ID, address); texture.DecrementReferenceCount(); // Refetch the range. Changes since the last check could have been lost // as the cache entry was not restored (required to queue mapping change). range = _channel.MemoryManager.GetPhysicalRegions(address, texture.Size); if (!range.Equals(texture.Range)) { // Part of the texture was mapped or unmapped. Replace the range and regenerate tracking handles. if (!_channel.MemoryManager.Physical.TextureCache.UpdateMapping(texture, range)) { // Texture could not be remapped due to a collision, just delete it. if (Interlocked.Exchange(ref Items[request.ID], null) != null) { // If this is null, a request was already queued to decrement reference. texture.DecrementReferenceCount(this, request.ID); } continue; } } if (request.ID == id) { return texture; } } else { texture.DecrementReferenceCount(); } RemoveAliasList(texture); } return null; } /// /// Implementation of the texture pool range invalidation. /// /// Start address of the range of the texture pool /// Size of the range being invalidated protected override void InvalidateRangeImpl(ulong address, ulong size) { ProcessDereferenceQueue(); ulong endAddress = address + size; for (; address < endAddress; address += DescriptorSize) { int id = (int)((address - Address) / DescriptorSize); Texture texture = Items[id]; if (texture != null) { ref TextureDescriptor cachedDescriptor = ref DescriptorCache[id]; ref readonly TextureDescriptor descriptor = ref GetDescriptorRefAddress(address); // If the descriptors are the same, the texture is the same, // we don't need to remove as it was not modified. Just continue. if (descriptor.Equals(ref cachedDescriptor)) { continue; } if (texture.HasOneReference()) { _channel.MemoryManager.Physical.TextureCache.AddShortCache(texture, ref cachedDescriptor); } if (Interlocked.Exchange(ref Items[id], null) != null) { texture.DecrementReferenceCount(this, id); RemoveAliasList(texture); } } } } /// /// Gets texture information from a texture descriptor. /// /// The texture descriptor /// Layer size for textures using a sub-range of mipmap levels, otherwise 0 /// The texture information private static TextureInfo GetInfo(in TextureDescriptor descriptor, out int layerSize) { int depthOrLayers = descriptor.UnpackDepth(); int levels = descriptor.UnpackLevels(); TextureMsaaMode msaaMode = descriptor.UnpackTextureMsaaMode(); int samplesInX = msaaMode.SamplesInX(); int samplesInY = msaaMode.SamplesInY(); int stride = descriptor.UnpackStride(); TextureDescriptorType descriptorType = descriptor.UnpackTextureDescriptorType(); bool isLinear = descriptorType == TextureDescriptorType.Linear; Target target = descriptor.UnpackTextureTarget().Convert((samplesInX | samplesInY) != 1); int width = target == Target.TextureBuffer ? descriptor.UnpackBufferTextureWidth() : descriptor.UnpackWidth(); int height = descriptor.UnpackHeight(); if (target == Target.Texture2DMultisample || target == Target.Texture2DMultisampleArray) { // This is divided back before the backend texture is created. width *= samplesInX; height *= samplesInY; } // We use 2D targets for 1D textures as that makes texture cache // management easier. We don't know the target for render target // and copies, so those would normally use 2D targets, which are // not compatible with 1D targets. By doing that we also allow those // to match when looking for compatible textures on the cache. if (target == Target.Texture1D) { target = Target.Texture2D; height = 1; } else if (target == Target.Texture1DArray) { target = Target.Texture2DArray; height = 1; } uint format = descriptor.UnpackFormat(); bool srgb = descriptor.UnpackSrgb(); ulong gpuVa = descriptor.UnpackAddress(); if (!FormatTable.TryGetTextureFormat(format, srgb, out FormatInfo formatInfo)) { if (gpuVa != 0 && format != 0) { Logger.Error?.Print(LogClass.Gpu, $"Invalid texture format 0x{format:X} (sRGB: {srgb})."); } formatInfo = FormatInfo.Default; } int gobBlocksInY = descriptor.UnpackGobBlocksInY(); int gobBlocksInZ = descriptor.UnpackGobBlocksInZ(); int gobBlocksInTileX = descriptor.UnpackGobBlocksInTileX(); layerSize = 0; int minLod = descriptor.UnpackBaseLevel(); int maxLod = descriptor.UnpackMaxLevelInclusive(); // Linear textures don't support mipmaps, so we don't handle this case here. if ((minLod != 0 || maxLod + 1 != levels) && target != Target.TextureBuffer && !isLinear) { int depth = TextureInfo.GetDepth(target, depthOrLayers); int layers = TextureInfo.GetLayers(target, depthOrLayers); SizeInfo sizeInfo = SizeCalculator.GetBlockLinearTextureSize( width, height, depth, levels, layers, formatInfo.BlockWidth, formatInfo.BlockHeight, formatInfo.BytesPerPixel, gobBlocksInY, gobBlocksInZ, gobBlocksInTileX); layerSize = sizeInfo.LayerSize; if (minLod != 0 && minLod < levels) { // If the base level is not zero, we additionally add the mip level offset // to the address, this allows the texture manager to find the base level from the // address if there is a overlapping texture on the cache that can contain the new texture. gpuVa += (ulong)sizeInfo.GetMipOffset(minLod); width = Math.Max(1, width >> minLod); height = Math.Max(1, height >> minLod); if (target == Target.Texture3D) { depthOrLayers = Math.Max(1, depthOrLayers >> minLod); } (gobBlocksInY, gobBlocksInZ) = SizeCalculator.GetMipGobBlockSizes(height, depth, formatInfo.BlockHeight, gobBlocksInY, gobBlocksInZ, minLod); } levels = (maxLod - minLod) + 1; } levels = ClampLevels(target, width, height, depthOrLayers, levels); SwizzleComponent swizzleR = descriptor.UnpackSwizzleR().Convert(); SwizzleComponent swizzleG = descriptor.UnpackSwizzleG().Convert(); SwizzleComponent swizzleB = descriptor.UnpackSwizzleB().Convert(); SwizzleComponent swizzleA = descriptor.UnpackSwizzleA().Convert(); DepthStencilMode depthStencilMode = GetDepthStencilMode( formatInfo.Format, swizzleR, swizzleG, swizzleB, swizzleA); if (formatInfo.Format.IsDepthOrStencil()) { swizzleR = SwizzleComponent.Red; swizzleG = SwizzleComponent.Red; swizzleB = SwizzleComponent.Red; if (depthStencilMode == DepthStencilMode.Depth) { swizzleA = SwizzleComponent.One; } else { swizzleA = SwizzleComponent.Red; } } return new TextureInfo( gpuVa, width, height, depthOrLayers, levels, samplesInX, samplesInY, stride, isLinear, gobBlocksInY, gobBlocksInZ, gobBlocksInTileX, target, formatInfo, depthStencilMode, swizzleR, swizzleG, swizzleB, swizzleA); } /// /// Clamps the amount of mipmap levels to the maximum allowed for the given texture dimensions. /// /// Number of texture dimensions (1D, 2D, 3D, Cube, etc) /// Width of the texture /// Height of the texture, ignored for 1D textures /// Depth of the texture for 3D textures, otherwise ignored /// Original amount of mipmap levels /// Clamped mipmap levels private static int ClampLevels(Target target, int width, int height, int depthOrLayers, int levels) { int maxSize = width; if (target != Target.Texture1D && target != Target.Texture1DArray) { maxSize = Math.Max(maxSize, height); } if (target == Target.Texture3D) { maxSize = Math.Max(maxSize, depthOrLayers); } int maxLevels = BitOperations.Log2((uint)maxSize) + 1; return Math.Min(levels, maxLevels); } /// /// Gets the texture depth-stencil mode, based on the swizzle components of each color channel. /// The depth-stencil mode is determined based on how the driver sets those parameters. /// /// The format of the texture /// The texture swizzle components /// The depth-stencil mode private static DepthStencilMode GetDepthStencilMode(Format format, params SwizzleComponent[] components) { // R = Depth, G = Stencil. // On 24-bits depth formats, this is inverted (Stencil is R etc). // NVN setup: // For depth, A is set to 1.0f, the other components are set to Depth. // For stencil, all components are set to Stencil. SwizzleComponent component = components[0]; for (int index = 1; index < 4 && !IsRG(component); index++) { component = components[index]; } if (!IsRG(component)) { return DepthStencilMode.Depth; } if (format == Format.D24UnormS8Uint) { return component == SwizzleComponent.Red ? DepthStencilMode.Stencil : DepthStencilMode.Depth; } else { return component == SwizzleComponent.Red ? DepthStencilMode.Depth : DepthStencilMode.Stencil; } } /// /// Checks if the swizzle component is equal to the red or green channels. /// /// The swizzle component to check /// True if the swizzle component is equal to the red or green, false otherwise private static bool IsRG(SwizzleComponent component) { return component == SwizzleComponent.Red || component == SwizzleComponent.Green; } /// /// Changes the format on the texture information structure, and also adjusts the width for the new format if needed. /// /// Texture information /// New format /// Texture information with the new format private static TextureInfo ChangeFormat(in TextureInfo info, FormatInfo dstFormat) { int width = info.Width; if (info.FormatInfo.BytesPerPixel != dstFormat.BytesPerPixel) { int stride = width * info.FormatInfo.BytesPerPixel; width = stride / dstFormat.BytesPerPixel; } return new TextureInfo( info.GpuAddress, width, info.Height, info.DepthOrLayers, info.Levels, info.SamplesInX, info.SamplesInY, info.Stride, info.IsLinear, info.GobBlocksInY, info.GobBlocksInZ, info.GobBlocksInTileX, info.Target, dstFormat, info.DepthStencilMode, info.SwizzleR, info.SwizzleG, info.SwizzleB, info.SwizzleA); } /// /// Removes all aliases for a texture. /// /// Texture to have the aliases removed private void RemoveAliasList(Texture texture) { if (_aliasLists.TryGetValue(texture, out TextureAliasList aliasList)) { _aliasLists.Remove(texture); aliasList.Destroy(); } } /// /// Decrements the reference count of the texture. /// This indicates that the texture pool is not using it anymore. /// /// The texture to be deleted protected override void Delete(Texture item) { if (item != null) { item.DecrementReferenceCount(this); RemoveAliasList(item); } } public override void Dispose() { ProcessDereferenceQueue(); base.Dispose(); } } }