using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Engine.Types; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Shader; using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Image { /// /// Texture bindings array cache. /// class TextureBindingsArrayCache { /// /// Minimum timestamp delta until texture array can be removed from the cache. /// private const int MinDeltaForRemoval = 20000; private readonly GpuContext _context; private readonly GpuChannel _channel; private readonly bool _isCompute; /// /// Array cache entry key. /// private readonly struct CacheEntryKey : IEquatable { /// /// Whether the entry is for an image. /// public readonly bool IsImage; /// /// Texture or image target type. /// public readonly Target Target; /// /// Word offset of the first handle on the constant buffer. /// public readonly int HandleIndex; /// /// Number of entries of the array. /// public readonly int ArrayLength; private readonly TexturePool _texturePool; private readonly SamplerPool _samplerPool; private readonly BufferBounds _textureBufferBounds; /// /// Creates a new array cache entry. /// /// Whether the entry is for an image /// Binding information for the array /// Texture pool where the array textures are located /// Sampler pool where the array samplers are located /// Constant buffer bounds with the texture handles public CacheEntryKey( bool isImage, TextureBindingInfo bindingInfo, TexturePool texturePool, SamplerPool samplerPool, ref BufferBounds textureBufferBounds) { IsImage = isImage; Target = bindingInfo.Target; HandleIndex = bindingInfo.Handle; ArrayLength = bindingInfo.ArrayLength; _texturePool = texturePool; _samplerPool = samplerPool; _textureBufferBounds = textureBufferBounds; } /// /// Checks if the texture and sampler pools matches the cached pools. /// /// Texture pool instance /// Sampler pool instance /// True if the pools match, false otherwise private bool MatchesPools(TexturePool texturePool, SamplerPool samplerPool) { return _texturePool == texturePool && _samplerPool == samplerPool; } /// /// Checks if the cached constant buffer address and size matches. /// /// New buffer address and size /// True if the address and size matches, false otherwise private bool MatchesBufferBounds(BufferBounds textureBufferBounds) { return _textureBufferBounds.Equals(textureBufferBounds); } public bool Equals(CacheEntryKey other) { return IsImage == other.IsImage && Target == other.Target && HandleIndex == other.HandleIndex && ArrayLength == other.ArrayLength && MatchesPools(other._texturePool, other._samplerPool) && MatchesBufferBounds(other._textureBufferBounds); } public override bool Equals(object obj) { return obj is CacheEntryKey other && Equals(other); } public override int GetHashCode() { return _textureBufferBounds.Range.GetHashCode(); } } /// /// Array cache entry. /// private class CacheEntry { /// /// Key for this entry on the cache. /// public readonly CacheEntryKey Key; /// /// Linked list node used on the texture bindings array cache. /// public LinkedListNode CacheNode; /// /// Timestamp set on the last use of the array by the cache. /// public int CacheTimestamp; /// /// All cached textures, along with their invalidated sequence number as value. /// public readonly Dictionary Textures; /// /// All pool texture IDs along with their textures. /// public readonly Dictionary TextureIds; /// /// All pool sampler IDs along with their samplers. /// public readonly Dictionary SamplerIds; /// /// Backend texture array if the entry is for a texture, otherwise null. /// public readonly ITextureArray TextureArray; /// /// Backend image array if the entry is for an image, otherwise null. /// public readonly IImageArray ImageArray; private readonly TexturePool _texturePool; private readonly SamplerPool _samplerPool; private int _texturePoolSequence; private int _samplerPoolSequence; private int[] _cachedTextureBuffer; private int[] _cachedSamplerBuffer; private int _lastSequenceNumber; /// /// Creates a new array cache entry. /// /// Key for this entry on the cache /// Texture pool where the array textures are located /// Sampler pool where the array samplers are located private CacheEntry(ref CacheEntryKey key, TexturePool texturePool, SamplerPool samplerPool) { Key = key; Textures = new Dictionary(); TextureIds = new Dictionary(); SamplerIds = new Dictionary(); _texturePool = texturePool; _samplerPool = samplerPool; _lastSequenceNumber = -1; } /// /// Creates a new array cache entry. /// /// Key for this entry on the cache /// Backend texture array /// Texture pool where the array textures are located /// Sampler pool where the array samplers are located public CacheEntry(ref CacheEntryKey key, ITextureArray array, TexturePool texturePool, SamplerPool samplerPool) : this(ref key, texturePool, samplerPool) { TextureArray = array; } /// /// Creates a new array cache entry. /// /// Key for this entry on the cache /// Backend image array /// Texture pool where the array textures are located /// Sampler pool where the array samplers are located public CacheEntry(ref CacheEntryKey key, IImageArray array, TexturePool texturePool, SamplerPool samplerPool) : this(ref key, texturePool, samplerPool) { ImageArray = array; } /// /// Synchronizes memory for all textures in the array. /// /// Indicates if the texture may be modified by the access /// Indicates if the texture should be blacklisted for scaling public void SynchronizeMemory(bool isStore, bool blacklistScale) { foreach (Texture texture in Textures.Keys) { texture.SynchronizeMemory(); if (isStore) { texture.SignalModified(); } if (blacklistScale && texture.ScaleMode != TextureScaleMode.Blacklisted) { // Scaling textures used on arrays is currently not supported. texture.BlacklistScale(); } } } /// /// Clears all cached texture instances. /// public void Reset() { Textures.Clear(); TextureIds.Clear(); SamplerIds.Clear(); } /// /// Updates the cached constant buffer data. /// /// Constant buffer data with the texture handles (and sampler handles, if they are combined) /// Constant buffer data with the sampler handles /// Whether and comes from different buffers public void UpdateData(ReadOnlySpan cachedTextureBuffer, ReadOnlySpan cachedSamplerBuffer, bool separateSamplerBuffer) { _cachedTextureBuffer = cachedTextureBuffer.ToArray(); _cachedSamplerBuffer = separateSamplerBuffer ? cachedSamplerBuffer.ToArray() : _cachedTextureBuffer; } /// /// Checks if any texture has been deleted since the last call to this method. /// /// True if one or more textures have been deleted, false otherwise public bool ValidateTextures() { foreach ((Texture texture, int invalidatedSequence) in Textures) { if (texture.InvalidatedSequence != invalidatedSequence) { return false; } } return true; } /// /// Checks if the cached texture or sampler pool has been modified since the last call to this method. /// /// True if any used entries of the pools might have been modified, false otherwise public bool PoolsModified() { bool texturePoolModified = _texturePool.WasModified(ref _texturePoolSequence); bool samplerPoolModified = _samplerPool.WasModified(ref _samplerPoolSequence); // If both pools were not modified since the last check, we have nothing else to check. if (!texturePoolModified && !samplerPoolModified) { return false; } // If the pools were modified, let's check if any of the entries we care about changed. // Check if any of our cached textures changed on the pool. foreach ((int textureId, Texture texture) in TextureIds) { if (_texturePool.GetCachedItem(textureId) != texture) { return true; } } // Check if any of our cached samplers changed on the pool. foreach ((int samplerId, Sampler sampler) in SamplerIds) { if (_samplerPool.GetCachedItem(samplerId) != sampler) { return true; } } return false; } /// /// Checks if the sequence number matches the one used on the last call to this method. /// /// Current sequence number /// True if the sequence numbers match, false otherwise public bool MatchesSequenceNumber(int currentSequenceNumber) { if (_lastSequenceNumber == currentSequenceNumber) { return true; } _lastSequenceNumber = currentSequenceNumber; return false; } /// /// Checks if the buffer data matches the cached data. /// /// New texture buffer data /// New sampler buffer data /// Whether and comes from different buffers /// Word offset of the sampler constant buffer handle that is used /// True if the data matches, false otherwise public bool MatchesBufferData( ReadOnlySpan cachedTextureBuffer, ReadOnlySpan cachedSamplerBuffer, bool separateSamplerBuffer, int samplerWordOffset) { if (_cachedTextureBuffer != null && cachedTextureBuffer.Length > _cachedTextureBuffer.Length) { cachedTextureBuffer = cachedTextureBuffer[.._cachedTextureBuffer.Length]; } if (!_cachedTextureBuffer.AsSpan().SequenceEqual(cachedTextureBuffer)) { return false; } if (separateSamplerBuffer) { if (_cachedSamplerBuffer == null || _cachedSamplerBuffer.Length <= samplerWordOffset || cachedSamplerBuffer.Length <= samplerWordOffset) { return false; } int oldValue = _cachedSamplerBuffer[samplerWordOffset]; int newValue = cachedSamplerBuffer[samplerWordOffset]; return oldValue == newValue; } return true; } } private readonly Dictionary _cache; private readonly LinkedList _lruCache; private int _currentTimestamp; /// /// Creates a new instance of the texture bindings array cache. /// /// GPU context /// GPU channel /// Whether the bindings will be used for compute or graphics pipelines public TextureBindingsArrayCache(GpuContext context, GpuChannel channel, bool isCompute) { _context = context; _channel = channel; _isCompute = isCompute; _cache = new Dictionary(); _lruCache = new LinkedList(); } /// /// Updates a texture array bindings and textures. /// /// Texture pool /// Sampler pool /// Shader stage where the array is used /// Shader stage index where the array is used /// Texture constant buffer index /// Sampler handles source /// Array binding information public void UpdateTextureArray( TexturePool texturePool, SamplerPool samplerPool, ShaderStage stage, int stageIndex, int textureBufferIndex, SamplerIndex samplerIndex, TextureBindingInfo bindingInfo) { Update(texturePool, samplerPool, stage, stageIndex, textureBufferIndex, isImage: false, samplerIndex, bindingInfo); } /// /// Updates a image array bindings and textures. /// /// Texture pool /// Shader stage where the array is used /// Shader stage index where the array is used /// Texture constant buffer index /// Array binding information public void UpdateImageArray(TexturePool texturePool, ShaderStage stage, int stageIndex, int textureBufferIndex, TextureBindingInfo bindingInfo) { Update(texturePool, null, stage, stageIndex, textureBufferIndex, isImage: true, SamplerIndex.ViaHeaderIndex, bindingInfo); } /// /// Updates a texture or image array bindings and textures. /// /// Texture pool /// Sampler pool /// Shader stage where the array is used /// Shader stage index where the array is used /// Texture constant buffer index /// Whether the array is a image or texture array /// Sampler handles source /// Array binding information private void Update( TexturePool texturePool, SamplerPool samplerPool, ShaderStage stage, int stageIndex, int textureBufferIndex, bool isImage, SamplerIndex samplerIndex, TextureBindingInfo bindingInfo) { (textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, textureBufferIndex); bool separateSamplerBuffer = textureBufferIndex != samplerBufferIndex; ref BufferBounds textureBufferBounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex); ref BufferBounds samplerBufferBounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex); CacheEntry entry = GetOrAddEntry( texturePool, samplerPool, bindingInfo, isImage, ref textureBufferBounds, out bool isNewEntry); bool poolsModified = entry.PoolsModified(); bool isStore = bindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore); bool resScaleUnsupported = bindingInfo.Flags.HasFlag(TextureUsageFlags.ResScaleUnsupported); ReadOnlySpan cachedTextureBuffer; ReadOnlySpan cachedSamplerBuffer; if (!poolsModified && !isNewEntry && entry.ValidateTextures()) { if (entry.MatchesSequenceNumber(_context.SequenceNumber)) { entry.SynchronizeMemory(isStore, resScaleUnsupported); if (isImage) { _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); } else { _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); } return; } cachedTextureBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(textureBufferBounds.Range)); if (separateSamplerBuffer) { cachedSamplerBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(samplerBufferBounds.Range)); } else { cachedSamplerBuffer = cachedTextureBuffer; } (_, int samplerWordOffset, _) = TextureHandle.UnpackOffsets(bindingInfo.Handle); if (entry.MatchesBufferData(cachedTextureBuffer, cachedSamplerBuffer, separateSamplerBuffer, samplerWordOffset)) { entry.SynchronizeMemory(isStore, resScaleUnsupported); if (isImage) { _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); } else { _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); } return; } } else { cachedTextureBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(textureBufferBounds.Range)); if (separateSamplerBuffer) { cachedSamplerBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(samplerBufferBounds.Range)); } else { cachedSamplerBuffer = cachedTextureBuffer; } } if (!isNewEntry) { entry.Reset(); } entry.UpdateData(cachedTextureBuffer, cachedSamplerBuffer, separateSamplerBuffer); Format[] formats = isImage ? new Format[bindingInfo.ArrayLength] : null; ISampler[] samplers = isImage ? null : new ISampler[bindingInfo.ArrayLength]; ITexture[] textures = new ITexture[bindingInfo.ArrayLength]; for (int index = 0; index < bindingInfo.ArrayLength; index++) { int handleIndex = bindingInfo.Handle + index * (Constants.TextureHandleSizeInBytes / sizeof(int)); int packedId = TextureHandle.ReadPackedId(handleIndex, cachedTextureBuffer, cachedSamplerBuffer); int textureId = TextureHandle.UnpackTextureId(packedId); int samplerId; if (samplerIndex == SamplerIndex.ViaHeaderIndex) { samplerId = textureId; } else { samplerId = TextureHandle.UnpackSamplerId(packedId); } ref readonly TextureDescriptor descriptor = ref texturePool.GetForBinding(textureId, out Texture texture); if (texture != null) { entry.Textures[texture] = texture.InvalidatedSequence; if (isStore) { texture.SignalModified(); } if (resScaleUnsupported && texture.ScaleMode != TextureScaleMode.Blacklisted) { // Scaling textures used on arrays is currently not supported. texture.BlacklistScale(); } } Sampler sampler = samplerPool?.Get(samplerId); entry.TextureIds[textureId] = texture; entry.SamplerIds[samplerId] = sampler; ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); ISampler hostSampler = sampler?.GetHostSampler(texture); Format format = bindingInfo.Format; if (hostTexture != null && texture.Target == Target.TextureBuffer) { // Ensure that the buffer texture is using the correct buffer as storage. // Buffers are frequently re-created to accommodate larger data, so we need to re-bind // to ensure we're not using a old buffer that was already deleted. if (isImage) { if (format == 0 && texture != null) { format = texture.Format; } _channel.BufferManager.SetBufferTextureStorage(entry.ImageArray, hostTexture, texture.Range, bindingInfo, index, format); } else { _channel.BufferManager.SetBufferTextureStorage(entry.TextureArray, hostTexture, texture.Range, bindingInfo, index, format); } } else if (isImage) { if (format == 0 && texture != null) { format = texture.Format; } formats[index] = format; textures[index] = hostTexture; } else { samplers[index] = hostSampler; textures[index] = hostTexture; } } if (isImage) { entry.ImageArray.SetFormats(0, formats); entry.ImageArray.SetImages(0, textures); _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); } else { entry.TextureArray.SetSamplers(0, samplers); entry.TextureArray.SetTextures(0, textures); _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); } } /// /// Gets a cached texture entry, or creates a new one if not found. /// /// Texture pool /// Sampler pool /// Array binding information /// Whether the array is a image or texture array /// Constant buffer bounds with the texture handles /// Whether a new entry was created, or an existing one was returned /// Cache entry private CacheEntry GetOrAddEntry( TexturePool texturePool, SamplerPool samplerPool, TextureBindingInfo bindingInfo, bool isImage, ref BufferBounds textureBufferBounds, out bool isNew) { CacheEntryKey key = new CacheEntryKey( isImage, bindingInfo, texturePool, samplerPool, ref textureBufferBounds); isNew = !_cache.TryGetValue(key, out CacheEntry entry); if (isNew) { int arrayLength = bindingInfo.ArrayLength; if (isImage) { IImageArray array = _context.Renderer.CreateImageArray(arrayLength, bindingInfo.Target == Target.TextureBuffer); _cache.Add(key, entry = new CacheEntry(ref key, array, texturePool, samplerPool)); } else { ITextureArray array = _context.Renderer.CreateTextureArray(arrayLength, bindingInfo.Target == Target.TextureBuffer); _cache.Add(key, entry = new CacheEntry(ref key, array, texturePool, samplerPool)); } } if (entry.CacheNode != null) { _lruCache.Remove(entry.CacheNode); _lruCache.AddLast(entry.CacheNode); } else { entry.CacheNode = _lruCache.AddLast(entry); } entry.CacheTimestamp = ++_currentTimestamp; RemoveLeastUsedEntries(); return entry; } /// /// Remove entries from the cache that have not been used for some time. /// private void RemoveLeastUsedEntries() { LinkedListNode nextNode = _lruCache.First; while (nextNode != null && _currentTimestamp - nextNode.Value.CacheTimestamp >= MinDeltaForRemoval) { LinkedListNode toRemove = nextNode; nextNode = nextNode.Next; _cache.Remove(toRemove.Value.Key); _lruCache.Remove(toRemove); } } } }