// Copyright Epic Games, Inc. All Rights Reserved. #pragma once #include "D3D12RHICommon.h" #include "D3D12Descriptors.h" #include "RHIDefinitions.h" #include "RHIDescriptorAllocator.h" #include "Templates/RefCounting.h" class FRHICommandListBase; class FD3D12SamplerState; class FD3D12ShaderResourceView; class FD3D12UnorderedAccessView; struct FD3D12Payload; #if PLATFORM_SUPPORTS_BINDLESS_RENDERING #include COMPILED_PLATFORM_HEADER(D3D12BindlessDescriptors.h) namespace UE::D3D12BindlessDescriptors { FD3D12DescriptorHeap* CreateCpuHeap(FD3D12Device* InDevice, ERHIDescriptorHeapType InType, uint32 InNewNumDescriptorsPerHeap); FD3D12DescriptorHeap* CreateGpuHeap(FD3D12Device* InDevice, ERHIDescriptorHeapType InType, uint32 InNewNumDescriptorsPerHeap); void DeferredFreeHeap(FD3D12Device* InDevice, FD3D12DescriptorHeap* InHeap); } /** Manager for configuration settings and shared descriptor allocators, stored on the adapter. */ class FD3D12BindlessDescriptorAllocator : public FD3D12AdapterChild { public: FD3D12BindlessDescriptorAllocator() = delete; FD3D12BindlessDescriptorAllocator(FD3D12Adapter* InParent); void Init(); ERHIBindlessConfiguration GetConfiguration() const { return BindlessConfiguration; } bool AreResourcesBindless() const { return ResourceAllocator != nullptr; } bool AreSamplersBindless() const { return SamplerAllocator != nullptr; } // Bindless descriptor allocators are stored in the adapter, so descriptor handles can be allocated once and shared for multi-GPU objects FRHIDescriptorHandle AllocateSamplerHandle(); FRHIDescriptorHandle AllocateResourceHandle(); void FreeSamplerHandle(FRHIDescriptorHandle Handle); void FreeResourceHandle(FRHIDescriptorHandle Handle); FCriticalSection& GetResourceHeapsCS() { return ResourceHeapsCS; } uint32 GetResourceCapacity() const { return ResourceAllocator->GetCapacity(); } uint32 GetSamplerCapacity() const { return SamplerAllocator->GetCapacity(); } bool GetResourceAllocatedRange(FRHIDescriptorAllocatorRange& OutAllocatedRange) const { return ResourceAllocator->GetAllocatedRange(OutAllocatedRange); } #if D3D12RHI_CUSTOM_BINDLESS_RESOURCE_MANAGER FRHIHeapDescriptorAllocator* GetResourceAllocator() { return ResourceAllocator; } #endif private: ERHIBindlessConfiguration BindlessConfiguration{}; uint32 MaxResourceHeapSize = 0; uint32 MaxSamplerHeapSize = 0; FCriticalSection ResourceHeapsCS; FRHIHeapDescriptorAllocator* ResourceAllocator = nullptr; FRHIHeapDescriptorAllocator* SamplerAllocator = nullptr; }; /** Manager specifically for bindless sampler descriptors. */ class FD3D12BindlessSamplerManager : public FD3D12DeviceChild { public: FD3D12BindlessSamplerManager() = delete; FD3D12BindlessSamplerManager(FD3D12Device* InDevice, FD3D12BindlessDescriptorAllocator& InAllocator); void CleanupResources(); void InitializeDescriptor(FRHIDescriptorHandle DstHandle, FD3D12SamplerState* SamplerState); void OpenCommandList(FD3D12CommandContext& Context); void CloseCommandList(FD3D12CommandContext& Context); FD3D12DescriptorHeap* GetExplicitHeapForContext(FD3D12CommandContext& Context) const; FD3D12DescriptorHeap* GetHeap() const { return GpuHeap.GetReference(); } ERHIBindlessConfiguration GetConfiguration() const { return Configuration; } private: FD3D12DescriptorHeapPtr GpuHeap; ERHIBindlessConfiguration Configuration; }; #if !D3D12RHI_CUSTOM_BINDLESS_RESOURCE_MANAGER #define D3D12RHI_BINDLESS_RESOURCE_MANAGER_SUPPORTS_RESIZING 1 // Helper container for all context related bindless state. struct FD3D12ContextBindlessState { FD3D12DescriptorHeapPtr CurrentGpuHeap; bool bRefreshHeap = false; FD3D12ContextBindlessState() = default; ~FD3D12ContextBindlessState() { check(!bRefreshHeap); } void RefreshDescriptorHeap() { bRefreshHeap = true; } }; /** Simple helper class to compute moving max in given amount of values. */ template class FMovingWindowMax { public: FMovingWindowMax() : RemoveNextIdx(0) , NumValuesUsed(0) { static_assert(ArraySize > 0, "ArraySize must be greater than zero"); } void PushValue(T Value) { if (ArraySize == NumValuesUsed) { ValuesArray[RemoveNextIdx] = Value; RemoveNextIdx = (RemoveNextIdx + 1) % ArraySize; } else { ValuesArray[NumValuesUsed] = Value; ++NumValuesUsed; } } T GetMax() const { T Max = static_cast(0); for (int32 Index = 0; Index < NumValuesUsed; ++Index) { Max = FMath::Max(Max, ValuesArray[Index]); } return Max; } private: TStaticArray ValuesArray; /** The array Index of the next item to remove when the moving window is full */ int32 RemoveNextIdx; int32 NumValuesUsed; }; /** Manager specifically for bindless resource descriptors. Has to handle renames on command lists. */ class FD3D12BindlessResourceManager : public FD3D12DeviceChild { public: FD3D12BindlessResourceManager() = delete; FD3D12BindlessResourceManager(FD3D12Device* InDevice, FD3D12BindlessDescriptorAllocator& InAllocator); void CleanupResources(); void GarbageCollect(); void Recycle(FD3D12DescriptorHeap* DescriptorHeap); void InitializeDescriptor(FRHIDescriptorHandle DstHandle, FD3D12View* View); void UpdateDescriptor(FD3D12ContextArray const& Contexts, FRHIDescriptorHandle DstHandle, FD3D12View* View); void FlushPendingDescriptorUpdates(FD3D12CommandContext& Context); void OpenCommandList(FD3D12CommandContext& Context); void CloseCommandList(FD3D12CommandContext& Context); void FinalizeContext(FD3D12CommandContext& Context); FD3D12DescriptorHeap* GetHeap(ERHIPipeline Pipeline) const; FD3D12DescriptorHeap* GetExplicitHeapForContext(FD3D12CommandContext& Context); ERHIBindlessConfiguration GetConfiguration() const { return Configuration; } // Called from FD3D12Adapter::AllocateBindlessResourceHandle void GrowCPUHeap(uint32 OriginalNumDescriptors, uint32 NewNumDescriptors); private: void CopyCpuHeap(FD3D12DescriptorHeap* DestinationHeap); void AssignHeapToState(FD3D12ContextBindlessState& State); void FinalizeHeapOnState(FD3D12ContextBindlessState& State); void CheckRequestNewActiveGPUHeap(); int AddActiveGPUHeap(); void ReleaseGPUHeaps(); void UpdateInUseGPUHeaps(bool bInUse); // Critical section shared across devices FCriticalSection& HeapsCS; FD3D12DescriptorHeapPtr CpuHeap; const ERHIBindlessConfiguration Configuration; uint64 GarbageCollectCycle = 0; uint64 LastUsedExplicitHeapCycle = 0; bool bRequestNewActiveGpuHeap = false; bool bCPUHeapResized = false; uint32 InUseGPUHeaps = 0; uint32 MaxInUseGPUHeaps = 0; FMovingWindowMax MovingWindowMaxInUseGPUHeaps; struct FGpuHeapData { FD3D12DescriptorHeapPtr GpuHeap; TArray UpdatedHandles; bool bInUse = true; uint64 LastUsedGarbageCollectCycle = 0; }; int32 ActiveGpuHeapIndex = -1; TArray ActiveGpuHeaps; TArray PooledGpuHeaps; }; #endif struct FD3D12DescriptorHeapPair { FD3D12DescriptorHeap* SamplerHeap; FD3D12DescriptorHeap* ResourceHeap; }; /** Manager for descriptors used in bindless rendering. */ class FD3D12BindlessDescriptorManager : public FD3D12DeviceChild { public: FD3D12BindlessDescriptorManager(FD3D12Device* InDevice, FD3D12BindlessDescriptorAllocator& InAllocator); ~FD3D12BindlessDescriptorManager(); void Init(); void CleanupResources(); FD3D12BindlessDescriptorAllocator& GetAllocator() { return Allocator; } FD3D12BindlessResourceManager* GetResourceManager() const { return ResourceManager.Get(); } FD3D12BindlessSamplerManager* GetSamplerManager() const { return SamplerManager.Get(); } ERHIBindlessConfiguration GetConfiguration() const { return Configuration; } void ImmediateFree(FRHIDescriptorHandle InHandle); void DeferredFreeFromDestructor(FRHIDescriptorHandle InHandle); void GarbageCollect(); void Recycle(FD3D12DescriptorHeap* DescriptorHeap); void InitializeDescriptor(FRHIDescriptorHandle DstHandle, FD3D12SamplerState* SamplerState); void InitializeDescriptor(FRHIDescriptorHandle DstHandle, FD3D12View* View); void UpdateDescriptor(FD3D12ContextArray const& Contexts, FRHIDescriptorHandle DstHandle, FD3D12View* SourceView); void FinalizeContext(FD3D12CommandContext& Context); void OpenCommandList(FD3D12CommandContext& Context); void CloseCommandList(FD3D12CommandContext& Context); void FlushPendingDescriptorUpdates(FD3D12CommandContext& Context); void SetHeapsForRayTracing(FD3D12CommandContext& Context); FD3D12DescriptorHeapPair GetExplicitHeapsForContext(FD3D12CommandContext& Context, ERHIBindlessConfiguration InConfiguration); #if D3D12RHI_USE_CONSTANT_BUFFER_VIEWS TRHIPipelineArray AllocateResourceHeapsForAllPipelines(int32 InSize); #endif private: FD3D12BindlessDescriptorAllocator& Allocator; TUniquePtr ResourceManager; TUniquePtr SamplerManager; ERHIBindlessConfiguration Configuration{}; }; #endif // !PLATFORM_SUPPORTS_BINDLESS_RENDERING