// Copyright Epic Games, Inc. All Rights Reserved. /*============================================================================= D3D12Resources.cpp: D3D RHI utility implementation. =============================================================================*/ #include "D3D12Resources.h" #include "D3D12RHIPrivate.h" #include "D3D12IntelExtensions.h" #include "D3D12RayTracing.h" #include "EngineModule.h" #include "HAL/LowLevelMemTracker.h" #include "ProfilingDebugging/MemoryTrace.h" #include "ProfilingDebugging/AssetMetadataTrace.h" #include "RHICoreStats.h" static TAutoConsoleVariable CVarD3D12ReservedResourceHeapSizeMB( TEXT("d3d12.ReservedResourceHeapSizeMB"), 16, TEXT("Size of the backing heaps for reserved resources in megabytes (default 16MB)."), ECVF_ReadOnly ); ///////////////////////////////////////////////////////////////////// // ID3D12ResourceAllocator ///////////////////////////////////////////////////////////////////// void ID3D12ResourceAllocator::AllocateTexture(uint32 GPUIndex, D3D12_HEAP_TYPE InHeapType, const FD3D12ResourceDesc& InDesc, EPixelFormat InUEFormat, ED3D12ResourceStateMode InResourceStateMode, D3D12_RESOURCE_STATES InCreateState, const D3D12_CLEAR_VALUE* InClearValue, const TCHAR* InName, FD3D12ResourceLocation& ResourceLocation) { // Check if texture can be 4K aligned FD3D12ResourceDesc Desc = InDesc; bool b4KAligment = FD3D12Texture::CanBe4KAligned(Desc, InUEFormat); Desc.Alignment = b4KAligment ? D3D12_SMALL_RESOURCE_PLACEMENT_ALIGNMENT : D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; // Get the size and alignment for the allocation D3D12_RESOURCE_ALLOCATION_INFO Info = FD3D12DynamicRHI::GetD3DRHI()->GetAdapter().GetDevice(GPUIndex)->GetResourceAllocationInfo(Desc); AllocateResource(GPUIndex, InHeapType, Desc, Info.SizeInBytes, Info.Alignment, InResourceStateMode, InCreateState, InClearValue, InName, ResourceLocation); } #if D3D12RHI_SUPPORTS_UNCOMPRESSED_UAV TArray> FD3D12ResourceDesc::GetCastableFormats() const { TArray> Result; // We have to add the 'implied' castable formats for SRVs. Since we don't have any sRGB flags here, just add both formats. Result.Emplace(UE::DXGIUtilities::FindShaderResourceFormat(Format, true)); Result.Emplace(UE::DXGIUtilities::FindShaderResourceFormat(Format, false)); if (UAVPixelFormat != PF_Unknown) { // Add the uncompressed UAV format we want Result.Emplace((DXGI_FORMAT)GPixelFormats[UAVPixelFormat].PlatformFormat); } return Result; } #endif // D3D12RHI_SUPPORTS_UNCOMPRESSED_UAV ///////////////////////////////////////////////////////////////////// // FD3D12 Resource ///////////////////////////////////////////////////////////////////// #if UE_BUILD_DEBUG int64 FD3D12Resource::TotalResourceCount = 0; int64 FD3D12Resource::NoStateTrackingResourceCount = 0; #endif FD3D12Resource::FD3D12Resource(FD3D12Device* ParentDevice, FRHIGPUMask VisibleNodes, ID3D12Resource* InResource, D3D12_RESOURCE_STATES InInitialState, const FD3D12ResourceDesc& InDesc, FD3D12Heap* InHeap, D3D12_HEAP_TYPE InHeapType) : FD3D12Resource(ParentDevice, VisibleNodes, InResource, InInitialState, ED3D12ResourceStateMode::Default, D3D12_RESOURCE_STATE_TBD, InDesc, InHeap, InHeapType) { } FD3D12Resource::FD3D12Resource(FD3D12Device* ParentDevice, FRHIGPUMask VisibleNodes, ID3D12Resource* InResource, D3D12_RESOURCE_STATES InInitialState, ED3D12ResourceStateMode InResourceStateMode, D3D12_RESOURCE_STATES InDefaultResourceState, const FD3D12ResourceDesc& InDesc, FD3D12Heap* InHeap, D3D12_HEAP_TYPE InHeapType) : FD3D12DeviceChild(ParentDevice) , FD3D12MultiNodeGPUObject(ParentDevice->GetGPUMask(), VisibleNodes) , Resource(InResource) , Heap(InHeap) , Desc(InDesc) , HeapType(InHeapType) , PlaneCount(UE::DXGIUtilities::GetPlaneCount(Desc.Format)) , bRequiresResourceStateTracking(true) , bRequiresResidencyTracking(bool(ENABLE_RESIDENCY_MANAGEMENT)) , bDepthStencil(false) , bDeferDelete(true) { #if UE_BUILD_DEBUG FPlatformAtomics::InterlockedIncrement(&TotalResourceCount); #endif D3D12_HEAP_DESC HeapDesc = {}; D3D12_HEAP_PROPERTIES* HeapProps = nullptr; if (InHeap) { HeapDesc = InHeap->GetHeapDesc(); HeapProps = &HeapDesc.Properties; } #if ENABLE_RESIDENCY_MANAGEMENT // Residency tracking is only used for GPU-only resources owned by the Engine. // Back buffers may be referenced outside of command lists (during presents), however D3DX12Residency.h library // uses fences tied to command lists to detect when it's safe to evict a resource, which is wrong for back buffers. // External/shared resources may be referenced by command buffers in third-party code. bRequiresResidencyTracking = IsGPUOnly(InHeapType, HeapProps) && !Desc.bExternal && !Desc.bBackBuffer; #endif // On Windows it's sadly enough not possible to get the GPU virtual address from the resource directly if (Resource #if PLATFORM_WINDOWS && Desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER #endif ) { GPUVirtualAddress = Resource->GetGPUVirtualAddress(); } InitalizeResourceState(InInitialState, InResourceStateMode, InDefaultResourceState); #if NV_AFTERMATH AftermathHandle = UE::RHICore::Nvidia::Aftermath::D3D12::RegisterResource(InResource); #endif if (Desc.bReservedResource) { checkf(Heap == nullptr, TEXT("Reserved resources are not expected to have a heap")); ReservedResourceData = MakeUnique(); } } FD3D12Resource::~FD3D12Resource() { #if ENABLE_RESIDENCY_MANAGEMENT if (D3DX12Residency::IsInitialized(ResidencyHandle)) { D3DX12Residency::EndTrackingObject(GetParentDevice()->GetResidencyManager(), *ResidencyHandle); } delete ResidencyHandle; #endif // ENABLE_RESIDENCY_MANAGEMENT #if NV_AFTERMATH UE::RHICore::Nvidia::Aftermath::D3D12::UnregisterResource(AftermathHandle); #endif if (Desc.bBackBuffer) { // Don't make the windows association call and release back buffer at the same time (see notes on critical section) FScopeLock Lock(&FD3D12Viewport::DXGIBackBufferLock); Resource.SafeRelease(); } // Update reserved resources' physical memory stats. if (ReservedResourceData.IsValid() && ReservedResourceData->NumCommittedTiles > 0) { bool bBuffer = Desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER; const uint64 DecommitBytes = GRHIGlobals.ReservedResources.TileSizeInBytes * ReservedResourceData->NumCommittedTiles; UE::RHICore::UpdateReservedResourceStatsOnCommit(DecommitBytes, bBuffer, false /* Decommit */); // The backing heaps are going to be released once this resource is destroyed. for (const TRefCountPtr& BackingHeap : ReservedResourceData->BackingHeaps) { DEC_MEMORY_STAT_BY(STAT_D3D12ReservedResourcePhysical, BackingHeap->GetHeapDesc().SizeInBytes); } } } struct FD3D12UpdateTileMappingsParams { D3D12_TILE_RANGE_FLAGS RangeFlags = D3D12_TILE_RANGE_FLAG_NONE; D3D12_TILED_RESOURCE_COORDINATE Coord = {}; D3D12_TILE_REGION_SIZE Size = {}; ID3D12Heap* Heap = nullptr; uint32 HeapOffsetInTiles = 0; }; void FD3D12Resource::CommitReservedResource(ID3D12CommandQueue* D3DCommandQueue, uint64 RequiredCommitSizeInBytes) { TRACE_CPUPROFILER_EVENT_SCOPE(CommitReservedResource); static constexpr uint64 TileSizeInBytes = GRHIGlobals.ReservedResources.TileSizeInBytes; static_assert(TileSizeInBytes == 65536, "Reserved resource tiles are expected to always be 64KB"); check(Desc.bReservedResource); check(ReservedResourceData.IsValid()); LLM_REALLOC_SCOPE(ReservedResourceData.Get()); UE_MEMSCOPE_PTR(ReservedResourceData.Get()); checkf(GRHIGlobals.ReservedResources.Supported, TEXT("Current RHI does not support reserved resources")); if (Desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE3D) { checkf(GRHIGlobals.ReservedResources.SupportsVolumeTextures, TEXT("Current RHI does not support reserved volume textures")); } uint32 D3DResourceNumTiles = 0; D3D12_PACKED_MIP_INFO PackedMipDesc = {}; D3D12_TILE_SHAPE TileShape = {}; const uint32 FirstSubresource = 0; const bool bBuffer = Desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER; const uint32 NumSubresources = SubresourceCount; const uint32 NumMipLevels = GetMipLevels(); const uint32 NumArraySlices = GetArraySize(); uint32 NumSubresourceTilings = NumMipLevels; TArray> MipTilingInfo; check(NumSubresourceTilings >= 1); MipTilingInfo.SetNum(NumSubresourceTilings); ID3D12Device* D3DDevice = GetParentDevice()->GetDevice(); FD3D12Adapter* Adapter = GetParentDevice()->GetParentAdapter(); D3DDevice->GetResourceTiling(GetResource(), &D3DResourceNumTiles, &PackedMipDesc, &TileShape, &NumSubresourceTilings, FirstSubresource, MipTilingInfo.GetData()); if (bBuffer) { // Buffers obviously don't have mips, but we can pretend they do to make the code below agnostic to resource type PackedMipDesc.NumStandardMips = 1; } check(MipTilingInfo.Num() == PackedMipDesc.NumStandardMips + PackedMipDesc.NumPackedMips); const uint32 NumPackedTilesPerArraySlice = PackedMipDesc.NumTilesForPackedMips; const uint32 NumTotalPackedMipTiles = NumPackedTilesPerArraySlice * NumArraySlices; const uint32 NumTotalStandardMipTiles = D3DResourceNumTiles - NumTotalPackedMipTiles; const uint64 TotalSize = D3DResourceNumTiles * TileSizeInBytes; RequiredCommitSizeInBytes = FMath::Min(RequiredCommitSizeInBytes, TotalSize); RequiredCommitSizeInBytes = AlignArbitrary(RequiredCommitSizeInBytes, TileSizeInBytes); const uint64 MaxHeapSize = uint64(CVarD3D12ReservedResourceHeapSizeMB.GetValueOnAnyThread()) * 1024 * 1024; const uint64 NumHeaps = FMath::DivideAndRoundUp(TotalSize, MaxHeapSize); ReservedResourceData->BackingHeaps.Reserve(NumHeaps); const uint32 MaxTilesPerHeap = uint32(MaxHeapSize / TileSizeInBytes); // Set high residency priority based on the same heuristics as D3D12 committed resources, // i.e. normal priority unless it's a UAV/RT/DS texture. const bool bRenderOrDepthTarget = EnumHasAnyFlags(Desc.Flags, D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL); const bool bHighPriorityResource = bRenderOrDepthTarget || EnumHasAnyFlags(Desc.Flags, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS); const uint32 GPUIndex = GetParentDevice()->GetGPUIndex(); D3D12_HEAP_PROPERTIES BackingHeapProps = {}; BackingHeapProps.Type = D3D12_HEAP_TYPE_DEFAULT; BackingHeapProps.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; BackingHeapProps.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; BackingHeapProps.CreationNodeMask = GetGPUMask().GetNative(); BackingHeapProps.VisibleNodeMask = GetVisibilityMask().GetNative(); uint32 NumStandardTilesPerArraySlice = 0; uint32 NumTotalTiles = 0; if (bBuffer) { NumTotalTiles = NumStandardTilesPerArraySlice = D3DResourceNumTiles; checkf(D3DResourceNumTiles == MipTilingInfo[0].WidthInTiles, TEXT("Reserved buffers are expected to have trivial tiling configuration: single 1D subresource that contains all tiles.")); } else { NumStandardTilesPerArraySlice = NumTotalStandardMipTiles / NumArraySlices; NumTotalTiles = (NumStandardTilesPerArraySlice + NumPackedTilesPerArraySlice) * NumArraySlices; } const uint32 NumTotalTilesPerArraySlice = NumStandardTilesPerArraySlice + NumPackedTilesPerArraySlice; checkf(D3DResourceNumTiles == NumTotalTiles, TEXT("D3D resource size in tiles: %d, computed size in tiles: %d"), D3DResourceNumTiles, NumTotalTiles); const uint32 NumRequiredCommitTiles = RequiredCommitSizeInBytes / TileSizeInBytes; auto GetTiledResourceCoordinate = [&MipTilingInfo, &PackedMipDesc, D3DResourceNumTiles, NumTotalTilesPerArraySlice, NumSubresources, MaxTilesPerHeap] (uint32 OffsetInTiles, uint32 NumTiles) -> D3D12_TILED_RESOURCE_COORDINATE { check(OffsetInTiles < D3DResourceNumTiles); const uint32 ArraySliceIndex = OffsetInTiles / NumTotalTilesPerArraySlice; const uint32 TileIndexInArraySlice = OffsetInTiles % NumTotalTilesPerArraySlice; const uint32 NumTotalMips = MipTilingInfo.Num(); uint32 MipLevel = 0; { uint32 NextMipTileThreshold = 0; while (MipLevel < PackedMipDesc.NumStandardMips) { const D3D12_SUBRESOURCE_TILING& CurrentMipTiling = MipTilingInfo[MipLevel]; NextMipTileThreshold += CurrentMipTiling.WidthInTiles * CurrentMipTiling.HeightInTiles * CurrentMipTiling.DepthInTiles; if (TileIndexInArraySlice < NextMipTileThreshold) { break; } MipLevel += 1; } } D3D12_TILED_RESOURCE_COORDINATE ResourceCoordinate = {}; // Coordinates are in tiles, not pixels ResourceCoordinate.Subresource = MipLevel + ArraySliceIndex * NumTotalMips; const D3D12_SUBRESOURCE_TILING& CurrentMipTiling = MipTilingInfo[MipLevel]; if (MipLevel < PackedMipDesc.NumStandardMips) { // Standard mip level case check(CurrentMipTiling.StartTileIndexInOverallResource != ~0u) const uint32 NumTilesPerVolumeSlice = CurrentMipTiling.WidthInTiles * CurrentMipTiling.HeightInTiles; const uint32 TileIndexInMipLevel = TileIndexInArraySlice - CurrentMipTiling.StartTileIndexInOverallResource; ResourceCoordinate.X = TileIndexInMipLevel % CurrentMipTiling.WidthInTiles; ResourceCoordinate.Y = (TileIndexInMipLevel / CurrentMipTiling.WidthInTiles) % CurrentMipTiling.HeightInTiles; ResourceCoordinate.Z = TileIndexInMipLevel / NumTilesPerVolumeSlice; } else { // Packed mip level case checkf(NumTiles <= MaxTilesPerHeap, TEXT("Reserved texture packed mip level requires tiles: %d, maximum supported tiles: %d. ") TEXT("Increase d3d12.ReservedResourceHeapSizeMB or avoid packed mips by using a larger texture dimensions."), NumTiles, MaxTilesPerHeap); // Entire packed mip chain must be covered in one map operation, so mapping origin is always 0 ResourceCoordinate.X = 0; ResourceCoordinate.Y = 0; ResourceCoordinate.Z = 0; } return ResourceCoordinate; }; TArray MappingParams; TArray UsedResidencyHandles; const uint32 NumPreviousCommittedTiles = ReservedResourceData->NumCommittedTiles; if (ReservedResourceData->NumCommittedTiles > NumRequiredCommitTiles) // Decommit / shrink case { check(!ReservedResourceData->BackingHeaps.IsEmpty()); // Iterate through heaps in reverse order, unmap ranges and release heaps if they are completely unused while (ReservedResourceData->NumCommittedTiles > NumRequiredCommitTiles) { TRefCountPtr& LastHeap = ReservedResourceData->BackingHeaps.Last(); const uint32 NumTotalTilesInHeap = LastHeap->GetHeapDesc().SizeInBytes / TileSizeInBytes; check(ReservedResourceData->NumSlackTiles <= NumTotalTilesInHeap); const uint32 NumUsedTilesInHeap = NumTotalTilesInHeap - ReservedResourceData->NumSlackTiles; check(NumUsedTilesInHeap <= ReservedResourceData->NumCommittedTiles); const uint32 HeapFirstTile = ReservedResourceData->NumCommittedTiles - NumUsedTilesInHeap; const uint32 RegionEnd = ReservedResourceData->NumCommittedTiles; const uint32 RegionBegin = FMath::Max(HeapFirstTile, NumRequiredCommitTiles); D3D12_TILE_REGION_SIZE RegionSize = {}; RegionSize.UseBox = false; RegionSize.NumTiles = RegionEnd - RegionBegin; // Coordinates are in tiles, not pixels D3D12_TILED_RESOURCE_COORDINATE ResourceCoordinate = GetTiledResourceCoordinate(RegionBegin, RegionSize.NumTiles); FD3D12UpdateTileMappingsParams Params = {}; Params.RangeFlags = D3D12_TILE_RANGE_FLAG_NULL; Params.Coord = ResourceCoordinate; Params.Size = RegionSize; MappingParams.Add(Params); if (HeapFirstTile == RegionBegin) { // All tiles from this heap were unmapped, so it can be dropped DEC_MEMORY_STAT_BY(STAT_D3D12ReservedResourcePhysical, LastHeap->GetHeapDesc().SizeInBytes); LastHeap->DeferDelete(); ReservedResourceData->BackingHeaps.Pop(); int32 NumResidencyHandles = ReservedResourceData->NumResidencyHandlesPerHeap.Last(); ReservedResourceData->NumResidencyHandlesPerHeap.Pop(); while (NumResidencyHandles != 0) { ReservedResourceData->ResidencyHandles.Pop(); --NumResidencyHandles; } ReservedResourceData->NumSlackTiles = 0; } else { // Heap remains referenced, but now contains some free tiles at the end (which we just unmapped) ReservedResourceData->NumSlackTiles += RegionSize.NumTiles; check(ReservedResourceData->NumSlackTiles <= NumTotalTilesInHeap); } check(ReservedResourceData->NumCommittedTiles >= RegionSize.NumTiles); ReservedResourceData->NumCommittedTiles -= RegionSize.NumTiles; } } else // Commit / grow case { bool bForceGetGPUAddress = false; // See FD3D12Buffer::UpdateAllocationTags: we might need to rebase the allocation tag and we need the gpu address for that LLM_IF_ENABLED(bForceGetGPUAddress=true); while (ReservedResourceData->NumCommittedTiles < NumRequiredCommitTiles) { const uint32 NumRemainingTiles = NumRequiredCommitTiles - ReservedResourceData->NumCommittedTiles; ID3D12Heap* D3DHeap = nullptr; uint32 HeapRangeStartOffsetInTiles = 0; D3D12_TILE_REGION_SIZE RegionSize = {}; RegionSize.UseBox = false; if (ReservedResourceData->NumSlackTiles) { // Consume any heap slack space before allocating a new heap const TRefCountPtr& LastHeap = ReservedResourceData->BackingHeaps.Last(); const uint32 NumTotalTilesInHeap = LastHeap->GetHeapDesc().SizeInBytes / TileSizeInBytes; RegionSize.NumTiles = FMath::Min(ReservedResourceData->NumSlackTiles, NumRemainingTiles); HeapRangeStartOffsetInTiles = NumTotalTilesInHeap - ReservedResourceData->NumSlackTiles; D3DHeap = LastHeap->GetHeap(); check(RegionSize.NumTiles <= ReservedResourceData->NumSlackTiles); ReservedResourceData->NumSlackTiles -= RegionSize.NumTiles; UsedResidencyHandles.Append(LastHeap->GetResidencyHandles()); } else { // Create a new heap to service the commit request RegionSize.NumTiles = FMath::Min(MaxTilesPerHeap, NumRemainingTiles); HeapRangeStartOffsetInTiles = 0; #if NAME_OBJECTS const int32 HeapIndex = ReservedResourceData->BackingHeaps.Num(); FString HeapName = FString::Printf(TEXT("%s.Heap[%d]"), DebugName.IsValid() ? *DebugName.ToString() : TEXT("UNKNOWN"), HeapIndex); const TCHAR* HeapNameChars = *HeapName; #else const TCHAR* HeapNameChars = TEXT("ReservedResourceBackingHeap"); #endif // NAME_OBJECTS const D3D12_HEAP_FLAGS TextureHeapFlags = bRenderOrDepthTarget ? D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES : D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES; const D3D12_HEAP_FLAGS HeapFlags = bBuffer ? D3D12_HEAP_FLAG_ALLOW_ONLY_BUFFERS : TextureHeapFlags; static_assert((D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES) == D3D12_HEAP_FLAG_ALLOW_ONLY_NON_RT_DS_TEXTURES); static_assert((D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES) == D3D12_HEAP_FLAG_ALLOW_ONLY_RT_DS_TEXTURES); const uint32 ThisHeapSize = RegionSize.NumTiles * TileSizeInBytes; D3D12_HEAP_DESC NewHeapDesc = {}; NewHeapDesc.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; NewHeapDesc.Flags = HeapFlags; NewHeapDesc.SizeInBytes = ThisHeapSize; NewHeapDesc.Properties = BackingHeapProps; VERIFYD3D12RESULT(D3DDevice->CreateHeap(&NewHeapDesc, IID_PPV_ARGS(&D3DHeap))); INC_MEMORY_STAT_BY(STAT_D3D12ReservedResourcePhysical, NewHeapDesc.SizeInBytes); if (bHighPriorityResource) { Adapter->SetResidencyPriority(D3DHeap, D3D12_RESIDENCY_PRIORITY_HIGH, GPUIndex); } TRefCountPtr NewHeap = new FD3D12Heap(GetParentDevice(), GetVisibilityMask()); NewHeap->SetHeap(D3DHeap, HeapNameChars, true /*bTrack*/, bForceGetGPUAddress); NewHeap->BeginTrackingResidency(ThisHeapSize); TConstArrayView HeapResidencyHandles = NewHeap->GetResidencyHandles(); ReservedResourceData->ResidencyHandles.Append(HeapResidencyHandles); ReservedResourceData->NumResidencyHandlesPerHeap.Add(HeapResidencyHandles.Num()); ReservedResourceData->BackingHeaps.Add(MoveTemp(NewHeap)); UsedResidencyHandles.Append(HeapResidencyHandles); } // Coordinates are in tiles, not pixels D3D12_TILED_RESOURCE_COORDINATE ResourceCoordinate = GetTiledResourceCoordinate(ReservedResourceData->NumCommittedTiles, RegionSize.NumTiles); FD3D12UpdateTileMappingsParams Params = {}; Params.RangeFlags = D3D12_TILE_RANGE_FLAG_NONE; Params.Coord = ResourceCoordinate; Params.Size = RegionSize; Params.Heap = D3DHeap; Params.HeapOffsetInTiles = HeapRangeStartOffsetInTiles; MappingParams.Add(Params); ReservedResourceData->NumCommittedTiles += RegionSize.NumTiles; } } #if ENABLE_RESIDENCY_MANAGEMENT FD3D12ResidencyManager& ResidencyManager = GetParentDevice()->GetResidencyManager(); if (GEnableResidencyManagement && !UsedResidencyHandles.IsEmpty()) { HRESULT HR = S_OK; FD3D12ResidencySet* ResidencySet = ResidencyManager.CreateResidencySet(); HR = ResidencySet->Open(); checkf(SUCCEEDED(HR), TEXT("Failed to open residency set. Error code: 0x%08x."), uint32(HR)); for (FD3D12ResidencyHandle* Handle : UsedResidencyHandles) { if (D3DX12Residency::IsInitialized(Handle)) { ResidencySet->Insert(Handle); } } HR = ResidencySet->Close(); checkf(SUCCEEDED(HR), TEXT("Failed to close residency set. Error code: 0x%08x."), uint32(HR)); // NOTE: ResidencySet ownership is taken over by the residency manager. // It is destroyed when paging work completes, which may happen async on another thread in some cases. HR = ResidencyManager.MakeResident(D3DCommandQueue, MoveTemp(ResidencySet)); checkf(SUCCEEDED(HR), TEXT("Failed to process residency set. Error code: 0x%08x."), uint32(HR)); } #endif // ENABLE_RESIDENCY_MANAGEMENT for (const FD3D12UpdateTileMappingsParams& Params : MappingParams) { D3DCommandQueue->UpdateTileMappings(GetResource(), 1 /*NumRegions*/, &Params.Coord, &Params.Size, Params.Heap, 1 /*NumRanges*/, &Params.RangeFlags, &Params.HeapOffsetInTiles, &Params.Size.NumTiles, D3D12_TILE_MAPPING_FLAG_NONE); } #if ENABLE_RESIDENCY_MANAGEMENT if (GEnableResidencyManagement && !UsedResidencyHandles.IsEmpty()) { // Signal the fence for this queue after UpdateTileMappings complete. // This is analogous to executing a command list that references a set of resources. ResidencyManager.SignalFence(D3DCommandQueue); } #endif // ENABLE_RESIDENCY_MANAGEMENT checkf(ReservedResourceData->NumCommittedTiles == NumRequiredCommitTiles, TEXT("Reserved resource was not fully processed while committing physical memory. Expected to process tiles: %d, actually processed: %d"), D3DResourceNumTiles, ReservedResourceData->NumCommittedTiles); if (ReservedResourceData->NumCommittedTiles != NumPreviousCommittedTiles) { int64 CommitDeltaInBytes = TileSizeInBytes * FMath::Abs((int32)ReservedResourceData->NumCommittedTiles - (int32)NumPreviousCommittedTiles); UE::RHICore::UpdateReservedResourceStatsOnCommit(CommitDeltaInBytes, bBuffer, ReservedResourceData->NumCommittedTiles > NumPreviousCommittedTiles); } } ID3D12Pageable* FD3D12Resource::GetPageable() { if (IsPlacedResource()) { return (ID3D12Pageable*)(GetHeap()->GetHeap()); } else { return (ID3D12Pageable*)GetResource(); } } void FD3D12Resource::StartTrackingForResidency() { #if ENABLE_RESIDENCY_MANAGEMENT if (!bRequiresResidencyTracking) { return; } checkf(IsGPUOnly(HeapType), TEXT("Residency tracking is not expected for CPU-accessible resources")); checkf(!Desc.bBackBuffer, TEXT("Residency tracking is not expected for back buffers")); checkf(!Desc.bExternal, TEXT("Residency tracking is not expected for externally-owned resources")); if (!IsPlacedResource() && !IsReservedResource()) { checkf(!ResidencyHandle, TEXT("Residency tracking is already initialzied for this resource")); ResidencyHandle = new FD3D12ResidencyHandle; const D3D12_RESOURCE_ALLOCATION_INFO Info = GetParentDevice()->GetResourceAllocationInfoUncached(Desc); D3DX12Residency::Initialize(*ResidencyHandle, Resource.GetReference(), Info.SizeInBytes, this); D3DX12Residency::BeginTrackingObject(GetParentDevice()->GetResidencyManager(), *ResidencyHandle); } #endif // ENABLE_RESIDENCY_MANAGEMENT } void FD3D12Resource::DeferDelete() { FD3D12DynamicRHI::GetD3DRHI()->DeferredDelete(this); } ///////////////////////////////////////////////////////////////////// // FD3D12 Heap ///////////////////////////////////////////////////////////////////// FD3D12Heap::FD3D12Heap(FD3D12Device* Parent, FRHIGPUMask VisibleNodes, HeapId InTraceParentHeapId) : FD3D12DeviceChild(Parent), FD3D12MultiNodeGPUObject(Parent->GetGPUMask(), VisibleNodes), TraceParentHeapId(InTraceParentHeapId) { } FD3D12Heap::~FD3D12Heap() { #if UE_MEMORY_TRACE_ENABLED if (GPUVirtualAddress != 0) { MemoryTrace_UnmarkAllocAsHeap(GPUVirtualAddress, TraceHeapId); MemoryTrace_Free(GPUVirtualAddress, EMemoryTraceRootHeap::VideoMemory); } #endif #if TRACK_RESOURCE_ALLOCATIONS FD3D12Adapter* Adapter = GetParentDevice()->GetParentAdapter(); if (GPUVirtualAddress != 0 && bTrack) { Adapter->ReleaseTrackedHeap(this); } #endif // TRACK_RESOURCE_ALLOCATIONS #if ENABLE_RESIDENCY_MANAGEMENT if (D3DX12Residency::IsInitialized(ResidencyHandle)) { D3DX12Residency::EndTrackingObject(GetParentDevice()->GetResidencyManager(), *ResidencyHandle); } delete ResidencyHandle; #endif // ENABLE_RESIDENCY_MANAGEMENT // Release actual d3d object Heap.SafeRelease(); } void FD3D12Heap::DeferDelete() { // ProcessDeferredDeletionQueue() performs final Release(), but deletion queue itself only holds a raw pointer, so explicit addref is required. AddRef(); FD3D12DynamicRHI::GetD3DRHI()->DeferredDelete(this); } void FD3D12Heap::SetHeap(ID3D12Heap* HeapIn, const TCHAR* const InName, bool bInTrack, bool bForceGetGPUAddress) { *Heap.GetInitReference() = HeapIn; bTrack = bInTrack; HeapName = InName; HeapDesc = Heap->GetDesc(); SetD3D12ObjectName(HeapIn, InName); #if ENABLE_RESIDENCY_MANAGEMENT bRequiresResidencyTracking = IsGPUOnly(HeapDesc.Properties.Type, &HeapDesc.Properties); #endif // ENABLE_RESIDENCY_MANAGEMENT // Create a buffer placed resource on the heap to extract the gpu virtual address // if we are tracking all allocations FD3D12Adapter* Adapter = GetParentDevice()->GetParentAdapter(); if ((bForceGetGPUAddress || Adapter->IsTrackingAllAllocations()) && !(HeapDesc.Flags & D3D12_HEAP_FLAG_DENY_BUFFERS) && HeapDesc.Properties.Type == D3D12_HEAP_TYPE_DEFAULT) { uint64 HeapSize = HeapDesc.SizeInBytes; TRefCountPtr TempResource; const D3D12_RESOURCE_DESC BufDesc = CD3DX12_RESOURCE_DESC::Buffer(HeapSize, D3D12_RESOURCE_FLAG_NONE); VERIFYD3D12RESULT(Adapter->GetD3DDevice()->CreatePlacedResource(Heap, 0, &BufDesc, D3D12_RESOURCE_STATE_COMMON, nullptr, IID_PPV_ARGS(TempResource.GetInitReference()))); GPUVirtualAddress = TempResource->GetGPUVirtualAddress(); #if UE_MEMORY_TRACE_ENABLED TraceHeapId = MemoryTrace_HeapSpec(TraceParentHeapId, *(FString(InName) + TEXT(" D3D12Heap"))); // Calling GetResourceAllocationInfo is not trivial, only do it if memory trace is enabled if (UE_TRACE_CHANNELEXPR_IS_ENABLED(MemAllocChannel)) { const D3D12_RESOURCE_DESC ResourceDesc = TempResource->GetDesc(); const D3D12_RESOURCE_ALLOCATION_INFO Info = Adapter->GetD3DDevice()->GetResourceAllocationInfo(0, 1, &ResourceDesc); MemoryTrace_Alloc(GPUVirtualAddress, Info.SizeInBytes, Info.Alignment, EMemoryTraceRootHeap::VideoMemory); MemoryTrace_MarkAllocAsHeap(GPUVirtualAddress, TraceHeapId); } #endif #if TRACK_RESOURCE_ALLOCATIONS if (bTrack) { Adapter->TrackHeapAllocation(this); } #endif } } void FD3D12Heap::DisallowTrackingResidency() { #if ENABLE_RESIDENCY_MANAGEMENT checkf(ResidencyHandle == nullptr, TEXT("Can't disallow residency tracking after it has started. Call this function instead of BeginTrackingResidency().")); bRequiresResidencyTracking = false; #endif // ENABLE_RESIDENCY_MANAGEMENT } void FD3D12Heap::BeginTrackingResidency(uint64 Size) { #if ENABLE_RESIDENCY_MANAGEMENT checkf(bRequiresResidencyTracking, TEXT("Residency tracking is not expected for this resource")); checkf(!ResidencyHandle, TEXT("Residency tracking is already initialzied for this resource")); ResidencyHandle = new FD3D12ResidencyHandle; D3DX12Residency::Initialize(*ResidencyHandle, Heap.GetReference(), Size, this); D3DX12Residency::BeginTrackingObject(GetParentDevice()->GetResidencyManager(), *ResidencyHandle); #endif // ENABLE_RESIDENCY_MANAGEMENT } ///////////////////////////////////////////////////////////////////// // FD3D12 Adapter ///////////////////////////////////////////////////////////////////// HRESULT FD3D12Adapter::CreateCommittedResource(const FD3D12ResourceDesc& InDesc, FRHIGPUMask CreationNode, const D3D12_HEAP_PROPERTIES& HeapProps, D3D12_RESOURCE_STATES InInitialState, ED3D12ResourceStateMode InResourceStateMode, D3D12_RESOURCE_STATES InDefaultState, const D3D12_CLEAR_VALUE* ClearValue, FD3D12Resource** ppOutResource, const TCHAR* Name, bool bVerifyHResult) { if (!ppOutResource) { return E_POINTER; } TRACE_CPUPROFILER_EVENT_SCOPE(CreateCommittedResource); LLM_PLATFORM_SCOPE(ELLMTag::GraphicsPlatform); TRefCountPtr pResource; const bool bRequiresInitialization = (InDesc.Flags & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)) != 0; D3D12_HEAP_FLAGS HeapFlags = (bHeapNotZeroedSupported && !bRequiresInitialization) ? FD3D12_HEAP_FLAG_CREATE_NOT_ZEROED : D3D12_HEAP_FLAG_NONE; FD3D12ResourceDesc LocalDesc = InDesc; if (InDesc.Flags & D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS) { HeapFlags |= D3D12_HEAP_FLAG_SHARED; // Simultaneous access flag is used to detect shared heap requirement but can't be used when allocating buffer resource if (InDesc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) { LocalDesc.Flags &= ~D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS; } } #if D3D12_RHI_RAYTRACING if (InDefaultState == D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE) { LocalDesc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; } #endif // D3D12_RHI_RAYTRACING #if D3D12_WITH_CUSTOM_TEXTURE_LAYOUT ApplyCustomTextureLayout(LocalDesc, *this); #endif HRESULT hr = S_OK; #if INTEL_EXTENSIONS if (InDesc.bRequires64BitAtomicSupport && IsRHIDeviceIntel() && GDX12INTCAtomicUInt64Emulation) { INTC_D3D12_RESOURCE_DESC_0001 IntelLocalDesc{}; IntelLocalDesc.pD3D12Desc = &LocalDesc; IntelLocalDesc.EmulatedTyped64bitAtomics = true; hr = INTC_D3D12_CreateCommittedResource(FD3D12DynamicRHI::GetD3DRHI()->GetIntelExtensionContext(), &HeapProps, HeapFlags, &IntelLocalDesc, InInitialState, ClearValue, IID_PPV_ARGS(pResource.GetInitReference())); } else #endif #if D3D12RHI_SUPPORTS_UNCOMPRESSED_UAV if (InDesc.SupportsUncompressedUAV()) { // Convert the desc to the version required by CreateCommittedResource3 const CD3DX12_RESOURCE_DESC1 LocalDesc1(LocalDesc); // Common layout is the required starting state for any "legacy" transitions const D3D12_BARRIER_LAYOUT InitialLayout = D3D12_BARRIER_LAYOUT_COMMON; checkf(InInitialState == D3D12_RESOURCE_STATE_COMMON, TEXT("RESOURCE_STATE_COMMON is required for castable resources (Given: %d)"), InInitialState); ID3D12ProtectedResourceSession* ProtectedSession = nullptr; const TArray> CastableFormats = InDesc.GetCastableFormats(); hr = RootDevice12->CreateCommittedResource3( &HeapProps, HeapFlags, &LocalDesc1, InitialLayout, ClearValue, ProtectedSession, CastableFormats.Num(), CastableFormats.GetData(), IID_PPV_ARGS(pResource.GetInitReference())); } else #endif { hr = RootDevice->CreateCommittedResource(&HeapProps, HeapFlags, &LocalDesc, InInitialState, ClearValue, IID_PPV_ARGS(pResource.GetInitReference())); } if (SUCCEEDED(hr)) { // Set the output pointer *ppOutResource = new FD3D12Resource(GetDevice(CreationNode.ToIndex()), CreationNode, pResource, InInitialState, InResourceStateMode, InDefaultState, LocalDesc, nullptr, HeapProps.Type); (*ppOutResource)->AddRef(); // Set a default name (can override later). SetD3D12ResourceName(*ppOutResource, Name); (*ppOutResource)->StartTrackingForResidency(); TraceMemoryAllocation(*ppOutResource); } else { UE_LOG(LogD3D12RHI, Display, TEXT("D3D12 CreateCommittedResource failed with params:\n\tHeap Type: %d\n\tHeap Flags: %d\n\tResource Dimension: %d\n\tResource Width: %llu\n\tResource Height: %u\n\tArray Size: %u\n\tMip Levels: %u\n\tFormat: %d\n\tResource Flags: %d"), HeapProps.Type, HeapFlags, LocalDesc.Dimension, LocalDesc.Width, LocalDesc.Height, LocalDesc.DepthOrArraySize, LocalDesc.MipLevels, LocalDesc.PixelFormat, LocalDesc.Flags); if (bVerifyHResult) { VERIFYD3D12RESULT_EX(hr, RootDevice); } } return hr; } HRESULT FD3D12Adapter::CreateReservedResource(const FD3D12ResourceDesc& InDesc, FRHIGPUMask CreationNode, D3D12_RESOURCE_STATES InInitialState, ED3D12ResourceStateMode InResourceStateMode, D3D12_RESOURCE_STATES InDefaultState, const D3D12_CLEAR_VALUE* ClearValue, FD3D12Resource** ppOutResource, const TCHAR* Name, bool bVerifyHResult) { if (!ppOutResource) { return E_POINTER; } TRACE_CPUPROFILER_EVENT_SCOPE(CreateReservedResource); LLM_PLATFORM_SCOPE(ELLMTag::GraphicsPlatform); TRefCountPtr pResource; FD3D12ResourceDesc LocalDesc = InDesc; checkf(LocalDesc.bReservedResource, TEXT("FD3D12ResourceDesc is expected to be initialized as a reserved resource. See FD3D12DynamicRHI::GetResourceDesc().")); if (LocalDesc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE1D || LocalDesc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE2D || LocalDesc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE3D) { checkf(LocalDesc.Layout == D3D12_TEXTURE_LAYOUT_64KB_UNDEFINED_SWIZZLE, TEXT("Reserved textures are expected to have layout %d (64KB_UNDEFINED_SWIZZLE), but have %d. See FD3D12DynamicRHI::GetResourceDesc()."), uint32(D3D12_TEXTURE_LAYOUT_64KB_UNDEFINED_SWIZZLE), uint32(LocalDesc.Layout)); } checkf(LocalDesc.Alignment == 0 || LocalDesc.Alignment == 65536, TEXT("Reserved resources must use either 64KB alignment or 0 (unspecified/default), but have %d. See FD3D12DynamicRHI::GetResourceDesc()."), uint32(LocalDesc.Alignment)); #if D3D12_RHI_RAYTRACING if (InDefaultState == D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE) { LocalDesc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; } #endif // D3D12_RHI_RAYTRACING HRESULT hr = S_OK; #if D3D12RHI_SUPPORTS_UNCOMPRESSED_UAV if (InDesc.SupportsUncompressedUAV()) { checkf(InInitialState == D3D12_RESOURCE_STATE_COMMON, TEXT("RESOURCE_STATE_COMMON is required for castable resources (Given: %d)"), InInitialState); // Common layout is the require starting state for any "legacy" transitions const D3D12_BARRIER_LAYOUT InitialLayout = D3D12_BARRIER_LAYOUT_COMMON; ID3D12ProtectedResourceSession* ProtectedSession = nullptr; const TArray> CastableFormats = InDesc.GetCastableFormats(); hr = RootDevice12->CreateReservedResource2( &LocalDesc, InitialLayout, ClearValue, ProtectedSession, CastableFormats.Num(), CastableFormats.GetData(), IID_PPV_ARGS(pResource.GetInitReference())); } else #endif { hr = RootDevice->CreateReservedResource(&LocalDesc, InInitialState, ClearValue, IID_PPV_ARGS(pResource.GetInitReference())); } if (SUCCEEDED(hr)) { // Set the output pointer *ppOutResource = new FD3D12Resource(GetDevice(CreationNode.ToIndex()), CreationNode, pResource, InInitialState, InResourceStateMode, InDefaultState, LocalDesc, nullptr /*Heap*/, D3D12_HEAP_TYPE_DEFAULT); (*ppOutResource)->AddRef(); // Set a default name (can override later). SetD3D12ResourceName(*ppOutResource, Name); // NOTE: reserved resource residency is not tracked/managed by the engine, so we don't need to call StartTrackingForResidency(). } else { UE_LOG(LogD3D12RHI, Display, TEXT("D3D12 CreateReservedResource failed with params:\n\tResource Dimension: %d\n\tResource Width: %llu\n\tResource Height: %u\n\tFormat: %d\n\tResource Flags: %d"), LocalDesc.Dimension, LocalDesc.Width, LocalDesc.Height, LocalDesc.PixelFormat, LocalDesc.Flags); if (bVerifyHResult) { VERIFYD3D12RESULT_EX(hr, RootDevice); } } return hr; } HRESULT FD3D12Adapter::CreatePlacedResource(const FD3D12ResourceDesc& InDesc, FD3D12Heap* BackingHeap, uint64 HeapOffset, D3D12_RESOURCE_STATES InInitialState, ED3D12ResourceStateMode InResourceStateMode, D3D12_RESOURCE_STATES InDefaultState, const D3D12_CLEAR_VALUE* ClearValue, FD3D12Resource** ppOutResource, const TCHAR* Name, bool bVerifyHResult) { if (!ppOutResource) { return E_POINTER; } ID3D12Heap* Heap = BackingHeap->GetHeap(); TRefCountPtr pResource; HRESULT hr = S_OK; #if INTEL_EXTENSIONS if (InDesc.bRequires64BitAtomicSupport && IsRHIDeviceIntel() && GDX12INTCAtomicUInt64Emulation) { FD3D12ResourceDesc LocalDesc = InDesc; INTC_D3D12_RESOURCE_DESC_0001 IntelLocalDesc{}; IntelLocalDesc.pD3D12Desc = &LocalDesc; IntelLocalDesc.EmulatedTyped64bitAtomics = true; hr = INTC_D3D12_CreatePlacedResource(FD3D12DynamicRHI::GetD3DRHI()->GetIntelExtensionContext(), Heap, HeapOffset, &IntelLocalDesc, InInitialState, ClearValue, IID_PPV_ARGS(pResource.GetInitReference())); } else #endif #if D3D12RHI_SUPPORTS_UNCOMPRESSED_UAV if (InDesc.SupportsUncompressedUAV()) { checkf(InInitialState == D3D12_RESOURCE_STATE_COMMON, TEXT("RESOURCE_STATE_COMMON is required for castable resources (Given: %d)"), InInitialState); // Convert the desc to the version required by CreatePlacedResource2 const CD3DX12_RESOURCE_DESC1 LocalDesc(InDesc); // Common layout is the required starting state for any "legacy" transitions const D3D12_BARRIER_LAYOUT InitialLayout = D3D12_BARRIER_LAYOUT_COMMON; const TArray> CastableFormats = InDesc.GetCastableFormats(); hr = RootDevice10->CreatePlacedResource2( Heap, HeapOffset, &LocalDesc, InitialLayout, ClearValue, CastableFormats.Num(), CastableFormats.GetData(), IID_PPV_ARGS(pResource.GetInitReference())); } else #endif { hr = RootDevice->CreatePlacedResource(Heap, HeapOffset, &InDesc, InInitialState, ClearValue, IID_PPV_ARGS(pResource.GetInitReference())); } if (SUCCEEDED(hr)) { FD3D12Device* Device = BackingHeap->GetParentDevice(); const D3D12_HEAP_DESC HeapDesc = Heap->GetDesc(); // Set the output pointer *ppOutResource = new FD3D12Resource(Device, Device->GetVisibilityMask(), pResource, InInitialState, InResourceStateMode, InDefaultState, InDesc, BackingHeap, HeapDesc.Properties.Type); #if PLATFORM_WINDOWS if (IsTrackingAllAllocations() && BackingHeap->GetHeapDesc().Properties.Type == D3D12_HEAP_TYPE_DEFAULT) { // Manually set the GPU virtual address from the heap gpu virtual address & offset if (InDesc.Dimension != D3D12_RESOURCE_DIMENSION_BUFFER) { check(BackingHeap->GetGPUVirtualAddress() != 0); (*ppOutResource)->SetGPUVirtualAddress(BackingHeap->GetGPUVirtualAddress() + HeapOffset); } else { check((*ppOutResource)->GetGPUVirtualAddress() != 0); check((*ppOutResource)->GetGPUVirtualAddress() == BackingHeap->GetGPUVirtualAddress() + HeapOffset); } } #endif // Don't track resources allocated on transient heaps if (!BackingHeap->GetIsTransient()) { TraceMemoryAllocation(*ppOutResource); } // Set a default name (can override later). SetD3D12ResourceName(*ppOutResource, Name); (*ppOutResource)->AddRef(); } else { UE_LOG(LogD3D12RHI, Display, TEXT("D3D12 CreatePlacedResource failed with params:\n\tHeap Type: %d\n\tHeap Flags: %d\n\tResource Dimension: %d\n\tResource Width: %llu\n\tResource Height: %u\n\tFormat: %d\n\tResource Flags: %d"), BackingHeap->GetHeapDesc().Properties.Type, BackingHeap->GetHeapDesc().Flags, InDesc.Dimension, InDesc.Width, InDesc.Height, InDesc.PixelFormat, InDesc.Flags); if (bVerifyHResult) { VERIFYD3D12RESULT_EX(hr, RootDevice); } } return hr; } void FD3D12Adapter::TraceMemoryAllocation(FD3D12Resource* Resource) { #if UE_MEMORY_TRACE_ENABLED // Calling GetResourceAllocationInfo is not cheap so check memory allocation tracking is enabled if (UE_TRACE_CHANNELEXPR_IS_ENABLED(MemAllocChannel)) { const D3D12_RESOURCE_ALLOCATION_INFO Info = Resource->GetParentDevice()->GetResourceAllocationInfo(Resource->GetDesc()); D3D12_GPU_VIRTUAL_ADDRESS GPUAddress = Resource->GetGPUVirtualAddress(); // Textures don't have valid GPUVirtualAddress when IsTrackingAllAllocations() is false, so don't do memory trace in this case. if (IsTrackingAllAllocations() && GPUAddress != 0) { MemoryTrace_Alloc(GPUAddress, Info.SizeInBytes, Info.Alignment, EMemoryTraceRootHeap::VideoMemory); } } #endif } HRESULT FD3D12Adapter::CreateBuffer(D3D12_HEAP_TYPE HeapType, FRHIGPUMask CreationNode, FRHIGPUMask VisibleNodes, uint64 HeapSize, FD3D12Resource** ppOutResource, const TCHAR* Name, D3D12_RESOURCE_FLAGS Flags) { const D3D12_HEAP_PROPERTIES HeapProps = CD3DX12_HEAP_PROPERTIES(HeapType, CreationNode.GetNative(), VisibleNodes.GetNative()); const D3D12_RESOURCE_STATES InitialState = DetermineInitialResourceState(HeapProps.Type, &HeapProps); return CreateBuffer(HeapProps, CreationNode, InitialState, ED3D12ResourceStateMode::Default, D3D12_RESOURCE_STATE_TBD, HeapSize, ppOutResource, Name, Flags); } HRESULT FD3D12Adapter::CreateBuffer(D3D12_HEAP_TYPE HeapType, FRHIGPUMask CreationNode, FRHIGPUMask VisibleNodes, D3D12_RESOURCE_STATES InitialState, ED3D12ResourceStateMode ResourceStateMode, uint64 HeapSize, FD3D12Resource** ppOutResource, const TCHAR* Name, D3D12_RESOURCE_FLAGS Flags) { const D3D12_HEAP_PROPERTIES HeapProps = CD3DX12_HEAP_PROPERTIES(HeapType, CreationNode.GetNative(), VisibleNodes.GetNative()); return CreateBuffer(HeapProps, CreationNode, InitialState, ResourceStateMode, InitialState, HeapSize, ppOutResource, Name, Flags); } HRESULT FD3D12Adapter::CreateBuffer(const D3D12_HEAP_PROPERTIES& HeapProps, FRHIGPUMask CreationNode, D3D12_RESOURCE_STATES InitialState, ED3D12ResourceStateMode ResourceStateMode, D3D12_RESOURCE_STATES InDefaultState, uint64 HeapSize, FD3D12Resource** ppOutResource, const TCHAR* Name, D3D12_RESOURCE_FLAGS Flags) { if (!ppOutResource) { return E_POINTER; } const D3D12_RESOURCE_DESC BufDesc = CD3DX12_RESOURCE_DESC::Buffer(HeapSize, Flags); return CreateCommittedResource(BufDesc, CreationNode, HeapProps, InitialState, ResourceStateMode, InDefaultState, nullptr, ppOutResource, Name); } #if !D3D12RHI_SUPPORTS_UNCOMPRESSED_UAV void FD3D12Adapter::CreateUAVAliasResourceDesc(const FD3D12ResourceLocation& Location) { FD3D12Resource* SourceResource = Location.GetResource(); const FD3D12ResourceDesc& SourceDesc = SourceResource->GetDesc(); const EPixelFormat SourceFormat = SourceDesc.PixelFormat; const EPixelFormat AliasTextureFormat = SourceDesc.UAVPixelFormat; if (ensure(SourceFormat != PF_Unknown) && (SourceFormat != AliasTextureFormat)) { D3D12_RESOURCE_DESC AliasTextureDesc = SourceDesc; AliasTextureDesc.Format = (DXGI_FORMAT)GPixelFormats[AliasTextureFormat].PlatformFormat; AliasTextureDesc.Width = SourceDesc.Width / GPixelFormats[SourceFormat].BlockSizeX; AliasTextureDesc.Height = SourceDesc.Height / GPixelFormats[SourceFormat].BlockSizeY; // layout of UAV must match source resource AliasTextureDesc.Layout = SourceResource->GetResource()->GetDesc().Layout; EnumAddFlags(AliasTextureDesc.Flags, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS); SourceResource->SetUAVAccessResourceDesc(AliasTextureDesc); } } #endif ///////////////////////////////////////////////////////////////////// // FD3D12 Resource Location ///////////////////////////////////////////////////////////////////// FD3D12ResourceLocation::FD3D12ResourceLocation(FD3D12Device* Parent) : FD3D12DeviceChild(Parent) , Allocator(nullptr) { FMemory::Memzero(AllocatorData); } FD3D12ResourceLocation::~FD3D12ResourceLocation() { ReleaseResource(); } void FD3D12ResourceLocation::Clear() { InternalClear(); } template void FD3D12ResourceLocation::InternalClear(); template void FD3D12ResourceLocation::InternalClear(); template void FD3D12ResourceLocation::InternalClear() { if (bReleaseResource) { ReleaseResource(); } // Reset members Type = ResourceLocationType::eUndefined; UnderlyingResource = nullptr; MappedBaseAddress = nullptr; GPUVirtualAddress = 0; Size = 0; OffsetFromBaseOfResource = 0; FMemory::Memzero(AllocatorData); Allocator = nullptr; AllocatorType = AT_Unknown; } void FD3D12ResourceLocation::TransferOwnership(FD3D12ResourceLocation& Destination, FD3D12ResourceLocation& Source) { // The bTransient field is not preserved check(!Destination.bTransient && !Source.bTransient); // Preserve the owner fields FD3D12BaseShaderResource* DstOwner = Destination.Owner; FD3D12BaseShaderResource* SrcOwner = Source.Owner; // Clear out the destination Destination.Clear(); FMemory::Memmove(&Destination, &Source, sizeof(FD3D12ResourceLocation)); if (Source.GetAllocatorType() == FD3D12ResourceLocation::AT_Pool) { Source.GetPoolAllocator()->TransferOwnership(Source, Destination); } // update tracked allocation #if !PLATFORM_WINDOWS && ENABLE_LOW_LEVEL_MEM_TRACKER if (Source.GetType() == ResourceLocationType::eSubAllocation && Source.AllocatorType != AT_SegList) { FLowLevelMemTracker::Get().OnLowLevelAllocMoved(ELLMTracker::Default, Destination.GetAddressForLLMTracking(), Source.GetAddressForLLMTracking()); } #endif // Destroy the source but don't invoke any resource destruction Source.InternalClear(); Destination.Owner = DstOwner; Source.Owner = SrcOwner; } void FD3D12ResourceLocation::Alias(FD3D12ResourceLocation & Destination, FD3D12ResourceLocation & Source) { // Should not be linked list allocated - otherwise internal linked list data needs to be updated as well in a threadsafe way check(Source.GetAllocatorType() != FD3D12ResourceLocation::AT_Pool); check(Source.GetResource() != nullptr); Destination.Clear(); FMemory::Memmove(&Destination, &Source, sizeof(FD3D12ResourceLocation)); Destination.SetType(ResourceLocationType::eAliased); Source.SetType(ResourceLocationType::eAliased); // Addref the source as another resource location references it Source.GetResource()->AddRef(); } void FD3D12ResourceLocation::ReferenceNode(FD3D12Device* DestinationDevice, FD3D12ResourceLocation& Destination, FD3D12ResourceLocation& Source) { check(Source.GetResource() != nullptr); Destination.Clear(); FMemory::Memmove(&Destination, &Source, sizeof(FD3D12ResourceLocation)); Destination.SetType(ResourceLocationType::eNodeReference); Destination.Parent = DestinationDevice; // Addref the source as another resource location references it Source.GetResource()->AddRef(); if (Source.GetAllocatorType() == FD3D12ResourceLocation::AT_Pool) { Source.GetPoolAllocatorPrivateData().PoolData.AddAlias( &Destination.GetPoolAllocatorPrivateData().PoolData); } } void FD3D12ResourceLocation::ReleaseResource() { #if TRACK_RESOURCE_ALLOCATIONS if (IsTransient()) { FD3D12Adapter* Adapter = GetParentDevice()->GetParentAdapter(); if (Adapter->IsTrackingAllAllocations()) { bool bDefragFree = false; Adapter->ReleaseTrackedAllocationData(this, bDefragFree); } } #endif switch (Type) { case ResourceLocationType::eStandAlone: { bool bIncrement = false; UpdateStandAloneStats(bIncrement); // Multi-GPU support : When the resource enters this point for the first time the number of references should be the same as number of GPUs. // Shouldn't queue deferred deletion until all references are released as this could cause issues at the end of the pipe. // Instead reduce number of references until nothing else holds the resource. if (GNumExplicitGPUsForRendering > 1 && UnderlyingResource->GetRefCount() > 1) { check(UnderlyingResource->GetRefCount() <= GNumExplicitGPUsForRendering) UnderlyingResource->Release(); } else { check(UnderlyingResource->GetRefCount() == 1); if (UnderlyingResource->ShouldDeferDelete()) { UnderlyingResource->DeferDelete(); } else { UnderlyingResource->Release(); } } break; } case ResourceLocationType::eSubAllocation: { check(Allocator != nullptr); if (AllocatorType == AT_SegList) { SegListAllocator->Deallocate( GetResource(), GetSegListAllocatorPrivateData().Offset, GetSize()); } else if (AllocatorType == AT_Pool) { // Unlink any aliases -- the contents of aliases are cleaned up separately elsewhere via iteration over // the FD3D12LinkedAdapterObject. for (FRHIPoolAllocationData* Alias = GetPoolAllocatorPrivateData().PoolData.GetFirstAlias(); Alias; Alias = GetPoolAllocatorPrivateData().PoolData.GetFirstAlias()) { Alias->RemoveAlias(); } PoolAllocator->DeallocateResource(*this); } else { Allocator->Deallocate(*this); } break; } case ResourceLocationType::eNodeReference: case ResourceLocationType::eAliased: { if (GetAllocatorType() == FD3D12ResourceLocation::AT_Pool) { GetPoolAllocatorPrivateData().PoolData.RemoveAlias(); } if (UnderlyingResource->ShouldDeferDelete() && UnderlyingResource->GetRefCount() == 1) { UnderlyingResource->DeferDelete(); } else { UnderlyingResource->Release(); } break; } case ResourceLocationType::eHeapAliased: { check(UnderlyingResource->GetRefCount() == 1); if (UnderlyingResource->ShouldDeferDelete()) { UnderlyingResource->DeferDelete(); } else { UnderlyingResource->Release(); } break; } case ResourceLocationType::eFastAllocation: case ResourceLocationType::eUndefined: default: // Fast allocations are volatile by default so no work needs to be done. break; } } void FD3D12ResourceLocation::UpdateStandAloneStats(bool bIncrement) { if (UnderlyingResource->GetHeapType() == D3D12_HEAP_TYPE_DEFAULT) { FD3D12ResourceDesc Desc = UnderlyingResource->GetDesc(); bool bIsBuffer = (Desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER); bool bIsRenderTarget = (Desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET || Desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL); bool bIsUAV = (Desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS) > 0; if (bIsBuffer) { // Simultaneous access flag is used to detect shared heap requirement but can't be used for buffers on device calls Desc.Flags &= ~D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS; } // Get the desired size and allocated size for stand alone resources - allocated are very slow anyway D3D12_RESOURCE_ALLOCATION_INFO Info = UnderlyingResource->GetParentDevice()->GetResourceAllocationInfoUncached(Desc); int64 SizeInBytes = bIncrement ? Info.SizeInBytes : -(int64)Info.SizeInBytes; int32 Count = bIncrement ? 1 : -1; if (bIsBuffer) { if (bIsUAV) { INC_DWORD_STAT_BY(STAT_D3D12UAVBufferStandAloneCount, Count); INC_MEMORY_STAT_BY(STAT_D3D12UAVBufferStandAloneAllocated, SizeInBytes); } else { INC_DWORD_STAT_BY(STAT_D3D12BufferStandAloneCount, Count); INC_MEMORY_STAT_BY(STAT_D3D12BufferStandAloneAllocated, SizeInBytes); } } else { if (bIsRenderTarget) { INC_DWORD_STAT_BY(STAT_D3D12RenderTargetStandAloneCount, Count); INC_MEMORY_STAT_BY(STAT_D3D12RenderTargetStandAloneAllocated, SizeInBytes); } else if (bIsUAV) { INC_DWORD_STAT_BY(STAT_D3D12UAVTextureStandAloneCount, Count); INC_MEMORY_STAT_BY(STAT_D3D12UAVTextureStandAloneAllocated, SizeInBytes); } else { INC_DWORD_STAT_BY(STAT_D3D12TextureStandAloneCount, Count); INC_MEMORY_STAT_BY(STAT_D3D12TextureStandAloneAllocated, SizeInBytes); } } // Track all committed resource allocations if (bIncrement) { bool bCollectCallstack = true; UnderlyingResource->GetParentDevice()->GetParentAdapter()->TrackAllocationData(this, Info.SizeInBytes, bCollectCallstack); } else { bool bDefragFree = false; UnderlyingResource->GetParentDevice()->GetParentAdapter()->ReleaseTrackedAllocationData(this, bDefragFree); } } } void FD3D12ResourceLocation::SetResource(FD3D12Resource* Value) { check(UnderlyingResource == nullptr); GPUVirtualAddress = Value->GetGPUVirtualAddress(); UnderlyingResource = Value; } void FD3D12ResourceLocation::AsStandAlone(FD3D12Resource* Resource, uint64 InSize, bool bInIsTransient, const D3D12_HEAP_PROPERTIES* CustomHeapProperties) { SetType(FD3D12ResourceLocation::ResourceLocationType::eStandAlone); SetResource(Resource); SetSize(InSize); if (IsCPUAccessible(Resource->GetHeapType(), CustomHeapProperties)) { D3D12_RANGE range = { 0, IsCPUWritable(Resource->GetHeapType()) ? 0 : InSize }; SetMappedBaseAddress(Resource->Map(&range)); } SetGPUVirtualAddress(Resource->GetGPUVirtualAddress()); SetTransient(bInIsTransient); bool bIncrement = true; UpdateStandAloneStats(bIncrement); } bool FD3D12ResourceLocation::OnAllocationMoved(FD3D12ContextArray const& Contexts, FRHIPoolAllocationData* InNewData, D3D12_RESOURCE_STATES& OutCreateState) { // Assume linked list allocated for now - only defragging allocator FRHIPoolAllocationData& AllocationData = GetPoolAllocatorPrivateData().PoolData; check(InNewData == &AllocationData); check(AllocationData.IsAllocated()); // Should be allocated check(AllocationData.GetSize() == Size); // Same size check(Type == ResourceLocationType::eSubAllocation); // Suballocated check(GetMappedBaseAddress() == nullptr); // And VRAM only // Get the resource and the actual new allocator FD3D12Resource* CurrentResource = GetResource(); FD3D12PoolAllocator* NewAllocator = GetPoolAllocator(); UE_TRACE_METADATA_SCOPE_ASSET_FNAME(CurrentResource->GetName(), FName(TEXT("FD3D12ResourceLocation::OnAllocationMoved")), NAME_None); // Textures don't have valid GPUVirtualAddress when IsTrackingAllAllocations() is false, so don't do memory trace in this case. const bool bTrackingAllAllocations = GetParentDevice()->GetParentAdapter()->IsTrackingAllAllocations(); const bool bMemoryTrace = bTrackingAllAllocations || GPUVirtualAddress != 0; // If sub allocated and not placed only update the internal data if (NewAllocator->GetAllocationStrategy() == EResourceAllocationStrategy::kManualSubAllocation) { check(!CurrentResource->IsPlacedResource()); D3D12_GPU_VIRTUAL_ADDRESS OldGPUAddress = GPUVirtualAddress; OffsetFromBaseOfResource = AllocationData.GetOffset(); UnderlyingResource = NewAllocator->GetBackingResource(*this); GPUVirtualAddress = UnderlyingResource->GetGPUVirtualAddress() + OffsetFromBaseOfResource; #if UE_MEMORY_TRACE_ENABLED if (bMemoryTrace) { MemoryTrace_ReallocFree(OldGPUAddress, EMemoryTraceRootHeap::VideoMemory); MemoryTrace_ReallocAlloc(GPUVirtualAddress, AllocationData.GetSize(), AllocationData.GetAlignment(), EMemoryTraceRootHeap::VideoMemory); } #endif } else { check(CurrentResource->IsPlacedResource()); check(OffsetFromBaseOfResource == 0); #if UE_MEMORY_TRACE_ENABLED if (bMemoryTrace) { // CreatePlacedResource function below calls MemoryTrace_Alloc to track new memory, so call MemoryTrace_Free to match (instead of calling MemoryTrace_ReallocFree/MemoryTrace_ReallocAlloc). MemoryTrace_Free(GPUVirtualAddress, EMemoryTraceRootHeap::VideoMemory); } #endif // recreate the placed resource (ownership of current resource is already handled during the internal move) FD3D12HeapAndOffset HeapAndOffset = NewAllocator->GetBackingHeapAndAllocationOffsetInBytes(*this); D3D12_RESOURCE_STATES CreateState; ED3D12ResourceStateMode ResourceStateMode; if (CurrentResource->RequiresResourceStateTracking()) { // The newly created placed resource will be copied into by the defragger. Create it in the COPY_DEST to avoid an additional transition. // Standard resource state tracking will handle transitioning the resource out of this state as required. CreateState = D3D12_RESOURCE_STATE_COPY_DEST; ResourceStateMode = ED3D12ResourceStateMode::MultiState; } else { CreateState = CurrentResource->GetDefaultResourceState(); ResourceStateMode = ED3D12ResourceStateMode::Default; } // TODO: fix retrieval of ClearValue from owner (currently not a problem because not defragging RT/DS resource yet) D3D12_CLEAR_VALUE* ClearValue = nullptr; FName Name = CurrentResource->GetName(); FD3D12Resource* NewResource = nullptr; VERIFYD3D12RESULT(CurrentResource->GetParentDevice()->GetParentAdapter()->CreatePlacedResource(CurrentResource->GetDesc(), HeapAndOffset.Heap, HeapAndOffset.Offset, CreateState, ResourceStateMode, D3D12_RESOURCE_STATE_TBD, ClearValue, &NewResource, *Name.ToString())); UnderlyingResource = NewResource; GPUVirtualAddress = UnderlyingResource->GetGPUVirtualAddress() + OffsetFromBaseOfResource; OutCreateState = CreateState == D3D12_RESOURCE_STATE_TBD ? CurrentResource->GetWritableState() : CreateState; } // Refresh aliases for (FRHIPoolAllocationData* OtherAlias = AllocationData.GetFirstAlias(); OtherAlias; OtherAlias = OtherAlias->GetNext()) { FD3D12ResourceLocation* OtherResourceLocation = (FD3D12ResourceLocation*)OtherAlias->GetOwner(); OtherResourceLocation->OffsetFromBaseOfResource = OffsetFromBaseOfResource; OtherResourceLocation->UnderlyingResource = UnderlyingResource; OtherResourceLocation->GPUVirtualAddress = GPUVirtualAddress; } // Notify all the dependent resources about the change Owner->ResourceRenamed(Contexts); if (OutCreateState == D3D12_RESOURCE_STATE_TBD) { ERHIAccess ResourceAccess = ERHIAccess::Unknown; FD3D12Texture* CurrentResourceRHITexture = nullptr; if (CurrentResource->GetDesc().Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) { FD3D12Buffer* CurrentResourceRHIBuffer = (FD3D12Buffer*)Owner; ResourceAccess = CurrentResourceRHIBuffer->GetTrackedAccess_Unsafe(); } else { CurrentResourceRHITexture = (FD3D12Texture*)Owner; ResourceAccess = CurrentResourceRHITexture->GetTrackedAccess_Unsafe(); } OutCreateState = ResourceAccess == ERHIAccess::Unknown ? CurrentResource->GetInitialResourceState() : GetD3D12ResourceState(ResourceAccess, ED3D12QueueType::Direct, CurrentResourceRHITexture); } return true; } void FD3D12ResourceLocation::UnlockPoolData() { if (AllocatorType == AT_Pool) { GetPoolAllocatorPrivateData().PoolData.Unlock(); } } bool FD3D12ResourceLocation::IsStandaloneOrPooledPlacedResource() const { bool bStandalone = Type == ResourceLocationType::eStandAlone; bool bPoolPlacedResource = (!bStandalone && AllocatorType == AT_Pool) ? PoolAllocator->GetAllocationStrategy() == EResourceAllocationStrategy::kPlacedResource : false; return bStandalone || bPoolPlacedResource; } ///////////////////////////////////////////////////////////////////// // FD3D12 Resource Barrier Batcher ///////////////////////////////////////////////////////////////////// // Add a UAV barrier to the batch. Ignoring the actual resource for now. void FD3D12ResourceBarrierBatcher::AddUAV() { FD3D12ResourceBarrier& Barrier = Barriers.Emplace_GetRef(); Barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV; Barrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; Barrier.UAV.pResource = nullptr; // Ignore the resource ptr for now. HW doesn't do anything with it. } // Add a transition resource barrier to the batch. Returns the number of barriers added, which may be negative if an existing barrier was cancelled. int32 FD3D12ResourceBarrierBatcher::AddTransition(FD3D12Resource* pResource, D3D12_RESOURCE_STATES Before, D3D12_RESOURCE_STATES After, uint32 Subresource) { check(Before != After); if (Barriers.Num()) { // Check if we are simply reverting the last transition. In that case, we can just remove both transitions. // This happens fairly frequently due to resource pooling since different RHI buffers can point to the same underlying D3D buffer. // Instead of ping-ponging that underlying resource between COPY_DEST and GENERIC_READ, several copies can happen without a ResourceBarrier() in between. // Doing this check also eliminates a D3D debug layer warning about multiple transitions of the same subresource. const FD3D12ResourceBarrier& Last = Barriers.Last(); if (Last.Type == D3D12_RESOURCE_BARRIER_TYPE_TRANSITION && pResource->GetResource() == Last.Transition.pResource && Subresource == Last.Transition.Subresource && Before == Last.Transition.StateAfter && After == Last.Transition.StateBefore ) { Barriers.RemoveAt(Barriers.Num() - 1); return -1; } } check(IsValidD3D12ResourceState(Before) && IsValidD3D12ResourceState(After)); FD3D12ResourceBarrier& Barrier = Barriers.Emplace_GetRef(CD3DX12_RESOURCE_BARRIER::Transition(pResource->GetResource(), Before, After, Subresource)); if (pResource->IsBackBuffer() && EnumHasAnyFlags(After, BackBufferBarrierWriteTransitionTargets)) { Barrier.Flags |= BarrierFlag_CountAsIdleTime; } return 1; } void FD3D12ResourceBarrierBatcher::AddAliasingBarrier(ID3D12Resource* InResourceBefore, ID3D12Resource* InResourceAfter) { FD3D12ResourceBarrier& Barrier = Barriers.Emplace_GetRef(); Barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_ALIASING; Barrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; Barrier.Aliasing.pResourceBefore = InResourceBefore; Barrier.Aliasing.pResourceAfter = InResourceAfter; } void FD3D12ResourceBarrierBatcher::FlushIntoCommandList(FD3D12CommandList& CommandList, FD3D12QueryAllocator& TimestampAllocator) { auto InsertTimestamp = [&](bool bBegin) { #if RHI_NEW_GPU_PROFILER if (bBegin) { auto& Event = CommandList.EmplaceProfilerEvent(); CommandList.EndQuery(TimestampAllocator.Allocate(ED3D12QueryType::ProfilerTimestampBOP, &Event.GPUTimestampBOP)); } else { // CPUTimestamp is filled in at submission time in FlushProfilerEvents auto& Event = CommandList.EmplaceProfilerEvent(0); CommandList.EndQuery(TimestampAllocator.Allocate(ED3D12QueryType::ProfilerTimestampTOP, &Event.GPUTimestampTOP)); } #else ED3D12QueryType Type = bBegin ? ED3D12QueryType::IdleBegin : ED3D12QueryType::IdleEnd; CommandList.EndQuery(TimestampAllocator.Allocate(Type, nullptr)); #endif }; for (int32 BatchStart = 0, BatchEnd = 0; BatchStart < Barriers.Num(); BatchStart = BatchEnd) { // Gather a range of barriers that all have the same idle flag bool const bIdle = Barriers[BatchEnd].HasIdleFlag(); while (BatchEnd < Barriers.Num() && bIdle == Barriers[BatchEnd].HasIdleFlag()) { // Clear the idle flag since its not a valid D3D bit. Barriers[BatchEnd++].ClearIdleFlag(); } // Insert an idle begin/end timestamp around the barrier batch if required. if (bIdle) { InsertTimestamp(true); } #if DEBUG_RESOURCE_STATES TArrayView SubsetBarriers(Barriers.GetData() + BatchStart, BatchEnd - BatchStart); TConstArrayView ConstArrayView( reinterpret_cast(SubsetBarriers.GetData()), SubsetBarriers.Num()); LogResourceBarriers(ConstArrayView, CommandList.Interfaces.CommandList.GetReference(), CommandList.QueueType, FString(DX12_RESOURCE_NAME_TO_LOG)); #endif CommandList.GraphicsCommandList()->ResourceBarrier(BatchEnd - BatchStart, &Barriers[BatchStart]); #if DEBUG_RESOURCE_STATES // Keep track of all the resource barriers that have been submitted to the current command list. for(int i = BatchStart; i < BatchEnd - BatchStart - 1; i++) { CommandList.State.ResourceBarriers.Emplace(Barriers[i]); } #endif // #if DEBUG_RESOURCE_STATES if (bIdle) { InsertTimestamp(false); } } Barriers.Reset(); } void FD3D12DynamicRHI::RHIReplaceResources(FRHICommandListBase& RHICmdList, TArray&& ReplaceInfos) { RHICmdList.EnqueueLambdaMultiPipe(GetEnabledRHIPipelines(), FRHICommandListBase::EThreadFence::Enabled, TEXT("FD3D12DynamicRHI::RHIReplaceResources"), [ReplaceInfos = MoveTemp(ReplaceInfos)](FD3D12ContextArray const& Contexts) { for (FRHIResourceReplaceInfo const& Info : ReplaceInfos) { switch (Info.GetType()) { default: checkNoEntry(); break; case FRHIResourceReplaceInfo::EType::Buffer: { FD3D12Buffer* Dst = ResourceCast(Info.GetBuffer().Dst); FD3D12Buffer* Src = ResourceCast(Info.GetBuffer().Src); if (Src) { // The source buffer should not have any associated views. check(!Src->HasLinkedViews()); Dst->TakeOwnership(*Src); } else { Dst->ReleaseOwnership(); } Dst->ResourceRenamed(Contexts); } break; #if D3D12_RHI_RAYTRACING case FRHIResourceReplaceInfo::EType::RTGeometry: { FD3D12RayTracingGeometry* Src = ResourceCast(Info.GetRTGeometry().Src); FD3D12RayTracingGeometry* Dst = ResourceCast(Info.GetRTGeometry().Dst); if (Src) { Dst->Swap(*Src); } else { Dst->ReleaseUnderlyingResource(); } } break; #endif // D3D12_RHI_RAYTRACING } } } ); }