CGNICO/UnrealEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c11d382a6f0acddb0bc1e95ab9bc5f8fc3570b55
UnrealEngine/Engine/Source/Runtime/D3D12RHI/Private/D3D12Buffer.cpp
Jeffreytsai1004 c11d382a6f ue5-main
2025-05-18 13:04:45 +08:00

913 lines
38 KiB
C++
Raw Blame History

// Copyright Epic Games, Inc. All Rights Reserved.
/*=============================================================================
D3D12Buffer.cpp: D3D Common code for buffers.
=============================================================================*/
#include "D3D12RHIPrivate.h"
#include "HAL/LowLevelMemStats.h"
#include "ProfilingDebugging/AssetMetadataTrace.h"
#include "RHICoreBufferInitializer.h"
#include "RHICoreStats.h"
extern int32 GD3D12BindResourceLabels;
FD3D12Buffer::~FD3D12Buffer()
{
if (EnumHasAnyFlags(GetUsage(), EBufferUsageFlags::VertexBuffer) && GetParentDevice())
{
FD3D12CommandContext& DefaultContext = GetParentDevice()->GetDefaultCommandContext();
DefaultContext.StateCache.ClearVertexBuffer(&ResourceLocation);
}
bool bTransient = ResourceLocation.IsTransient();
if (!bTransient)
{
D3D12BufferStats::UpdateBufferStats(*this, false);
}
}
void FD3D12Buffer::UploadResourceData(FD3D12CommandContext& CommandContext, D3D12_RESOURCE_STATES InDestinationState, FD3D12ResourceLocation& DestinationResourceLocation, const FD3D12ResourceLocation& SourceResourceLocation, uint32 Size)
{
FD3D12Resource* Destination = DestinationResourceLocation.GetResource();
// Copy from the temporary upload heap to the default resource
// if resource doesn't require state tracking then transition to copy dest here (could have been suballocated from shared resource) - not very optimal and should be batched
if (!Destination->RequiresResourceStateTracking())
{
CommandContext.AddTransitionBarrier(Destination, Destination->GetDefaultResourceState(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES);
}
CommandContext.FlushResourceBarriers();
CommandContext.CopyBufferRegionChecked(
Destination->GetResource(), Destination->GetName(),
DestinationResourceLocation.GetOffsetFromBaseOfResource(),
SourceResourceLocation.GetResource()->GetResource(), FName(),
SourceResourceLocation.GetOffsetFromBaseOfResource(),
Size);
// Update the resource state after the copy has been done (will take care of updating the residency as well)
if (InDestinationState != D3D12_RESOURCE_STATE_COPY_DEST)
{
CommandContext.AddTransitionBarrier(Destination, D3D12_RESOURCE_STATE_COPY_DEST, InDestinationState, D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES);
}
CommandContext.UpdateResidency(SourceResourceLocation.GetResource());
CommandContext.ConditionalSplitCommandList();
// If the resource is untracked, the destination state must match the default state of the resource.
check(Destination->RequiresResourceStateTracking() || (Destination->GetDefaultResourceState() == InDestinationState));
// Buffer is now written and ready, so unlock the block (locked after creation and can be defragmented if needed)
DestinationResourceLocation.UnlockPoolData();
}
static FD3D12ResourceLocation AllocateUploadMemory(FD3D12Device* Device, uint32 Size, uint32 Alignment)
{
FD3D12ResourceLocation Location(Device);
if (!IsInRHIThread() && !IsInRenderingThread())
{
Device->GetParentAdapter()->GetUploadHeapAllocator(Device->GetGPUIndex()).AllocUploadResource(Size, Alignment, Location);
}
else
{
Device->GetDefaultFastAllocator().Allocate(Size, Alignment, &Location);
}
return Location;
}
static FD3D12ResourceLocation AllocateUploadMemory(FD3D12Buffer* Buffer, const FRHIBufferDesc& Desc)
{
return AllocateUploadMemory(Buffer->GetParentDevice(), Desc.Size, Buffer->BufferAlignment);
}
void FD3D12CommandContext::CopyBufferRegionChecked(
ID3D12Resource* DestResource, const FName& DestName, uint64 DestOffset,
ID3D12Resource* SourceResource, const FName& SourceName, uint64 SourceOffset,
uint32 ByteCount
)
{
#if ENABLE_COPY_BUFFER_REGION_CHECK
RHI_BREADCRUMB_CHECK_SHIPPINGF(*this, DestOffset + ByteCount <= DestResource->GetDesc().Width, TEXT("Dest byte range out of bounds for: '%s'"), *DestName.ToString());
RHI_BREADCRUMB_CHECK_SHIPPINGF(*this, SourceOffset + ByteCount <= SourceResource->GetDesc().Width, TEXT("Source byte range out of bounds for: '%s'"), *SourceName.ToString());
#endif // ENABLE_COPY_BUFFER_REGION_CHECK
// Pass down callchain
GraphicsCommandList()->CopyBufferRegion(
DestResource, DestOffset,
SourceResource, SourceOffset,
ByteCount
);
}
FD3D12SyncPointRef FD3D12Buffer::UploadResourceDataViaCopyQueue(FD3D12CommandContext& OwningContext, FResourceArrayUploadInterface* InResourceArray)
{
TRACE_CPUPROFILER_EVENT_SCOPE(UploadResourceDataViaCopyQueue);
// assume not dynamic and not on async thread (probably fine but untested)
check(IsInRHIThread() || IsInRenderingThread());
check(!(GetUsage() & EBufferUsageFlags::AnyDynamic));
uint32 BufferSize = GetSize();
// Get an upload heap and copy the data
FD3D12ResourceLocation SrcResourceLoc(GetParentDevice());
void* pData = GetParentDevice()->GetDefaultFastAllocator().Allocate(BufferSize, 4UL, &SrcResourceLoc);
check(pData);
{
TRACE_CPUPROFILER_EVENT_SCOPE(CopyToUploadMemory);
FMemory::Memcpy(pData, InResourceArray->GetResourceData(), BufferSize);
}
// Allocate copy queue command list and perform the copy op
FD3D12Device* Device = SrcResourceLoc.GetParentDevice();
FD3D12SyncPointRef SyncPoint;
{
FD3D12CopyScope CopyScope(Device, ED3D12SyncPointType::GPUOnly);
SyncPoint = CopyScope.GetSyncPoint();
RHI_BREADCRUMB_CHECK_SHIPPING(OwningContext, ResourceLocation.GetOffsetFromBaseOfResource() + BufferSize <= ResourceLocation.GetResource()->GetDesc().Width);
RHI_BREADCRUMB_CHECK_SHIPPING(OwningContext, SrcResourceLoc.GetOffsetFromBaseOfResource() + BufferSize <= SrcResourceLoc.GetResource()->GetDesc().Width);
// Perform actual copy op
CopyScope.Context.CopyCommandList()->CopyBufferRegion(
ResourceLocation.GetResource()->GetResource(),
ResourceLocation.GetOffsetFromBaseOfResource(),
SrcResourceLoc.GetResource()->GetResource(),
SrcResourceLoc.GetOffsetFromBaseOfResource(), BufferSize);
// Residency update needed since it's just been created?
CopyScope.Context.UpdateResidency(ResourceLocation.GetResource());
}
// Buffer is now written and ready, so unlock the block
ResourceLocation.UnlockPoolData();
// Discard the resource array's contents.
InResourceArray->Discard();
return SyncPoint;
}
void FD3D12Adapter::AllocateBuffer(FD3D12Device* Device,
const D3D12_RESOURCE_DESC& InDesc,
uint32 Size,
EBufferUsageFlags InUsage,
ED3D12ResourceStateMode InResourceStateMode,
D3D12_RESOURCE_STATES InCreateState,
uint32 Alignment,
FD3D12Buffer* Buffer,
FD3D12ResourceLocation& ResourceLocation,
ID3D12ResourceAllocator* ResourceAllocator,
const TCHAR* InDebugName,
const FName& OwnerName)
{
TRACE_CPUPROFILER_EVENT_SCOPE(D3D12RHI::AllocateBuffer);
// Explicitly check that the size is nonzero before allowing CreateBuffer to opaquely fail.
if (Size == 0)
{
UE_LOG(
LogD3D12RHI, Fatal,
TEXT("Attempt to create zero-sized buffer '%s', owner '%s', usage 0x%x"),
InDebugName ? InDebugName : TEXT("(null)"), *OwnerName.ToString(), static_cast<uint32>(InUsage)
);
}
if (EnumHasAnyFlags(InUsage, EBufferUsageFlags::AnyDynamic))
{
check(ResourceAllocator == nullptr);
check(InResourceStateMode != ED3D12ResourceStateMode::MultiState);
check(InCreateState == D3D12_RESOURCE_STATE_GENERIC_READ);
GetUploadHeapAllocator(Device->GetGPUIndex()).AllocUploadResource(Size, Alignment, ResourceLocation);
check(ResourceLocation.GetSize() >= Size);
}
else
{
if (ResourceAllocator)
{
ResourceAllocator->AllocateResource(Device->GetGPUIndex(), D3D12_HEAP_TYPE_DEFAULT, InDesc, InDesc.Width, Alignment, InResourceStateMode, InCreateState, nullptr, InDebugName, ResourceLocation);
}
else
{
Device->GetDefaultBufferAllocator().AllocDefaultResource(D3D12_HEAP_TYPE_DEFAULT, InDesc, InUsage, InResourceStateMode, InCreateState, ResourceLocation, Alignment, InDebugName);
}
ResourceLocation.SetOwner(Buffer);
check(ResourceLocation.GetSize() >= Size);
}
}
FD3D12Buffer* FD3D12Adapter::CreateRHIBuffer(
const D3D12_RESOURCE_DESC& InDesc,
uint32 Alignment,
const FRHIBufferCreateDesc& CreateDesc,
ED3D12ResourceStateMode InResourceStateMode,
D3D12_RESOURCE_STATES InCreateState,
bool bKeepUnlocked,
ID3D12ResourceAllocator* ResourceAllocator)
{
TRACE_CPUPROFILER_EVENT_SCOPE(D3D12RHI::CreateRHIBuffer);
SCOPE_CYCLE_COUNTER(STAT_D3D12CreateBufferTime);
LLM_SCOPE_DYNAMIC_STAT_OBJECTPATH_FNAME(CreateDesc.OwnerName, ELLMTagSet::Assets);
LLM_SCOPE_DYNAMIC_STAT_OBJECTPATH_FNAME(CreateDesc.GetTraceClassName(), ELLMTagSet::AssetClasses);
UE_TRACE_METADATA_SCOPE_ASSET_FNAME(CreateDesc.DebugName, CreateDesc.GetTraceClassName(), CreateDesc.OwnerName);
check(InDesc.Width >= CreateDesc.Size);
const bool bDynamic = EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::AnyDynamic);
FD3D12Buffer* BufferOut = nullptr;
// Theoretically, we could assert if GPUMask isn't correct, but at the moment the RDG and RHI buffer descriptions don't include the
// GPU mask, so there's no way for the caller to configure it (only the lower level CreateInfo includes it). Note that differentiation
// for NNE (DirectML) is required beyond just setting the mask anyway, in the sense of forcing separate GPU0 visible only heaps, not just
// filtering which GPU copies are allocated. Because this is necessary to solve a crash, it's higher priority than GPUMask support,
// which may be added in the future.
FRHIGPUMask GPUMask = EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::NNE) ? FRHIGPUMask::GPU0() : CreateDesc.GPUMask;
const uint32 FirstGPUIndex = GPUMask.GetFirstIndex();
FD3D12Buffer* NewBuffer0 = nullptr;
BufferOut = CreateLinkedObject<FD3D12Buffer>(GPUMask, [&](FD3D12Device* Device, FD3D12Buffer* FirstLinkedObject)
{
FD3D12Buffer* NewBuffer = new FD3D12Buffer(Device, CreateDesc);
NewBuffer->BufferAlignment = Alignment;
if (!bDynamic || (Device->GetGPUIndex() == FirstGPUIndex) || EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::MultiGPUAllocate))
{
AllocateBuffer(Device, InDesc, CreateDesc.Size, CreateDesc.Usage, InResourceStateMode, InCreateState, Alignment, NewBuffer, NewBuffer->ResourceLocation, ResourceAllocator, CreateDesc.DebugName, CreateDesc.OwnerName);
NewBuffer0 = NewBuffer;
}
else
{
check(NewBuffer0);
FD3D12ResourceLocation::ReferenceNode(Device, NewBuffer->ResourceLocation, NewBuffer0->ResourceLocation);
}
// Unlock immediately if there is no initial data
if (!bDynamic && !bKeepUnlocked)
{
NewBuffer->ResourceLocation.UnlockPoolData();
}
return NewBuffer;
});
// Don't track transient buffer stats here
if (!BufferOut->ResourceLocation.IsTransient())
{
D3D12BufferStats::UpdateBufferStats(*BufferOut, true);
}
return BufferOut;
}
void FD3D12Buffer::Rename(FD3D12ContextArray const& Contexts, FD3D12ResourceLocation& NewLocation)
{
FD3D12ResourceLocation::TransferOwnership(ResourceLocation, NewLocation);
ResourceRenamed(Contexts);
}
void FD3D12Buffer::RenameLDAChain(FD3D12ContextArray const& Contexts, FD3D12ResourceLocation& NewLocation)
{
// Dynamic buffers use cross-node resources (with the exception of EBufferUsageFlags::MultiGPUAllocate)
//ensure(GetUsage() & EBufferUsageFlags::AnyDynamic);
Rename(Contexts, NewLocation);
if (GNumExplicitGPUsForRendering > 1)
{
ensure(GetParentDevice() == NewLocation.GetParentDevice());
if (EnumHasAnyFlags(GetUsage(), EBufferUsageFlags::MultiGPUAllocate) == false)
{
ensure(IsHeadLink());
// Update all of the resources in the LDA chain to reference this cross-node resource
for (auto NextBuffer = ++FLinkedObjectIterator(this); NextBuffer; ++NextBuffer)
{
FD3D12ResourceLocation::ReferenceNode(NextBuffer->GetParentDevice(), NextBuffer->ResourceLocation, ResourceLocation);
NextBuffer->ResourceRenamed(Contexts);
}
}
}
}
void FD3D12Buffer::TakeOwnership(FD3D12Buffer& Other)
{
check(!Other.LockedData.bLocked);
// Clean up any resource this buffer already owns
ReleaseOwnership();
// Transfer ownership of Other's resources to this instance
FRHIBuffer::TakeOwnership(Other);
FD3D12ResourceLocation::TransferOwnership(ResourceLocation, Other.ResourceLocation);
}
void FD3D12Buffer::ReleaseOwnership()
{
check(!LockedData.bLocked);
check(IsHeadLink());
FRHIBuffer::ReleaseOwnership();
if (!ResourceLocation.IsTransient())
{
D3D12BufferStats::UpdateBufferStats(*this, false);
}
ResourceLocation.Clear();
}
void FD3D12Buffer::GetResourceDescAndAlignment(const FRHIBufferCreateDesc& CreateDesc, D3D12_RESOURCE_DESC& ResourceDesc, uint32& Alignment)
{
ResourceDesc = CD3DX12_RESOURCE_DESC::Buffer(CreateDesc.Size);
// Align size to 16 so RAW buffer view can be created without loosing any data at the end when dividing num elements by 4
ResourceDesc.Width = Align(ResourceDesc.Width, RHI_RAW_VIEW_ALIGNMENT);
if (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::UnorderedAccess))
{
ResourceDesc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
}
if (!EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::ShaderResource | EBufferUsageFlags::AccelerationStructure))
{
ResourceDesc.Flags |= D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE;
}
if (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::DrawIndirect))
{
ResourceDesc.Flags |= D3D12RHI_RESOURCE_FLAG_ALLOW_INDIRECT_BUFFER;
}
if (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::Shared))
{
ResourceDesc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS;
}
if (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::ReservedResource))
{
checkf(CreateDesc.Stride <= GRHIGlobals.ReservedResources.TileSizeInBytes,
TEXT("Reserved buffer stride %d must not be greater than the reserved resource tile size %d"),
CreateDesc.Stride, GRHIGlobals.ReservedResources.TileSizeInBytes);
Alignment = GRHIGlobals.ReservedResources.TileSizeInBytes;
}
else
{
// Structured buffers, non-ByteAddress buffers, need to be aligned to their stride to ensure that they can be addressed correctly with element based offsets.
Alignment = (CreateDesc.Stride > 0) && (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::StructuredBuffer) || !EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::ByteAddressBuffer | EBufferUsageFlags::DrawIndirect)) ? FMath::LeastCommonMultiplier(CreateDesc.Stride, RHI_RAW_VIEW_ALIGNMENT) : RHI_RAW_VIEW_ALIGNMENT;
}
}
#if ENABLE_LOW_LEVEL_MEM_TRACKER || UE_MEMORY_TRACE_ENABLED
void FD3D12Buffer::UpdateAllocationTags() const
{
FD3D12Resource* D3D12Resource = GetResource();
if (D3D12Resource)
{
auto ExecuteUpdateAllocationTags = [](uint64 LLMPtr, uint64 TracePtr, uint64 Size, bool bVideoMemory=true)
{
// PLATFORM_WINDOWS does not perform per buffer LLM tracking currently
#if !PLATFORM_WINDOWS
LLM_IF_ENABLED(FLowLevelMemTracker::Get().OnLowLevelFree(ELLMTracker::Default, (const void*)LLMPtr));
LLM_IF_ENABLED(FLowLevelMemTracker::Get().OnLowLevelAlloc(ELLMTracker::Default, (const void*)LLMPtr, Size));
#endif // #if !PLATFORM_WINDOWS
#if D3D12RHI_PLATFORM_HAS_UNIFIED_MEMORY
bVideoMemory = false;
#endif
#if UE_MEMORY_TRACE_ENABLED
if (UE_TRACE_CHANNELEXPR_IS_ENABLED(MemAllocChannel))
{
MemoryTrace_UpdateAlloc(TracePtr, bVideoMemory ? EMemoryTraceRootHeap::VideoMemory : EMemoryTraceRootHeap::SystemMemory);
}
#endif
};
if (D3D12Resource->IsReservedResource())
{
D3D12Resource->GetBackingHeapsGpuAddresses(ExecuteUpdateAllocationTags);
// Done so that FD3D12Resource::CommitReservedResource is able to restore the proper tag with LLM_REALLOC_SCOPE
ExecuteUpdateAllocationTags((uint64)D3D12Resource->ReservedResourceData.Get(), (uint64)D3D12Resource->ReservedResourceData.Get(), sizeof(FD3D12Resource::FD3D12ReservedResourceData), false);
}
else
{
FD3D12ResourceLocation::EAllocatorType AllocatorType = ResourceLocation.GetAllocatorType();
uint64 AddressForLLM = (AllocatorType == FD3D12ResourceLocation::AT_Pool || AllocatorType == FD3D12ResourceLocation::AT_Default) ? (uint64)ResourceLocation.GetAddressForLLMTracking() : ResourceLocation.GetGPUVirtualAddress();
ExecuteUpdateAllocationTags(AddressForLLM, (uint64)ResourceLocation.GetGPUVirtualAddress(), ResourceLocation.GetSize());
}
}
}
void FD3D12DynamicRHI::RHIUpdateAllocationTags(FRHICommandListBase& RHICmdList, FRHIBuffer* BufferRHI)
{
check(RHICmdList.IsBottomOfPipe());
FD3D12DynamicRHI::ResourceCast(BufferRHI)->UpdateAllocationTags();
}
#endif // #if ENABLE_LOW_LEVEL_MEM_TRACKER || UE_MEMORY_TRACE_ENABLED
FD3D12DynamicRHI::FCreateBufferInternalResult FD3D12DynamicRHI::CreateBufferInternal(const FRHIBufferCreateDesc& CreateDesc, bool bHasInitialData, ID3D12ResourceAllocator* ResourceAllocator)
{
D3D12_RESOURCE_DESC Desc{};
uint32 Alignment{};
FD3D12Buffer::GetResourceDescAndAlignment(CreateDesc, Desc, Alignment);
ED3D12ResourceStateMode StateMode = EnumHasAllFlags(CreateDesc.Usage, EBufferUsageFlags::AccelerationStructure)
? ED3D12ResourceStateMode::SingleState
: ED3D12ResourceStateMode::Default;
const bool bIsDynamic = EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::AnyDynamic);
if (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::ReservedResource))
{
checkf(!bHasInitialData, TEXT("Reserved resources may not have initial data"));
checkf(!bIsDynamic, TEXT("Reserved resources may not be dynamic"));
checkf(!ResourceAllocator, TEXT("Reserved resources may not use a custom resource allocator"));
}
const D3D12_HEAP_TYPE HeapType = bIsDynamic ? D3D12_HEAP_TYPE_UPLOAD : D3D12_HEAP_TYPE_DEFAULT;
const FD3D12Resource::FD3D12ResourceTypeHelper Type(Desc, HeapType);
// Does this resource support tracking?
const bool bSupportResourceStateTracking = !bIsDynamic && FD3D12DefaultBufferAllocator::IsPlacedResource(Desc.Flags, StateMode, Alignment) && Type.bWritable;
// Initial state is derived from the InResourceState if it supports tracking
const D3D12_RESOURCE_STATES DesiredState = bSupportResourceStateTracking ? Type.GetOptimalInitialState(CreateDesc.InitialState, false) :
FD3D12DefaultBufferAllocator::GetDefaultInitialResourceState(HeapType, CreateDesc.Usage, StateMode);
// Setup the state at which the resource needs to be created - copy dest only supported for placed resources
const D3D12_RESOURCE_STATES CreateState = (bHasInitialData && bSupportResourceStateTracking) ? D3D12_RESOURCE_STATE_COPY_DEST : DesiredState;
FD3D12Buffer* Buffer = GetAdapter().CreateRHIBuffer(Desc, Alignment, CreateDesc, StateMode, CreateState, bHasInitialData, ResourceAllocator);
check(Buffer->ResourceLocation.IsValid());
return FCreateBufferInternalResult{ Buffer, DesiredState };
}
static FRHIBufferInitializer CreateBufferInitializerForWriting(FRHICommandListBase& RHICmdList, const FD3D12DynamicRHI::FCreateBufferInternalResult& CreateResult, const FRHIBufferCreateDesc& CreateDesc)
{
FD3D12Buffer* Buffer = CreateResult.Buffer;
D3D12_RESOURCE_STATES DesiredState = CreateResult.DesiredState;
if (EnumHasAnyFlags(CreateDesc.Usage, EBufferUsageFlags::AnyDynamic))
{
// Copy directly in mapped data
const FD3D12ResourceLocation& UploadLocation = Buffer->ResourceLocation;
return UE::RHICore::FDefaultBufferInitializer(RHICmdList, Buffer, UploadLocation.GetMappedBaseAddress(), UploadLocation.GetSize());
}
FD3D12ResourceLocation UploadLocation(AllocateUploadMemory(Buffer, CreateDesc));
// Note: we have to get this pointer before the Initializer is constructed because it will be moved before we get the pointer
void* WritableData = UploadLocation.GetMappedBaseAddress();
return UE::RHICore::FCustomBufferInitializer(RHICmdList, Buffer, WritableData, CreateDesc.Size,
[Buffer = TRefCountPtr<FD3D12Buffer>(Buffer), DesiredState, UploadLocation = MoveTemp(UploadLocation)](FRHICommandListBase& RHICmdList) mutable
{
RHICmdList.EnqueueLambda(
[Buffer = Buffer.GetReference(), DesiredState, UploadLocation = MoveTemp(UploadLocation)](FRHICommandListBase& ExecutingCmdList)
{
const FRHIGPUMask EffectiveMask = ExecutingCmdList.GetGPUMask();
for (uint32 GPUIndex : EffectiveMask)
{
FD3D12CommandContext& CommandContext = FD3D12CommandContext::Get(ExecutingCmdList, GPUIndex);
FD3D12Buffer* DeviceBuffer = CommandContext.RetrieveObject<FD3D12Buffer>(Buffer, GPUIndex);
Buffer->UploadResourceData(CommandContext, DesiredState, DeviceBuffer->ResourceLocation, UploadLocation, Buffer->GetSize());
}
});
return TRefCountPtr<FRHIBuffer>(MoveTemp(Buffer));
});
}
FRHIBufferInitializer FD3D12DynamicRHI::RHICreateBufferInitializer(FRHICommandListBase& RHICmdList, const FRHIBufferCreateDesc& CreateDesc)
{
LLM_SCOPE_DYNAMIC_STAT_OBJECTPATH_FNAME(CreateDesc.OwnerName, ELLMTagSet::Assets);
LLM_SCOPE_DYNAMIC_STAT_OBJECTPATH_FNAME(CreateDesc.GetTraceClassName(), ELLMTagSet::AssetClasses);
UE_TRACE_METADATA_SCOPE_ASSET_FNAME(CreateDesc.DebugName, CreateDesc.GetTraceClassName(), CreateDesc.OwnerName);
if (CreateDesc.IsNull())
{
FD3D12Buffer* Buffer = GetAdapter().CreateLinkedObject<FD3D12Buffer>(
CreateDesc.GPUMask,
[CreateDesc](FD3D12Device* Device, FD3D12Buffer* FirstLinkedObject)
{
return new FD3D12Buffer(Device, CreateDesc);
});
return UE::RHICore::FDefaultBufferInitializer(RHICmdList, Buffer);
}
#if D3D12RHI_PLATFORM_HAS_UNIFIED_MEMORY
// Unified platforms don't need to use the copy queue for uploads.
constexpr bool bCreateAsCopyDest = false;
FCreateBufferInternalResult CreateResult = CreateBufferInternal(CreateDesc, bCreateAsCopyDest, nullptr);
const FD3D12ResourceLocation& UploadLocation = CreateResult.Buffer->ResourceLocation;
void* const WritableData = reinterpret_cast<void*>(UploadLocation.GetGPUVirtualAddress());
const uint64 WritableSize = UploadLocation.GetSize();
return UE::RHICore::CreateUnifiedMemoryBufferInitializer(RHICmdList, CreateDesc, CreateResult.Buffer, WritableData);
#else
const bool bCreateAsCopyDest = (CreateDesc.InitAction != ERHIBufferInitAction::Default);
FCreateBufferInternalResult CreateResult = CreateBufferInternal(CreateDesc, bCreateAsCopyDest, nullptr);
if (CreateDesc.InitAction == ERHIBufferInitAction::Default)
{
// Just return the buffer with its default contents
return UE::RHICore::FDefaultBufferInitializer(RHICmdList, CreateResult.Buffer);
}
// TODO: the UploadLocation logic is too complex to "just" write a different code path for each initializer type, so we will just use a common Initializer.
if (CreateDesc.InitAction == ERHIBufferInitAction::ResourceArray)
{
FRHIBufferInitializer Initializer = CreateBufferInitializerForWriting(RHICmdList, CreateResult, CreateDesc);
// Resource array initialization goes through the same steps as external code that writes its own data.
FResourceArrayUploadInterface* InitialData = CreateDesc.InitialData;
Initializer.WriteData(InitialData->GetResourceData(), InitialData->GetResourceDataSize());
// Discard the resource array's contents.
InitialData->Discard();
return MoveTemp(Initializer);
}
if (CreateDesc.InitAction == ERHIBufferInitAction::Zeroed)
{
FRHIBufferInitializer Initializer = CreateBufferInitializerForWriting(RHICmdList, CreateResult, CreateDesc);
// TODO: write a custom Initializer method that enqueues zeroing on the upload context.
Initializer.FillWithValue(0);
return MoveTemp(Initializer);
}
if (CreateDesc.InitAction == ERHIBufferInitAction::Initializer)
{
return CreateBufferInitializerForWriting(RHICmdList, CreateResult, CreateDesc);
}
return UE::RHICore::HandleUnknownBufferInitializerInitAction(RHICmdList, CreateDesc);
#endif // D3D12RHI_PLATFORM_HAS_UNIFIED_MEMORY
}
void* FD3D12DynamicRHI::LockBuffer(FRHICommandListBase& RHICmdList, FD3D12Buffer* Buffer, uint32 BufferSize, EBufferUsageFlags BufferUsage, uint32 Offset, uint32 Size, EResourceLockMode LockMode)
{
SCOPE_CYCLE_COUNTER(STAT_D3D12LockBufferTime);
UE_TRACE_METADATA_SCOPE_ASSET_FNAME(Buffer->GetName(), Buffer->GetName(), Buffer->GetOwnerName());
checkf(Size <= BufferSize, TEXT("Requested lock size %u is larger than the total size %u for buffer '%s'."), Size, BufferSize, *Buffer->GetName().ToString());
FD3D12LockedResource& LockedData = Buffer->LockedData;
check(LockedData.bLocked == false);
FD3D12Adapter& Adapter = GetAdapter();
void* Data = nullptr;
// Determine whether the buffer is dynamic or not.
if (EnumHasAnyFlags(BufferUsage, EBufferUsageFlags::AnyDynamic))
{
check(LockMode == RLM_WriteOnly || LockMode == RLM_WriteOnly_NoOverwrite);
if (LockedData.bHasNeverBeenLocked || LockMode == RLM_WriteOnly_NoOverwrite)
{
// Buffers on upload heap are mapped right after creation
Data = Buffer->ResourceLocation.GetMappedBaseAddress();
check(!!Data);
}
else
{
FD3D12Device* Device = Buffer->GetParentDevice();
FD3D12ResourceLocation NewLocation(Device);
Data = Adapter.GetUploadHeapAllocator(Device->GetGPUIndex()).AllocUploadResource(BufferSize, Buffer->BufferAlignment, NewLocation);
RHICmdList.EnqueueLambdaMultiPipe(GetEnabledRHIPipelines(), FRHICommandListBase::EThreadFence::Enabled, TEXT("FD3D12DynamicRHI::LockBuffer"),
[
Resource = Buffer,
NewLocation = MoveTemp(NewLocation)
](FD3D12ContextArray const& Contexts) mutable
{
const static FLazyName ExecuteName(TEXT("FRHICommandRenameUploadBuffer::Execute"));
UE_TRACE_METADATA_SCOPE_ASSET_FNAME(Resource->GetName(), ExecuteName, Resource->GetOwnerName());
for (FD3D12Buffer& DeviceBuffer : *Resource)
{
for (FD3D12CommandContextBase* ContextBase : Contexts)
{
if (FD3D12CommandContext* Context = ContextBase ? ContextBase->GetSingleDeviceContext(DeviceBuffer.GetParentDevice()->GetGPUIndex()) : nullptr)
{
// Clear the resource if still bound to make sure the SRVs are rebound again on next operation. This needs to happen
// on the RHI timeline when this command runs at the top of the pipe (which can happen when locking buffers in
// RLM_WriteOnly_NoOverwrite mode).
Context->ConditionalClearShaderResource(&DeviceBuffer.ResourceLocation, EShaderParameterTypeMask::SRVMask);
}
}
}
#if UE_MEMORY_TRACE_ENABLED
// This memory trace happens before RenameLDAChain so the old & new GPU addresses are correct
MemoryTrace_ReallocFree(Resource->ResourceLocation.GetGPUVirtualAddress(), EMemoryTraceRootHeap::VideoMemory);
MemoryTrace_ReallocAlloc(NewLocation.GetGPUVirtualAddress(), Resource->ResourceLocation.GetSize(), Resource->BufferAlignment, EMemoryTraceRootHeap::VideoMemory);
#endif
Resource->RenameLDAChain(Contexts, NewLocation);
});
}
}
else
{
// Static and read only buffers only have one version of the content. Use the first related device.
FD3D12Device* Device = Buffer->GetParentDevice();
FD3D12Resource* pResource = Buffer->ResourceLocation.GetResource();
// Locking for read must occur immediately so we can't queue up the operations later.
if (LockMode == RLM_ReadOnly)
{
LockedData.bLockedForReadOnly = true;
// If the static buffer is being locked for reading, create a staging buffer.
FD3D12Resource* StagingBuffer = nullptr;
const FRHIGPUMask Node = Device->GetGPUMask();
VERIFYD3D12RESULT(Adapter.CreateBuffer(D3D12_HEAP_TYPE_READBACK, Node, Node, Offset + Size, &StagingBuffer, nullptr));
// Copy the contents of the buffer to the staging buffer.
RHICmdList.EnqueueLambda([Node, StagingBuffer, pResource, Buffer, Offset, Size](FRHICommandListBase& ExecutingCmdList)
{
FD3D12CommandContext& Context = FD3D12CommandContext::Get(ExecutingCmdList, Node.GetFirstIndex());
uint64 SubAllocOffset = Buffer->ResourceLocation.GetOffsetFromBaseOfResource();
FScopedResourceBarrier ScopeResourceBarrierSource(Context, pResource, &Buffer->ResourceLocation, D3D12_RESOURCE_STATE_COPY_SOURCE, 0);
// Don't need to transition upload heaps
Context.FlushResourceBarriers(); // Must flush so the desired state is actually set.
Context.UpdateResidency(StagingBuffer);
Context.UpdateResidency(pResource);
Context.CopyBufferRegionChecked(
StagingBuffer->GetResource(), StagingBuffer->GetName(),
0,
pResource->GetResource(), pResource->GetName(),
SubAllocOffset + Offset, Size);
});
RHICmdList.GetAsImmediate().SubmitAndBlockUntilGPUIdle();
LockedData.ResourceLocation.AsStandAlone(StagingBuffer, Size);
Data = LockedData.ResourceLocation.GetMappedBaseAddress();
}
else
{
// If the static buffer is being locked for writing, allocate memory for the contents to be written to.
Data = Adapter.GetUploadHeapAllocator(Device->GetGPUIndex()).AllocUploadResource(Size, D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT, LockedData.ResourceLocation);
}
}
LockedData.LockOffset = Offset;
LockedData.LockSize = Size;
LockedData.bLocked = true;
LockedData.bHasNeverBeenLocked = false;
// Return the offset pointer
check(Data != nullptr);
return Data;
}
void FD3D12DynamicRHI::UnlockBuffer(FRHICommandListBase& RHICmdList, FD3D12Buffer* Buffer, EBufferUsageFlags BufferUsage)
{
SCOPE_CYCLE_COUNTER(STAT_D3D12UnlockBufferTime);
FD3D12LockedResource& LockedData = Buffer->LockedData;
check(LockedData.bLocked == true);
// Determine whether the buffer is dynamic or not.
if (EnumHasAnyFlags(BufferUsage, EBufferUsageFlags::AnyDynamic))
{
// If the Buffer is dynamic, its upload heap memory can always stay mapped. Don't do anything.
}
else if (LockedData.bLockedForReadOnly)
{
// Nothing to do, just release the locked data at the end of the function
}
else
{
// Update all of the resources in the LDA chain
check(Buffer->IsHeadLink());
RHICmdList.EnqueueLambda([
RootBuffer = Buffer,
LockedData = MoveTemp(LockedData)
](FRHICommandListBase& ExecutingCmdList)
{
for (FD3D12Buffer& Buffer : *RootBuffer)
{
FD3D12CommandContext& Context = FD3D12CommandContext::Get(ExecutingCmdList, Buffer.GetParentDevice()->GetGPUIndex());
FD3D12Resource* SourceResource = LockedData.ResourceLocation.GetResource();
uint32 SourceFullOffset = LockedData.ResourceLocation.GetOffsetFromBaseOfResource();
FD3D12Resource* DestResource = Buffer.ResourceLocation.GetResource();
uint32 DestFullOffset = Buffer.ResourceLocation.GetOffsetFromBaseOfResource() + LockedData.LockOffset;
// Clear the resource if still bound to make sure the SRVs are rebound again on next operation (and get correct resource transitions enqueued)
Context.ConditionalClearShaderResource(&Buffer.ResourceLocation, EShaderParameterTypeMask::SRVMask);
FScopedResourceBarrier ScopeResourceBarrierDest(Context, DestResource, &Buffer.ResourceLocation, D3D12_RESOURCE_STATE_COPY_DEST, 0);
// Don't need to transition upload heaps
Context.FlushResourceBarriers();
Context.UpdateResidency(DestResource);
Context.UpdateResidency(SourceResource);
Context.CopyBufferRegionChecked(
DestResource->GetResource(), DestResource->GetName(),
DestFullOffset,
SourceResource->GetResource(), SourceResource->GetName(),
SourceFullOffset,
LockedData.LockSize
);
Context.ConditionalSplitCommandList();
DEBUG_RHI_EXECUTE_COMMAND_LIST(this);
}
});
}
LockedData.Reset();
}
void* FD3D12DynamicRHI::RHILockBuffer(FRHICommandListBase& RHICmdList, FRHIBuffer* BufferRHI, uint32 Offset, uint32 Size, EResourceLockMode LockMode)
{
// If you hit this assert, you should be using LockBufferMGPU and iterating over FRHIGPUMask::All() to initialize the resource separately for each GPU.
// "MultiGPUAllocate" only makes sense if a buffer must vary per GPU, for example if it's a buffer that includes GPU specific virtual addresses for ray
// tracing acceleration structures.
check(!EnumHasAnyFlags(BufferRHI->GetUsage(), EBufferUsageFlags::MultiGPUAllocate));
bool bNeedTransition = ( (!(EnumHasAnyFlags(BufferRHI->GetUsage(), BUF_AnyDynamic))) && (LockMode == EResourceLockMode::RLM_ReadOnly) );
if (RHICmdList.NeedsExtraTransitions() && bNeedTransition)
{
RHICmdList.TransitionInternal(FRHITransitionInfo(BufferRHI, ERHIAccess::Unknown, ERHIAccess::CopySrc, EResourceTransitionFlags::IgnoreAfterState), ERHITransitionCreateFlags::AllowDuringRenderPass);
}
FD3D12Buffer* Buffer = FD3D12DynamicRHI::ResourceCast(BufferRHI);
void* retval = LockBuffer(RHICmdList, Buffer, Buffer->GetSize(), Buffer->GetUsage(), Offset, Size, LockMode);
if (RHICmdList.NeedsExtraTransitions() && bNeedTransition)
{
RHICmdList.TransitionInternal(FRHITransitionInfo(BufferRHI, ERHIAccess::CopySrc, ERHIAccess::Unknown, EResourceTransitionFlags::IgnoreAfterState), ERHITransitionCreateFlags::AllowDuringRenderPass);
}
return retval;
}
void* FD3D12DynamicRHI::RHILockBufferMGPU(FRHICommandListBase& RHICmdList, FRHIBuffer* BufferRHI, uint32 GPUIndex, uint32 Offset, uint32 Size, EResourceLockMode LockMode)
{
// If you hit this assert, you should be using LockBuffer to initialize the resource, rather than this function. The MGPU version is only for resources
// with the MultiGPUAllocate flag, where it's necessary for the caller to initialize the buffer for each GPU. The other LockBuffer call initializes the
// resource on all GPUs with one call, due to driver mirroring of the underlying resource.
check(EnumHasAnyFlags(BufferRHI->GetUsage(), EBufferUsageFlags::MultiGPUAllocate));
FD3D12Buffer* Buffer = FD3D12DynamicRHI::ResourceCast(BufferRHI, GPUIndex);
return LockBuffer(RHICmdList, Buffer, Buffer->GetSize(), Buffer->GetUsage(), Offset, Size, LockMode);
}
void FD3D12DynamicRHI::RHIUnlockBuffer(FRHICommandListBase& RHICmdList, FRHIBuffer* BufferRHI)
{
check(!EnumHasAnyFlags(BufferRHI->GetUsage(), EBufferUsageFlags::MultiGPUAllocate));
FD3D12Buffer* Buffer = FD3D12DynamicRHI::ResourceCast(BufferRHI);
FD3D12LockedResource& LockedData = Buffer->LockedData;
const bool bNeedTransition = (!((EnumHasAnyFlags(Buffer->GetUsage(), BUF_AnyDynamic)) || LockedData.bLockedForReadOnly));
if (RHICmdList.NeedsExtraTransitions() && bNeedTransition)
{
RHICmdList.TransitionInternal(FRHITransitionInfo(Buffer, ERHIAccess::Unknown, ERHIAccess::CopyDest, EResourceTransitionFlags::IgnoreAfterState), ERHITransitionCreateFlags::AllowDuringRenderPass);
}
UnlockBuffer(RHICmdList, Buffer, Buffer->GetUsage());
if (RHICmdList.NeedsExtraTransitions() && bNeedTransition )
{
RHICmdList.TransitionInternal(FRHITransitionInfo(Buffer, ERHIAccess::CopyDest, ERHIAccess::Unknown, EResourceTransitionFlags::IgnoreAfterState), ERHITransitionCreateFlags::AllowDuringRenderPass);
}
}
void FD3D12DynamicRHI::RHIUnlockBufferMGPU(FRHICommandListBase& RHICmdList, FRHIBuffer* BufferRHI, uint32 GPUIndex)
{
check(EnumHasAnyFlags(BufferRHI->GetUsage(), EBufferUsageFlags::MultiGPUAllocate));
FD3D12Buffer* Buffer = FD3D12DynamicRHI::ResourceCast(BufferRHI, GPUIndex);
UnlockBuffer(RHICmdList, Buffer, Buffer->GetUsage());
}
void FD3D12DynamicRHI::RHIBindDebugLabelName(FRHICommandListBase& RHICmdList, FRHIBuffer* BufferRHI, const TCHAR* Name)
{
if (BufferRHI == nullptr || !GD3D12BindResourceLabels)
{
return;
}
#if RHI_USE_RESOURCE_DEBUG_NAME
// Also set on RHI object
BufferRHI->SetName(Name);
FD3D12Buffer* Buffer = FD3D12DynamicRHI::ResourceCast(BufferRHI);
// only rename the underlying d3d12 resource if it's not sub allocated (requires resource state tracking or stand alone allocated)
if (ShouldSetD3D12ResourceName(Buffer->ResourceLocation))
{
if (GNumExplicitGPUsForRendering > 1)
{
// Generate string of the form "Name (GPU #)" -- assumes GPU index is a single digit. This is called many times
// a frame, so we want to avoid any string functions which dynamically allocate, to reduce perf overhead.
static_assert(MAX_NUM_GPUS <= 10);
static const TCHAR NameSuffix[] = TEXT(" (GPU #)");
constexpr int32 NameSuffixLengthWithTerminator = (int32)UE_ARRAY_COUNT(NameSuffix);
constexpr int32 NameBufferLength = 256;
constexpr int32 GPUIndexSuffixOffset = 6; // Offset of '#' character
// Combine Name and suffix in our string buffer (clamping the length for bounds checking). We'll replace the GPU index
// with the appropriate digit in the loop.
int32 NameLength = FMath::Min(FCString::Strlen(Name), NameBufferLength - NameSuffixLengthWithTerminator);
int32 GPUIndexOffset = NameLength + GPUIndexSuffixOffset;
TCHAR DebugName[NameBufferLength];
FMemory::Memcpy(&DebugName[0], Name, NameLength * sizeof(TCHAR));
FMemory::Memcpy(&DebugName[NameLength], NameSuffix, NameSuffixLengthWithTerminator * sizeof(TCHAR));
for (FD3D12Buffer::FLinkedObjectIterator BufferIt(Buffer); BufferIt; ++BufferIt)
{
FD3D12Resource* Resource = BufferIt->GetResource();
DebugName[GPUIndexOffset] = TEXT('0') + BufferIt->GetParentDevice()->GetGPUIndex();
SetD3D12ResourceName(BufferIt->GetResource(), DebugName);
}
}
else
{
SetD3D12ResourceName(Buffer->GetResource(), Name);
}
}
#endif
}
void FD3D12CommandContext::RHICopyBufferRegion(FRHIBuffer* DestBufferRHI, uint64 DstOffset, FRHIBuffer* SourceBufferRHI, uint64 SrcOffset, uint64 NumBytes)
{
FD3D12Buffer* SourceBuffer = RetrieveObject<FD3D12Buffer>(SourceBufferRHI);
FD3D12Buffer* DestBuffer = RetrieveObject<FD3D12Buffer>(DestBufferRHI);
FD3D12Device* BufferDevice = SourceBuffer->GetParentDevice();
check(BufferDevice == DestBuffer->GetParentDevice());
check(BufferDevice == GetParentDevice());
FD3D12Resource* pSourceResource = SourceBuffer->ResourceLocation.GetResource();
D3D12_RESOURCE_DESC const& SourceBufferDesc = pSourceResource->GetDesc();
FD3D12Resource* pDestResource = DestBuffer->ResourceLocation.GetResource();
D3D12_RESOURCE_DESC const& DestBufferDesc = pDestResource->GetDesc();
checkf(pSourceResource != pDestResource, TEXT("CopyBufferRegion cannot be used on the same resource. This can happen when both the source and the dest are suballocated from the same resource."));
check(DstOffset + NumBytes <= DestBufferDesc.Width);
check(SrcOffset + NumBytes <= SourceBufferDesc.Width);
FScopedResourceBarrier ScopeResourceBarrierSrc(*this, pSourceResource, &SourceBuffer->ResourceLocation, D3D12_RESOURCE_STATE_COPY_SOURCE, 0);
FScopedResourceBarrier ScopeResourceBarrierDst(*this, pDestResource , &DestBuffer->ResourceLocation, D3D12_RESOURCE_STATE_COPY_DEST , 0);
FlushResourceBarriers();
CopyBufferRegionChecked(
pDestResource->GetResource(), pDestResource->GetName(),
DestBuffer->ResourceLocation.GetOffsetFromBaseOfResource() + DstOffset,
pSourceResource->GetResource(), pSourceResource->GetName(),
SourceBuffer->ResourceLocation.GetOffsetFromBaseOfResource() + SrcOffset,
NumBytes
);
UpdateResidency(pDestResource);
UpdateResidency(pSourceResource);
ConditionalSplitCommandList();
#if (RHI_NEW_GPU_PROFILER == 0)
BufferDevice->RegisterGPUWork(1);
#endif
}
Reference in New Issue View Git Blame Copy Permalink