// Copyright Epic Games, Inc. All Rights Reserved. #include "DerivedDataCacheKey.h" #include "Containers/Map.h" #include "Containers/Set.h" #include "Containers/StringConv.h" #include "DerivedDataCachePrivate.h" #include "Hash/xxhash.h" #include "IO/IoHash.h" #include "Math/UnrealMathUtility.h" #include "Misc/ScopeRWLock.h" #include "Misc/StringBuilder.h" #include "Serialization/CompactBinary.h" #include "Serialization/CompactBinarySerialization.h" #include "Serialization/CompactBinaryWriter.h" #include "String/Find.h" namespace UE::DerivedData::Private { class FCacheBucketOwner : public FCacheBucket { public: inline explicit FCacheBucketOwner(FUtf8StringView Bucket); inline FCacheBucketOwner(FCacheBucketOwner&& Bucket); inline ~FCacheBucketOwner(); FCacheBucketOwner(const FCacheBucketOwner&) = delete; FCacheBucketOwner& operator=(const FCacheBucketOwner&) = delete; using FCacheBucket::operator==; inline bool operator==(const FCacheBucketOwner& Other) const { return FCacheBucket::operator==(Other); } inline bool operator==(FUtf8StringView Bucket) const { return ToString() == Bucket; } }; inline FCacheBucketOwner::FCacheBucketOwner(FUtf8StringView Bucket) { checkf(FCacheBucket::IsValidName(Bucket), TEXT("A cache bucket name must be alphanumeric, non-empty, and contain at most %d code units. Name: '%s'"), FCacheBucket::MaxNameLen, *WriteToString<256>(Bucket)); static_assert(sizeof(ANSICHAR) == sizeof(UTF8CHAR)); const int32 BucketLen = Bucket.Len(); const int32 PrefixSize = FMath::DivideAndRoundUp(1, sizeof(ANSICHAR)); UTF8CHAR* Buffer = new UTF8CHAR[PrefixSize + BucketLen + 1]; Buffer += PrefixSize; Name = reinterpret_cast(Buffer); reinterpret_cast(Buffer)[LengthOffset] = static_cast(BucketLen); Bucket.CopyString(Buffer, BucketLen); Buffer += BucketLen; *Buffer = UTF8CHAR('\0'); } inline FCacheBucketOwner::FCacheBucketOwner(FCacheBucketOwner&& Bucket) : FCacheBucket(Bucket) { Bucket.Reset(); } inline FCacheBucketOwner::~FCacheBucketOwner() { if (Name) { const int32 PrefixSize = FMath::DivideAndRoundUp(1, sizeof(ANSICHAR)); delete[] (Name - PrefixSize); } } struct FCacheBucketOwnerKeyFuncs : DefaultKeyFuncs { static uint32 GetKeyHash(const FUtf8StringView Key) { const int32 Len = Key.Len(); check(Len <= FCacheBucket::MaxNameLen); UTF8CHAR LowerKey[FCacheBucket::MaxNameLen]; UTF8CHAR* LowerKeyIt = LowerKey; for (const UTF8CHAR& Char : Key) { *LowerKeyIt++ = TChar::ToLower(Char); } return uint32(FXxHash64::HashBuffer(LowerKey, Len).Hash); } static uint32 GetKeyHash(const FCacheBucketOwner& Key) { return GetKeyHash(Key.ToString()); } }; class FCacheBuckets { public: template inline FCacheBucket FindOrAdd(TStringView Name); inline void GetDisplayName(FCacheBucket Bucket, FStringBuilderBase& OutDisplayName); inline void SetDisplayName(FCacheBucket Bucket, FStringView DisplayName); private: FRWLock Lock; TSet Buckets; TMap DisplayNames; }; template inline FCacheBucket FCacheBuckets::FindOrAdd(const TStringView Name) { const auto NameCast = StringCast(Name.GetData(), Name.Len()); const FUtf8StringView NameView = NameCast; uint32 Hash = 0; if (NameView.Len() <= FCacheBucket::MaxNameLen) { Hash = FCacheBucketOwnerKeyFuncs::GetKeyHash(NameView); FReadScopeLock ReadLock(Lock); if (const FCacheBucketOwner* Bucket = Buckets.FindByHash(Hash, NameView)) { return *Bucket; } } FCacheBucketOwner LocalBucket(NameView); FWriteScopeLock WriteLock(Lock); return Buckets.FindOrAddByHash(Hash, MoveTemp(LocalBucket)); } void FCacheBuckets::GetDisplayName(FCacheBucket Bucket, FStringBuilderBase& OutDisplayName) { if (FReadScopeLock ReadLock(Lock); const FString* DisplayName = DisplayNames.Find(Bucket)) { OutDisplayName << *DisplayName; return; } FAnsiStringView BucketName = Bucket.ToString(); if (BucketName.StartsWith(ANSITEXTVIEW("Legacy"))) { BucketName.RightChopInline(TEXTVIEW("Legacy").Len()); } OutDisplayName << BucketName; } void FCacheBuckets::SetDisplayName(FCacheBucket Bucket, FStringView DisplayName) { if (FReadScopeLock ReadLock(Lock); const FString* ExistingDisplayName = DisplayNames.Find(Bucket)) { if (*ExistingDisplayName == DisplayName) { return; } } FWriteScopeLock WriteLock(Lock); DisplayNames.Emplace(Bucket, DisplayName); } static FCacheBuckets& GetCacheBuckets() { static FCacheBuckets Buckets; return Buckets; } } // UE::DerivedData::Private namespace UE::DerivedData { FCacheBucket::FCacheBucket(FUtf8StringView InName) : FCacheBucket(Private::GetCacheBuckets().FindOrAdd(InName)) { } FCacheBucket::FCacheBucket(FWideStringView InName) : FCacheBucket(Private::GetCacheBuckets().FindOrAdd(InName)) { } FCacheBucket::FCacheBucket(FUtf8StringView Name, FStringView DisplayName) : FCacheBucket(Name) { if (!DisplayName.IsEmpty()) { Private::GetCacheBuckets().SetDisplayName(*this, DisplayName); } } FCacheBucket::FCacheBucket(FWideStringView Name, FStringView DisplayName) : FCacheBucket(Name) { if (!DisplayName.IsEmpty()) { Private::GetCacheBuckets().SetDisplayName(*this, DisplayName); } } void FCacheBucket::ToDisplayName(FStringBuilderBase& OutDisplayName) const { Private::GetCacheBuckets().GetDisplayName(*this, OutDisplayName); } FCbWriter& operator<<(FCbWriter& Writer, const FCacheBucket Bucket) { Writer.AddString(Bucket.ToString()); return Writer; } bool LoadFromCompactBinary(FCbFieldView Field, FCacheBucket& OutBucket) { if (const FUtf8StringView Bucket = Field.AsString(); !Field.HasError() && FCacheBucket::IsValidName(Bucket)) { OutBucket = FCacheBucket(Bucket); return true; } OutBucket.Reset(); return false; } FCbWriter& operator<<(FCbWriter& Writer, const FCacheKey& Key) { Writer.BeginObject(); Writer << ANSITEXTVIEW("Bucket") << Key.Bucket; Writer << ANSITEXTVIEW("Hash") << Key.Hash; Writer.EndObject(); return Writer; } void SerializeForLog(FCbWriter& Writer, const FCacheKey& Key) { Writer.AddString(WriteToString<96>(Key)); } bool LoadFromCompactBinary(const FCbFieldView Field, FCacheKey& OutKey) { bool bOk = Field.IsObject(); bOk &= LoadFromCompactBinary(Field[ANSITEXTVIEW("Bucket")], OutKey.Bucket); bOk &= LoadFromCompactBinary(Field[ANSITEXTVIEW("Hash")], OutKey.Hash); return bOk; } FCacheKey ConvertLegacyCacheKey(const FStringView Key) { FTCHARToUTF8 Utf8Key(Key); TUtf8StringBuilder<64> Utf8Bucket; Utf8Bucket << ANSITEXTVIEW("Legacy"); if (const int32 BucketEnd = String::FindFirstChar(Utf8Key, '_'); BucketEnd != INDEX_NONE) { Utf8Bucket << FUtf8StringView(Utf8Key).Left(BucketEnd); } const FCacheBucket Bucket(Utf8Bucket); return {Bucket, FIoHash::HashBuffer(MakeMemoryView(Utf8Key))}; } } // UE::DerivedData