// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.IO.Enumeration; using System.Linq; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using EpicGames.UHT.Parsers; using EpicGames.UHT.Tables; using EpicGames.UHT.Tokenizer; using EpicGames.UHT.Types; using Microsoft.Extensions.Logging; namespace EpicGames.UHT.Utils { /// /// To support the testing framework, source files can be containing in other source files. /// A source fragment represents this possibility. /// public struct UhtSourceFragment { /// /// When not null, this source comes from another source file /// public UhtSourceFile? SourceFile { get; set; } /// /// The file path of the source /// public string FilePath { get; set; } /// /// The line number of the fragment in the containing source file. /// public int LineNumber { get; set; } /// /// Data of the source file /// public StringView Data { get; set; } } /// /// A structure that represents an engine version. /// public struct EngineVersion { /// /// The major engine version. /// public int MajorVersion { get; set; } = 0; /// /// The minor engine version. /// public int MinorVersion { get; set; } = 0; /// /// The patch engine version. /// public int PatchVersion { get; set; } = 0; /// /// Build a default engine version (0.0.0). /// public EngineVersion() { } /// /// Build a new engine version with the given numbers. /// public EngineVersion(int major, int minor, int patch) { MajorVersion = major; MinorVersion = minor; PatchVersion = patch; } /// /// Compares this engine version to another one. /// /// The other engine version /// Returns -1 if this version is less than the other, 1 if it is greater than the other, and 0 if they're equal public int CompareTo(EngineVersion other) { if (MajorVersion != other.MajorVersion) { return MajorVersion.CompareTo(other.MajorVersion); } if (MinorVersion != other.MinorVersion) { return MinorVersion.CompareTo(other.MinorVersion); } if (PatchVersion != other.PatchVersion) { return PatchVersion.CompareTo(other.PatchVersion); } return 0; } /// /// Generates a string representation of the engine version. /// /// public override string ToString() { return String.Format("{0}.{1}.{2}", MajorVersion, MinorVersion, PatchVersion); } } /// /// Implementation of the export factory /// class UhtExportFactory : IUhtExportFactory { public struct Output { public string FilePath { get; set; } public string TempFilePath { get; set; } public bool Saved { get; set; } } /// /// UHT session /// private readonly UhtSession _session; /// /// Module associated with the plugin /// private readonly UHTManifest.Module? _pluginModule; /// /// Limiter for the number of files being saved to the reference directory. /// The OS can get swamped on high core systems /// private static readonly Semaphore s_writeRefSemaphore = new(32, 32); /// /// Requesting exporter /// public readonly UhtExporter Exporter; /// /// UHT Session /// public UhtSession Session => _session; /// /// Plugin module /// public UHTManifest.Module? PluginModule => _pluginModule; /// /// Collection of error from mismatches with the reference files /// public Dictionary ReferenceErrorMessages { get; } = new Dictionary(); /// /// List of export outputs /// public List Outputs { get; } = new List(); /// /// Directory for the reference output /// public string ReferenceDirectory { get; set; } = String.Empty; /// /// Directory for the verify output /// public string VerifyDirectory { get; set; } = String.Empty; /// /// Collection of external dependencies /// public HashSet ExternalDependencies { get; } = new HashSet(); /// /// Create a new instance of the export factory /// /// UHT session /// Plugin module /// Exporter being run public UhtExportFactory(UhtSession session, UHTManifest.Module? pluginModule, UhtExporter exporter) { _session = session; _pluginModule = pluginModule; Exporter = exporter; if (Session.ReferenceMode != UhtReferenceMode.None) { ReferenceDirectory = Path.Combine(Session.ReferenceDirectory, Exporter.Name); VerifyDirectory = Path.Combine(Session.VerifyDirectory, Exporter.Name); Directory.CreateDirectory(Session.ReferenceMode == UhtReferenceMode.Reference ? ReferenceDirectory : VerifyDirectory); } } /// /// Commit the contents of the string builder as the output. /// If you have a string builder, use this method so that a /// temporary buffer can be used. /// /// Destination file path /// Source for the content public void CommitOutput(string filePath, StringBuilder builder) { using UhtRentedPoolBuffer borrowBuffer = builder.RentPoolBuffer(); string tempFilePath = filePath + ".tmp"; SaveIfChanged(filePath, tempFilePath, new StringView(borrowBuffer.Buffer.Memory)); } /// /// Commit the value of the string as the output /// /// Destination file path /// Output to commit public void CommitOutput(string filePath, StringView output) { string tempFilePath = filePath + ".tmp"; SaveIfChanged(filePath, tempFilePath, output); } /// /// Create a task to export two files /// /// Tasks that must be completed prior to this task running /// Action to be invoked to generate the output /// Task object or null if the task was immediately executed. public Task? CreateTask(List? prereqs, UhtExportTaskDelegate action) { if (Session.GoWide) { Task[]? prereqTasks = prereqs?.Where(x => x != null).Cast().ToArray(); if (prereqTasks != null && prereqTasks.Length > 0) { return Task.Factory.ContinueWhenAll(prereqTasks, (Task[] tasks) => { Session.TryNoErrorCheck(null, () => action(this)); }); } else { return Task.Factory.StartNew(() => { Session.TryNoErrorCheck(null, () => action(this)); }, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default); } } else { Session.TryNoErrorCheck(null, () => action(this)); return null; } } /// /// Create a task to export two files /// /// Action to be invoked to generate the output /// Task object or null if the task was immediately executed. public Task? CreateTask(UhtExportTaskDelegate action) { return CreateTask(null, action); } /// /// Given a header file, generate the output file name. /// /// Header file /// Suffix/extension to be added to the file name. /// Resulting file name public string MakePath(UhtHeaderFile headerFile, string suffix) { return MakePath(headerFile.Module.Module, headerFile.FileNameWithoutExtension, suffix); } /// /// Given a package file, generate the output file name /// /// Module /// Suffix/extension to be added to the file name. /// Resulting file name public string MakePath(UhtModule module, string suffix) { return MakePath(module.Module, module.ShortName, suffix); } /// /// Make a path for an output based on the package output directory. /// /// Name of the file /// Extension to add to the file /// Output file path public string MakePath(string fileName, string extension) { if (PluginModule == null) { throw new UhtIceException("MakePath with just a filename and extension can not be called from non-plugin exporters"); } return MakePath(PluginModule, fileName, extension); } /// /// Add an external dependency to the given file path /// /// External dependency to add public void AddExternalDependency(string filePath) { ExternalDependencies.Add(filePath); } /// public string GetModuleShortestIncludePath(UhtModule moduleObj, string filePath) { return GetShortestIncludePath(moduleObj.Module, filePath); } /// public string GetPluginShortestIncludePath(string filePath) { if (PluginModule == null) { throw new UhtIceException("Made a request for the plugin's shortest include path but there is no plugin"); } return GetShortestIncludePath(PluginModule, filePath); } public string GetShortestIncludePath(UHTManifest.Module moduleObj, string filePath) { string? relativePath = null; foreach (string includePath in moduleObj.IncludePaths) { if (filePath.StartsWith(includePath, StringComparison.Ordinal)) { if (relativePath == null || (includePath.Length - filePath.Length - 1) < relativePath.Length) { relativePath = filePath.Substring(includePath.Length + 1); } } } // This will create a "../" path which is not great if (relativePath == null) { string? includePath = moduleObj.IncludePaths.Find(includePath => includePath.StartsWith(moduleObj.BaseDirectory, StringComparison.Ordinal)); if (includePath == null) { Session.Logger.LogWarning("There are no include paths in module {ModuleName} for {FileName}. It will be included by a \"../\" path external to the module.", moduleObj.Name, filePath); includePath = moduleObj.IncludePaths.FirstOrDefault("."); } relativePath = Path.GetRelativePath(includePath!, filePath); } return relativePath!.Replace("\\", "/", StringComparison.Ordinal); } private string MakePath(UHTManifest.Module module, string fileName, string suffix) { if (PluginModule != null) { module = PluginModule; } return Path.Combine(module.OutputDirectory, fileName) + suffix; } /// /// Helper method to test to see if the output has changed. /// /// Name of the output file /// Name of the temporary file /// Exported contents of the file internal void SaveIfChanged(string filePath, string tempFilePath, StringView exported) { ReadOnlySpan exportedSpan = exported.Span; if (Session.ReferenceMode != UhtReferenceMode.None) { string fileName = Path.GetFileName(filePath); // Writing billions of files to the same directory causes issues. Use ourselves to throttle reference writes try { UhtExportFactory.s_writeRefSemaphore.WaitOne(); { string outPath = Path.Combine(Session.ReferenceMode == UhtReferenceMode.Reference ? ReferenceDirectory : VerifyDirectory, fileName); if (!Session.WriteSource(outPath, exported.Span)) { new UhtSimpleFileMessageSite(Session, outPath).LogWarning($"Unable to write reference file {outPath}"); } } } finally { UhtExportFactory.s_writeRefSemaphore.Release(); } // If we are verifying, read the existing file and check the contents if (Session.ReferenceMode == UhtReferenceMode.Verify) { string message = String.Empty; string refPath = Path.Combine(ReferenceDirectory, fileName); if (Session.ReadSource(refPath, out UhtPoolBuffer existingRef)) { ReadOnlySpan existingSpan = existingRef.Memory.Span; if (existingSpan.CompareTo(exportedSpan, StringComparison.Ordinal) != 0) { message = $"********************************* {fileName} has changed."; } UhtPoolBuffers.Return(existingRef); } else { message = $"********************************* {fileName} appears to be a new generated file."; } if (!String.IsNullOrEmpty(message)) { Session.Logger.LogInformation("{Message}", message); Session.SignalError(); lock (ReferenceErrorMessages) { ReferenceErrorMessages.Add(message, true); } } } } // Check to see if the contents have changed bool save = false; if (Session.ReadSource(filePath, out UhtPoolBuffer original)) { ReadOnlySpan originalSpan = original.Memory.Span; if (originalSpan.CompareTo(exportedSpan, StringComparison.Ordinal) != 0) { if (Session.FailIfGeneratedCodeChanges) { string conflictPath = filePath + ".conflict"; if (!Session.WriteSource(conflictPath, exported.Span)) { new UhtSimpleFileMessageSite(Session, filePath).LogError($"Changes to generated code are not allowed - conflicts written to '{conflictPath}'"); } } save = true; } UhtPoolBuffers.Return(original); } else { save = true; } // If changed of the original didn't exist, then save the new version if (save && !Session.NoOutput) { if (!Session.WriteSource(tempFilePath, exported.Span)) { new UhtSimpleFileMessageSite(Session, filePath).LogWarning($"Failed to save export file: '{tempFilePath}'"); } } else { save = false; } // Add this to the list of outputs lock (Outputs) { Outputs.Add(new Output { FilePath = filePath, TempFilePath = tempFilePath, Saved = save }); } } /// /// Run the output exporter /// public void Run() { // Invoke the exported via the delegate Exporter.Delegate(this); // These outputs are used to cull old outputs from the directories Dictionary> outputsByDirectory = new( Session.Modules.Where(x => x.Module.SaveExportedHeaders).Select(x => new KeyValuePair>(x.Module.OutputDirectory, new HashSet(StringComparer.OrdinalIgnoreCase))), StringComparer.OrdinalIgnoreCase ); // If outputs were exported if (Outputs.Count > 0) { List saves = new(); // Collect information about the outputs foreach (UhtExportFactory.Output output in Outputs) { // Add this output to the list of expected outputs by directory string? fileDirectory = Path.GetDirectoryName(output.FilePath); if (fileDirectory != null) { if (!outputsByDirectory.TryGetValue(fileDirectory, out HashSet? files)) { files = new HashSet(StringComparer.OrdinalIgnoreCase); outputsByDirectory.Add(fileDirectory, files); } files.Add(Path.GetFileName(output.FilePath)); } // Add the save task if (output.Saved) { saves.Add(output); } } // Perform the renames if (Session.GoWide) { Parallel.ForEach(saves, (UhtExportFactory.Output output) => { RenameSource(output); }); } else { foreach (UhtExportFactory.Output output in saves) { RenameSource(output); } } } // Perform the culling of the output directories if (Session.CullOutput && !Session.NoOutput && (Exporter.CppFilters != null || Exporter.HeaderFilters != null || Exporter.OtherFilters != null)) { if (Session.GoWide) { Parallel.ForEach(outputsByDirectory, (KeyValuePair> kvp) => { CullOutputDirectory(kvp.Key, kvp.Value); }); } else { foreach (KeyValuePair> kvp in outputsByDirectory) { CullOutputDirectory(kvp.Key, kvp.Value); } } } } /// /// Given an output, rename the output file from the temporary file name to the final file name. /// If there exists a current final file, it will be replaced. /// /// The output file to rename private void RenameSource(UhtExportFactory.Output output) { Session.RenameSource(output.TempFilePath, output.FilePath); } /// /// Given a directory and a list of known files, delete any unknown file that matches of the supplied filters /// /// Output directory to scan /// Collection of known output files not to be deleted private void CullOutputDirectory(string outputDirectory, HashSet knownOutputs) { foreach (string filePath in Directory.EnumerateFiles(outputDirectory)) { string fileName = Path.GetFileName(filePath); if (knownOutputs.Contains(fileName)) { continue; } if (IsFilterMatch(fileName, Exporter.CppFilters) || IsFilterMatch(fileName, Exporter.HeaderFilters) || IsFilterMatch(fileName, Exporter.OtherFilters)) { try { File.Delete(Path.Combine(outputDirectory, filePath)); } catch (Exception) { } } } } /// /// Test to see if the given filename (without directory), matches one of the given filters /// /// File name to test /// List of wildcard filters /// True if there is a match private static bool IsFilterMatch(string fileName, IEnumerable filters) { foreach (string filter in filters) { if (FileSystemName.MatchesSimpleExpression(filter, fileName, true)) { return true; } } return false; } } /// /// UHT supports the exporting of two reference output directories for testing. The reference version can be used to test /// modification to UHT and verify there are no output changes or just expected changes. /// public enum UhtReferenceMode { /// /// Do not export any reference output files /// None, /// /// Export the reference copy /// Reference, /// /// Export the verify copy and compare to the reference copy /// Verify, }; /// /// Session object that represents a UHT run /// public partial class UhtSession : IUhtMessageSite, IUhtMessageSession { /// /// Helper class for returning a sequence of auto-incrementing indices /// private class TypeCounter { /// /// Current number of types /// private int _count = 0; /// /// Get the next type index /// /// Index starting at zero public int GetNext() { return Interlocked.Increment(ref _count) - 1; } /// /// The number of times GetNext was called. /// public int Count => Interlocked.Add(ref _count, 0) + 1; } /// /// Pair that represents a specific value for an enumeration /// private struct EnumAndValue { public UhtEnum Enum { get; set; } public long Value { get; set; } } /// /// Collection of reserved names /// private static readonly HashSet s_reservedNames = new() { "none" }; #region Configurable settings /// /// Logger interface. /// public ILogger Logger { get; } /// /// Interface used to read/write files /// public IUhtFileManager? FileManager { get; set; } /// /// Location of the engine code /// public string? EngineDirectory { get; set; } /// /// If set, the name of the project file. /// public string? ProjectFile { get; set; } /// /// Optional location of the project /// public string? ProjectDirectory { get; set; } /// /// Root directory for the engine. This is usually just EngineDirectory without the Engine directory. /// public string? RootDirectory { get; set; } /// /// Directory to store the reference output /// public string ReferenceDirectory { get; set; } = String.Empty; /// /// Directory to store the verification output /// public string VerifyDirectory { get; set; } = String.Empty; /// /// Mode for generating and/or testing reference output /// public UhtReferenceMode ReferenceMode { get; set; } = UhtReferenceMode.None; /// /// If true, warnings are considered to be errors /// public bool WarningsAsErrors { get; set; } = false; /// /// If true, include relative file paths in the log file /// public bool RelativePathInLog { get; set; } = false; /// /// If true, use concurrent tasks to run UHT /// public bool GoWide { get; set; } = true; /// /// If any output file mismatches existing outputs, an error will be generated /// public bool FailIfGeneratedCodeChanges { get; set; } = false; /// /// If true, no output files will be saved /// public bool NoOutput { get; set; } = false; /// /// If true, cull the output for any extra files /// public bool CullOutput { get; set; } = true; /// /// If true, include extra output in code generation /// public bool IncludeDebugOutput { get; set; } = false; /// /// If true, disable all exporters which would normally be run by default /// public bool NoDefaultExporters { get; set; } = false; /// /// If true, cache any error messages until the end of processing. This is used by the testing /// harness to generate more stable console output. /// public bool CacheMessages { get; set; } = false; /// /// Collection of UHT tables /// public UhtTables? Tables { get; set; } /// /// Configuration for the session /// public IUhtConfig? Config { get; set; } #endregion /// /// Manifest file /// public UhtManifestFile? ManifestFile { get; set; } = null; /// /// Manifest data from the manifest file /// public UHTManifest? Manifest => ManifestFile?.Manifest; /// /// Collection of modules from the manifest /// public IReadOnlyList Modules => _modules; /// /// Collection of header files from the manifest. The header files will also appear as the children /// of the packages /// public IReadOnlyList HeaderFiles => _headerFiles; /// /// Collection of header files topologically sorted. This will not be populated until after header files /// are parsed and resolved. /// public IReadOnlyList SortedHeaderFiles => _sortedHeaderFiles; /// /// Dictionary of stripped file name to the header file /// public IReadOnlyDictionary HeaderFileDictionary => _headerFileDictionary; /// /// Gets the version of the engine this session belongs to. /// public EngineVersion EngineVersion => _engineVersion.Value; /// /// After headers are parsed, returns the UObject class. /// public UhtClass UObject { get { if (_uobject == null) { throw new UhtIceException("UObject was not defined."); } return _uobject; } } /// /// After headers are parsed, returns the UClass class. /// public UhtClass UClass { get { if (_uclass == null) { throw new UhtIceException("UClass was not defined."); } return _uclass; } } /// /// After headers are parsed, returns the UInterface class. /// public UhtClass UInterface { get { if (_uinterface == null) { throw new UhtIceException("UInterface was not defined."); } return _uinterface; } } /// /// After headers are parsed, returns the IInterface class. /// public UhtClass IInterface { get { if (_iinterface == null) { throw new UhtIceException("IInterface was not defined."); } return _iinterface; } } /// /// After headers are parsed, returns the AActor class. Unlike such properties as "UObject", there /// is no requirement for AActor to be defined. May be null. /// public UhtClass? AActor { get; set; } = null; /// /// After headers are parsed, return the INotifyFieldValueChanged interface. There is no requirement /// that this interface be defined. /// public UhtClass? INotifyFieldValueChanged { get; set; } = null; /// /// After headers are parsed, returns the FInstancedStruct script struct. Unlike such properties as /// "UObject", there is no requirement for FInstancedStruct to be defined. May be null. /// public UhtScriptStruct? FInstancedStruct { get; set; } = null; /// /// After headers are parsed, returns the FStateTreePropertyRef script struct. /// There is no requirement for FStateTreePropertyRef to be defined. May be null. /// public UhtScriptStruct? FStateTreePropertyRef { get; set; } = null; /// /// After headers are parsed, returns the EVerseNativeCallResult enumeration. /// There is no requirement for EVerseNativeCallResult to be defined. May be null. /// public UhtEnum? EVerseNativeCallResult { get; set; } = null; private readonly List _modules = new(); private readonly List _headerFiles = new(); private readonly List _sortedHeaderFiles = new(); private readonly Dictionary _headerFileDictionary = new(StringComparer.OrdinalIgnoreCase); private readonly Lazy _engineVersion; private long _errorCount = 0; private long _warningCount = 0; private readonly List _messages = new(); private Task? _messageTask = null; private UhtClass? _uobject = null; private UhtClass? _uclass = null; private UhtClass? _uinterface = null; private UhtClass? _iinterface = null; private readonly TypeCounter _typeCounter = new(); private readonly TypeCounter _packageTypeCount = new(); private readonly TypeCounter _headerFileTypeCount = new(); private readonly TypeCounter _objectTypeCount = new(); private UhtSymbolTable _sourceNameSymbolTable = new(0); private UhtSymbolTable _engineNameSymbolTable = new(0); private bool _symbolTablePopulated = false; private Task? _referenceDeleteTask = null; private readonly Dictionary _exporterStates = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary _fullEnumValueLookup = new(); private readonly Dictionary _shortEnumValueLookup = new(); private JsonDocument? _projectJson = null; /// /// The number of errors /// public long ErrorCount => Interlocked.Read(ref _errorCount); /// /// The number of warnings /// public long WarningCount => Interlocked.Read(ref _warningCount); /// /// True if any errors have occurred or warnings if warnings are to be treated as errors /// public bool HasErrors => ErrorCount > 0 || (WarningsAsErrors && WarningCount > 0); #region IUHTMessageSession implementation /// public IUhtMessageSession MessageSession => this; /// public IUhtMessageSource? MessageSource => null; /// public IUhtMessageLineNumber? MessageLineNumber => null; #endregion /// /// Constructor /// /// Logger for the session public UhtSession(ILogger logger) { Logger = logger; _engineVersion = new Lazy(() => LoadEngineVersion(), true); } /// /// Return the index for a newly defined type /// /// New index public int GetNextTypeIndex() { return _typeCounter.GetNext(); } /// /// Return the number of types that have been defined. This includes all types. /// public int TypeCount => _typeCounter.Count; /// /// Return the index for a newly defined packaging /// /// New index public int GetNextHeaderFileTypeIndex() { return _headerFileTypeCount.GetNext(); } /// /// Return the number of headers that have been defined /// public int HeaderFileTypeCount => _headerFileTypeCount.Count; /// /// Return the index for a newly defined package /// /// New index public int GetNextPackageTypeIndex() { return _packageTypeCount.GetNext(); } /// /// Return the number of UPackage types that have been defined /// public int PackageTypeCount => _packageTypeCount.Count; /// /// Return the index for a newly defined object /// /// New index public int GetNextObjectTypeIndex() { return _objectTypeCount.GetNext(); } /// /// Return the total number of UObject types that have been defined /// public int ObjectTypeCount => _objectTypeCount.Count; /// /// Return the collection of exporters /// public UhtExporterTable ExporterTable => Tables!.ExporterTable; /// /// Return the keyword table for the given table name /// /// Name of the table /// The requested table public UhtKeywordTable GetKeywordTable(string tableName) { return Tables!.KeywordTables.Get(tableName); } /// /// Return the specifier table for the given table name /// /// Name of the table /// The requested table public UhtSpecifierTable GetSpecifierTable(string tableName) { return Tables!.SpecifierTables.Get(tableName); } /// /// Return the specifier validator table for the given table name /// /// Name of the table /// The requested table public UhtSpecifierValidatorTable GetSpecifierValidatorTable(string tableName) { return Tables!.SpecifierValidatorTables.Get(tableName); } /// /// Generate an error for the given unhandled keyword /// /// Token reader /// Unhandled token public void LogUnhandledKeywordError(IUhtTokenReader tokenReader, UhtToken token) { Tables!.KeywordTables.LogUnhandledError(tokenReader, token); } /// /// Test to see if the given class name is a property /// /// Name of the class without the prefix /// True if the class name is a property. False if the class name isn't a property or isn't an engine class. public bool IsValidPropertyTypeName(StringView name) { return Tables!.EngineClassTable.IsValidPropertyTypeName(name); } /// /// Return the loc text default value associated with the given name /// /// /// Loc text default value handler /// public bool TryGetLocTextDefaultValue(StringView name, out UhtLocTextDefaultValue locTextDefaultValue) { return Tables!.LocTextDefaultValueTable.TryGet(name, out locTextDefaultValue); } /// /// Return the default processor /// public UhtPropertyType DefaultPropertyType => Tables!.PropertyTypeTable.Default; /// /// Return the property type associated with the given name /// /// /// Property type if matched /// public bool TryGetPropertyType(StringView name, out UhtPropertyType propertyType) { return Tables!.PropertyTypeTable.TryGet(name, out propertyType); } /// /// Fetch the default sanitizer /// public UhtStructDefaultValue DefaultStructDefaultValue => Tables!.StructDefaultValueTable.Default; /// /// Return the structure default value associated with the given name /// /// /// Structure default value handler /// public bool TryGetStructDefaultValue(StringView name, out UhtStructDefaultValue structDefaultValue) { return Tables!.StructDefaultValueTable.TryGet(name, out structDefaultValue); } /// /// Run UHT on the given manifest. Use the bHasError property to see if process was successful. /// /// Path to the manifest file /// Any command line arguments public void Run(string manifestFilePath, CommandLineArguments? arguments = null) { if (FileManager == null) { SignalError(); Logger.LogError("No file manager supplied, aborting."); return; } if (Config == null) { SignalError(); Logger.LogError("No configuration supplied, aborting."); return; } if (Tables == null) { SignalError(); Logger.LogError("No parsing tables supplied, aborting."); return; } switch (ReferenceMode) { case UhtReferenceMode.None: break; case UhtReferenceMode.Reference: if (String.IsNullOrEmpty(ReferenceDirectory)) { Logger.LogError("WRITEREF requested but directory not set, ignoring"); ReferenceMode = UhtReferenceMode.None; } break; case UhtReferenceMode.Verify: if (String.IsNullOrEmpty(ReferenceDirectory) || String.IsNullOrEmpty(VerifyDirectory)) { Logger.LogError("VERIFYREF requested but directories not set, ignoring"); ReferenceMode = UhtReferenceMode.None; } break; } { string defaultPolicyName = Config.DefaultDocumentationPolicy; if (!String.IsNullOrEmpty(defaultPolicyName)) { if (!Config.DocumentationPolicies.TryGetValue(defaultPolicyName, out UhtDocumentationPolicy? policy)) { Logger.LogError("The default documentation policy '{DefaultPolicyName}' isn't known", defaultPolicyName); return; } } } if (ReferenceMode != UhtReferenceMode.None) { string directoryToDelete = ReferenceMode == UhtReferenceMode.Reference ? ReferenceDirectory : VerifyDirectory; _referenceDeleteTask = Task.Factory.StartNew(() => { try { Directory.Delete(directoryToDelete, true); } catch (Exception) { } }, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default); } StepReadManifestFile(manifestFilePath); if (arguments != null) { foreach (UhtExporter Exporter in ExporterTable) { if (arguments.HasOption($"-{Exporter.Name}")) { SetExporterStatus(Exporter.Name, true); } else if (arguments.HasOption($"-no{Exporter.Name}")) { SetExporterStatus(Exporter.Name, false); } } } StepPrepareModules(); StepPrepareHeaders(); StepParseHeaders(); StepPopulateTypeTable(); StepResolveInvalidCheck(); StepBindSuperAndBases(); RecursiveStructCheck(); StepResolveBases(); StepResolveProperties(); StepResolveFinal(); StepResolveValidate(); StepCollectReferences(); TopologicalSortHeaderFiles(); // If we are deleting the reference directory, then wait for that task to complete if (_referenceDeleteTask != null) { Logger.LogTrace("Step - Waiting for reference output to be cleared."); _referenceDeleteTask.Wait(); } Logger.LogTrace("Step - Starting exporters."); StepExport(); } /// /// Try the given action. If an exception occurs that doesn't have the required /// context, use the supplied context to generate the message. If a previous error /// has occured, the action will not be executed /// /// Message context for when the exception doesn't contain a context. /// The lambda to be invoked public void Try(IUhtMessageSource? messageSource, Action action) { if (!HasErrors) { TryNoErrorCheck(messageSource, action); } } /// /// Try the given action. If an exception occurs that doesn't have the required /// context, use the supplied context to generate the message. /// /// Message context for when the exception doesn't contain a context. /// The lambda to be invoked public void TryNoErrorCheck(IUhtMessageSource? messageSource, Action action) { try { action(); } catch (Exception e) { HandleException(messageSource, e); } } /// /// Execute the given action on all the headers /// /// Action to be executed /// If true, go wide if enabled. Otherwise single threaded public void ForEachHeader(Action action, bool allowGoWide) { if (!HasErrors) { ForEachHeaderNoErrorCheck(action, allowGoWide); } } /// /// Execute the given action on all the headers /// /// Action to be executed /// If true, go wide if enabled. Otherwise single threaded public void ForEachHeaderNoErrorCheck(Action action, bool allowGoWide) { if (GoWide && allowGoWide) { Parallel.ForEach(_headerFiles, headerFile => { TryHeader(action, headerFile); }); } else { foreach (UhtHeaderFile headerFile in _headerFiles) { TryHeader(action, headerFile); } } } /// /// Read the given source file /// /// Full or relative file path /// Information about the read source public UhtSourceFragment ReadSource(string filePath) { if (FileManager!.ReadSource(filePath, out UhtSourceFragment fragment)) { return fragment; } throw new UhtException($"File not found '{filePath}'"); } /// /// Read the given source file /// /// Full or relative file path /// Buffer containing the read data or null if not found. The returned buffer must be returned to the cache via a call to UhtBuffer.Return [Obsolete("Use the ReadSource method with the UhtPoolBuffer output parameter")] public UhtBuffer? ReadSourceToBuffer(string filePath) { return FileManager!.ReadOutput(filePath); } /// /// Read the given source file /// /// Full or relative file path /// Read source file /// True if the file was read public bool ReadSource(string filePath, out UhtPoolBuffer output) { return FileManager!.ReadOutput(filePath, out output); } /// /// Write the given contents to the file /// /// Path to write to /// Contents to write /// True if the source was written internal bool WriteSource(string filePath, ReadOnlySpan contents) { return FileManager!.WriteOutput(filePath, contents); } /// /// Rename the given file /// /// Old file path name /// New file path name public void RenameSource(string oldFilePath, string newFilePath) { if (!FileManager!.RenameOutput(oldFilePath, newFilePath)) { new UhtSimpleFileMessageSite(this, newFilePath).LogError($"Failed to rename exported file: '{oldFilePath}'"); } } /// /// Given the name of a regular enum value, return the enum type /// /// Enum value /// Associated regular enum type or null if not found or enum isn't a regular enum. public UhtEnum? FindRegularEnumValue(string name) { //COMPATIBILITY-TODO - See comment below on a more rebust version of the enum lookup //if (RegularEnumValueLookup.TryGetValue(name, out UhtEnum? enumObj)) //{ // return enumObj; //} if (_fullEnumValueLookup.TryGetValue(name, out EnumAndValue value)) { if (value.Value != -1) { return value.Enum; } } if (!name.Contains("::", StringComparison.Ordinal) && _shortEnumValueLookup.TryGetValue(name, out UhtEnum? enumObj)) { return enumObj; } return null; } /// /// Find the given type in the type hierarchy /// /// Starting point for searches /// Options controlling what is searched /// Name of the type. /// If supplied, then a error message will be generated if the type can not be found /// Source code line number requesting the lookup. /// The located type of null if not found public UhtType? FindType(UhtType? startingType, UhtFindOptions options, string name, IUhtMessageSite? messageSite = null, int lineNumber = -1) { ValidateFindOptions(options); UhtType? type = FindTypeInternal(startingType, options, name); if (type == null && messageSite != null) { FindTypeError(messageSite, lineNumber, options, name); } return type; } /// /// Find the given type in the type hierarchy /// /// Starting point for searches /// Options controlling what is searched /// Name of the type. /// If supplied, then a error message will be generated if the type can not be found /// The located type of null if not found public UhtType? FindType(UhtType? startingType, UhtFindOptions options, ref UhtToken name, IUhtMessageSite? messageSite = null) { ValidateFindOptions(options); UhtType? type = FindTypeInternal(startingType, options, name.Value.ToString()); if (type == null && messageSite != null) { FindTypeError(messageSite, name.InputLine, options, name.Value.ToString()); } return type; } /// /// Find the given type in the type hierarchy /// /// If specified, this represents the starting type to use when searching base/owner chain for a match /// Options controlling what is searched /// Enumeration of identifiers. /// If supplied, then a error message will be generated if the type can not be found /// Source code line number requesting the lookup. /// The located type of null if not found public UhtType? FindType(UhtType? startingType, UhtFindOptions options, UhtTokenList identifiers, IUhtMessageSite? messageSite = null, int lineNumber = -1) { ValidateFindOptions(options); if (identifiers.Next != null && identifiers.Next.Next != null) { if (messageSite != null) { //messageSite.LogError(lineNumber, "UnrealHeaderTool only supports C++ identifiers of two or less identifiers"); string fullIdentifier = identifiers.Join("::"); FindTypeError(messageSite, lineNumber, options, fullIdentifier); return null; } } UhtType? type; if (identifiers.Next != null) { type = FindTypeTwoNamesInternal(startingType, options, identifiers.Token.Value.ToString(), identifiers.Next.Token.Value.ToString()); } else { type = FindTypeInternal(startingType, options, identifiers.Token.Value.ToString()); } if (type == null && messageSite != null) { string fullIdentifier = identifiers.Join("::"); FindTypeError(messageSite, lineNumber, options, fullIdentifier); } return type; } /// /// Find the given type in the type hierarchy /// /// If specified, this represents the starting type to use when searching base/owner chain for a match /// Options controlling what is searched /// Enumeration of identifiers. /// If supplied, then a error message will be generated if the type can not be found /// Source code line number requesting the lookup. /// The located type of null if not found public UhtType? FindType(UhtType? startingType, UhtFindOptions options, UhtToken[] identifiers, IUhtMessageSite? messageSite = null, int lineNumber = -1) { ValidateFindOptions(options); if (identifiers.Length == 0) { throw new UhtIceException("Empty identifier array"); } if (identifiers.Length > 2) { if (messageSite != null) { //messageSite.LogError(lineNumber, "UnrealHeaderTool only supports C++ identifiers of two or less identifiers"); string fullIdentifier = String.Join("::", identifiers); FindTypeError(messageSite, lineNumber, options, fullIdentifier); return null; } } UhtType? type; if (identifiers.Length == 0) { type = FindTypeTwoNamesInternal(startingType, options, identifiers[0].Value.ToString(), identifiers[1].Value.ToString()); } else { type = FindTypeInternal(startingType, options, identifiers[0].Value.ToString()); } if (type == null && messageSite != null) { string fullIdentifier = String.Join("::", identifiers); FindTypeError(messageSite, lineNumber, options, fullIdentifier); } return type; } /// /// Find the given type in the type hierarchy /// /// Starting point for searches /// Options controlling what is searched /// First name of the type. /// Second name used by delegates in classes and namespace enumerations /// The located type of null if not found private UhtType? FindTypeTwoNamesInternal(UhtType? startingType, UhtFindOptions options, string firstName, string secondName) { // If we have two names if (secondName.Length > 0) { if (options.HasAnyFlags(UhtFindOptions.DelegateFunction | UhtFindOptions.Enum)) { UhtFindOptions subOptions = UhtFindOptions.NoParents | (options & ~UhtFindOptions.TypesMask) | (options & UhtFindOptions.Enum); if (options.HasAnyFlags(UhtFindOptions.DelegateFunction)) { subOptions |= UhtFindOptions.Class; } UhtType? type = FindTypeInternal(startingType, subOptions, firstName); if (type == null) { return null; } if (type is UhtEnum) { return type; } if (type is UhtClass) { //TODO - Old UHT compatibility. In UWidget, it references USlateAccessibleWidgetData::FGetText. However, since UWidget has a FGetText, that // is returned first. UhtType? compatType = FindTypeInternal(startingType, UhtFindOptions.DelegateFunction | UhtFindOptions.NoParents | (options & ~UhtFindOptions.TypesMask), secondName); if (compatType != null) { return compatType; } return FindTypeInternal(type, UhtFindOptions.DelegateFunction | UhtFindOptions.NoParents | (options & ~UhtFindOptions.TypesMask), secondName); } } // We can't match anything at this point return null; } // Perform the lookup for just a single name return FindTypeInternal(startingType, options, firstName); } /// /// Find the given type in the type hierarchy /// /// Starting point for searches /// Options controlling what is searched /// Name of the type. /// The located type of null if not found public UhtType? FindTypeInternal(UhtType? startingType, UhtFindOptions options, string name) { if (options.HasAnyFlags(UhtFindOptions.EngineName)) { if (options.HasAnyFlags(UhtFindOptions.CaseCompare)) { return _engineNameSymbolTable.FindCasedType(startingType, options, name); } else { return _engineNameSymbolTable.FindCaselessType(startingType, options, name); } } else if (options.HasAnyFlags(UhtFindOptions.SourceName)) { if (options.HasAnyFlags(UhtFindOptions.CaselessCompare)) { return _sourceNameSymbolTable.FindCaselessType(startingType, options, name); } else { return _sourceNameSymbolTable.FindCasedType(startingType, options, name); } } else { throw new UhtIceException("Either EngineName or SourceName must be specified in the options"); } } /// /// Verify that the options are valid. Will also check to make sure the symbol table has been populated. /// /// Find options private void ValidateFindOptions(UhtFindOptions options) { if (!options.HasAnyFlags(UhtFindOptions.EngineName | UhtFindOptions.SourceName)) { throw new UhtIceException("Either EngineName or SourceName must be specified in the options"); } if (options.HasAnyFlags(UhtFindOptions.CaseCompare) && options.HasAnyFlags(UhtFindOptions.CaselessCompare)) { throw new UhtIceException("Both CaseCompare and CaselessCompare can't be specified as FindType options"); } UhtFindOptions typeOptions = options & UhtFindOptions.TypesMask; if (typeOptions == 0) { throw new UhtIceException("No type options specified"); } if (!_symbolTablePopulated) { throw new UhtIceException("Symbol table has not been populated, don't call FindType until headers are parsed."); } } /// /// Generate an error message for when a given symbol wasn't found. The text will contain the list of types that the symbol must be /// /// Destination for the message /// Line number generating the error /// Collection of required types /// The name of the symbol private static void FindTypeError(IUhtMessageSite messageSite, int lineNumber, UhtFindOptions options, string name) { List types = new(); if (options.HasAnyFlags(UhtFindOptions.Enum)) { types.Add("'enum'"); } if (options.HasAnyFlags(UhtFindOptions.ScriptStruct)) { types.Add("'struct'"); } if (options.HasAnyFlags(UhtFindOptions.Class)) { types.Add("'class'"); } if (options.HasAnyFlags(UhtFindOptions.DelegateFunction)) { types.Add("'delegate'"); } if (options.HasAnyFlags(UhtFindOptions.Function)) { types.Add("'function'"); } if (options.HasAnyFlags(UhtFindOptions.Property)) { types.Add("'property'"); } messageSite.LogError(lineNumber, $"Unable to find {UhtUtilities.MergeTypeNames(types, "or")} with name '{name}'"); } /// /// Search for the given header file by just the file name /// /// Name to be found /// public UhtHeaderFile? FindHeaderFile(string name) { if (_headerFileDictionary.TryGetValue(name, out UhtHeaderFile? headerFile)) { return headerFile; } return null; } #region IUHTMessageSource implementation /// /// Add a message to the collection of output messages /// /// Message being added public void AddMessage(UhtMessage message) { if (message.MessageType == UhtMessageType.Deprecation && !Config!.ShowDeprecations) { return; } lock (_messages) { _messages.Add(message); // If we aren't caching messages and this is the first message, // start a task to flush the messages. if (!CacheMessages && _messageTask == null) { _messageTask = Task.Factory.StartNew(() => FlushMessages(), CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default); } } switch (message.MessageType) { case UhtMessageType.Error: case UhtMessageType.Ice: SignalError(); break; case UhtMessageType.Warning: Interlocked.Increment(ref _warningCount); break; case UhtMessageType.Info: case UhtMessageType.Trace: case UhtMessageType.Deprecation: break; } } #endregion /// /// Register than an error has happened /// public void SignalError() { Interlocked.Increment(ref _errorCount); } /// /// Log all the collected messages to the log/console. If messages aren't being /// cached, then this just waits until the flush task has completed. If messages /// are being cached, they are sorted by file name and line number to ensure the /// output is stable. /// public void LogMessages() { Task? messageTask = null; lock (_messages) { messageTask = _messageTask; } messageTask?.Wait(); foreach (UhtMessage message in FetchOrderedMessages()) { LogMessage(message); } } /// /// Flush all pending messages to the logger /// private void FlushMessages() { while (true) { UhtMessage[]? messageArray = null; lock (_messages) { messageArray = _messages.ToArray(); _messages.Clear(); if (messageArray.Length == 0) { _messageTask = null; return; } } foreach (UhtMessage message in messageArray) { LogMessage(message); } } } /// /// Log the given message /// /// The message to be logged private void LogMessage(UhtMessage msg) { LogLevel logLevel; switch (msg.MessageType) { default: case UhtMessageType.Error: case UhtMessageType.Ice: logLevel = LogLevel.Error; break; case UhtMessageType.Warning: logLevel = LogLevel.Warning; break; case UhtMessageType.Deprecation: logLevel = LogLevel.Information; break; case UhtMessageType.Info: logLevel = LogLevel.Information; break; case UhtMessageType.Trace: logLevel = LogLevel.Trace; break; } GetMessageParts(msg, out string filePath, out int lineNumber, out string fragmentPath, out string severity, out string message); (Logger??Log.Logger).Log(logLevel, KnownLogEvents.UHT, "{File}({Line}){FileFragment}: {Severity}: {Message}", new FileReference(filePath), new LogValue(LogValueType.LineNumber, lineNumber.ToString()), fragmentPath, new LogValue(LogValueType.Severity, severity), new LogValue(LogValueType.Message, message)); } /// /// Return all of the messages into a list /// /// List of all the messages public List CollectMessages() { List messages = new(); foreach (UhtMessage msg in FetchOrderedMessages()) { GetMessageParts(msg, out string filePath, out int lineNumber, out string fragmentPath, out string severity, out string message); messages.Add($"{filePath}({lineNumber}){fragmentPath}: {severity}: {message}"); } return messages; } /// /// Given an existing and a new instance, replace the given type in the symbol table. /// This is used by the property resolution system to replace properties created during /// the parsing phase that couldn't be resoled until after all headers are parsed. /// /// /// public void ReplaceTypeInSymbolTable(UhtType oldType, UhtType newType) { _sourceNameSymbolTable.Replace(oldType, newType, oldType.SourceName); if (oldType.EngineType.HasEngineName()) { _engineNameSymbolTable.Replace(oldType, newType, oldType.EngineName); } } /// /// Hide the given type in the symbol table /// /// public void HideTypeInSymbolTable(UhtType typeToHide) { _sourceNameSymbolTable.Hide(typeToHide, typeToHide.SourceName); if (typeToHide.EngineType.HasEngineName()) { _engineNameSymbolTable.Hide(typeToHide, typeToHide.EngineName); } } /// /// Return an ordered enumeration of all messages. /// /// Enumerator private IOrderedEnumerable FetchOrderedMessages() { List messages = new(); lock (_messages) { messages.AddRange(_messages); _messages.Clear(); } return messages.OrderBy(context => context.FilePath).ThenBy(context => context.LineNumber + context.MessageSource?.MessageFragmentLineNumber); } /// /// Extract the part of a message /// /// Message being consider /// Full path of file where error happened /// Line number where error happened /// For multi-part files (test harness), the inner file /// Severity of the error /// Text of the message private void GetMessageParts(UhtMessage msg, out string filePath, out int lineNumber, out string fragmentPath, out string severity, out string message) { fragmentPath = ""; lineNumber = msg.LineNumber; if (msg.FilePath != null) { filePath = msg.FilePath; } else if (msg.MessageSource != null) { if (msg.MessageSource.MessageIsFragment) { if (RelativePathInLog) { filePath = msg.MessageSource.MessageFragmentFilePath; } else { filePath = msg.MessageSource.MessageFragmentFullFilePath; } fragmentPath = $"[{msg.MessageSource.MessageFilePath}]"; lineNumber += msg.MessageSource.MessageFragmentLineNumber; } else { if (RelativePathInLog) { filePath = msg.MessageSource.MessageFilePath; } else { filePath = msg.MessageSource.MessageFullFilePath; } } } else { filePath = "UnknownSource"; } switch (msg.MessageType) { case UhtMessageType.Error: severity = "Error"; break; case UhtMessageType.Warning: severity = "Warning"; break; case UhtMessageType.Info: severity = "Info"; break; case UhtMessageType.Trace: severity = "Trace"; break; case UhtMessageType.Deprecation: severity = "Deprecation"; break; default: case UhtMessageType.Ice: severity = "Internal Compiler Error"; break; } message = msg.Message; } /// /// Handle the given exception with the provided message context /// /// Context for the exception. Required to handled all exceptions other than UHTException /// Exception being handled private void HandleException(IUhtMessageSource? messageSource, Exception e) { switch (e) { case UhtException uhtException: UhtMessage message = uhtException.UhtMessage; message.MessageSource ??= messageSource; AddMessage(message); break; case JsonException jsonException: AddMessage(UhtMessage.MakeMessage(UhtMessageType.Error, messageSource, null, (int)(jsonException.LineNumber + 1 ?? 1), jsonException.Message)); break; default: //Log.TraceInformation("{0}", E.StackTrace); AddMessage(UhtMessage.MakeMessage(UhtMessageType.Ice, messageSource, null, 1, $"{e.GetType()} - {e.Message}")); break; } } [GeneratedRegex(@"^\#define\s+ENGINE_(?[A-Z]+)_VERSION\s+(?\d+)\s*$")] private static partial Regex GetEngineVersionFragmentRegex(); private EngineVersion LoadEngineVersion() { if (EngineDirectory == null) { throw new UhtException("Can't parse engine version without an engine directory"); } string versionHeader = Path.Combine(EngineDirectory, "Source", "Runtime", "Launch", "Resources", "Version.h"); Log.Logger.LogDebug("Reading engine version from file: {VersionHeader}", versionHeader); string[] lines; try { lines = File.ReadAllLines(versionHeader); } catch (Exception ex) { throw new UhtException(String.Format("Error reading engine version header '{0}': {1}", versionHeader, ex)); } EngineVersion loadedVersion = new EngineVersion(); bool gotMajor = false, gotMinor = false, gotPatch = false; Regex versionRegex = GetEngineVersionFragmentRegex(); foreach (string line in lines) { Match match = versionRegex.Match(line); if (match.Success) { int matchNumber = Int32.Parse(match.Groups["Number"].Value); switch (match.Groups["Name"].Value) { case "MAJOR": loadedVersion.MajorVersion = matchNumber; gotMajor = true; break; case "MINOR": loadedVersion.MinorVersion = matchNumber; gotMinor = true; break; case "PATCH": loadedVersion.PatchVersion = matchNumber; gotPatch = true; break; } if (gotMajor && gotMinor && gotPatch) { break; } } } if (!gotMajor || !gotMinor || !gotPatch) { throw new UhtException(String.Format("Got incomplete engine version {0}", loadedVersion)); } Log.Logger.LogDebug("Got engine version {EngineVersion}", loadedVersion); return loadedVersion; } #region Run steps private void StepReadManifestFile(string manifestFilePath) { ManifestFile = new UhtManifestFile(this, manifestFilePath); Try(ManifestFile.MessageSource, () => { Log.Logger.LogTrace("Step - Read Manifest File"); ManifestFile.Read(); if (Manifest != null && Tables != null) { Tables.AddPlugins(Manifest.UhtPlugins); } }); } private void StepPrepareModules() { if (ManifestFile == null || ManifestFile.Manifest == null) { return; } Try(ManifestFile.MessageSource, () => { Log.Logger.LogTrace("Step - Prepare Modules"); foreach (UHTManifest.Module manifestModule in ManifestFile.Manifest.Modules) { UhtModule module = new(this, manifestModule); _modules.Add(module); } }); } private void StepPrepareHeaders() { if (ManifestFile == null) { return; } Try(ManifestFile.MessageSource, () => { Log.Logger.LogTrace("Step - Prepare Headers"); foreach (UhtModule module in Modules) { module.PrepareHeaders(headerFile => { _headerFiles.Add(headerFile); _headerFileDictionary.Add(Path.GetFileName(headerFile.FilePath), headerFile); }); } // Locate the NoExportTypes.h file and add it to every other header file if (_headerFileDictionary.TryGetValue("NoExportTypes.h", out UhtHeaderFile? noExportTypes)) { foreach (UhtHeaderFile headerFile in _headerFiles) { if (headerFile != noExportTypes) { headerFile.AddReferencedHeader(noExportTypes, UhtHeaderReferenceType.Passive); } } } }); } private void StepParseHeaders() { if (HasErrors) { return; } Log.Logger.LogTrace("Step - Parse Headers"); ForEachHeader(headerFile => { headerFile.Read(); UhtHeaderFileParser.Parse(headerFile); }, true); } private void StepPopulateTypeTable() { Try(null, () => { Log.Logger.LogTrace("Step - Populate symbol table"); _sourceNameSymbolTable = new UhtSymbolTable(TypeCount); _engineNameSymbolTable = new UhtSymbolTable(TypeCount); PopulateSymbolTable(); _uobject = (UhtClass?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Class, "UObject"); _uclass = (UhtClass?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Class, "UClass"); _uinterface = (UhtClass?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Class, "UInterface"); _iinterface = (UhtClass?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Class, "IInterface"); EVerseNativeCallResult = (UhtEnum?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Enum, "EVerseNativeCallResult"); AActor = (UhtClass?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Class, "AActor"); INotifyFieldValueChanged = (UhtClass?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.Class, "INotifyFieldValueChanged"); FInstancedStruct = (UhtScriptStruct?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.ScriptStruct, "FInstancedStruct"); FStateTreePropertyRef = (UhtScriptStruct?)FindType(null, UhtFindOptions.SourceName | UhtFindOptions.ScriptStruct, "FStateTreePropertyRef"); }); } private void StepBindSuperAndBases() { Log.Logger.LogTrace("Step - Bind super and bases"); ForEachHeader(headerFile => headerFile.BindSuperAndBases(), true); } private void StepResolveBases() { Log.Logger.LogTrace("Step - Resolve bases"); ResolveAllHeaders(UhtResolvePhase.Bases); } private void StepResolveInvalidCheck() { Log.Logger.LogTrace("Step - Resolve invalid check"); ResolveAllHeaders(UhtResolvePhase.InvalidCheck); } private void StepResolveProperties() { Log.Logger.LogTrace("Step - Resolve properties"); ResolveAllHeaders(UhtResolvePhase.Properties); } private void StepResolveFinal() { Log.Logger.LogTrace("Step - Resolve final"); ResolveAllHeaders(UhtResolvePhase.Final); } private void StepResolveValidate() { Log.Logger.LogTrace("Step - Resolve validate"); ForEachHeader(headerFile => headerFile.Validate(UhtValidationOptions.None), true); } private void StepCollectReferences() { Log.Logger.LogTrace("Step - Collect references"); ForEachHeader(headerFile => headerFile.CollectReferences(), true); } private void ResolveAllHeaders(UhtResolvePhase resolvePhase) { ForEachHeader(headerFile => headerFile.Resolve(resolvePhase), resolvePhase.IsMultiThreadedResolvePhase()); if (resolvePhase == UhtResolvePhase.InvalidCheck) { foreach (UhtModule module in Modules) { foreach (UhtPackage package in module.Packages) { package.RemoveInvalidChildren(); } } } } private void TryHeader(Action action, UhtHeaderFile headerFile) { try { action(headerFile); } catch (Exception e) { HandleException(headerFile.MessageSource, e); } } #endregion #region Symbol table initialization private void PopulateSymbolTable() { foreach (UhtHeaderFile headerFile in _headerFiles) { foreach (UhtType child in headerFile.Children) { if (child.Outer is UhtPackage package) { package.AddChildDirectly(child); } else { throw new UhtIceException($"Type \"{child.SourceName}\" is a root level type but does not have a package outer"); } AddTypeToSymbolTable(child); } } _symbolTablePopulated = true; } private void AddTypeToSymbolTable(UhtType type) { UhtEngineType engineExtendedType = type.EngineType; if (type is UhtEnum enumObj) { //COMPATIBILITY-TODO: We can get more reliable results by only adding regular enums to the table // and then in the lookup code in the property system to look for the '::' and just lookup // the raw enum name. In UHT we only care about the enum and not the value. // // The current algorithm has issues with two cases: // // EnumNamespaceName::EnumTypeName::Value - Where the enum type name is included with a namespace enum // EnumName::Value - Where the value is defined in terms that can't be parsed. The -1 check causes it // to be kicked out. //if (Enum.CppForm == UhtEnumCppForm.Regular) //{ // foreach (UhtEnumValue Value in Enum.EnumValues) // { // RegularEnumValueLookup.Add(Value.Name.ToString(), Enum); // } //} bool addShortNames = enumObj.CppForm == UhtEnumCppForm.Namespaced || enumObj.CppForm == UhtEnumCppForm.EnumClass; string checkName = $"{enumObj.SourceName}::"; foreach (UhtEnumValue value in enumObj.EnumValues) { if (!_fullEnumValueLookup.TryAdd(value.Name, new EnumAndValue { Enum = enumObj, Value = value.Value })) { //TODO - add a warning } if (addShortNames) { if (value.Name.StartsWith(checkName, StringComparison.Ordinal)) { _shortEnumValueLookup.TryAdd(value.Name[checkName.Length..], enumObj); } } } } if (engineExtendedType.FindOptions() != 0) { if (engineExtendedType.MustNotBeReserved()) { if (s_reservedNames.Contains(type.EngineName)) { type.HeaderFile.LogError(type.LineNumber, $"{engineExtendedType.CapitalizedText()} '{type.EngineName}' uses a reserved type name."); } } if (engineExtendedType.HasEngineName() && engineExtendedType.MustBeUnique()) { UhtType? existingType = _engineNameSymbolTable.FindCaselessType(null, engineExtendedType.MustBeUniqueFindOptions(), type.EngineName); if (existingType != null) { type.HeaderFile.LogError(type.LineNumber, $"{engineExtendedType.CapitalizedText()} '{type.SourceName}' shares engine name '{type.EngineName}' with " + $"{existingType.EngineType.LowercaseText()} '{existingType.SourceName}' in {existingType.HeaderFile.FilePath}({existingType.LineNumber})"); } } _sourceNameSymbolTable.Add(type, type.SourceName); if (engineExtendedType.HasEngineName()) { _engineNameSymbolTable.Add(type, type.EngineName); } } if (engineExtendedType.AddChildrenToSymbolTable()) { foreach (UhtType child in type.Children) { AddTypeToSymbolTable(child); } } } #endregion #region Topological testing of headers and structs private enum TopologicalState { Unmarked, Temporary, Permanent, } private void TopologicalHeaderVisit(List states, UhtHeaderFile visit, List headerStack) { headerStack.Add(visit); switch (states[visit.HeaderFileTypeIndex]) { case TopologicalState.Unmarked: states[visit.HeaderFileTypeIndex] = TopologicalState.Temporary; foreach (UhtHeaderFile referenced in visit.ReferencedHeadersNoLock) { TopologicalHeaderVisit(states, referenced, headerStack); } states[visit.HeaderFileTypeIndex] = TopologicalState.Permanent; _sortedHeaderFiles.Add(visit); break; case TopologicalState.Temporary: { int index = headerStack.IndexOf(visit); if (index == -1 || index == headerStack.Count - 1) { throw new UhtIceException("Error locating include file loop"); } index++; visit.LogError("Circular dependency detected:"); UhtHeaderFile previous = visit; for (int loopIndex = index; loopIndex < headerStack.Count; loopIndex++) { UhtHeaderFile next = headerStack[loopIndex]; previous.LogError($"includes/requires '{next.FilePath}'"); previous = next; } } break; case TopologicalState.Permanent: break; default: throw new UhtIceException("Unknown topological state"); } headerStack.RemoveAt(headerStack.Count - 1); } private void TopologicalSortHeaderFiles() { Try(null, () => { Log.Logger.LogTrace("Step - Topological Sort Header Files"); // Initialize a scratch table for topological states _sortedHeaderFiles.Capacity = HeaderFileTypeCount; List states = new(HeaderFileTypeCount); for (int index = 0; index < HeaderFileTypeCount; ++index) { states.Add(TopologicalState.Unmarked); } List headerStack = new(32); // arbitrary capacity foreach (UhtHeaderFile headerFile in HeaderFiles) { if (states[headerFile.HeaderFileTypeIndex] == TopologicalState.Unmarked) { TopologicalHeaderVisit(states, headerFile, headerStack); } } }); } private void TopologicalStructVisitChildren(List states, UhtHeaderFile visit, List structStack) { foreach (UhtType child in visit.Children) { if (child is UhtStruct childStruct) { TopologicalStructVisit(states, childStruct, structStack); } } } private void TopologicalStructVisitChildren(List states, UhtType visit, List structStack) { foreach (UhtType child in visit.Children) { if (child is UhtStruct childStruct) { TopologicalStructVisit(states, childStruct, structStack); } } } private void TopologicalStructVisit(List states, UhtStruct visit, List structStack) { structStack.Add(visit); switch (states[visit.ObjectTypeIndex]) { case TopologicalState.Unmarked: states[visit.ObjectTypeIndex] = TopologicalState.Temporary; if (visit.Super != null) { TopologicalStructVisit(states, visit.Super, structStack); } foreach (UhtStruct baseStruct in visit.Bases) { TopologicalStructVisit(states, baseStruct, structStack); } if (visit is UhtClass classObj) { foreach (UhtStruct verseInterface in classObj.VerseInterfaces) { TopologicalStructVisit(states, verseInterface, structStack); } } TopologicalStructVisitChildren(states, visit, structStack); states[visit.ObjectTypeIndex] = TopologicalState.Permanent; break; case TopologicalState.Temporary: { int index = structStack.IndexOf(visit); if (index == -1 || index == structStack.Count - 1) { throw new UhtIceException("Error locating struct loop"); } index++; visit.LogError("Recursive class/struct definition:"); UhtStruct previous = visit; for (int loopIndex = index; loopIndex < structStack.Count; loopIndex++) { UhtStruct next = structStack[loopIndex]; previous.LogError($"'{previous.SourceName}' inherits '{next.SourceName}'"); previous = next; } } break; case TopologicalState.Permanent: break; default: throw new UhtIceException("Unknown topological state"); } structStack.RemoveAt(structStack.Count - 1); } private void RecursiveStructCheck() { Try(null, () => { Log.Logger.LogTrace("Step - Check for recursive structs"); // Initialize a scratch table for topological states List states = new(ObjectTypeCount); for (int index = 0; index < ObjectTypeCount; ++index) { states.Add(TopologicalState.Unmarked); } List structStack = new(32); // arbitrary capacity foreach (UhtHeaderFile headerFile in HeaderFiles) { TopologicalStructVisitChildren(states, headerFile, structStack); } }); } #endregion #region Validation helpers private readonly HashSet _scriptStructsValidForNet = new(); /// /// Validate that the given referenced script structure is valid for network operations. If the structure /// is valid, then the result will be cached. It not valid, errors will be generated each time the structure /// is referenced. /// /// The property referencing a structure /// The script structure being referenced /// public bool ValidateScriptStructOkForNet(UhtProperty referencingProperty, UhtScriptStruct referencedScriptStruct) { // Check for existing value lock (_scriptStructsValidForNet) { if (_scriptStructsValidForNet.Contains(referencedScriptStruct)) { return true; } } bool isStructValid = true; // Check the super chain structure UhtScriptStruct? superScriptStruct = referencedScriptStruct.SuperScriptStruct; if (superScriptStruct != null) { if (!ValidateScriptStructOkForNet(referencingProperty, superScriptStruct)) { isStructValid = false; } } // Check the structure properties foreach (UhtProperty property in referencedScriptStruct.Properties) { if (!property.ValidateStructPropertyOkForNet(referencingProperty)) { isStructValid = false; break; } } // Save the results if (isStructValid) { lock (_scriptStructsValidForNet) { _scriptStructsValidForNet.Add(referencedScriptStruct); } } return isStructValid; } #endregion #region Exporting /// /// Enable/Disable an exporter. This overrides the default state of the exporter. /// /// Name of the exporter /// If true, the exporter is to be enabled public void SetExporterStatus(string name, bool enabled) { _exporterStates[name] = enabled; } /// /// Test to see if the given exporter plugin is enabled. /// /// Name of the plugin /// If true, include a target check /// True if enabled public bool IsPluginEnabled(string pluginName, bool includeTargetCheck) { if (_projectJson == null && ProjectDirectory != null && ProjectFile != null) { if (ReadSource(ProjectFile, out UhtPoolBuffer contents)) { _projectJson = JsonDocument.Parse(contents.Memory); UhtPoolBuffers.Return(contents); } } if (_projectJson == null) { return false; } JsonObject rootObject = new(_projectJson.RootElement); if (rootObject.TryGetObjectArrayField("Plugins", out JsonObject[]? plugins)) { foreach (JsonObject plugin in plugins) { if (!plugin.TryGetStringField("Name", out string? testPluginName) || !String.Equals(pluginName, testPluginName, StringComparison.OrdinalIgnoreCase)) { continue; } if (!plugin.TryGetBoolField("Enabled", out bool enabled) || !enabled) { return false; } if (includeTargetCheck && Manifest != null) { if (plugin.TryGetStringArrayField("TargetAllowList", out string[]? allowList)) { if (allowList.Contains(Manifest.TargetName, StringComparer.OrdinalIgnoreCase)) { return true; } } if (plugin.TryGetStringArrayField("TargetDenyList", out string[]? denyList)) { if (denyList.Contains(Manifest.TargetName, StringComparer.OrdinalIgnoreCase)) { return false; } } } return true; } } return false; } private void StepExport() { HashSet externalDependencies = new(); long totalWrittenFiles = 0; Try(null, () => { Logger.LogDebug("Step - Exports"); foreach (UhtExporter exporter in ExporterTable) { if (!_exporterStates.TryGetValue(exporter.Name, out bool run)) { run = Config!.IsExporterEnabled(exporter.Name) || (exporter.Options.HasAnyFlags(UhtExporterOptions.Default) && !NoDefaultExporters); } UHTManifest.Module? pluginModule = null; if (!String.IsNullOrEmpty(exporter.ModuleName)) { foreach (UHTManifest.Module module in Manifest!.Modules) { if (String.Equals(module.Name, exporter.ModuleName, StringComparison.OrdinalIgnoreCase)) { pluginModule = module; break; } } if (pluginModule == null) { Logger.LogWarning("Exporter \"{ExporterName}\" skipped because module \"{ModuleName}\" was not found in manifest", exporter.Name, exporter.ModuleName); continue; } } if (run) { Logger.LogDebug(" Running exporter {ExporterName}", exporter.Name); UhtExportFactory factory = new(this, pluginModule, exporter); factory.Run(); foreach (UhtExportFactory.Output output in factory.Outputs) { if (output.Saved) { totalWrittenFiles++; } } foreach (string dep in factory.ExternalDependencies) { externalDependencies.Add(dep); } } else { Logger.LogDebug(" Exporter {ExporterName} skipped", exporter.Name); } } // Save the collected external dependencies if (!String.IsNullOrEmpty(Manifest!.ExternalDependenciesFile)) { using StreamWriter output = new(Manifest!.ExternalDependenciesFile); foreach (string dep in externalDependencies) { output.WriteLine(dep); } } }); Logger.LogInformation("Total of {NumFiles} written", totalWrittenFiles); } #endregion #region Other helper methods /// /// Test to see if the given property is an incomplete return type /// /// /// public bool IsIncompleteReturn(UhtProperty? property) { return property is UhtEnumProperty enumProperty && enumProperty.Enum == EVerseNativeCallResult; } #endregion } }