// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using EpicGames.Core; using Microsoft.Extensions.Logging; using OpenTracing.Util; using UnrealBuildBase; namespace UnrealBuildTool { /// /// Class which compiles (and caches) rules assemblies for different folders. /// public class RulesCompiler { /// /// /// const string FrameworkAssemblyExtension = ".dll"; /// /// /// const BuildSettingsVersion DefaultEngineBuildSettingsVersion = BuildSettingsVersion.Latest; /// /// Find all the module rules files under a given directory /// /// The directory to search under /// The module context for each found rules instance /// Map of module files to their context private static void AddModuleRulesWithContext(DirectoryReference BaseDirectory, ModuleRulesContext ModuleContext, Dictionary ModuleFileToContext) { IReadOnlyList RulesFiles = Rules.FindAllRulesFiles(BaseDirectory, Rules.RulesFileType.Module); foreach (FileReference RulesFile in RulesFiles) { ModuleFileToContext[RulesFile] = ModuleContext; } } /// /// Find all the module rules files under a given directory /// /// The directory to search under /// Name of the subdirectory to look under /// The module context for each found rules instance /// The UHT module type /// Map of module files to their context private static void AddEngineModuleRulesWithContext(DirectoryReference BaseDirectory, string SubDirectoryName, ModuleRulesContext BaseModuleContext, UHTModuleType? DefaultUHTModuleType, Dictionary ModuleFileToContext) { DirectoryReference Directory = DirectoryReference.Combine(BaseDirectory, SubDirectoryName); if (DirectoryLookupCache.DirectoryExists(Directory)) { ModuleRulesContext ModuleContext = new ModuleRulesContext(BaseModuleContext) { DefaultUHTModuleType = DefaultUHTModuleType, bCanHotReload = true }; AddModuleRulesWithContext(Directory, ModuleContext, ModuleFileToContext); } } /// /// The cached rules assembly for engine modules and targets. /// private static RulesAssembly? EngineRulesAssembly; /// /// Map of assembly names we've already compiled and loaded to their Assembly and list of game folders. This is used to prevent /// trying to recompile the same assembly when ping-ponging between different types of targets /// private static ConcurrentDictionary LoadedAssemblyMap = new ConcurrentDictionary(); /// /// Creates the engine rules assembly /// /// Whether to use a precompiled engine /// Whether to skip compilation for this assembly /// Whether to always compile this assembly /// Logger for output /// New rules assembly public static RulesAssembly CreateEngineRulesAssembly(bool bUsePrecompiled, bool bSkipCompile, bool bForceCompile, ILogger Logger) { // Prevent multiple conflicting processes building rules assembly at the same time string MutexName = GlobalSingleInstanceMutex.GetUniqueMutexForPath("UnrealBuildTool_CreateTargetRulesAssembly", Unreal.EngineDirectory); using (new GlobalSingleInstanceMutex(MutexName, true)) { if (EngineRulesAssembly == null) { List EnginePlugins = new List(); List MarketplacePlugins = new List(); DirectoryReference MarketplaceDirectory = DirectoryReference.Combine(Unreal.EngineDirectory, "Plugins", "Marketplace"); foreach (PluginInfo PluginInfo in Plugins.ReadEnginePlugins(Unreal.EngineDirectory)) { if (PluginInfo.File.IsUnderDirectory(MarketplaceDirectory)) { MarketplacePlugins.Add(PluginInfo); } else { EnginePlugins.Add(PluginInfo); } } EngineRulesAssembly = CreateEngineRulesAssemblyInternal(Unreal.GetExtensionDirs(Unreal.EngineDirectory), ProjectFileGenerator.EngineRulesAssemblyName, EnginePlugins, Unreal.IsEngineInstalled() || bUsePrecompiled, bSkipCompile, bForceCompile, null, Logger); if (MarketplacePlugins.Count > 0) { EngineRulesAssembly = CreateMarketplaceRulesAssembly(MarketplacePlugins, Unreal.IsEngineInstalled() || bUsePrecompiled, bSkipCompile, bForceCompile, EngineRulesAssembly, Logger); } } return EngineRulesAssembly; } } /// /// Creates a rules assembly /// /// The root directories to create rules for /// A prefix for the assembly file name /// List of plugins to include in this assembly /// Whether the assembly should be marked as installed /// Whether to skip compilation for this assembly /// Whether to always compile this assembly /// The parent rules assembly /// Logger for output /// New rules assembly private static RulesAssembly CreateEngineRulesAssemblyInternal(List RootDirectories, string AssemblyPrefix, IReadOnlyList Plugins, bool bReadOnly, bool bSkipCompile, bool bForceCompile, RulesAssembly? Parent, ILogger Logger) { // Scope hierarchy RulesScope Scope = new RulesScope("Engine", null); RulesScope PluginsScope = new RulesScope("Engine Plugins", Scope); RulesScope ProgramsScope = new RulesScope("Engine Programs", PluginsScope); // Find the shared modules, excluding the programs directory. These are used to create an assembly with the bContainsEngineModules flag set to true. Dictionary ModuleFileToContext = new Dictionary(); ModuleRulesContext DefaultModuleContext = new ModuleRulesContext(Scope, RootDirectories[0]); List ProgramTargetFiles = new List(); foreach (DirectoryReference RootDirectory in RootDirectories) { using (GlobalTracer.Instance.BuildSpan("Finding engine modules").StartActive()) { DirectoryReference SourceDirectory = DirectoryReference.Combine(RootDirectory, "Source"); AddEngineModuleRulesWithContext(SourceDirectory, "Runtime", DefaultModuleContext, UHTModuleType.EngineRuntime, ModuleFileToContext); AddEngineModuleRulesWithContext(SourceDirectory, "Developer", DefaultModuleContext, UHTModuleType.EngineDeveloper, ModuleFileToContext); AddEngineModuleRulesWithContext(SourceDirectory, "Editor", DefaultModuleContext, UHTModuleType.EngineEditor, ModuleFileToContext); AddEngineModuleRulesWithContext(SourceDirectory, "ThirdParty", DefaultModuleContext, UHTModuleType.EngineThirdParty, ModuleFileToContext); AddEngineModuleRulesWithContext(RootDirectory, "Shaders", DefaultModuleContext, UHTModuleType.EngineThirdParty, ModuleFileToContext); } } // Add all the plugin modules too (don't need to loop over RootDirectories since the plugins come in already found using (GlobalTracer.Instance.BuildSpan("Finding plugin modules").StartActive()) { ModuleRulesContext PluginsModuleContext = new ModuleRulesContext(PluginsScope, RootDirectories[0]); PluginsModuleContext.bCanHotReload = true; FindModuleRulesForPlugins(Plugins, PluginsModuleContext, ModuleFileToContext); // Plugin test target rules only FindTestRulesForPlugins(Plugins, PluginsModuleContext, ModuleFileToContext, ProgramTargetFiles); } // Create the assembly DirectoryReference AssemblyDir = RootDirectories[0]; FileReference EngineAssemblyFileName = FileReference.Combine(AssemblyDir, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "Rules" + FrameworkAssemblyExtension); RulesAssembly EngineAssembly = new RulesAssembly(Scope, RootDirectories, Plugins, ModuleFileToContext, new List(), EngineAssemblyFileName, bContainsEngineModules: true, DefaultBuildSettings: DefaultEngineBuildSettingsVersion, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, bForceCompile: bForceCompile, Parent: Parent, Logger: Logger); Dictionary ProgramModuleFiles = new Dictionary(); foreach (DirectoryReference RootDirectory in RootDirectories) { DirectoryReference SourceDirectory = DirectoryReference.Combine(RootDirectory, "Source"); DirectoryReference ProgramsDirectory = DirectoryReference.Combine(SourceDirectory, "Programs"); // Also create a scope for them, and update the UHT module type ModuleRulesContext ProgramsModuleContext = new ModuleRulesContext(ProgramsScope, RootDirectory); ProgramsModuleContext.DefaultUHTModuleType = UHTModuleType.Program; using (GlobalTracer.Instance.BuildSpan("Finding program modules").StartActive()) { // Find all the rules files AddModuleRulesWithContext(ProgramsDirectory, ProgramsModuleContext, ProgramModuleFiles); } using (GlobalTracer.Instance.BuildSpan("Finding program targets").StartActive()) { ProgramTargetFiles.AddRange(Rules.FindAllRulesFiles(SourceDirectory, Rules.RulesFileType.Target)); } } // Create a path to the assembly that we'll either load or compile FileReference ProgramAssemblyFileName = FileReference.Combine(AssemblyDir, "Intermediate", "Build", "BuildRules", AssemblyPrefix + "ProgramRules" + FrameworkAssemblyExtension); RulesAssembly ProgramAssembly = new RulesAssembly(ProgramsScope, RootDirectories, new List().AsReadOnly(), ProgramModuleFiles, ProgramTargetFiles, ProgramAssemblyFileName, bContainsEngineModules: false, DefaultBuildSettings: DefaultEngineBuildSettingsVersion, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, bForceCompile: bForceCompile, Parent: EngineAssembly, Logger: Logger); // Return the combined assembly return ProgramAssembly; } /// /// Creates a rules assembly /// /// List of plugins to include in this assembly /// Whether the assembly should be marked as installed /// Whether to skip compilation for this assembly /// Whether to always compile this assembly /// The parent rules assembly /// /// New rules assembly private static RulesAssembly CreateMarketplaceRulesAssembly(IReadOnlyList Plugins, bool bReadOnly, bool bSkipCompile, bool bForceCompile, RulesAssembly Parent, ILogger Logger) { RulesScope MarketplaceScope = new RulesScope("Marketplace", Parent.Scope); // Add all the plugin modules too (don't need to loop over RootDirectories since the plugins come in already found Dictionary ModuleFileToContext = new Dictionary(); using (GlobalTracer.Instance.BuildSpan("Finding marketplace plugin modules").StartActive()) { ModuleRulesContext PluginsModuleContext = new ModuleRulesContext(MarketplaceScope, Unreal.EngineDirectory); FindModuleRulesForPlugins(Plugins, PluginsModuleContext, ModuleFileToContext); } // Create the assembly RulesAssembly Result = Parent; if (ModuleFileToContext.Count > 0) { FileReference AssemblyFileName = FileReference.Combine(Unreal.WritableEngineDirectory, "Intermediate", "Build", "BuildRules", "MarketplaceRules.dll"); Result = new RulesAssembly(MarketplaceScope, new List { DirectoryReference.Combine(Unreal.EngineDirectory, "Plugins", "Marketplace") }, Plugins, ModuleFileToContext, new List(), AssemblyFileName, bContainsEngineModules: true, DefaultBuildSettings: null, bReadOnly: bReadOnly, bSkipCompile: bSkipCompile, bForceCompile: bForceCompile, Parent: Parent, Logger: Logger); } return Result; } /// /// Creates a rules assembly with the given parameters. /// /// The project file to create rules for. Null for the engine. /// Whether to use a precompiled engine /// Whether to skip compilation for this assembly /// Whether to always compile this assembly /// Logger for output /// New rules assembly public static RulesAssembly CreateProjectRulesAssembly(FileReference ProjectFileName, bool bUsePrecompiled, bool bSkipCompile, bool bForceCompile, ILogger Logger) { // Prevent multiple conflicting processes building rules assembly at the same time string MutexName = GlobalSingleInstanceMutex.GetUniqueMutexForPath("UnrealBuildTool_CreateTargetRulesAssembly", ProjectFileName); using (new GlobalSingleInstanceMutex(MutexName, true)) { // Check if there's an existing assembly for this project return LoadedAssemblyMap.GetOrAdd(ProjectFileName, _ => { Logger.LogTrace("Creating project rules assembly for {Project}...", ProjectFileName); ProjectDescriptor Project = ProjectDescriptor.FromFile(ProjectFileName); // Create the parent assembly RulesAssembly Parent = CreateEngineRulesAssembly(bUsePrecompiled, bSkipCompile, bForceCompile, Logger); DirectoryReference MainProjectDirectory = ProjectFileName.Directory; //DirectoryReference MainProjectSourceDirectory = DirectoryReference.Combine(MainProjectDirectory, "Source"); // Create a scope for things in this assembly RulesScope Scope = new RulesScope("Project", Parent.Scope); // Create a new context for modules created by this assembly ModuleRulesContext DefaultModuleContext = new ModuleRulesContext(Scope, MainProjectDirectory); DefaultModuleContext.bCanBuildDebugGame = true; DefaultModuleContext.bCanHotReload = true; DefaultModuleContext.bClassifyAsGameModuleForUHT = true; DefaultModuleContext.bCanUseForSharedPCH = false; // gather modules from project and platforms Dictionary ModuleFiles = new Dictionary(); List TargetFiles = new List(); // Find all the project directories List ProjectDirectories = Unreal.GetExtensionDirs(ProjectFileName.Directory); if (Project.AdditionalRootDirectories != null) { ProjectDirectories.AddRange(Project.AdditionalRootDirectories); } // Find all the rules/plugins under the project source directories foreach (DirectoryReference ProjectDirectory in ProjectDirectories) { DirectoryReference ProjectSourceDirectory = DirectoryReference.Combine(ProjectDirectory, "Source"); AddModuleRulesWithContext(ProjectSourceDirectory, DefaultModuleContext, ModuleFiles); TargetFiles.AddRange(Rules.FindAllRulesFiles(ProjectSourceDirectory, Rules.RulesFileType.Target)); } // Find all the project plugins List ProjectPlugins = new List(); ProjectPlugins.AddRange(Plugins.ReadProjectPlugins(MainProjectDirectory)); // Add the project's additional plugin directories plugins too if (Project.AdditionalPluginDirectories != null) { foreach (DirectoryReference AdditionalPluginDirectory in Project.AdditionalPluginDirectories) { ProjectPlugins.AddRange(Plugins.ReadAdditionalPlugins(AdditionalPluginDirectory, Logger)); } } Logger.LogTrace(" Found {Count} Plugins:", ProjectPlugins.Count); ProjectPlugins.ForEach(x => { Logger.LogTrace(" {Plugin}", x.File); }); // Find all the plugin module rules as well as plugin test target and module rules FindModuleRulesForPlugins(ProjectPlugins, DefaultModuleContext, ModuleFiles); FindTestRulesForPlugins(ProjectPlugins, DefaultModuleContext, ModuleFiles, TargetFiles); Logger.LogTrace(" Found {Count} Modules:", ModuleFiles.Count); foreach (KeyValuePair Item in ModuleFiles) { Logger.LogTrace(" {Module}", Item.Key); } // Add the games project's intermediate source folder DirectoryReference ProjectIntermediateSourceDirectory = DirectoryReference.Combine(MainProjectDirectory, "Intermediate", "Source"); if (DirectoryReference.Exists(ProjectIntermediateSourceDirectory)) { AddModuleRulesWithContext(ProjectIntermediateSourceDirectory, DefaultModuleContext, ModuleFiles); TargetFiles.AddRange(Rules.FindAllRulesFiles(ProjectIntermediateSourceDirectory, Rules.RulesFileType.Target)); } RulesAssembly ProjectRulesAssembly; // Compile the assembly. If there are no module or target files, just use the parent assembly. FileReference AssemblyFileName = FileReference.Combine(MainProjectDirectory, "Intermediate", "Build", "BuildRules", ProjectFileName.GetFileNameWithoutExtension() + "ModuleRules" + FrameworkAssemblyExtension); if (ModuleFiles.Count == 0 && TargetFiles.Count == 0 && ProjectPlugins.Count == 0) { ProjectRulesAssembly = Parent; } else { ProjectRulesAssembly = new RulesAssembly(Scope, new List { MainProjectDirectory }, ProjectPlugins, ModuleFiles, TargetFiles, AssemblyFileName, bContainsEngineModules: false, DefaultBuildSettings: null, bReadOnly: Unreal.IsProjectInstalled(), bSkipCompile: bSkipCompile, bForceCompile: bForceCompile, Parent: Parent, Logger: Logger); } return ProjectRulesAssembly; }); } } /// /// Creates a rules assembly with the given parameters. /// /// The plugin file to create rules for /// Whether to skip compilation for this assembly /// Whether to always compile this assembly /// The parent rules assembly /// Whether the plugin should be built as though it is a local plugin. /// Whether the plugin contains engine modules. Used to initialize the default value for ModuleRules.bTreatAsEngineModule. /// Logger for ouptut /// The new rules assembly public static RulesAssembly CreatePluginRulesAssembly(FileReference PluginFileName, bool bSkipCompile, bool bForceCompile, RulesAssembly Parent, bool bBuildPluginAsLocal, bool bContainsEngineModules, ILogger Logger) { // Prevent multiple conflicting processes building rules assembly at the same time string MutexName = GlobalSingleInstanceMutex.GetUniqueMutexForPath("UnrealBuildTool_CreateTargetRulesAssembly", PluginFileName); using (new GlobalSingleInstanceMutex(MutexName, true)) { // Check if there's an existing assembly for this project return LoadedAssemblyMap.GetOrAdd(PluginFileName, _ => { Logger.LogTrace("Creating plugin rules assembly for {Plugin}...", PluginFileName); // Find all the rules source files Dictionary ModuleFiles = new Dictionary(); List TargetFiles = new List(); // Create a list of plugins for this assembly. We need to override the parent plugin, if it exists, due to overriding the // setting for bClassifyAsGameModuleForUHT below. List ForeignPlugins = new List(); if (!Parent.EnumeratePlugins().Any(x => x.ChoiceVersion != null && x.ChoiceVersion.File == PluginFileName && x.ChoiceVersion.Type == PluginType.Engine)) { PluginInfo ForeignPluginInfo = new PluginInfo(PluginFileName, PluginType.External) { bExplicitPluginTarget = true }; ForeignPlugins.Add(ForeignPluginInfo); } Logger.LogTrace(" Found {Count} ForeignPlugins:", ForeignPlugins.Count); ForeignPlugins.ForEach(x => { Logger.LogTrace(" {ForeignPlugin}", x.File); }); // Create a new scope for the plugin. It should not reference anything else. RulesScope Scope = new RulesScope("Plugin", Parent.Scope); // Find all the modules ModuleRulesContext PluginModuleContext = new ModuleRulesContext(Scope, PluginFileName.Directory); PluginModuleContext.bClassifyAsGameModuleForUHT = !bContainsEngineModules; if (bBuildPluginAsLocal) { PluginModuleContext.bCanBuildDebugGame = true; PluginModuleContext.bCanHotReload = true; PluginModuleContext.bClassifyAsGameModuleForUHT = true; PluginModuleContext.bCanUseForSharedPCH = false; } FindModuleRulesForPlugins(ForeignPlugins, PluginModuleContext, ModuleFiles); Logger.LogTrace(" Found {Count} Modules:", ModuleFiles.Count); foreach (KeyValuePair Item in ModuleFiles) { Logger.LogTrace(" {Module}", Item.Key); } // Compile the assembly FileReference AssemblyFileName = FileReference.Combine(PluginFileName.Directory, "Intermediate", "Build", "BuildRules", Path.GetFileNameWithoutExtension(PluginFileName.FullName) + "ModuleRules" + FrameworkAssemblyExtension); return new RulesAssembly(Scope, new List { PluginFileName.Directory }, ForeignPlugins, ModuleFiles, TargetFiles, AssemblyFileName, bContainsEngineModules, DefaultBuildSettings: null, bReadOnly: false, bSkipCompile: bSkipCompile, bForceCompile: bForceCompile, Parent: Parent, Logger: Logger); }); } } /// /// Compile a rules assembly for the current target /// /// The project file being compiled /// The target being built /// Whether to skip compiling any rules assemblies /// Whether to always compile all rules assemblies /// Whether to use a precompiled engine build /// Foreign plugin to be compiled /// Whether the plugin should be built as though it is a local plugin /// Logger for output /// The compiled rules assembly public static RulesAssembly CreateTargetRulesAssembly(FileReference? ProjectFile, string TargetName, bool bSkipRulesCompile, bool bForceRulesCompile, bool bUsePrecompiled, FileReference? ForeignPlugin, bool bBuildPluginAsLocal, ILogger Logger) { RulesAssembly RulesAssembly; if (ProjectFile != null) { RulesAssembly = CreateProjectRulesAssembly(ProjectFile, bUsePrecompiled, bSkipRulesCompile, bForceRulesCompile, Logger); } else { RulesAssembly = CreateEngineRulesAssembly(bUsePrecompiled, bSkipRulesCompile, bForceRulesCompile, Logger); } if (ForeignPlugin != null) { RulesAssembly = CreatePluginRulesAssembly(ForeignPlugin, bSkipRulesCompile, bForceRulesCompile, RulesAssembly, bBuildPluginAsLocal, true, Logger); } return RulesAssembly; } /// /// Finds all the module rules for plugins under the given directory. /// /// The list of plugins to search modules for /// The default context for any files that are enumerated /// Dictionary which is filled with mappings from the module file to its corresponding context private static void FindModuleRulesForPlugins(IReadOnlyList Plugins, ModuleRulesContext DefaultContext, Dictionary ModuleFileToContext) { Rules.PrefetchRulesFiles(Plugins.Select(x => DirectoryReference.Combine(x.Directory, "Source"))); foreach (PluginInfo Plugin in Plugins) { FindModuleRulesForPluginInFolder(Plugin, "Source", DefaultContext, ModuleFileToContext); } } /// /// Finds all the module and target rules for plugins' tests. /// /// The list of plugins to search test rules for /// The default context for any files that are enumerated /// Dictionary which is filled with mappings from the module file to its corresponding context /// List of target files to add test target rules to private static void FindTestRulesForPlugins(IReadOnlyList Plugins, ModuleRulesContext DefaultContext, Dictionary ModuleFileToContext, List TargetFiles) { Rules.PrefetchRulesFiles(Plugins.Select(x => DirectoryReference.Combine(x.Directory, "Tests"))); foreach (PluginInfo Plugin in Plugins) { FindModuleRulesForPluginInFolder(Plugin, "Tests", DefaultContext, ModuleFileToContext); TargetFiles.AddRange(Rules.FindAllRulesFiles(DirectoryReference.Combine(Plugin.Directory, "Tests"), Rules.RulesFileType.Target)); } } /// /// Finds all the module rules for a given plugin in a specified folder. /// /// The plugin to search module rules for /// The folder relative to the plugin root to look into, usually "Source" or "Tests" /// The default context for any files that are enumerated /// Dictionary which is filled with mappings from the module file to its corresponding context private static void FindModuleRulesForPluginInFolder(PluginInfo Plugin, string Folder, ModuleRulesContext DefaultContext, Dictionary ModuleFileToContext) { List PluginModuleFiles = Rules.FindAllRulesFiles(DirectoryReference.Combine(Plugin.Directory, Folder), Rules.RulesFileType.Module).ToList(); foreach (FileReference ChildFile in Plugin.ChildFiles) { PluginModuleFiles.AddRange(Rules.FindAllRulesFiles(DirectoryReference.Combine(ChildFile.Directory, Folder), Rules.RulesFileType.Module)); } foreach (FileReference ModuleFile in PluginModuleFiles) { ModuleRulesContext PluginContext = new ModuleRulesContext(DefaultContext); PluginContext.DefaultOutputBaseDir = Plugin.Directory; PluginContext.Plugin = Plugin; ModuleFileToContext[ModuleFile] = PluginContext; } } /// /// Gets the filename that declares the given type. /// /// The type to search for. /// The filename that declared the given type, or null public static string? GetFileNameFromType(Type ExistingType) { FileReference? FileName; if (EngineRulesAssembly != null && EngineRulesAssembly.TryGetFileNameFromType(ExistingType, out FileName)) { return FileName.FullName; } foreach (RulesAssembly RulesAssembly in LoadedAssemblyMap.Values) { if (RulesAssembly.TryGetFileNameFromType(ExistingType, out FileName)) { return FileName.FullName; } } return null; } } }