/// <summary>
/// Constructor.
/// </summary>
/// <param name="InAdd"></param>
/// <param name="InFilter"></param>
/// <param name="InBaseDirectory"></param>
public OutputTask(FileFilter InAddFiles, FileFilter InFilterFiles, string InBaseDirectory)
{
AddFiles = InAddFiles;
FilterFiles = InFilterFiles;
BaseDirectory = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, InBaseDirectory);
}
/// <summary>
/// Runs UBT with the specified commandline. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to UBT.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine)
{
if (!FileExists(UBTExecutable))
{
throw new AutomationException("Unable to find UBT executable: " + UBTExecutable);
}
if (GlobalCommandLine.VS2015)
{
CommandLine += " -2015";
}
if (!IsBuildMachine && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Mac)
{
CommandLine += " -nocreatestub";
}
CommandLine += " -NoHotReload";
if (bJunkDeleted || GlobalCommandLine.IgnoreJunk)
{
// UBT has already deleted junk files, make sure it doesn't do it again
CommandLine += " -ignorejunk";
}
else
{
// UBT will delete junk on first run
bJunkDeleted = true;
}
string BaseLogName = String.Format("UBT-{0}", String.Join("-", SharedUtils.ParseCommandLine(CommandLine).Where(x => !x.Contains('/') && !x.Contains('\\') && !x.StartsWith("-"))));
string LogName;
for (int Attempt = 1;; Attempt++)
{
LogName = String.Format("{0}.txt", (Attempt == 1)? BaseLogName : String.Format("{0}_{1}", BaseLogName, Attempt));
FileReference LogLocation = FileReference.Combine(new DirectoryReference(Env.LogFolder), LogName);
if (!FileReference.Exists(LogLocation))
{
CommandLine += String.Format(" -log=\"{0}\"", LogLocation);
break;
}
if (Attempt >= 50)
{
throw new AutomationException("Unable to find name for UBT log file after {0} attempts", Attempt);
}
}
IProcessResult Result = Run(UBTExecutable, CommandLine, Options: ERunOptions.AllowSpew | ERunOptions.NoStdOutCapture);
if (Result.ExitCode != 0)
{
throw new AutomationException((ExitCode)Result.ExitCode, "UnrealBuildTool failed. See log for more details. ({0})", CommandUtils.CombinePaths(Env.FinalLogFolder, LogName));
}
}
public static int Main()
{
var CommandLine = SharedUtils.ParseCommandLine();
LogUtils.InitLogging(CommandLine);
ErrorCodes ReturnCode = ErrorCodes.Error_Success;
try
{
HostPlatform.Initialize();
Log.TraceVerbose("Running on {0} as a {1}-bit process.", HostPlatform.Current.GetType().Name, Environment.Is64BitProcess ? 64 : 32);
XmlConfigLoader.Init();
// Log if we're running from the launcher
var ExecutingAssemblyLocation = CommandUtils.CombinePaths(Assembly.GetExecutingAssembly().Location);
if (String.Compare(ExecutingAssemblyLocation, CommandUtils.CombinePaths(InternalUtils.ExecutingAssemblyLocation), true) != 0)
{
Log.WriteLine(TraceEventType.Information, "Executed from AutomationToolLauncher ({0})", ExecutingAssemblyLocation);
}
Log.WriteLine(TraceEventType.Information, "CWD={0}", Environment.CurrentDirectory);
// Hook up exit callbacks
var Domain = AppDomain.CurrentDomain;
Domain.ProcessExit += Domain_ProcessExit;
Domain.DomainUnload += Domain_ProcessExit;
HostPlatform.Current.SetConsoleCtrlHandler(ProgramCtrlHandler);
var Version = InternalUtils.ExecutableVersion;
Log.WriteLine(TraceEventType.Verbose, "{0} ver. {1}", Version.ProductName, Version.ProductVersion);
// Don't allow simultaneous execution of AT (in the same branch)
InternalUtils.RunSingleInstance(MainProc, CommandLine);
}
catch (Exception Ex)
{
// Catch all exceptions and propagate the ErrorCode if we are given one.
Log.WriteLine(TraceEventType.Error, "AutomationTool terminated with exception:");
Log.WriteLine(TraceEventType.Error, LogUtils.FormatException(Ex));
// set the exit code of the process
if (Ex is AutomationException)
{
ReturnCode = (Ex as AutomationException).ErrorCode;
}
else
{
ReturnCode = ErrorCodes.Error_Unknown;
}
}
finally
{
// In all cases, do necessary shut down stuff, but don't let any additional exceptions leak out while trying to shut down.
// Make sure there's no directories on the stack.
NoThrow(() => CommandUtils.ClearDirStack(), "Clear Dir Stack");
// Try to kill process before app domain exits to leave the other KillAll call to extreme edge cases
NoThrow(() => { if (ShouldKillProcesses && !Utils.IsRunningOnMono)
{
ProcessManager.KillAll();
}
}, "Kill All Processes");
Log.WriteLine(TraceEventType.Information, "AutomationTool exiting with ExitCode={0}", ReturnCode);
// Can't use NoThrow here because the code logs exceptions. We're shutting down logging!
LogUtils.ShutdownLogging();
}
// STOP: No code beyond the return statement should go beyond this point!
// Nothing should happen after the finally block above is finished.
return((int)ReturnCode);
}
private static void FindAndCompileScriptModules(string ScriptsForProjectFileName, List <string> AdditionalScriptsFolders)
{
Log.TraceInformation("Compiling scripts.");
var OldCWD = Environment.CurrentDirectory;
var UnrealBuildToolCWD = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Source");
Environment.CurrentDirectory = UnrealBuildToolCWD;
// Configure the rules compiler
// Get all game folders and convert them to build subfolders.
List <DirectoryReference> AllGameFolders;
if (ScriptsForProjectFileName == null)
{
AllGameFolders = UProjectInfo.FilterGameProjects(true, null).Select(x => x.Folder).ToList();
}
else
{
AllGameFolders = new List <DirectoryReference> {
new DirectoryReference(Path.GetDirectoryName(ScriptsForProjectFileName))
};
}
var AllAdditionalScriptFolders = new List <DirectoryReference>(AdditionalScriptsFolders.Select(x => new DirectoryReference(x)));
foreach (var Folder in AllGameFolders)
{
var GameBuildFolder = DirectoryReference.Combine(Folder, "Build");
if (DirectoryReference.Exists(GameBuildFolder))
{
AllAdditionalScriptFolders.Add(GameBuildFolder);
}
}
Log.TraceVerbose("Discovering game folders.");
var DiscoveredModules = UnrealBuildTool.RulesCompiler.FindAllRulesSourceFiles(UnrealBuildTool.RulesCompiler.RulesFileType.AutomationModule, GameFolders: AllGameFolders, ForeignPlugins: null, AdditionalSearchPaths: AllAdditionalScriptFolders);
var ModulesToCompile = new List <string>(DiscoveredModules.Count);
foreach (var ModuleFilename in DiscoveredModules)
{
if (HostPlatform.Current.IsScriptModuleSupported(ModuleFilename.GetFileNameWithoutAnyExtensions()))
{
ModulesToCompile.Add(ModuleFilename.FullName);
}
else
{
CommandUtils.LogVerbose("Script module {0} filtered by the Host Platform and will not be compiled.", ModuleFilename);
}
}
if ((UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Win64) ||
(UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Win32))
{
string Modules = string.Join(";", ModulesToCompile.ToArray());
var UATProj = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine\Source\Programs\AutomationTool\Scripts\UAT.proj");
var CmdLine = String.Format("\"{0}\" /p:Modules=\"{1}\" /p:Configuration={2} /verbosity:minimal /nologo", UATProj, Modules, BuildConfig);
// suppress the run command because it can be long and intimidating, making the logs around this code harder to read.
var Result = CommandUtils.Run(CommandUtils.CmdEnv.MsBuildExe, CmdLine, Options: CommandUtils.ERunOptions.Default | CommandUtils.ERunOptions.NoLoggingOfRunCommand | CommandUtils.ERunOptions.LoggingOfRunDuration);
if (Result.ExitCode != 0)
{
throw new AutomationException(String.Format("Failed to build \"{0}\":{1}{2}", UATProj, Environment.NewLine, Result.Output));
}
}
else
{
CompileModules(ModulesToCompile);
}
Environment.CurrentDirectory = OldCWD;
}
/// <summary>
/// Detects root paths for the specified client.
/// </summary>
/// <param name="UATLocation">AutomationTool.exe location</param>
/// <param name="ThisClient">Client to detect the root paths for</param>
/// <param name="BuildRootPath">Build root</param>
/// <param name="LocalRootPath">Local root</param>
/// <param name="ClientRootPath">Client root</param>
private static void DetectRootPaths(P4Connection Connection, string LocalRootPath, P4ClientInfo ThisClient, out string BuildRootPath, out string ClientRootPath)
{
if (!String.IsNullOrEmpty(ThisClient.Stream))
{
BuildRootPath = ThisClient.Stream;
ClientRootPath = String.Format("//{0}", ThisClient.Name);
}
else
{
// Figure out the build root
string KnownFilePathFromRoot = CommandEnvironment.KnownFileRelativeToRoot;
string KnownLocalPath = CommandUtils.MakePathSafeToUseWithCommandLine(CommandUtils.CombinePaths(PathSeparator.Slash, LocalRootPath, KnownFilePathFromRoot));
IProcessResult P4Result = Connection.P4(String.Format("files -m 1 {0}", KnownLocalPath), AllowSpew: false);
string KnownFileDepotMapping = P4Result.Output;
// Get the build root
Log.TraceVerbose("Looking for {0} in {1}", KnownFilePathFromRoot, KnownFileDepotMapping);
int EndIdx = KnownFileDepotMapping.IndexOf(KnownFilePathFromRoot, StringComparison.CurrentCultureIgnoreCase);
if (EndIdx < 0)
{
EndIdx = KnownFileDepotMapping.IndexOf(CommandUtils.ConvertSeparators(PathSeparator.Slash, KnownFilePathFromRoot), StringComparison.CurrentCultureIgnoreCase);
}
// Get the root path without the trailing path separator
BuildRootPath = KnownFileDepotMapping.Substring(0, EndIdx - 1);
// Get the client root
if (LocalRootPath.StartsWith(CommandUtils.CombinePaths(PathSeparator.Slash, ThisClient.RootPath, "/"), StringComparison.InvariantCultureIgnoreCase) || LocalRootPath == CommandUtils.CombinePaths(PathSeparator.Slash, ThisClient.RootPath))
{
ClientRootPath = CommandUtils.CombinePaths(PathSeparator.Depot, String.Format("//{0}", ThisClient.Name), LocalRootPath.Substring(ThisClient.RootPath.Length));
}
else
{
throw new AutomationException("LocalRootPath ({0}) does not start with the client root path ({1})", LocalRootPath, ThisClient.RootPath);
}
}
}
/// <summary>
/// Attempts to autodetect project properties.
/// </summary>
/// <param name="RawProjectPath">Full project path.</param>
/// <returns>Project properties.</returns>
private static ProjectProperties DetectProjectProperties(FileReference RawProjectPath, List <UnrealTargetPlatform> ClientTargetPlatforms, bool AssetNativizationRequested)
{
var Properties = new ProjectProperties();
Properties.RawProjectPath = RawProjectPath;
// detect if the project is content only, but has non-default build settings
List <string> ExtraSearchPaths = null;
if (RawProjectPath != null)
{
string TempTargetDir = CommandUtils.CombinePaths(Path.GetDirectoryName(RawProjectPath.FullName), "Intermediate", "Source");
if (RequiresTempTarget(RawProjectPath, ClientTargetPlatforms, AssetNativizationRequested))
{
GenerateTempTarget(RawProjectPath);
Properties.bWasGenerated = true;
ExtraSearchPaths = new List <string>();
ExtraSearchPaths.Add(TempTargetDir);
}
else if (File.Exists(Path.Combine(Path.GetDirectoryName(RawProjectPath.FullName), "Intermediate", "Source", Path.GetFileNameWithoutExtension(RawProjectPath.FullName) + ".Target.cs")))
{
File.Delete(Path.Combine(Path.GetDirectoryName(RawProjectPath.FullName), "Intermediate", "Source", Path.GetFileNameWithoutExtension(RawProjectPath.FullName) + ".Target.cs"));
}
// in case the RulesCompiler (what we use to find all the
// Target.cs files) has already cached the contents of this
// directory, then we need to invalidate that cache (so
// it'll find/use the new Target.cs file)
RulesCompiler.InvalidateRulesFileCache(TempTargetDir);
}
if (CommandUtils.CmdEnv.HasCapabilityToCompile)
{
DetectTargetsForProject(Properties, ExtraSearchPaths);
Properties.bIsCodeBasedProject = !CommandUtils.IsNullOrEmpty(Properties.Targets) || !CommandUtils.IsNullOrEmpty(Properties.Programs);
}
else
{
// should never ask for engine targets if we can't compile
if (RawProjectPath == null)
{
throw new AutomationException("Cannot determine engine targets if we can't compile.");
}
Properties.bIsCodeBasedProject = Properties.bWasGenerated;
// if there's a Source directory with source code in it, then mark us as having source code
string SourceDir = CommandUtils.CombinePaths(Path.GetDirectoryName(RawProjectPath.FullName), "Source");
if (Directory.Exists(SourceDir))
{
string[] CppFiles = Directory.GetFiles(SourceDir, "*.cpp", SearchOption.AllDirectories);
string[] HFiles = Directory.GetFiles(SourceDir, "*.h", SearchOption.AllDirectories);
Properties.bIsCodeBasedProject |= (CppFiles.Length > 0 || HFiles.Length > 0);
}
}
// check to see if the uproject loads modules, only if we haven't already determined it is a code based project
if (!Properties.bIsCodeBasedProject && RawProjectPath != null)
{
string uprojectStr = File.ReadAllText(RawProjectPath.FullName);
Properties.bIsCodeBasedProject = uprojectStr.Contains("\"Modules\"");
}
// Get all ini files
if (RawProjectPath != null)
{
CommandUtils.LogVerbose("Loading ini files for {0}", RawProjectPath);
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
var Config = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, RawProjectPath.Directory, TargetPlatformType);
Properties.EngineConfigs.Add(TargetPlatformType, Config);
}
}
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
var Config = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, RawProjectPath.Directory, TargetPlatformType);
Properties.GameConfigs.Add(TargetPlatformType, Config);
}
}
}
return(Properties);
}
/// <summary>
/// Entry point for the commandlet
/// </summary>
public override void ExecuteBuild()
{
string OutputDir = ParseParamValue("OutputDir");
string ContentOnlyPlatformsString = ParseParamValue("ContentOnlyPlatforms");
IEnumerable <UnrealTargetPlatform> ContentOnlyPlatforms = Enumerable.Empty <UnrealTargetPlatform>();
if (!String.IsNullOrWhiteSpace(ContentOnlyPlatformsString))
{
ContentOnlyPlatforms = ContentOnlyPlatformsString.Split(';').Where(x => !String.IsNullOrWhiteSpace(x)).Select(x => UnrealTargetPlatform.Parse(x));
}
string AnalyticsTypeOverride = ParseParamValue("AnalyticsTypeOverride");
// Write InstalledBuild.txt to indicate Engine is installed
string InstalledBuildFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/InstalledBuild.txt");
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(InstalledBuildFile));
CommandUtils.WriteAllText(InstalledBuildFile, "");
// Write InstalledBuild.txt to indicate Engine is installed
string Project = ParseParamValue("Project");
if (Project != null)
{
string InstalledProjectBuildFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/InstalledProjectBuild.txt");
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(InstalledProjectBuildFile));
CommandUtils.WriteAllText(InstalledProjectBuildFile, new FileReference(Project).MakeRelativeTo(new DirectoryReference(OutputDir)));
}
string OutputEnginePath = Path.Combine(OutputDir, "Engine");
string OutputBaseEnginePath = Path.Combine(OutputEnginePath, "Config", "BaseEngine.ini");
FileAttributes OutputAttributes = FileAttributes.ReadOnly;
List <String> IniLines = new List <String>();
// Should always exist but if not, we don't need extra line
if (File.Exists(OutputBaseEnginePath))
{
OutputAttributes = File.GetAttributes(OutputBaseEnginePath);
IniLines.Add("");
}
else
{
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(OutputBaseEnginePath));
CommandUtils.WriteAllText(OutputBaseEnginePath, "");
OutputAttributes = File.GetAttributes(OutputBaseEnginePath) | OutputAttributes;
}
// Create list of platform configurations installed in a Rocket build
List <InstalledPlatformInfo.InstalledPlatformConfiguration> InstalledConfigs = new List <InstalledPlatformInfo.InstalledPlatformConfiguration>();
// Add the editor platform, otherwise we'll never be able to run UAT
string EditorArchitecture = PlatformExports.GetDefaultArchitecture(HostPlatform.Current.HostEditorPlatform, null);
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(UnrealTargetConfiguration.Development, HostPlatform.Current.HostEditorPlatform, TargetRules.TargetType.Editor, EditorArchitecture, "", EProjectType.Unknown, false));
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(UnrealTargetConfiguration.DebugGame, HostPlatform.Current.HostEditorPlatform, TargetRules.TargetType.Editor, EditorArchitecture, "", EProjectType.Unknown, false));
foreach (UnrealTargetPlatform CodeTargetPlatform in UnrealTargetPlatform.GetValidPlatforms())
{
if (PlatformExports.IsPlatformAvailable(CodeTargetPlatform))
{
string Architecture = PlatformExports.GetDefaultArchitecture(CodeTargetPlatform, null);
// Try to parse additional Architectures from the command line
string Architectures = ParseParamValue(CodeTargetPlatform.ToString() + "Architectures");
string GPUArchitectures = ParseParamValue(CodeTargetPlatform.ToString() + "GPUArchitectures");
// Build a list of pre-compiled architecture combinations for this platform if any
List <string> AllArchNames;
if (!String.IsNullOrWhiteSpace(Architectures) && !String.IsNullOrWhiteSpace(GPUArchitectures))
{
AllArchNames = (from Arch in Architectures.Split('+')
from GPUArch in GPUArchitectures.Split('+')
select "-" + Arch + "-" + GPUArch).ToList();
}
else if (!String.IsNullOrWhiteSpace(Architectures))
{
AllArchNames = Architectures.Split('+').ToList();
}
// if there aren't any, use the default
else
{
AllArchNames = new List <string>()
{
Architecture
};
}
// Check whether this platform should only be used for content based projects
EProjectType ProjectType = ContentOnlyPlatforms.Contains(CodeTargetPlatform) ? EProjectType.Content : EProjectType.Any;
// Allow Content only platforms to be shown as options in all projects
bool bCanBeDisplayed = ProjectType == EProjectType.Content;
foreach (UnrealTargetConfiguration CodeTargetConfiguration in Enum.GetValues(typeof(UnrealTargetConfiguration)))
{
Dictionary <String, TargetType> Targets = new Dictionary <string, TargetType>()
{
{ "UE4Game", TargetType.Game },
{ "UE4Client", TargetType.Client },
{ "UE4Server", TargetType.Server }
};
foreach (KeyValuePair <string, TargetType> Target in Targets)
{
string CurrentTargetName = Target.Key;
TargetType CurrentTargetType = Target.Value;
// Need to check for development receipt as we use that for the Engine code in DebugGame
UnrealTargetConfiguration EngineConfiguration = (CodeTargetConfiguration == UnrealTargetConfiguration.DebugGame) ? UnrealTargetConfiguration.Development : CodeTargetConfiguration;
// Android has multiple architecture flavors built without receipts, so use the default arch target instead
if (CodeTargetPlatform == UnrealTargetPlatform.Android)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(new DirectoryReference(OutputEnginePath), CurrentTargetName, CodeTargetPlatform, EngineConfiguration, Architecture);
if (FileReference.Exists(ReceiptFileName))
{
// Strip the output folder so that this can be used on any machine
string RelativeReceiptFileName = ReceiptFileName.MakeRelativeTo(new DirectoryReference(OutputDir));
// Blindly append all of the architecture names
if (AllArchNames.Count > 0)
{
foreach (string Arch in AllArchNames)
{
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, CurrentTargetType, Arch, RelativeReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
// if for some reason we didn't specify any flavors, just add the default one.
else
{
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, CurrentTargetType, Architecture, RelativeReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
}
// If we're not Android, check the existence of the target receipts for each architecture specified.
else
{
foreach (string Arch in AllArchNames)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(new DirectoryReference(OutputEnginePath), CurrentTargetName, CodeTargetPlatform, EngineConfiguration, Arch);
if (FileReference.Exists(ReceiptFileName))
{
string RelativeReceiptFileName = ReceiptFileName.MakeRelativeTo(new DirectoryReference(OutputDir));
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, CurrentTargetType, Arch, RelativeReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
}
}
}
}
}
UnrealBuildTool.InstalledPlatformInfo.WriteConfigFileEntries(InstalledConfigs, ref IniLines);
if (!String.IsNullOrEmpty(AnalyticsTypeOverride))
{
// Write Custom Analytics type setting
IniLines.Add("");
IniLines.Add("[Analytics]");
IniLines.Add(String.Format("UE4TypeOverride=\"{0}\"", AnalyticsTypeOverride));
}
// Make sure we can write to the the config file
File.SetAttributes(OutputBaseEnginePath, OutputAttributes & ~FileAttributes.ReadOnly);
File.AppendAllLines(OutputBaseEnginePath, IniLines);
File.SetAttributes(OutputBaseEnginePath, OutputAttributes);
}
public static int Main()
{
var CommandLine = SharedUtils.ParseCommandLine();
HostPlatform.Initialize();
LogUtils.InitLogging(CommandLine);
Log.WriteLine(TraceEventType.Information, "Running on {0}", HostPlatform.Current.GetType().Name);
XmlConfigLoader.Init();
// Log if we're running from the launcher
var ExecutingAssemblyLocation = CommandUtils.CombinePaths(Assembly.GetExecutingAssembly().Location);
if (String.Compare(ExecutingAssemblyLocation, CommandUtils.CombinePaths(InternalUtils.ExecutingAssemblyLocation), true) != 0)
{
Log.WriteLine(TraceEventType.Information, "Executed from AutomationToolLauncher ({0})", ExecutingAssemblyLocation);
}
Log.WriteLine(TraceEventType.Information, "CWD={0}", Environment.CurrentDirectory);
// Hook up exit callbacks
var Domain = AppDomain.CurrentDomain;
Domain.ProcessExit += Domain_ProcessExit;
Domain.DomainUnload += Domain_ProcessExit;
HostPlatform.Current.SetConsoleCtrlHandler(ProgramCtrlHandler);
var Version = InternalUtils.ExecutableVersion;
Log.WriteLine(TraceEventType.Verbose, "{0} ver. {1}", Version.ProductName, Version.ProductVersion);
try
{
// Don't allow simultaneous execution of AT (in the same branch)
ReturnCode = InternalUtils.RunSingleInstance(MainProc, CommandLine);
}
catch (Exception Ex)
{
Log.WriteLine(TraceEventType.Error, "AutomationTool terminated with exception:");
Log.WriteLine(TraceEventType.Error, LogUtils.FormatException(Ex));
Log.WriteLine(TraceEventType.Error, Ex.Message);
if (ReturnCode == 0)
{
ReturnCode = (int)ErrorCodes.Error_Unknown;
}
}
// Make sure there's no directiories on the stack.
CommandUtils.ClearDirStack();
Environment.ExitCode = ReturnCode;
// Try to kill process before app domain exits to leave the other KillAll call to extreme edge cases
if (ShouldKillProcesses && !Utils.IsRunningOnMono)
{
ProcessManager.KillAll();
}
Log.WriteLine(TraceEventType.Information, "AutomationTool exiting with ExitCode={0}", ReturnCode);
LogUtils.CloseFileLogging();
return(ReturnCode);
}
/// <summary>
/// Compiles all script modules.
/// </summary>
/// <param name="Modules">Module project filenames.</param>
private static void CompileModules(List <string> Modules)
{
string DependencyFile = Path.Combine(CommandUtils.CmdEnv.EngineSavedFolder, "UATModuleHashes.xml");
if (AreDependenciesUpToDate(Modules, DependencyFile) && !GlobalCommandLine.IgnoreDependencies)
{
Log.TraceInformation("Dependencies are up to date. Skipping compile.");
return;
}
Log.TraceInformation("Dependencies are out of date. Compiling scripts....");
// clean old assemblies
CleanupScriptsAssemblies();
DateTime StartTime = DateTime.Now;
string BuildTool = CommandUtils.CmdEnv.MsBuildExe;
// msbuild (standard on windows, in mono >=5.0 is preferred due to speed and parallel compilation)
bool UseParallelMsBuild = Path.GetFileNameWithoutExtension(BuildTool).ToLower() == "msbuild";
if (UseParallelMsBuild)
{
string ModulesList = string.Join(";", Modules);
// Mono has an issue where arugments with semicolons or commas can't be passed through to
// as arguments so we need to manually construct a temp file with the list of modules
// see (https://github.com/Microsoft/msbuild/issues/471)
var UATProjTemplate = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine\Source\Programs\AutomationTool\Scripts\UAT.proj");
var UATProjFile = Path.Combine(CommandUtils.CmdEnv.EngineSavedFolder, "UATTempProj.proj");
string ProjContents = File.ReadAllText(UATProjTemplate);
ProjContents = ProjContents.Replace("$(Modules)", ModulesList);
Directory.CreateDirectory(Path.GetDirectoryName(UATProjFile));
File.WriteAllText(UATProjFile, ProjContents);
string MsBuildVerbosity = Log.OutputLevel >= LogEventType.Verbose ? "minimal" : "quiet";
var CmdLine = String.Format("\"{0}\" /p:Configuration={1} /verbosity:{2} /nologo", UATProjFile, BuildConfig, MsBuildVerbosity);
// suppress the run command because it can be long and intimidating, making the logs around this code harder to read.
var Result = CommandUtils.Run(BuildTool, CmdLine, Options: CommandUtils.ERunOptions.Default | CommandUtils.ERunOptions.NoLoggingOfRunCommand | CommandUtils.ERunOptions.LoggingOfRunDuration);
if (Result.ExitCode != 0)
{
throw new AutomationException(String.Format("Failed to build \"{0}\":{1}{2}", UATProjFile, Environment.NewLine, Result.Output));
}
}
else
{
// Make sure DefaultScriptsDLLName is compiled first
var DefaultScriptsProjName = Path.ChangeExtension(DefaultScriptsDLLName, "csproj");
// Primary modules must be built first
List <string> PrimaryModules = Modules.Where(M => M.IndexOf(DefaultScriptsProjName, StringComparison.InvariantCultureIgnoreCase) >= 0).ToList();
foreach (var ModuleName in PrimaryModules)
{
Log.TraceInformation("Building script module: {0}", ModuleName);
try
{
CompileScriptModule(ModuleName);
}
catch (Exception Ex)
{
CommandUtils.LogError(LogUtils.FormatException(Ex));
throw new AutomationException("Failed to compile module {0}", ModuleName);
}
break;
}
// Second pass, compile everything else
List <string> SecondaryModules = Modules.Where(M => !PrimaryModules.Contains(M)).ToList();
// Non-parallel method
foreach (var ModuleName in SecondaryModules)
{
Log.TraceInformation("Building script module: {0}", ModuleName);
try
{
CompileScriptModule(ModuleName);
}
catch (Exception Ex)
{
CommandUtils.LogError(LogUtils.FormatException(Ex));
throw new AutomationException("Failed to compile module {0}", ModuleName);
}
}
}
TimeSpan Duration = DateTime.Now - StartTime;
Log.TraceInformation("Compiled {0} modules in {1} secs", Modules.Count, Duration.TotalSeconds);
HashCollection NewHashes = HashModules(Modules);
if (NewHashes == null)
{
Log.TraceWarning("Failed to save dependency info!");
}
else
{
NewHashes.SaveToFile(DependencyFile);
Log.TraceVerbose("Wrote depencencies to {0}", DependencyFile);
}
}
private void WriteJson(List <BuildNode> OrderedToDo, bool bSkipTriggers)
{
using (JsonWriter JsonWriter = new JsonWriter(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "JobSteps.json")))
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteArrayStart("JobSteps");
string CurrentAgentGroup = "";
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
// Write the agent group object
string AgentGroup = GetAgentGroupOrSticky(NodeToDo);
if (AgentGroup != CurrentAgentGroup)
{
if (CurrentAgentGroup != "")
{
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
CurrentAgentGroup = AgentGroup;
if (CurrentAgentGroup != "")
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Group", AgentGroup);
JsonWriter.WriteValue("AgentPlatform", NodeToDo.AgentPlatform.ToString());
JsonWriter.WriteArrayStart("JobSteps");
}
}
// Add this node
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Node", NodeToDo.Name);
if (CurrentAgentGroup == "")
{
JsonWriter.WriteValue("AgentPlatform", NodeToDo.AgentPlatform.ToString());
}
if (!String.IsNullOrEmpty(NodeToDo.AgentRequirements))
{
JsonWriter.WriteValue("AgentRequirements", NodeToDo.AgentRequirements);
}
// Write all the dependencies
StringBuilder DependsOnList = new StringBuilder();
foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo))
{
if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
if (DependsOnList.Length > 0)
{
DependsOnList.Append(";");
}
string DepAgentGroup = GetAgentGroupOrSticky(Dep);
if (DepAgentGroup != "")
{
DependsOnList.AppendFormat("{0}/", DepAgentGroup);
}
DependsOnList.Append(Dep.Name);
}
}
JsonWriter.WriteValue("DependsOn", DependsOnList.ToString());
// Add any trigger-specific settings
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
if (TriggerNodeToDo != null && !TriggerNodeToDo.IsTriggered)
{
JsonWriter.WriteValue("TriggerState", TriggerNodeToDo.StateName);
JsonWriter.WriteValue("TriggerActionText", TriggerNodeToDo.ActionText);
JsonWriter.WriteValue("TriggerDescriptionText", TriggerNodeToDo.DescriptionText);
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
JsonWriter.WriteValue("TriggerEmailRecipients", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
}
}
JsonWriter.WriteObjectEnd();
}
}
if (CurrentAgentGroup != "")
{
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
/// <summary>
/// Execute the node
/// </summary>
/// <param name="BuildProducts">List of build products to run</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(List <string> BuildProducts)
{
if (Language == ScriptTaskLanguage.Batch)
{
// Write the script to a batch file
string FileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Intermediate", "GUBP", "Temp.bat");
CommandUtils.CreateDirectory(Path.GetDirectoryName(FileName));
CommandUtils.WriteAllText(FileName, Text);
// Run it through the command processor
int ExitCode;
CommandUtils.RunAndLog(Path.Combine(Environment.SystemDirectory, "cmd.exe"), "/Q /C " + CommandUtils.MakePathSafeToUseWithCommandLine(FileName), out ExitCode);
return(ExitCode == 0);
}
else if (Language == ScriptTaskLanguage.CSharp)
{
// Build the full C# source file containing the given fragment. We assume it's a function body.
StringBuilder FullScript = new StringBuilder();
FullScript.AppendLine("using System;");
FullScript.AppendLine("using System.IO;");
FullScript.AppendLine("using System.Collections.Generic;");
FullScript.AppendLine("using System.Linq;");
FullScript.AppendLine("using AutomationTool;");
FullScript.AppendLine();
FullScript.AppendLine("public static class TaskHarness");
FullScript.AppendLine("{");
FullScript.AppendLine("\tpublic static void Execute()");
FullScript.AppendLine("\t{");
FullScript.AppendLine(Text);
FullScript.AppendLine("\t}");
FullScript.AppendLine("}");
// Compile it into an in-memory assembly
CompilerParameters Parameters = new CompilerParameters();
Parameters.GenerateExecutable = false;
Parameters.GenerateInMemory = true;
Parameters.ReferencedAssemblies.Add("mscorlib.dll");
Parameters.ReferencedAssemblies.Add("System.dll");
Parameters.ReferencedAssemblies.Add("System.Core.dll");
Parameters.ReferencedAssemblies.Add("System.Data.Linq.dll");
Parameters.ReferencedAssemblies.Add(typeof(CommandUtils).Assembly.Location);
CSharpCodeProvider Compiler = new CSharpCodeProvider();
CompilerResults Results = Compiler.CompileAssemblyFromSource(Parameters, FullScript.ToString());
if (Results.Errors.HasErrors)
{
throw new AutomationException("Failed to compile C# script:\n" + String.Join("\n", Results.Errors.OfType <CompilerError>().Select(x => x.ToString())));
}
// Execute the code
Type CompiledType = Results.CompiledAssembly.GetType("TaskHarness");
try
{
CompiledType.GetMethod("Execute").Invoke(null, new object[0]);
}
catch (Exception Ex)
{
throw new AutomationException(Ex, "Task script threw an exception.\n");
}
return(true);
}
else
{
throw new NotImplementedException(String.Format("Script language {0} has not been implemented", Language));
}
}
/// <summary>
/// Runs UBT with the specified commandline. Automatically creates a logfile. When
/// no LogName is specified, the executable name is used as logfile base name.
/// </summary>
/// <param name="Env">Environment to use.</param>
/// <param name="CommandLine">Commandline to pass on to UBT.</param>
/// <param name="LogName">Optional logfile name.</param>
public static void RunUBT(CommandEnvironment Env, string UBTExecutable, string CommandLine)
{
if (!FileExists(UBTExecutable))
{
throw new AutomationException("Unable to find UBT executable: " + UBTExecutable);
}
string BaseLogName = String.Format("UBT-{0}", String.Join("-", SharedUtils.ParseCommandLine(CommandLine).Where(x => !x.Contains('/') && !x.Contains('\\') && !x.StartsWith("-"))));
string LogName;
for (int Attempt = 1;; Attempt++)
{
LogName = String.Format("{0}.txt", (Attempt == 1)? BaseLogName : String.Format("{0}_{1}", BaseLogName, Attempt));
FileReference LogLocation = FileReference.Combine(new DirectoryReference(Env.LogFolder), LogName);
if (!FileReference.Exists(LogLocation))
{
CommandLine += String.Format(" -log=\"{0}\"", LogLocation);
break;
}
if (Attempt >= 50)
{
throw new AutomationException("Unable to find name for UBT log file after {0} attempts", Attempt);
}
}
IProcessResult Result = Run(UBTExecutable, CommandLine, Options: ERunOptions.AllowSpew | ERunOptions.NoStdOutCapture);
if (Result.ExitCode != 0)
{
throw new AutomationException((ExitCode)Result.ExitCode, "UnrealBuildTool failed. See log for more details. ({0})", CommandUtils.CombinePaths(Env.FinalLogFolder, LogName))
{
OutputFormat = AutomationExceptionOutputFormat.Minimal
};
}
}
private static void FindAndCompileScriptModules(List <string> AdditionalScriptsFolders)
{
var OldCWD = Environment.CurrentDirectory;
var UnrealBuildToolCWD = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Source");
// Convert script folders to be relative to UnrealBuildTool's expected CWD
var RemappedAdditionalScriptFolders = new List <string>();
foreach (var CurFolder in AdditionalScriptsFolders)
{
RemappedAdditionalScriptFolders.Add(UnrealBuildTool.Utils.MakePathRelativeTo(CurFolder, UnrealBuildToolCWD));
}
Environment.CurrentDirectory = UnrealBuildToolCWD;
// Configure the rules compiler
// Get all game folders and convert them to build subfolders.
var AllGameFolders = UnrealBuildTool.UEBuildTarget.DiscoverAllGameFolders();
var BuildFolders = new List <string>(AllGameFolders.Count);
foreach (var Folder in AllGameFolders)
{
var GameBuildFolder = CommandUtils.CombinePaths(Folder, "Build");
if (Directory.Exists(GameBuildFolder))
{
BuildFolders.Add(GameBuildFolder);
}
}
RemappedAdditionalScriptFolders.AddRange(BuildFolders);
Log.TraceVerbose("Discovering game folders.");
UnrealBuildTool.RulesCompiler.SetAssemblyNameAndGameFolders("UnrealAutomationToolRules", AllGameFolders);
var DiscoveredModules = UnrealBuildTool.RulesCompiler.FindAllRulesSourceFiles(UnrealBuildTool.RulesCompiler.RulesFileType.AutomationModule, RemappedAdditionalScriptFolders);
var ModulesToCompile = new List <string>(DiscoveredModules.Count);
foreach (var ModuleFilename in DiscoveredModules)
{
if (HostPlatform.Current.IsScriptModuleSupported(CommandUtils.GetFilenameWithoutAnyExtensions(ModuleFilename)))
{
ModulesToCompile.Add(ModuleFilename);
}
else
{
CommandUtils.LogVerbose("Script module {0} filtered by the Host Platform and will not be compiled.", ModuleFilename);
}
}
if ((UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Win64) ||
(UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealBuildTool.UnrealTargetPlatform.Win32))
{
string Modules = string.Join(";", ModulesToCompile.ToArray());
var UATProj = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine\Source\Programs\AutomationTool\Scripts\UAT.proj");
var CmdLine = String.Format("\"{0}\" /p:Modules=\"{1}\" /p:Configuration={2} /verbosity:minimal", UATProj, Modules, BuildConfig);
Log.TraceInformation("Building Automation projects in parallel...");
// supress the run command because it can be long and intimidating, making the logs around this code harder to read.
var Result = CommandUtils.Run(CommandUtils.CmdEnv.MsBuildExe, CmdLine, Options: CommandUtils.ERunOptions.Default | CommandUtils.ERunOptions.NoLoggingOfRunCommand | CommandUtils.ERunOptions.LoggingOfRunDuration);
if (Result.ExitCode != 0)
{
throw new AutomationException(String.Format("Failed to build \"{0}\":{1}{2}", UATProj, Environment.NewLine, Result.Output));
}
}
else
{
CompileModules(ModulesToCompile);
}
Environment.CurrentDirectory = OldCWD;
}
/// <summary>
/// Entry point for the commandlet
/// </summary>
public override void ExecuteBuild()
{
string OutputDir = ParseParamValue("OutputDir");
string ContentOnlyPlatformsString = ParseParamValue("ContentOnlyPlatforms");
IEnumerable <UnrealTargetPlatform> ContentOnlyPlatforms = Enumerable.Empty <UnrealTargetPlatform>();
if (!String.IsNullOrWhiteSpace(ContentOnlyPlatformsString))
{
ContentOnlyPlatforms = ContentOnlyPlatformsString.Split(';').Where(x => !String.IsNullOrWhiteSpace(x)).Select(x => (UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), x));
}
string AnalyticsTypeOverride = ParseParamValue("AnalyticsTypeOverride");
// Write InstalledBuild.txt to indicate Engine is installed
string InstalledBuildFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/InstalledBuild.txt");
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(InstalledBuildFile));
CommandUtils.WriteAllText(InstalledBuildFile, "");
// Write InstalledBuild.txt to indicate Engine is installed
string Project = ParseParamValue("Project");
if (Project != null)
{
string InstalledProjectBuildFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/InstalledProjectBuild.txt");
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(InstalledProjectBuildFile));
CommandUtils.WriteAllText(InstalledProjectBuildFile, new FileReference(Project).MakeRelativeTo(new DirectoryReference(OutputDir)));
}
string OutputEnginePath = Path.Combine(OutputDir, "Engine");
string OutputBaseEnginePath = Path.Combine(OutputEnginePath, "Config", "BaseEngine.ini");
FileAttributes OutputAttributes = FileAttributes.ReadOnly;
List <String> IniLines = new List <String>();
// Should always exist but if not, we don't need extra line
if (File.Exists(OutputBaseEnginePath))
{
OutputAttributes = File.GetAttributes(OutputBaseEnginePath);
IniLines.Add("");
}
else
{
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(OutputBaseEnginePath));
CommandUtils.WriteAllText(OutputBaseEnginePath, "");
OutputAttributes = File.GetAttributes(OutputBaseEnginePath) | OutputAttributes;
}
// Create list of platform configurations installed in a Rocket build
List <InstalledPlatformInfo.InstalledPlatformConfiguration> InstalledConfigs = new List <InstalledPlatformInfo.InstalledPlatformConfiguration>();
foreach (UnrealTargetPlatform CodeTargetPlatform in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (PlatformExports.IsPlatformAvailable(CodeTargetPlatform))
{
string Architecture = PlatformExports.GetDefaultArchitecture(CodeTargetPlatform, null);
// Try to parse additional Architectures from the command line
string Architectures = ParseParamValue(CodeTargetPlatform.ToString() + "Architectures");
string GPUArchitectures = ParseParamValue(CodeTargetPlatform.ToString() + "GPUArchitectures");
// Build a list of pre-compiled architecture combinations for this platform if any
List <string> AllArchNames;
if (!String.IsNullOrWhiteSpace(Architectures) && !String.IsNullOrWhiteSpace(GPUArchitectures))
{
AllArchNames = (from Arch in Architectures.Split('+')
from GPUArch in GPUArchitectures.Split('+')
select "-" + Arch + "-" + GPUArch).ToList();
}
else if (!String.IsNullOrWhiteSpace(Architectures))
{
AllArchNames = Architectures.Split('+').ToList();
}
else
{
AllArchNames = new List <string>();
}
// Check whether this platform should only be used for content based projects
EProjectType ProjectType = ContentOnlyPlatforms.Contains(CodeTargetPlatform) ? EProjectType.Content : EProjectType.Any;
// Allow Content only platforms to be shown as options in all projects
bool bCanBeDisplayed = ProjectType == EProjectType.Content;
foreach (UnrealTargetConfiguration CodeTargetConfiguration in Enum.GetValues(typeof(UnrealTargetConfiguration)))
{
// Need to check for development receipt as we use that for the Engine code in DebugGame
UnrealTargetConfiguration EngineConfiguration = (CodeTargetConfiguration == UnrealTargetConfiguration.DebugGame) ? UnrealTargetConfiguration.Development : CodeTargetConfiguration;
string ReceiptFileName = TargetReceipt.GetDefaultPath(OutputEnginePath, "UE4Game", CodeTargetPlatform, EngineConfiguration, Architecture);
if (File.Exists(ReceiptFileName))
{
// Strip the output folder so that this can be used on any machine
ReceiptFileName = new FileReference(ReceiptFileName).MakeRelativeTo(new DirectoryReference(OutputDir));
// If we have pre-compiled architectures for this platform then add an entry for each of these -
// there isn't a receipt for each architecture like some other platforms
if (AllArchNames.Count > 0)
{
foreach (string Arch in AllArchNames)
{
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, TargetType.Game, Arch, ReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
else
{
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, TargetType.Game, Architecture, ReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
}
}
}
UnrealBuildTool.InstalledPlatformInfo.WriteConfigFileEntries(InstalledConfigs, ref IniLines);
if (!String.IsNullOrEmpty(AnalyticsTypeOverride))
{
// Write Custom Analytics type setting
IniLines.Add("");
IniLines.Add("[Analytics]");
IniLines.Add(String.Format("UE4TypeOverride=\"{0}\"", AnalyticsTypeOverride));
}
// Make sure we can write to the the config file
File.SetAttributes(OutputBaseEnginePath, OutputAttributes & ~FileAttributes.ReadOnly);
File.AppendAllLines(OutputBaseEnginePath, IniLines);
File.SetAttributes(OutputBaseEnginePath, OutputAttributes);
}
/// <summary>
/// Runs the task, filtering the list of build products
/// </summary>
/// <param name="BuildProducts"></param>
/// <returns>True if the task succeeds</returns>
public override bool Execute(List<string> BuildProducts)
{
if(AddFiles != null)
{
BuildProducts.AddRange(AddFiles.ApplyToDirectory(BaseDirectory, true).Select(x => CommandUtils.CombinePaths(BaseDirectory, x)));
}
if(FilterFiles != null)
{
BuildProducts.RemoveAll(x => UnrealBuildTool.Utils.IsFileUnderDirectory(x, BaseDirectory) && !FilterFiles.Matches(UnrealBuildTool.Utils.StripBaseDirectory(x, BaseDirectory)));
}
return true;
}
/// <summary>
/// Attempts to autodetect project properties.
/// </summary>
/// <param name="RawProjectPath">Full project path.</param>
/// <returns>Project properties.</returns>
private static ProjectProperties DetectProjectProperties(string RawProjectPath)
{
var Properties = new ProjectProperties();
Properties.RawProjectPath = RawProjectPath;
if (CommandUtils.CmdEnv.HasCapabilityToCompile)
{
DetectTargetsForProject(Properties);
Properties.bIsCodeBasedProject = !CommandUtils.IsNullOrEmpty(Properties.Targets) || !CommandUtils.IsNullOrEmpty(Properties.Programs);
}
else
{
// should never ask for engine targets if we can't compile
if (String.IsNullOrEmpty(RawProjectPath))
{
throw new AutomationException("Cannot dtermine engine targets if we can't compile.");
}
Properties.bIsCodeBasedProject = false;
// if there's a Source directory with source code in it, then mark us as having source code
string SourceDir = CommandUtils.CombinePaths(Path.GetDirectoryName(RawProjectPath), "Source");
if (Directory.Exists(SourceDir))
{
string[] CppFiles = Directory.GetFiles(SourceDir, "*.cpp", SearchOption.AllDirectories);
string[] HFiles = Directory.GetFiles(SourceDir, "*.h", SearchOption.AllDirectories);
Properties.bIsCodeBasedProject = CppFiles.Length > 0 || HFiles.Length > 0;
}
}
// check to see if the uproject loads modules, only if we haven't already determined it is a code based project
if (!Properties.bIsCodeBasedProject)
{
string uprojectStr = File.ReadAllText(RawProjectPath);
Properties.bIsCodeBasedProject = uprojectStr.Contains("\"Modules\"");
}
// Get all ini files
if (!String.IsNullOrWhiteSpace(RawProjectPath))
{
CommandUtils.Log("Loading ini files for {0}", RawProjectPath);
var EngineDirectory = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine");
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
var Config = new ConfigCacheIni(TargetPlatformType, "Engine", Path.GetDirectoryName(RawProjectPath), EngineDirectory);
Properties.EngineConfigs.Add(TargetPlatformType, Config);
}
}
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
var Config = new ConfigCacheIni(TargetPlatformType, "Game", Path.GetDirectoryName(RawProjectPath));
Properties.GameConfigs.Add(TargetPlatformType, Config);
}
}
}
return(Properties);
}
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail)
{
List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates);
Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables);
Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes);
if (OrdereredToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
List <string> ECProps = new List <string>();
ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum)));
}
foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value));
}
foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions)
{
ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name))));
}
foreach (AggregateNode Node in SeparatePromotables)
{
ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, ""));
}
List <string> ECJobProps = new List <string>();
if (ExplicitTrigger != null)
{
ECJobProps.Add("IsRoot=0");
}
else
{
ECJobProps.Add("IsRoot=1");
}
// here we are just making sure everything before the explicit trigger is completed.
if (ExplicitTrigger != null)
{
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
}
}
}
BuildNode LastSticky = null;
bool HitNonSticky = false;
bool bHaveECNodes = false;
List <string> StepList = new List <string>();
StepList.Add("use strict;");
StepList.Add("use diagnostics;");
StepList.Add("use ElectricCommander();");
StepList.Add("my $ec = new ElectricCommander;");
StepList.Add("$ec->setTimeout(600);");
StepList.Add("my $batch = $ec->newBatch(\"serial\");");
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (NodeToDo.IsSticky)
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput"))
{
string ParentPath = Command.ParseParamValue("ParentPath");
string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);
bool bHasNoop = false;
if (LastSticky == null && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
StepList.Add(Args);
bHasNoop = true;
}
Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >();
List <BuildNode> StickyChain = new List <BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = NodeToDo.AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> {
NodeToDo
});
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
if (NodeToDo.IsSticky)
{
if (!StickyChain.Contains(NodeToDo))
{
StickyChain.Add(NodeToDo);
}
}
}
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
List <string> NodeProps = GetECPropsForNode(NodeToDo);
ECProps.AddRange(NodeProps);
bool Sticky = NodeToDo.IsSticky;
if (NodeToDo.IsSticky)
{
if (NodeToDo.AgentSharingGroup != "")
{
throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
}
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
}
if (NodeToDo.AgentRequirements != "")
{
throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
}
}
string ProcedureInfix = "";
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
}
bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >();
Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name));
Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0"));
if (NodeToDo.AgentSharingGroup != "")
{
Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup));
}
string Procedure;
if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered)
{
if (NodeToDo.IsParallelAgentShareEditor)
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
}
else
{
Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
if (!NodeToDo.IsSticky)
{
Procedure += "_Parallel";
}
if (NodeToDo.AgentSharingGroup != "")
{
Procedure += "_AgentShare";
}
}
if (NodeToDo.IsSticky && NodeToDo == LastSticky)
{
Procedure += "_Release";
}
}
else
{
if (TriggerNodeToDo.RequiresRecursiveWorkflow)
{
Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
}
else
{
Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
}
Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName));
Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText));
Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText));
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)));
}
}
string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2)));
string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs);
List <BuildNode> UncompletedEcDeps = new List <BuildNode>();
{
foreach (BuildNode Dep in NodeToDo.AllDirectDependencies)
{
if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
UncompletedEcDeps.Add(Dep);
}
}
}
string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);
string MyAgentGroup = NodeToDo.AgentSharingGroup;
bool bDoNestedJobstep = false;
bool bDoFirstNestedJobstep = false;
string NodeParentPath = ParentPath;
if (MyAgentGroup != "")
{
bDoNestedJobstep = true;
NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]";
List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex <= 0)
{
bDoFirstNestedJobstep = bDoNestedJobstep;
}
}
if (bDoNestedJobstep)
{
if (bDoFirstNestedJobstep)
{
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
ParentPath, MyAgentGroup);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix,
String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
MyAgentGroup, NodeToDo.Name);
{
NestArgs = NestArgs + ", precondition => ";
NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
NestArgs = NestArgs + ") true;]\"";
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
PreCondition = "\"\\$\" . \"[/javascript if(";
PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
PreCondition = PreCondition + ") true;]\"";
}
}
Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
}
if (!String.IsNullOrEmpty(PreCondition))
{
Args = Args + ", precondition => " + PreCondition;
}
if (!String.IsNullOrEmpty(RunCondition))
{
Args = Args + ", condition => " + RunCondition;
}
#if false
// this doesn't work because it includes precondition time
if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
{
Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
}
#endif
if (Sticky && NodeToDo == LastSticky)
{
Args = Args + ", releaseMode => 'release'";
}
Args = Args + "});";
StepList.Add(Args);
if (MyAgentGroup != "" && !bDoNestedJobstep)
{
List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex == MyChain.Count - 1)
{
string RelPreCondition = "\"\\$\" . \"[/javascript if(";
// this runs "parallel", but we a precondition to serialize it
RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'";
RelPreCondition = RelPreCondition + ") true;]\"";
// we need to release the resource
string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
StepList.Add(RelArgs);
}
}
}
}
WriteECPerl(StepList);
}
bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest());
RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
{
ECProps.Add("GUBP_LoadedProps=1");
string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
}
{
ECProps.Add("GUBP_LoadedJobProps=1");
string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
}
}
/// <summary>
/// Detects the current code changelist the workspace is synced to.
/// </summary>
/// <param name="ClientRootPath">Workspace path.</param>
/// <returns>Changelist number as a string.</returns>
private static string DetectCurrentCodeCL(P4Connection Connection, string ClientRootPath)
{
CommandUtils.LogVerbose("uebp_CodeCL not set, detecting last code CL...");
// Retrieve the current changelist
string P4Cmd = String.Format("changes -m 1 \"{0}/....cpp#have\" \"{0}/....h#have\" \"{0}/....inl#have\" \"{0}/....cs#have\" \"{0}/....usf#have\"", CommandUtils.CombinePaths(PathSeparator.Depot, ClientRootPath));
IProcessResult P4Result = Connection.P4(P4Cmd, AllowSpew: false);
// Loop through all the lines of the output. Even though we requested one result, we'll get one for each search pattern.
int CL = 0;
foreach (string Line in P4Result.Output.Split('\n'))
{
string[] Tokens = Line.Trim().Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (Tokens.Length >= 2)
{
int LineCL = Int32.Parse(Tokens[1]);
CL = Math.Max(CL, LineCL);
}
}
return(CL.ToString());
}
/// <summary>
/// Finds all targets for the project.
/// </summary>
/// <param name="Properties">Project properties.</param>
/// <param name="ExtraSearchPaths">Additional search paths.</param>
private static void DetectTargetsForProject(ProjectProperties Properties, List <string> ExtraSearchPaths = null)
{
Properties.Targets = new Dictionary <TargetType, SingleTargetProperties>();
FileReference TargetsDllFilename;
string FullProjectPath = null;
var GameFolders = new List <DirectoryReference>();
var RulesFolder = new DirectoryReference(GetRulesAssemblyFolder());
if (Properties.RawProjectPath != null)
{
CommandUtils.LogVerbose("Looking for targets for project {0}", Properties.RawProjectPath);
TargetsDllFilename = FileReference.Combine(RulesFolder, String.Format("UATRules{0}.dll", Properties.RawProjectPath.GetHashCode()));
FullProjectPath = CommandUtils.GetDirectoryName(Properties.RawProjectPath.FullName);
GameFolders.Add(new DirectoryReference(FullProjectPath));
CommandUtils.LogVerbose("Searching for target rule files in {0}", FullProjectPath);
}
else
{
TargetsDllFilename = FileReference.Combine(RulesFolder, String.Format("UATRules{0}.dll", "_BaseEngine_"));
}
// the UBT code assumes a certain CWD, but artists don't have this CWD.
var SourceDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Source");
bool DirPushed = false;
if (CommandUtils.DirectoryExists_NoExceptions(SourceDir))
{
CommandUtils.PushDir(SourceDir);
DirPushed = true;
}
var ExtraSearchDirectories = (ExtraSearchPaths == null)? null : ExtraSearchPaths.Select(x => new DirectoryReference(x)).ToList();
var TargetScripts = RulesCompiler.FindAllRulesSourceFiles(RulesCompiler.RulesFileType.Target, GameFolders: GameFolders, ForeignPlugins: null, AdditionalSearchPaths: ExtraSearchDirectories, bIncludeEnterprise: false);
if (DirPushed)
{
CommandUtils.PopDir();
}
if (!CommandUtils.IsNullOrEmpty(TargetScripts))
{
// We only care about project target script so filter out any scripts not in the project folder, or take them all if we are just doing engine stuff
var ProjectTargetScripts = new List <FileReference>();
foreach (var TargetScript in TargetScripts)
{
if (FullProjectPath == null || TargetScript.IsUnderDirectory(new DirectoryReference(FullProjectPath)))
{
ProjectTargetScripts.Add(TargetScript);
}
}
TargetScripts = ProjectTargetScripts;
}
if (!CommandUtils.IsNullOrEmpty(TargetScripts))
{
CommandUtils.LogVerbose("Found {0} target rule files:", TargetScripts.Count);
foreach (var Filename in TargetScripts)
{
CommandUtils.LogVerbose(" {0}", Filename);
}
// Check if the scripts require compilation
bool DoNotCompile = false;
if (!CommandUtils.IsBuildMachine && !CheckIfScriptAssemblyIsOutOfDate(TargetsDllFilename, TargetScripts))
{
Log.TraceVerbose("Targets DLL {0} is up to date.", TargetsDllFilename);
DoNotCompile = true;
}
if (!DoNotCompile && CommandUtils.FileExists_NoExceptions(TargetsDllFilename.FullName))
{
if (!CommandUtils.DeleteFile_NoExceptions(TargetsDllFilename.FullName, true))
{
DoNotCompile = true;
CommandUtils.LogVerbose("Could not delete {0} assuming it is up to date and reusable for a recursive UAT call.", TargetsDllFilename);
}
}
CompileAndLoadTargetsAssembly(Properties, TargetsDllFilename, DoNotCompile, TargetScripts);
}
}
/// <summary>
/// Attempts to autodetect project properties.
/// </summary>
/// <param name="RawProjectPath">Full project path.</param>
/// <returns>Project properties.</returns>
private static ProjectProperties DetectProjectProperties(string RawProjectPath, List <UnrealTargetPlatform> ClientTargetPlatforms)
{
var Properties = new ProjectProperties();
Properties.RawProjectPath = RawProjectPath;
// detect if the project is content only, but has non-default build settings
List <string> ExtraSearchPaths = null;
if (!string.IsNullOrEmpty(RawProjectPath))
{
if (RequiresTempTarget(RawProjectPath, ClientTargetPlatforms))
{
GenerateTempTarget(RawProjectPath);
Properties.bWasGenerated = true;
ExtraSearchPaths = new List <string>();
ExtraSearchPaths.Add(CommandUtils.CombinePaths(Path.GetDirectoryName(RawProjectPath), "Intermediate", "Source"));
}
else if (File.Exists(Path.Combine(Path.GetDirectoryName(RawProjectPath), "Intermediate", "Source", Path.GetFileNameWithoutExtension(RawProjectPath) + ".Target.cs")))
{
File.Delete(Path.Combine(Path.GetDirectoryName(RawProjectPath), "Intermediate", "Source", Path.GetFileNameWithoutExtension(RawProjectPath) + ".Target.cs"));
}
}
if (CommandUtils.CmdEnv.HasCapabilityToCompile)
{
DetectTargetsForProject(Properties, ExtraSearchPaths);
Properties.bIsCodeBasedProject = !CommandUtils.IsNullOrEmpty(Properties.Targets) || !CommandUtils.IsNullOrEmpty(Properties.Programs);
}
else
{
// should never ask for engine targets if we can't compile
if (String.IsNullOrEmpty(RawProjectPath))
{
throw new AutomationException("Cannot dtermine engine targets if we can't compile.");
}
Properties.bIsCodeBasedProject = Properties.bWasGenerated;
// if there's a Source directory with source code in it, then mark us as having source code
string SourceDir = CommandUtils.CombinePaths(Path.GetDirectoryName(RawProjectPath), "Source");
if (Directory.Exists(SourceDir))
{
string[] CppFiles = Directory.GetFiles(SourceDir, "*.cpp", SearchOption.AllDirectories);
string[] HFiles = Directory.GetFiles(SourceDir, "*.h", SearchOption.AllDirectories);
Properties.bIsCodeBasedProject |= (CppFiles.Length > 0 || HFiles.Length > 0);
}
}
// check to see if the uproject loads modules, only if we haven't already determined it is a code based project
if (!Properties.bIsCodeBasedProject)
{
string uprojectStr = File.ReadAllText(RawProjectPath);
Properties.bIsCodeBasedProject = uprojectStr.Contains("\"Modules\"");
}
// Get all ini files
if (!String.IsNullOrWhiteSpace(RawProjectPath))
{
CommandUtils.Log("Loading ini files for {0}", RawProjectPath);
var EngineDirectory = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine");
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
var Config = new ConfigCacheIni(TargetPlatformType, "Engine", Path.GetDirectoryName(RawProjectPath), EngineDirectory);
Properties.EngineConfigs.Add(TargetPlatformType, Config);
}
}
foreach (UnrealTargetPlatform TargetPlatformType in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (TargetPlatformType != UnrealTargetPlatform.Unknown)
{
var Config = new ConfigCacheIni(TargetPlatformType, "Game", Path.GetDirectoryName(RawProjectPath));
Properties.GameConfigs.Add(TargetPlatformType, Config);
}
}
}
return(Properties);
}
public override string GetUE4ExePath(string UE4Exe)
{
return(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, RelativeBinariesFolder, UE4Exe));
}
/// <summary>
/// Finds all targets for the project.
/// </summary>
/// <param name="Properties">Project properties.</param>
/// <param name="ExtraSearchPaths">Additional search paths.</param>
private static void DetectTargetsForProject(ProjectProperties Properties, List <string> ExtraSearchPaths = null)
{
Properties.Targets = new Dictionary <TargetRules.TargetType, SingleTargetProperties>();
string TargetsDllFilename;
string FullProjectPath = null;
var GameFolders = new List <string>();
var RulesFolder = GetRulesAssemblyFolder();
if (!String.IsNullOrEmpty(Properties.RawProjectPath))
{
CommandUtils.Log("Looking for targets for project {0}", Properties.RawProjectPath);
TargetsDllFilename = CommandUtils.CombinePaths(RulesFolder, String.Format("UATRules{0}.dll", Properties.RawProjectPath.GetHashCode()));
FullProjectPath = CommandUtils.GetDirectoryName(Properties.RawProjectPath);
GameFolders.Add(FullProjectPath);
CommandUtils.Log("Searching for target rule files in {0}", FullProjectPath);
}
else
{
TargetsDllFilename = CommandUtils.CombinePaths(RulesFolder, String.Format("UATRules{0}.dll", "_BaseEngine_"));
}
RulesCompiler.SetAssemblyNameAndGameFolders(TargetsDllFilename, GameFolders);
// the UBT code assumes a certain CWD, but artists don't have this CWD.
var SourceDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Source");
bool DirPushed = false;
if (CommandUtils.DirectoryExists_NoExceptions(SourceDir))
{
CommandUtils.PushDir(SourceDir);
DirPushed = true;
}
var TargetScripts = RulesCompiler.FindAllRulesSourceFiles(RulesCompiler.RulesFileType.Target, ExtraSearchPaths);
if (DirPushed)
{
CommandUtils.PopDir();
}
if (!CommandUtils.IsNullOrEmpty(TargetScripts))
{
// We only care about project target script so filter out any scripts not in the project folder, or take them all if we are just doing engine stuff
var ProjectTargetScripts = new List <string>();
foreach (var Filename in TargetScripts)
{
var FullScriptPath = CommandUtils.CombinePaths(Path.GetFullPath(Filename));
if (FullProjectPath == null || FullScriptPath.StartsWith(FullProjectPath, StringComparison.InvariantCultureIgnoreCase))
{
ProjectTargetScripts.Add(FullScriptPath);
}
}
TargetScripts = ProjectTargetScripts;
}
if (!CommandUtils.IsNullOrEmpty(TargetScripts))
{
CommandUtils.LogVerbose("Found {0} target rule files:", TargetScripts.Count);
foreach (var Filename in TargetScripts)
{
CommandUtils.LogVerbose(" {0}", Filename);
}
// Check if the scripts require compilation
bool DoNotCompile = false;
if (!CommandUtils.IsBuildMachine && !CheckIfScriptAssemblyIsOutOfDate(TargetsDllFilename, TargetScripts))
{
Log.TraceInformation("Targets DLL {0} is up to date.", TargetsDllFilename);
DoNotCompile = true;
}
if (!DoNotCompile && CommandUtils.FileExists_NoExceptions(TargetsDllFilename))
{
if (!CommandUtils.DeleteFile_NoExceptions(TargetsDllFilename, true))
{
DoNotCompile = true;
CommandUtils.Log("Could not delete {0} assuming it is up to date and reusable for a recursive UAT call.", TargetsDllFilename);
}
}
CompileAndLoadTargetsAssembly(Properties, TargetsDllFilename, DoNotCompile, TargetScripts);
}
}
private static string GetScriptAssemblyFolder()
{
return(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Binaries", "DotNET", "AutomationScripts"));
}
/// <summary>
/// Finds and/or compiles all script files and assemblies.
/// </summary>
/// <param name="ScriptsForProjectFileName">Path to the current project. May be null, in which case we compile scripts for all projects.</param>
/// <param name="AdditionalScriptsFolders">Additional script fodlers to look for source files in.</param>
public void FindAndCompileAllScripts(string ScriptsForProjectFileName, List <string> AdditionalScriptsFolders)
{
bool DoCompile = false;
if (GlobalCommandLine.Compile)
{
DoCompile = true;
}
// Change to Engine\Source (if exists) to properly discover all UBT classes
var OldCWD = Environment.CurrentDirectory;
var UnrealBuildToolCWD = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Source");
if (Directory.Exists(UnrealBuildToolCWD))
{
Environment.CurrentDirectory = UnrealBuildToolCWD;
}
// Register all the classes inside UBT
Environment.CurrentDirectory = OldCWD;
// Compile only if not disallowed.
if (DoCompile && !String.IsNullOrEmpty(CommandUtils.CmdEnv.MsBuildExe))
{
CleanupScriptsAssemblies();
FindAndCompileScriptModules(ScriptsForProjectFileName, AdditionalScriptsFolders);
}
var ScriptAssemblies = new List <Assembly>();
LoadPreCompiledScriptAssemblies(ScriptAssemblies);
// Setup platforms
Platform.InitializePlatforms(ScriptAssemblies.ToArray());
// Instantiate all the automation classes for interrogation
Log.TraceVerbose("Creating commands.");
ScriptCommands = new CaselessDictionary <Type>();
foreach (var CompiledScripts in ScriptAssemblies)
{
try
{
foreach (var ClassType in CompiledScripts.GetTypes())
{
if (ClassType.IsSubclassOf(typeof(BuildCommand)) && ClassType.IsAbstract == false)
{
if (ScriptCommands.ContainsKey(ClassType.Name) == false)
{
ScriptCommands.Add(ClassType.Name, ClassType);
}
else
{
Log.TraceWarning("Unable to add command {0} twice. Previous: {1}, Current: {2}", ClassType.Name,
ClassType.AssemblyQualifiedName, ScriptCommands[ClassType.Name].AssemblyQualifiedName);
}
}
}
}
catch (Exception Ex)
{
throw new AutomationException("Failed to add commands from {0}. {1}", CompiledScripts, Ex);
}
}
}
/// <summary>
/// Initializes the environment.
/// </summary>
internal CommandEnvironment()
{
// Get the path to the UAT executable
UATExe = Assembly.GetEntryAssembly().GetOriginalLocation();
if (!CommandUtils.FileExists(UATExe))
{
throw new AutomationException("Could not find AutomationTool.exe. Reflection indicated it was here: {0}", UATExe);
}
// Find the root directory (containing the Engine folder)
LocalRoot = CommandUtils.GetEnvVar(EnvVarNames.LocalRoot);
if (String.IsNullOrEmpty(LocalRoot))
{
LocalRoot = CommandUtils.ConvertSeparators(PathSeparator.Slash, Path.GetFullPath(Path.Combine(Path.GetDirectoryName(UATExe), "..", "..", "..")));
CommandUtils.ConditionallySetEnvVar(EnvVarNames.LocalRoot, LocalRoot);
}
string SavedPath = CommandUtils.GetEnvVar(EnvVarNames.EngineSavedFolder);
if (String.IsNullOrEmpty(SavedPath))
{
SavedPath = CommandUtils.CombinePaths(PathSeparator.Slash, LocalRoot, "Engine", "Programs", "AutomationTool", "Saved");
CommandUtils.SetEnvVar(EnvVarNames.EngineSavedFolder, SavedPath);
}
EngineSavedFolder = CommandUtils.GetEnvVar(EnvVarNames.EngineSavedFolder);
CSVFile = CommandUtils.GetEnvVar(EnvVarNames.CSVFile);
LogFolder = CommandUtils.GetEnvVar(EnvVarNames.LogFolder);
if (String.IsNullOrEmpty(LogFolder))
{
if (GlobalCommandLine.Installed)
{
LogFolder = GetInstalledLogFolder();
}
else
{
LogFolder = CommandUtils.CombinePaths(PathSeparator.Slash, EngineSavedFolder, "Logs");
}
CommandUtils.SetEnvVar(EnvVarNames.LogFolder, LogFolder);
}
// clear the logfolder if we're the only running instance
if (InternalUtils.IsSoleInstance)
{
ClearLogFolder(LogFolder);
}
FinalLogFolder = CommandUtils.GetEnvVar(EnvVarNames.FinalLogFolder);
if (String.IsNullOrEmpty(FinalLogFolder))
{
FinalLogFolder = LogFolder;
CommandUtils.SetEnvVar(EnvVarNames.FinalLogFolder, FinalLogFolder);
}
RobocopyExe = GetSystemExePath("robocopy.exe");
MountExe = GetSystemExePath("mount.exe");
CmdExe = Utils.IsRunningOnMono ? "/bin/sh" : GetSystemExePath("cmd.exe");
MallocNanoZone = "0";
CommandUtils.SetEnvVar(EnvVarNames.MacMallocNanoZone, MallocNanoZone);
int IsChildInstanceInt;
int.TryParse(CommandUtils.GetEnvVar("uebp_UATChildInstance", "0"), out IsChildInstanceInt);
IsChildInstance = (IsChildInstanceInt != 0);
// Setup the timestamp string
DateTime LocalTime = DateTime.Now;
string TimeStamp = LocalTime.Year + "-"
+ LocalTime.Month.ToString("00") + "-"
+ LocalTime.Day.ToString("00") + "_"
+ LocalTime.Hour.ToString("00") + "."
+ LocalTime.Minute.ToString("00") + "."
+ LocalTime.Second.ToString("00");
TimestampAsString = TimeStamp;
SetupBuildEnvironment();
LogSettings();
}
/// <summary>
/// Initializes the environement.
/// </summary>
protected virtual void InitEnvironment()
{
SetUATLocation();
LocalRoot = CommandUtils.GetEnvVar(EnvVarNames.LocalRoot);
if (String.IsNullOrEmpty(CommandUtils.GetEnvVar(EnvVarNames.EngineSavedFolder)))
{
SetUATSavedPath();
}
if (LocalRoot.EndsWith(":"))
{
LocalRoot += Path.DirectorySeparatorChar;
}
EngineSavedFolder = CommandUtils.GetEnvVar(EnvVarNames.EngineSavedFolder);
CSVFile = CommandUtils.GetEnvVar(EnvVarNames.CSVFile);
LogFolder = CommandUtils.GetEnvVar(EnvVarNames.LogFolder);
RobocopyExe = CommandUtils.CombinePaths(Environment.SystemDirectory, "robocopy.exe");
MountExe = CommandUtils.CombinePaths(Environment.SystemDirectory, "mount.exe");
CmdExe = Utils.IsRunningOnMono ? "/bin/sh" : CommandUtils.CombinePaths(Environment.SystemDirectory, "cmd.exe");
if (String.IsNullOrEmpty(LogFolder))
{
throw new AutomationException("Environment is not set up correctly: LogFolder is not set!");
}
if (String.IsNullOrEmpty(LocalRoot))
{
throw new AutomationException("Environment is not set up correctly: LocalRoot is not set!");
}
if (String.IsNullOrEmpty(EngineSavedFolder))
{
throw new AutomationException("Environment is not set up correctly: EngineSavedFolder is not set!");
}
// Make sure that the log folder exists
var LogFolderInfo = new DirectoryInfo(LogFolder);
if (!LogFolderInfo.Exists)
{
LogFolderInfo.Create();
}
// Setup the timestamp string
DateTime LocalTime = DateTime.Now;
string TimeStamp = LocalTime.Year + "-"
+ LocalTime.Month.ToString("00") + "-"
+ LocalTime.Day.ToString("00") + "_"
+ LocalTime.Hour.ToString("00") + "."
+ LocalTime.Minute.ToString("00") + "."
+ LocalTime.Second.ToString("00");
TimestampAsString = TimeStamp;
SetupBuildEnvironment();
LogSettings();
}