public override void ExecuteBuild()
{
// Parse the target name
string Target = ParseParamValue("Target");
if (Target == null)
{
throw new AutomationException("Missing -Target=... argument");
}
// Parse the platform
string PlatformParam = ParseParamValue("Platform");
if (PlatformParam == null)
{
throw new AutomationException("Missing -Platform=... argument");
}
UnrealTargetPlatform Platform;
if (!Enum.TryParse(PlatformParam, true, out Platform))
{
throw new AutomationException("Invalid platform '{0}'", PlatformParam);
}
// Parse the configuration
string ConfigurationParam = ParseParamValue("Configuration");
if (ConfigurationParam == null)
{
throw new AutomationException("Missing -Configuration=... argument");
}
UnrealTargetConfiguration Configuration;
if (!Enum.TryParse(ConfigurationParam, true, out Configuration))
{
throw new AutomationException("Invalid configuration '{0}'", ConfigurationParam);
}
// Parse the project
string Project = ParseParamValue("Project");
if (Project != null && !File.Exists(Project))
{
throw new AutomationException("Specified project file '{0}' was not found", Project);
}
// Parse the architecture
string Architecture = ParseParamValue("Architecture");
// Check the receipt exists
DirectoryReference ProjectDir = null;
if (Project != null)
{
ProjectDir = new FileReference(Project).Directory;
}
FileReference ReceiptFile = TargetReceipt.GetDefaultPath(ProjectDir, Target, Platform, Configuration, Architecture);
if (!FileReference.Exists(ReceiptFile))
{
throw new AutomationException("FortniteEditor receipt not found ({0})", ReceiptFile);
}
LogInformation("Found {0}", ReceiptFile);
}
public DeploymentContext(
FileReference RawProjectPathOrName,
DirectoryReference InLocalRoot,
DirectoryReference BaseStageDirectory,
DirectoryReference BaseArchiveDirectory,
Platform InSourcePlatform,
Platform InTargetPlatform,
List <UnrealTargetConfiguration> InTargetConfigurations,
IEnumerable <StageTarget> InStageTargets,
List <String> InStageExecutables,
bool InServer,
bool InCooked,
bool InStageCrashReporter,
bool InStage,
bool InCookOnTheFly,
bool InArchive,
bool InProgram,
bool IsClientInsteadOfNoEditor,
bool InForceChunkManifests,
bool InSeparateDebugStageDirectory
)
{
bStageCrashReporter = InStageCrashReporter;
RawProjectPath = RawProjectPathOrName;
DedicatedServer = InServer;
LocalRoot = InLocalRoot;
CookSourcePlatform = InSourcePlatform;
StageTargetPlatform = InTargetPlatform;
StageTargetConfigurations = new List <UnrealTargetConfiguration>(InTargetConfigurations);
StageTargets = new List <StageTarget>(InStageTargets);
StageExecutables = InStageExecutables;
IsCodeBasedProject = ProjectUtils.IsCodeBasedUProjectFile(RawProjectPath, StageTargetConfigurations);
ShortProjectName = ProjectUtils.GetShortProjectName(RawProjectPath);
Stage = InStage;
Archive = InArchive;
if (CookSourcePlatform != null && InCooked)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(DedicatedServer, IsClientInsteadOfNoEditor);
}
else if (CookSourcePlatform != null && InProgram)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(false, false);
}
else
{
CookPlatform = "";
}
if (StageTargetPlatform != null && InCooked)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(DedicatedServer, IsClientInsteadOfNoEditor);
}
else if (StageTargetPlatform != null && InProgram)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(false, false);
}
else
{
FinalCookPlatform = "";
}
PlatformDir = StageTargetPlatform.PlatformType.ToString();
if (BaseStageDirectory != null)
{
StageDirectory = DirectoryReference.Combine(BaseStageDirectory, FinalCookPlatform);
DebugStageDirectory = InSeparateDebugStageDirectory? DirectoryReference.Combine(BaseStageDirectory, FinalCookPlatform + "Debug") : StageDirectory;
}
if (BaseArchiveDirectory != null)
{
ArchiveDirectory = DirectoryReference.Combine(BaseArchiveDirectory, FinalCookPlatform);
}
if (!FileReference.Exists(RawProjectPath))
{
throw new AutomationException("Can't find uproject file {0}.", RawProjectPathOrName);
}
EngineRoot = DirectoryReference.Combine(LocalRoot, "Engine");
ProjectRoot = RawProjectPath.Directory;
RelativeProjectRootForStage = new StagedDirectoryReference(ShortProjectName);
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(RawProjectPath.FullName);
CookSourceRuntimeRootDir = RuntimeRootDir = LocalRoot;
RuntimeProjectRootDir = ProjectRoot;
if (Stage)
{
CommandUtils.CreateDirectory(StageDirectory.FullName);
RuntimeRootDir = StageDirectory;
CookSourceRuntimeRootDir = DirectoryReference.Combine(BaseStageDirectory, CookPlatform);
RuntimeProjectRootDir = DirectoryReference.Combine(StageDirectory, RelativeProjectRootForStage.Name);
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(UProjectCommandLineArgInternalRoot + RelativeProjectRootForStage.Name + "/" + ShortProjectName + ".uproject");
}
if (Archive)
{
CommandUtils.CreateDirectory(ArchiveDirectory.FullName);
}
ProjectArgForCommandLines = ProjectArgForCommandLines.Replace("\\", "/");
ProjectBinariesFolder = DirectoryReference.Combine(ProjectUtils.GetClientProjectBinariesRootPath(RawProjectPath, TargetType.Game, IsCodeBasedProject), PlatformDir);
// Build a list of restricted folder names. This will comprise all other restricted platforms, plus standard restricted folder names such as NoRedist, NotForLicensees, etc...
RestrictedFolderNames.UnionWith(PlatformExports.GetPlatformFolderNames());
foreach (UnrealTargetPlatform StagePlatform in StageTargetPlatform.GetStagePlatforms())
{
RestrictedFolderNames.ExceptWith(PlatformExports.GetIncludedFolderNames(StagePlatform));
}
RestrictedFolderNames.UnionWith(RestrictedFolders.Names);
RestrictedFolderNames.Remove(StageTargetPlatform.IniPlatformType.ToString());
// Read the game config files
ConfigHierarchy GameConfig = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, ProjectRoot, InTargetPlatform.PlatformType);
// Read the list of directories to remap when staging
List <string> RemapDirectoriesList;
if (GameConfig.GetArray("Staging", "RemapDirectories", out RemapDirectoriesList))
{
foreach (string RemapDirectory in RemapDirectoriesList)
{
Dictionary <string, string> Properties;
if (!ConfigHierarchy.TryParse(RemapDirectory, out Properties))
{
throw new AutomationException("Unable to parse '{0}'", RemapDirectory);
}
string FromDir;
if (!Properties.TryGetValue("From", out FromDir))
{
throw new AutomationException("Missing 'From' property in '{0}'", RemapDirectory);
}
string ToDir;
if (!Properties.TryGetValue("To", out ToDir))
{
throw new AutomationException("Missing 'To' property in '{0}'", RemapDirectory);
}
RemapDirectories.Add(Tuple.Create(new StagedDirectoryReference(FromDir), new StagedDirectoryReference(ToDir)));
}
}
// Read the list of directories to whitelist from restricted folder warnings
List <string> WhitelistDirectoriesList;
if (GameConfig.GetArray("Staging", "WhitelistDirectories", out WhitelistDirectoriesList))
{
foreach (string WhitelistDirectory in WhitelistDirectoriesList)
{
WhitelistDirectories.Add(new StagedDirectoryReference(WhitelistDirectory));
}
}
List <string> BlacklistLocTargetsList;
if (GameConfig.GetArray("Staging", "BlacklistLocalizationTargets", out BlacklistLocTargetsList))
{
foreach (string BlacklistLocTarget in BlacklistLocTargetsList)
{
BlacklistLocalizationTargets.Add(BlacklistLocTarget);
}
}
// Read the list of files which are whitelisted to be staged
ReadConfigFileList(GameConfig, "Staging", "WhitelistConfigFiles", WhitelistConfigFiles);
ReadConfigFileList(GameConfig, "Staging", "BlacklistConfigFiles", BlacklistConfigFiles);
// If we were configured to use manifests across the whole project, then this platform should use manifests.
// Otherwise, read whether we are generating chunks from the ProjectPackagingSettings ini.
if (InForceChunkManifests)
{
PlatformUsesChunkManifests = true;
}
else
{
ConfigHierarchy GameIni = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, ProjectRoot, InTargetPlatform.PlatformType);
String IniPath = "/Script/UnrealEd.ProjectPackagingSettings";
bool bSetting = false;
if (GameIni.GetBool(IniPath, "bGenerateChunks", out bSetting))
{
PlatformUsesChunkManifests = bSetting;
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Set the Engine directory
DirectoryReference EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.EngineDir))
{
EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.EngineDir);
}
// Set the Project directory
DirectoryReference ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.ProjectDir))
{
ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.ProjectDir);
}
// Resolve the input list
IEnumerable <FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
HashSet <FileReference> Files = new HashSet <FileReference>();
HashSet <string> WildcardDependencies = new HashSet <string>();
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
CommandUtils.LogError("Invalid file passed to TagReceipt task ({0})", TargetFile.FullName);
continue;
}
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile.FullName, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
// Convert the paths to absolute
Receipt.ExpandPathVariables(EngineDir, ProjectDir);
if (Parameters.BuildProducts)
{
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if (String.IsNullOrEmpty(Parameters.BuildProductType) || BuildProduct.Type == BuildProductType)
{
Files.Add(new FileReference(BuildProduct.Path));
}
}
}
if (Parameters.RuntimeDependencies)
{
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
if (String.IsNullOrEmpty(Parameters.StagedFileType) || RuntimeDependency.Type == StagedFileType)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(RuntimeDependency.Path) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(RuntimeDependency.Path);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as RuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(RuntimeDependency.Path);
}
}
}
}
if (Parameters.PrecompiledBuildDependencies)
{
foreach (string PrecompiledBuildDependency in Receipt.PrecompiledBuildDependencies)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(PrecompiledBuildDependency) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(PrecompiledBuildDependency);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as PrecompiledBuildDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(PrecompiledBuildDependency);
}
}
}
if (Parameters.PrecompiledRuntimeDependencies)
{
foreach (string PrecompiledRuntimeDependency in Receipt.PrecompiledRuntimeDependencies)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(PrecompiledRuntimeDependency) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(PrecompiledRuntimeDependency);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as PrecompiledRuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(PrecompiledRuntimeDependency);
}
}
}
}
// Turn any wildcards into a file list
Files.UnionWith(ResolveFilespecWithExcludePatterns(CommandUtils.RootDirectory, WildcardDependencies.ToList(), new List <string>(), TagNameToFileSet));
// Apply the tag to all the matching files
FindOrAddTagSet(TagNameToFileSet, Parameters.With).UnionWith(Files);
return(true);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Find the directories we're going to rebase relative to
HashSet <DirectoryReference> RebaseDirs = new HashSet <DirectoryReference> {
CommandUtils.RootDirectory
};
if (Parameters.RebaseDir != null)
{
RebaseDirs.UnionWith(SplitDelimitedList(Parameters.RebaseDir).Select(x => ResolveDirectory(x)));
}
// Get the output parameter
FileReference OutputFile = ResolveFile(Parameters.Output);
// Check for a ResponseFile parameter
FileReference ResponseFile = null;
if (!String.IsNullOrEmpty(Parameters.ResponseFile))
{
ResponseFile = ResolveFile(Parameters.ResponseFile);
}
if (ResponseFile == null)
{
// Get a unique filename for the response file
ResponseFile = FileReference.Combine(new DirectoryReference(CommandUtils.CmdEnv.LogFolder), String.Format("PakList_{0}.txt", OutputFile.GetFileNameWithoutExtension()));
for (int Idx = 2; ResponseFile.Exists(); Idx++)
{
ResponseFile = FileReference.Combine(ResponseFile.Directory, String.Format("PakList_{0}_{1}.txt", OutputFile.GetFileNameWithoutExtension(), Idx));
}
// Write out the response file
HashSet <FileReference> Files = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
using (StreamWriter Writer = new StreamWriter(ResponseFile.FullName, false, new System.Text.UTF8Encoding(true)))
{
foreach (FileReference File in Files)
{
string RelativePath = FindShortestRelativePath(File, RebaseDirs);
if (RelativePath == null)
{
CommandUtils.LogError("Couldn't find relative path for '{0}' - not under any rebase directories", File.FullName);
return(false);
}
Writer.WriteLine("\"{0}\" \"{1}\"{2}", File.FullName, RelativePath, Parameters.Compress ? " -compress" : "");
}
}
}
// Format the command line
StringBuilder CommandLine = new StringBuilder();
CommandLine.AppendFormat("{0} -create={1}", CommandUtils.MakePathSafeToUseWithCommandLine(OutputFile.FullName), CommandUtils.MakePathSafeToUseWithCommandLine(ResponseFile.FullName));
if (Parameters.Sign != null)
{
CommandLine.AppendFormat(" -sign={0}", CommandUtils.MakePathSafeToUseWithCommandLine(ResolveFile(Parameters.Sign).FullName));
}
if (Parameters.Order != null)
{
CommandLine.AppendFormat(" -order={0}", CommandUtils.MakePathSafeToUseWithCommandLine(ResolveFile(Parameters.Order).FullName));
}
if (GlobalCommandLine.Installed)
{
CommandLine.Append(" -installed");
}
if (GlobalCommandLine.UTF8Output)
{
CommandLine.AppendFormat(" -UTF8Output");
}
// Get the executable path
FileReference UnrealPakExe;
if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Win64)
{
UnrealPakExe = ResolveFile("Engine/Binaries/Win64/UnrealPak.exe");
}
else
{
UnrealPakExe = ResolveFile(String.Format("Engine/Binaries/{0}/UnrealPak", HostPlatform.Current.HostEditorPlatform.ToString()));
}
// Run it
CommandUtils.Log("Running '{0} {1}'", CommandUtils.MakePathSafeToUseWithCommandLine(UnrealPakExe.FullName), CommandLine.ToString());
CommandUtils.RunAndLog(CommandUtils.CmdEnv, UnrealPakExe.FullName, CommandLine.ToString(), Options: CommandUtils.ERunOptions.Default | CommandUtils.ERunOptions.UTF8Output);
BuildProducts.Add(OutputFile);
// Apply the optional tag to the output file
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).Add(OutputFile);
}
return(true);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project file
HashSet <FileReference> ProjectFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Project, TagNameToFileSet);
foreach (FileReference ProjectFile in ProjectFiles)
{
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Couldn't find project file '{0}'", ProjectFile.FullName);
}
if (!ProjectFile.HasExtension(".csproj"))
{
throw new AutomationException("File '{0}' is not a C# project", ProjectFile.FullName);
}
}
// Get the default properties
Dictionary <string, string> Properties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
if (!String.IsNullOrEmpty(Parameters.Platform))
{
Properties["Platform"] = Parameters.Platform;
}
if (!String.IsNullOrEmpty(Parameters.Configuration))
{
Properties["Configuration"] = Parameters.Configuration;
}
// Build the arguments and run the build
if (!Parameters.EnumerateOnly)
{
List <string> Arguments = new List <string>();
foreach (KeyValuePair <string, string> PropertyPair in Properties)
{
Arguments.Add(String.Format("/property:{0}={1}", CommandUtils.MakePathSafeToUseWithCommandLine(PropertyPair.Key), CommandUtils.MakePathSafeToUseWithCommandLine(PropertyPair.Value)));
}
if (!String.IsNullOrEmpty(Parameters.Arguments))
{
Arguments.Add(Parameters.Arguments);
}
if (!String.IsNullOrEmpty(Parameters.Target))
{
Arguments.Add(String.Format("/target:{0}", CommandUtils.MakePathSafeToUseWithCommandLine(Parameters.Target)));
}
Arguments.Add("/verbosity:minimal");
Arguments.Add("/nologo");
foreach (FileReference ProjectFile in ProjectFiles)
{
CommandUtils.MsBuild(CommandUtils.CmdEnv, ProjectFile.FullName, String.Join(" ", Arguments), null);
}
}
// Try to figure out the output files
HashSet <FileReference> ProjectBuildProducts;
HashSet <FileReference> ProjectReferences;
FindBuildProductsAndReferences(ProjectFiles, Properties, out ProjectBuildProducts, out ProjectReferences);
// Apply the optional tag to the produced archive
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(ProjectBuildProducts);
}
// Apply the optional tag to any references
if (!String.IsNullOrEmpty(Parameters.TagReferences))
{
foreach (string TagName in FindTagNamesFromList(Parameters.TagReferences))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(ProjectReferences);
}
}
// Merge them into the standard set of build products
BuildProducts.UnionWith(ProjectBuildProducts);
BuildProducts.UnionWith(ProjectReferences);
}
public List <ISharedCookedBuild> FindBestBuilds()
{
// Attempt manifest searching first
ConfigHierarchy Hierarchy = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, DirectoryReference.FromFile(ProjectFile), UnrealTargetPlatform.Win64);
IReadOnlyList <string> RawSharedCookedSources = null;
Hierarchy.TryGetValues("SharedCookedBuildSettings", "SharedCookedSources", out RawSharedCookedSources);
if (RawSharedCookedSources == null)
{
throw new AutomationException("Unable to locate shared cooked builds. SharedCookedSources not set in Engine.ini [SharedCookedBuildSettings]");
}
List <Dictionary <string, string> > ParsedSharedCookSources = new List <Dictionary <string, string> >();
foreach (string RawConfig in RawSharedCookedSources)
{
Dictionary <string, string> ParsedSource = null;
if (ConfigHierarchy.TryParse(RawConfig, out ParsedSource))
{
ParsedSharedCookSources.Add(ParsedSource);
}
}
List <ISharedCookedBuild> CandidateBuilds = new List <ISharedCookedBuild>();
// If existing sync is present, stick to it. Read version out of sync file
foreach (string Platform in TargetPlatforms)
{
FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(InstallPath.FullName, Platform, SyncedBuildFileName));
if (FileReference.Exists(SyncedBuildFile))
{
string[] SyncedBuildInfo = FileReference.ReadAllLines(SyncedBuildFile);
int SyncedCL = int.Parse(SyncedBuildInfo[0]);
if (IsValidCL(SyncedCL, BuildType, LocalSync))
{
CandidateBuilds.Add(new ExistingSharedCookedBuild()
{
CL = SyncedCL, Platform = Platform
});
}
}
}
foreach (Dictionary <string, string> Source in ParsedSharedCookSources)
{
SharedCookSource SourceType = (SharedCookSource)Enum.Parse(typeof(SharedCookSource), Source["Type"], true);
foreach (string Platform in TargetPlatforms)
{
if (SourceType == SharedCookSource.Manifest)
{
CandidateBuilds.AddRange(FindValidManifestBuilds(Source["Path"], Platform));
}
else if (SourceType == SharedCookSource.LooseFiles)
{
CandidateBuilds.AddRange(FindValidLooseBuilds(Source["Path"], Platform));
}
}
}
// Strip all failed searches
CandidateBuilds.RemoveAll(x => x == null);
// Make sure we have a matching CL for all target platforms, regardless of source
List <int> OrderedDistinctCLs = CandidateBuilds.Select(x => x.CL).Distinct().OrderByDescending(i => i).ToList();
int BestCL = -1;
foreach (int CL in OrderedDistinctCLs)
{
// Ensure we have a platform for each
HashSet <string> CLPlatforms = new HashSet <string>(CandidateBuilds.Where(x => x.CL == CL).Select(x => x.Platform).ToList());
if (CLPlatforms.SetEquals(TargetPlatforms))
{
BestCL = CL;
break;
}
}
if (BestCL < 0)
{
CommandUtils.LogError("Could not locate valid shared cooked build for all target platforms");
CommandUtils.LogError("Current CL: {0}, Current Code CL: {1}", LocalSync.Changelist, LocalSync.CompatibleChangelist);
}
return(CandidateBuilds.Where(x => x.CL == BestCL).ToList());
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Set the Engine directory
DirectoryReference EngineDir = Parameters.EngineDir ?? CommandUtils.EngineDirectory;
// Resolve the input list
IEnumerable <FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
throw new AutomationException("Invalid file passed to TagReceipt task ({0})", TargetFile.FullName);
}
// Print the name of the file being scanned
Log.TraceInformation("Sanitizing {0}", TargetFile);
using (new LogIndentScope(" "))
{
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile, EngineDir, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
// Remove any build products that don't exist
List <BuildProduct> NewBuildProducts = new List <BuildProduct>(Receipt.BuildProducts.Count);
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if (FileReference.Exists(BuildProduct.Path))
{
NewBuildProducts.Add(BuildProduct);
}
else
{
Log.TraceInformation("Removing build product: {0}", BuildProduct.Path);
}
}
Receipt.BuildProducts = NewBuildProducts;
// Remove any runtime dependencies that don't exist
RuntimeDependencyList NewRuntimeDependencies = new RuntimeDependencyList();
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
if (FileReference.Exists(RuntimeDependency.Path))
{
NewRuntimeDependencies.Add(RuntimeDependency);
}
else
{
Log.TraceInformation("Removing runtime dependency: {0}", RuntimeDependency.Path);
}
}
Receipt.RuntimeDependencies = NewRuntimeDependencies;
// Save the new receipt
Receipt.Write(TargetFile, EngineDir);
}
}
}
/// <summary>
/// Creates and deserializes the dependency cache at the passed in location
/// </summary>
/// <param name="CachePath">Name of the cache file to deserialize</param>
public static DependencyCache Create(FileReference CacheFile)
{
// See whether the cache file exists.
if (CacheFile.Exists())
{
if (BuildConfiguration.bPrintPerformanceInfo)
{
Log.TraceInformation("Loading existing IncludeFileCache: " + CacheFile.FullName);
}
DateTime TimerStartTime = DateTime.UtcNow;
// Deserialize cache from disk if there is one.
DependencyCache Result = Load(CacheFile);
if (Result != null)
{
// Successfully serialize, create the transient variables and return cache.
Result.UpdateTimeUtc = DateTime.UtcNow;
TimeSpan TimerDuration = DateTime.UtcNow - TimerStartTime;
if (BuildConfiguration.bPrintPerformanceInfo)
{
Log.TraceInformation("Loading IncludeFileCache took " + TimerDuration.TotalSeconds + "s");
}
return Result;
}
}
// Fall back to a clean cache on error or non-existance.
return new DependencyCache(CacheFile);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Output a warning if the project directory is specified
if (Parameters.ProjectDir != null)
{
CommandUtils.LogWarning("The ProjectDir argument to the TagReceipt parameter is deprecated. This path is now determined automatically from the receipt.");
}
// Set the Engine directory
DirectoryReference EngineDir = Parameters.EngineDir ?? CommandUtils.EngineDirectory;
// Resolve the input list
IEnumerable <FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
HashSet <FileReference> Files = new HashSet <FileReference>();
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
throw new AutomationException("Invalid file passed to TagReceipt task ({0})", TargetFile.FullName);
}
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile, EngineDir, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
if (Parameters.BuildProducts)
{
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if (BuildProductType.HasValue && BuildProduct.Type != BuildProductType.Value)
{
continue;
}
if (StagedFileType.HasValue && TargetReceipt.GetStageTypeFromBuildProductType(BuildProduct) != StagedFileType.Value)
{
continue;
}
Files.Add(BuildProduct.Path);
}
}
if (Parameters.RuntimeDependencies)
{
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
// Skip anything that doesn't match the files we want
if (BuildProductType.HasValue)
{
continue;
}
if (StagedFileType.HasValue && RuntimeDependency.Type != StagedFileType.Value)
{
continue;
}
// Check which files exist, and warn about any that don't. Ignore debug files, as they are frequently excluded for size (eg. UE4 on GitHub). This matches logic during staging.
FileReference DependencyPath = RuntimeDependency.Path;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else if (RuntimeDependency.Type != UnrealBuildTool.StagedFileType.DebugNonUFS)
{
CommandUtils.LogWarning("File listed as RuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
}
// Apply the tag to all the matching files
FindOrAddTagSet(TagNameToFileSet, Parameters.With).UnionWith(Files);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = Parameters.Project;
if (Parameters.Project != null && !FileReference.Exists(ProjectFile))
{
throw new AutomationException("Couldn't find project '{0}'", ProjectFile.FullName);
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = CommandUtils.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = Parameters.ToDir;
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, SourceEngineDir, SourceProjectDir, out Receipt))
{
throw new AutomationException("Couldn't read receipt '{0}'", ReceiptFileName);
}
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
SourceFiles.Add(BuildProduct.Path);
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.Add(RuntimeDependency.Path);
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !FileReference.Exists(TargetFile))
{
DirectoryReference.CreateDirectory(TargetFile.Directory);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
// Force all destination files to not readonly.
CommandUtils.SetFileAttributes(TargetFile.FullName, ReadOnly: false);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
}
/// <summary>
/// Main command entry point
/// </summary>
/// <param name="Arguments">The command line arguments</param>
public override int Execute()
{
// Build a mapping from category to matching
Dictionary <string, Matcher> CategoryNameToMatcher = new Dictionary <string, Matcher>();
foreach (Matcher Matcher in Matchers)
{
CategoryNameToMatcher[Matcher.Category] = Matcher;
}
// Complete any interrupted operation to update the state file
CompleteStateTransaction(StateFile);
// Read the persistent data file
PersistentState State;
if (!bClean && FileReference.Exists(StateFile))
{
Log.TraceInformation("Reading persistent data from {0}", StateFile);
State = DeserializeJson <PersistentState>(StateFile);
}
else
{
Log.TraceInformation("Creating new persistent data");
State = new PersistentState();
}
// Fixup any issues loaded from disk
foreach (Issue Issue in State.Issues)
{
if (Issue.References == null)
{
Issue.References = new SortedSet <string>();
}
}
// Create the Perforce connection
PerforceConnection Perforce = new PerforceConnection(PerforcePort, PerforceUser, null);
// Process the input data
if (InputFile != null)
{
// Parse the input file
Log.TraceInformation("Reading build results from {0}", InputFile);
InputData InputData = DeserializeJson <InputData>(InputFile);
// Parse all the builds and add them to the persistent data
List <InputJob> InputJobs = InputData.Jobs.OrderBy(x => x.Change).ThenBy(x => x.Stream).ToList();
Stopwatch Timer = Stopwatch.StartNew();
foreach (InputJob InputJob in InputJobs)
{
// Add a new build for each job step
foreach (InputJobStep InputJobStep in InputJob.Steps)
{
IssueBuild NewBuild = new IssueBuild(InputJob.Change, InputJob.Name, InputJob.Url, InputJobStep.Name, InputJobStep.Url, null);
State.AddBuild(InputJob.Stream, NewBuild);
}
// Add all the job steps
List <InputJobStep> InputJobSteps = InputJob.Steps.OrderBy(x => x.Name).ToList();
foreach (InputJobStep InputJobStep in InputJobSteps)
{
if (InputJobStep.Diagnostics != null && InputJobStep.Diagnostics.Count > 0)
{
AddStep(Perforce, State, InputJob, InputJobStep);
}
}
// Remove any steps which are empty
InputJob.Steps.RemoveAll(x => x.Diagnostics == null || x.Diagnostics.Count == 0);
}
InputJobs.RemoveAll(x => x.Steps.Count == 0);
Log.TraceInformation("Added jobs in {0}s", Timer.Elapsed.TotalSeconds);
// If there are any unmatched issues, save out the current state and remaining input
if (SaveUnmatchedDir != null && InputJobs.Count > 0)
{
DirectoryReference.CreateDirectory(SaveUnmatchedDir);
if (FileReference.Exists(StateFile))
{
FileReference.Copy(StateFile, FileReference.Combine(SaveUnmatchedDir, "State.json"), true);
}
SerializeJson(FileReference.Combine(SaveUnmatchedDir, "Input.json"), InputData);
}
// Try to find the next successful build for each stream, so we can close it as part of updating the server
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
Issue Issue = State.Issues[Idx];
foreach (string Stream in Issue.Streams.Keys)
{
Dictionary <string, IssueHistory> StepNameToHistory = Issue.Streams[Stream];
foreach (string StepName in StepNameToHistory.Keys)
{
IssueHistory IssueHistory = StepNameToHistory[StepName];
if (IssueHistory.FailedBuilds.Count > 0 && IssueHistory.NextSuccessfulBuild == null)
{
// Find the successful build after this change
IssueBuild LastFailedBuild = IssueHistory.FailedBuilds[IssueHistory.FailedBuilds.Count - 1];
IssueHistory.NextSuccessfulBuild = State.FindBuildAfter(Stream, LastFailedBuild.Change, StepName);
}
}
}
}
// Find the change two days before the latest change being added
if (InputData.Jobs.Count > 0 && !bKeepHistory)
{
// Find all the unique change numbers for each stream
SortedSet <int> ChangeNumbers = new SortedSet <int>();
foreach (List <IssueBuild> Builds in State.Streams.Values)
{
ChangeNumbers.UnionWith(Builds.Select(x => x.Change));
}
// Get the latest change record
int LatestChangeNumber = InputData.Jobs.Min(x => x.Change);
ChangeRecord LatestChangeRecord = Perforce.GetChange(GetChangeOptions.None, LatestChangeNumber).Data;
// Step forward through all the changelists until we get to one we don't want to delete
int DeleteChangeNumber = -1;
foreach (int ChangeNumber in ChangeNumbers)
{
ChangeRecord ChangeRecord = Perforce.GetChange(GetChangeOptions.None, ChangeNumber).Data;
if (ChangeRecord.Date > LatestChangeRecord.Date - TimeSpan.FromDays(2))
{
break;
}
DeleteChangeNumber = ChangeNumber;
}
// Remove any builds we no longer want to track
foreach (List <IssueBuild> Builds in State.Streams.Values)
{
Builds.RemoveAll(x => x.Change <= DeleteChangeNumber);
}
}
}
// Mark any issues as resolved
foreach (Issue Issue in State.Issues)
{
if (Issue.IsResolved())
{
if (!Issue.ResolvedAt.HasValue)
{
Issue.ResolvedAt = DateTime.UtcNow;
}
}
else
{
if (Issue.ResolvedAt.HasValue)
{
Issue.ResolvedAt = null;
}
}
}
// If we're in read-only mode, don't write anything out
if (bReadOnly)
{
return(0);
}
// Save the persistent data
Log.TraceInformation("Writing persistent data to {0}", StateFile);
DirectoryReference.CreateDirectory(StateFile.Directory);
WriteState(StateFile, State);
// Synchronize with the server
if (ServerUrl != null)
{
// Post any issue updates
foreach (Issue Issue in State.Issues)
{
Matcher Matcher;
if (!CategoryNameToMatcher.TryGetValue(Issue.Category, out Matcher))
{
continue;
}
string Summary = Matcher.GetSummary(Issue);
if (Issue.Id == -1)
{
Log.TraceInformation("Adding issue: {0}", Issue);
if (Issue.PendingWatchers.Count == 0)
{
Log.TraceWarning("(No possible causers)");
}
ApiTypes.AddIssue IssueBody = new ApiTypes.AddIssue();
IssueBody.Project = Issue.Project;
IssueBody.Summary = Summary;
if (Issue.PendingWatchers.Count == 1)
{
IssueBody.Owner = Issue.PendingWatchers.First();
}
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues", ServerUrl), "POST", IssueBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add issue");
}
Issue.Id = ParseHttpResponse <ApiTypes.AddIssueResponse>(Response).Id;
}
Issue.PostedSummary = Summary;
WriteState(StateFile, State);
}
else if (Issue.PostedSummary == null || !String.Equals(Issue.PostedSummary, Summary, StringComparison.Ordinal))
{
Log.TraceInformation("Updating issue {0}", Issue.Id);
ApiTypes.UpdateIssue IssueBody = new ApiTypes.UpdateIssue();
IssueBody.Summary = Summary;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}", ServerUrl, Issue.Id), "PUT", IssueBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add issue");
}
}
Issue.PostedSummary = Summary;
WriteState(StateFile, State);
}
}
// Add any new builds associated with issues
Dictionary <string, long> JobStepUrlToId = new Dictionary <string, long>(StringComparer.Ordinal);
foreach (Issue Issue in State.Issues)
{
foreach (KeyValuePair <string, Dictionary <string, IssueHistory> > StreamPair in Issue.Streams)
{
foreach (IssueHistory StreamHistory in StreamPair.Value.Values)
{
foreach (IssueBuild Build in StreamHistory.Builds)
{
if (!Build.bPostedToServer)
{
Log.TraceInformation("Adding {0} to issue {1}", Build.JobStepUrl, Issue.Id);
ApiTypes.AddBuild AddBuild = new ApiTypes.AddBuild();
AddBuild.Stream = StreamPair.Key;
AddBuild.Change = Build.Change;
AddBuild.JobName = Build.JobName;
AddBuild.JobUrl = Build.JobUrl;
AddBuild.JobStepName = Build.JobStepName;
AddBuild.JobStepUrl = Build.JobStepUrl;
AddBuild.ErrorUrl = Build.ErrorUrl;
AddBuild.Outcome = (Build == StreamHistory.PrevSuccessfulBuild || Build == StreamHistory.NextSuccessfulBuild)? ApiTypes.Outcome.Success : ApiTypes.Outcome.Error;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/builds", ServerUrl, Issue.Id), "POST", AddBuild))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add build");
}
Build.Id = ParseHttpResponse <ApiTypes.AddBuildResponse>(Response).Id;
}
Build.bPostedToServer = true;
WriteState(StateFile, State);
}
if (Build.Id != -1)
{
JobStepUrlToId[Build.JobStepUrl] = Build.Id;
}
}
}
}
}
// Add any new diagnostics
foreach (Issue Issue in State.Issues)
{
foreach (IssueDiagnostic Diagnostic in Issue.Diagnostics)
{
if (!Diagnostic.bPostedToServer)
{
string Summary = Diagnostic.Message;
const int MaxLength = 40;
if (Summary.Length > MaxLength)
{
Summary = Summary.Substring(0, MaxLength).TrimEnd();
}
Log.TraceInformation("Adding diagnostic '{0}' to issue {1}", Summary, Issue.Id);
ApiTypes.AddDiagnostic AddDiagnostic = new ApiTypes.AddDiagnostic();
long BuildId;
if (Diagnostic.JobStepUrl != null && JobStepUrlToId.TryGetValue(Diagnostic.JobStepUrl, out BuildId))
{
AddDiagnostic.BuildId = BuildId;
}
else
{
Console.WriteLine("ERROR");
}
AddDiagnostic.Message = Diagnostic.Message;
AddDiagnostic.Url = Diagnostic.ErrorUrl;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/diagnostics", ServerUrl, Issue.Id), "POST", AddDiagnostic))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add build");
}
}
Diagnostic.bPostedToServer = true;
WriteState(StateFile, State);
}
}
}
// Close any issues which are complete
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
Issue Issue = State.Issues[Idx];
if (Issue.ResolvedAt.HasValue != Issue.bPostedResolved)
{
Log.TraceInformation("Setting issue {0} resolved flag to {1}", Issue.Id, Issue.ResolvedAt.HasValue);
ApiTypes.UpdateIssue UpdateBody = new ApiTypes.UpdateIssue();
UpdateBody.Resolved = Issue.ResolvedAt.HasValue;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}", ServerUrl, Issue.Id), "PUT", UpdateBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to delete issue");
}
}
Issue.bPostedResolved = Issue.ResolvedAt.HasValue;
WriteState(StateFile, State);
}
}
// Update watchers on any open builds
foreach (Issue Issue in State.Issues)
{
while (Issue.PendingWatchers.Count > 0)
{
ApiTypes.Watcher Watcher = new ApiTypes.Watcher();
Watcher.UserName = Issue.PendingWatchers.First();
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/watchers", ServerUrl, Issue.Id), "POST", Watcher))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add watcher");
}
}
Issue.PendingWatchers.Remove(Watcher.UserName);
Issue.Watchers.Add(Watcher.UserName);
WriteState(StateFile, State);
}
}
}
// Remove any issues which have been resolved for 24 hours. We have to keep information about issues that have been fixed for some time; we may be updating the same job
// multiple times while other steps are running, and we don't want to keep opening new issues for it. Also, it can take time for changes to propagate between streams.
DateTime RemoveIssueTime = DateTime.UtcNow - TimeSpan.FromHours(24.0);
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
Issue Issue = State.Issues[Idx];
if (Issue.ResolvedAt.HasValue && Issue.ResolvedAt.Value < RemoveIssueTime)
{
State.Issues.RemoveAt(Idx--);
WriteState(StateFile, State);
continue;
}
}
// TODO: VERIFY ISSUES ARE CLOSED
return(0);
}
public override void ExecuteBuild()
{
// just calling this DesiredTargetVersion because TargetVersion and TargetedVersion get super confusing.
string DesiredTargetVersionParam = this.ParseParamValue("TargetVersion", null);
if (string.IsNullOrEmpty(DesiredTargetVersionParam))
{
throw new AutomationException("-TargetVersion was not specified.");
}
string[] DesiredTargetVersions = DesiredTargetVersionParam.Split('+');
CommandUtils.LogInformation("Scanning for all csproj's...");
// Check for all csproj's in the engine dir
DirectoryReference EngineDir = CommandUtils.EngineDirectory;
// grab the targeted version.,
Regex FrameworkRegex = new Regex("<TargetFrameworkVersion>v(\\d\\.\\d\\.?\\d?)<\\/TargetFrameworkVersion>");
Regex PossibleAppConfigRegex = new Regex("<TargetFrameworkProfile>(.+)<\\/TargetFrameworkProfile>");
Regex AppConfigRegex = new Regex("<supportedRuntime version=\"v(\\d\\.\\d\\.?\\d?)\" sku=\"\\.NETFramework,Version=v(\\d\\.\\d\\.?\\d?),Profile=(.+)\"\\/>");
Regex DotNetCoreRegex = new Regex("<TargetFramework>(netcoreapp2.0|netstandard2.0)<\\/TargetFramework>");
foreach (FileReference CsProj in DirectoryReference.EnumerateFiles(EngineDir, "*.csproj", SearchOption.AllDirectories))
{
if (CsProj.ContainsName("ThirdParty", EngineDir) ||
(CsProj.ContainsName("UE4TemplateProject", EngineDir) && CsProj.GetFileName().Equals("ProjectTemplate.csproj")) ||
CsProj.GetFileNameWithoutExtension().ToLower().Contains("_mono") ||
CsProj.GetFileNameWithoutExtension().ToLower().Contains("unrealvs") ||
CsProj.GetFileNameWithoutExtension().ToLower().Contains("revit"))
{
continue;
}
// read in the file
string Contents = File.ReadAllText(CsProj.FullName);
Match Match = DotNetCoreRegex.Match(Contents);
// Check if we're a _NETCore app, ignore these.
if (Match.Success)
{
continue;
}
Match = FrameworkRegex.Match(Contents);
if (Match.Success)
{
string TargetedVersion = Match.Groups[1].Value;
// make sure we match, throw warning otherwise
if (!DesiredTargetVersions.Any(DesiredTargetVersion => DesiredTargetVersion.Equals(TargetedVersion, StringComparison.InvariantCultureIgnoreCase)))
{
CommandUtils.LogWarning("Targeted Framework version for project: {0} was not {1}! Targeted Version: {2}", CsProj, String.Join("/", DesiredTargetVersions), TargetedVersion);
}
}
// if we don't have a TargetFrameworkVersion, check for the existence of TargetFrameworkProfile.
else
{
Match = PossibleAppConfigRegex.Match(Contents);
if (!Match.Success)
{
CommandUtils.LogInformation("No TargetFrameworkVersion or TargetFrameworkProfile found for project {0}, is it a mono project? If not, does it compile properly?", CsProj);
continue;
}
// look for the app config
FileReference AppConfigFile = FileReference.Combine(CsProj.Directory, "app.config");
string Profile = Match.Groups[1].Value;
if (!FileReference.Exists(AppConfigFile))
{
CommandUtils.LogInformation("Found TargetFrameworkProfile but no associated app.config containing the version for project {0}.", CsProj);
continue;
}
// read in the app config
Contents = File.ReadAllText(AppConfigFile.FullName);
Match = AppConfigRegex.Match(Contents);
if (!Match.Success)
{
CommandUtils.LogInformation("Couldn't find a supportedRuntime match for the version in the app.config for project {0}.", CsProj);
continue;
}
// Version1 is the one that appears right after supportedRuntime
// Version2 is the one in the sku
// ProfileString should match the TargetFrameworkProfile from the csproj
string Version1String = Match.Groups[1].Value;
string Version2String = Match.Groups[2].Value;
string ProfileString = Match.Groups[3].Value;
// not sure how this is possible, but check for it anyway
if (!ProfileString.Equals(Profile, StringComparison.InvariantCultureIgnoreCase))
{
CommandUtils.LogWarning("The TargetFrameworkProfile in csproj {0} ({1}) doesn't match the sku in it's app.config ({2}).", CsProj, Profile, ProfileString);
continue;
}
// if the version numbers don't match the app.config is probably corrupt.
if (!Version1String.Equals(Version2String, StringComparison.InvariantCultureIgnoreCase))
{
CommandUtils.LogWarning("The supportedRunTimeVersion ({0}) and the sku version ({1}) in the app.config for project {2} don't match.", Version1String, Version2String, CsProj);
continue;
}
// make sure the versions match
if (!(DesiredTargetVersions.Any(DesiredTargetVersion => DesiredTargetVersion.Equals(Version1String, StringComparison.InvariantCultureIgnoreCase))))
{
CommandUtils.LogWarning("Targeted Framework version for project: {0} was not {1}! Targeted Version: {2}", CsProj, String.Join("/", DesiredTargetVersions), Version1String);
}
}
}
}
/// <summary>
/// Attempts to write an owner to a token file transactionally
/// </summary>
/// <returns>True if the lock was acquired, false otherwise</returns>
public bool WriteTokenFile(FileReference Location, string Signature)
{
// Check it doesn't already exist
if(Location.Exists())
{
return false;
}
// Make sure the directory exists
Location.Directory.CreateDirectory();
// Create a temp file containing the owner name
string TempFileName;
for(int Idx = 0;;Idx++)
{
TempFileName = String.Format("{0}.{1}.tmp", Location.FullName, Idx);
try
{
byte[] Bytes = Encoding.UTF8.GetBytes(Signature);
using (FileStream Stream = File.Open(TempFileName, FileMode.CreateNew, FileAccess.Write, FileShare.None))
{
Stream.Write(Bytes, 0, Bytes.Length);
}
break;
}
catch(IOException)
{
if(!File.Exists(TempFileName))
{
throw;
}
}
}
// Try to move the temporary file into place.
try
{
File.Move(TempFileName, Location.FullName);
return true;
}
catch
{
if(!File.Exists(TempFileName))
{
throw;
}
return false;
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
if (SC.bStageCrashReporter)
{
StagedDirectoryReference CrashReportClientPath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, "CrashReportClient.app");
StageAppBundle(SC, DirectoryReference.Combine(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir, "CrashReportClient.app"), CrashReportClientPath);
}
// Find the app bundle path
List <FileReference> Exes = GetExecutableNames(SC);
foreach (var Exe in Exes)
{
StagedDirectoryReference AppBundlePath = null;
if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeProjectRootDir, "Binaries", SC.PlatformDir)))
{
AppBundlePath = StagedDirectoryReference.Combine(SC.ShortProjectName, "Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
else if (Exe.IsUnderDirectory(DirectoryReference.Combine(SC.RuntimeRootDir, "Engine/Binaries", SC.PlatformDir)))
{
AppBundlePath = StagedDirectoryReference.Combine("Engine/Binaries", SC.PlatformDir, Path.GetFileNameWithoutExtension(Exe.FullName) + ".app");
}
// Copy the custom icon and Steam dylib, if needed
if (AppBundlePath != null)
{
FileReference AppIconsFile = FileReference.Combine(SC.ProjectRoot, "Build", "Mac", "Application.icns");
if (FileReference.Exists(AppIconsFile))
{
SC.StageFile(StagedFileType.NonUFS, AppIconsFile, StagedFileReference.Combine(AppBundlePath, "Contents", "Resources", "Application.icns"));
}
}
}
// Copy the splash screen, Mac specific
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.FullName.Replace("\\", "/").Contains("/" + TargetPlatformType.ToString() + "/"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("../../../{0}/{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Path.GetFileName(Executable.Path.FullName) + ".app";
}
else if (Params.IsCodeBasedProject)
{
// We want Mac-Shipping etc in the bundle name
BootstrapExeName = Path.GetFileName(Executable.Path.FullName) + ".app";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".app";
}
string AppSuffix = ".app" + Path.DirectorySeparatorChar;
string AppPath = Executable.Path.FullName.Substring(0, Executable.Path.FullName.LastIndexOf(AppSuffix) + AppSuffix.Length);
foreach (var DestPath in StagedFiles)
{
string AppRelativePath = DestPath.Name.Substring(0, DestPath.Name.LastIndexOf(AppSuffix) + AppSuffix.Length);
StageBootstrapExecutable(SC, BootstrapExeName, AppPath, AppRelativePath, BootstrapArguments);
}
}
}
}
}
// Copy the ShaderCache files, if they exist
FileReference DrawCacheFile = FileReference.Combine(SC.ProjectRoot, "Content", "DrawCache.ushadercache");
if (FileReference.Exists(DrawCacheFile))
{
SC.StageFile(StagedFileType.UFS, DrawCacheFile);
}
FileReference ByteCodeCacheFile = FileReference.Combine(SC.ProjectRoot, "Content", "ByteCodeCache.ushadercode");
if (FileReference.Exists(ByteCodeCacheFile))
{
SC.StageFile(StagedFileType.UFS, ByteCodeCacheFile);
}
{
// Stage any *.metallib files as NonUFS.
// Get the final output directory for cooked data
DirectoryReference CookOutputDir;
if (!String.IsNullOrEmpty(Params.CookOutputDir))
{
CookOutputDir = DirectoryReference.Combine(new DirectoryReference(Params.CookOutputDir), SC.CookPlatform);
}
else if (Params.CookInEditor)
{
CookOutputDir = DirectoryReference.Combine(SC.ProjectRoot, "Saved", "EditorCooked", SC.CookPlatform);
}
else
{
CookOutputDir = DirectoryReference.Combine(SC.ProjectRoot, "Saved", "Cooked", SC.CookPlatform);
}
if (DirectoryReference.Exists(CookOutputDir))
{
List <FileReference> CookedFiles = DirectoryReference.EnumerateFiles(CookOutputDir, "*.metallib", SearchOption.AllDirectories).ToList();
foreach (FileReference CookedFile in CookedFiles)
{
SC.StageFile(StagedFileType.NonUFS, CookedFile, new StagedFileReference(CookedFile.MakeRelativeTo(CookOutputDir)));
}
}
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
if (SC.bStageCrashReporter)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(CommandUtils.EngineDirectory, "CrashReportClient", SC.StageTargetPlatform.PlatformType, UnrealTargetConfiguration.Shipping, null);
if (FileReference.Exists(ReceiptFileName))
{
DirectoryReference EngineDir = CommandUtils.EngineDirectory;
DirectoryReference ProjectDir = DirectoryReference.FromFile(Params.RawProjectPath);
TargetReceipt Receipt = TargetReceipt.Read(ReceiptFileName, EngineDir, ProjectDir);
SC.StageBuildProductsFromReceipt(Receipt, true, false);
}
}
// Stage all the build products
Console.WriteLine("Staging all {0} build products", SC.StageTargets.Count);
int BuildProductIdx = 0;
foreach (StageTarget Target in SC.StageTargets)
{
Console.WriteLine(" Product {0}: {1}", BuildProductIdx, Target.Receipt.TargetName);
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
++BuildProductIdx;
}
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
string FullExecutablePath = Path.GetFullPath(Executable.Path.FullName);
if (Executable.Path.FullName.Replace("\\", "/").Contains("/" + TargetPlatformType.ToString() + "/"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("\\\"../../../{0}/{0}.uproject\\\"", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Path.GetFileName(Executable.Path.FullName);
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName;
}
else
{
BootstrapExeName = SC.ShortProjectName;
}
List <StagedFileReference> StagePaths = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
foreach (StagedFileReference StagePath in StagePaths)
{
StageBootstrapExecutable(SC, BootstrapExeName + ".sh", FullExecutablePath, StagePath.Name, BootstrapArguments);
}
}
}
}
}
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if(PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!PluginFile.Exists())
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (PackageDir.Exists())
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
PackageDir.CreateDirectory();
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!SourceFilterFile.Exists())
{
List<string> Lines = new List<string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
SourceFilterFile.Directory.CreateDirectory();
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.Log("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile, false);
// Compile the plugin for all the target platforms
List<UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List<UnrealTargetPlatform>() : new List<UnrealTargetPlatform> { BuildHostPlatform.Current.Platform };
List<UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform).Where(x => IsCodeTargetPlatform(BuildHostPlatform.Current.Platform, x)).ToList();
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, "");
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir);
// Remove the host project
if(!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null)
{
// Get the SYMSTORE.EXE path, using the latest SDK version we can find.
FileReference SymStoreExe = GetSymStoreExe();
List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList();
if (FilesToAdd.Count > 0)
{
DateTime Start = DateTime.Now;
DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore");
string TempFileName = Path.GetTempFileName();
try
{
File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII);
// Copy everything to the temp symstore
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = SymStoreExe.FullName;
StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\" /compress", TempFileName, TempSymStoreDir, Product);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0)
{
return(false);
}
}
finally
{
File.Delete(TempFileName);
}
DateTime CompressDone = DateTime.Now;
LogInformation("Took {0}s to compress the symbol files", (CompressDone - Start).TotalSeconds);
// Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files
foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => File.HasExtension(".dl_") || File.HasExtension(".ex_") || File.HasExtension(".pd_")))
{
string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir));
FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath);
// Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example)
if (!string.IsNullOrWhiteSpace(BuildVersion))
{
FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion));
// Attempt to create the file. Just continue if it fails.
try
{
DirectoryReference.CreateDirectory(BuildVersionFile.Directory);
FileReference.WriteAllText(BuildVersionFile, string.Empty);
}
catch (Exception Ex)
{
LogWarning("Failed to write the version file, reason {0}", Ex.ToString());
}
}
// Don't bother copying the temp file if the destination file is there already.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName);
continue;
}
FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString());
try
{
CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName);
}
catch (Exception Ex)
{
throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString());
}
// Move the file in the temp store over.
try
{
FileReference.Move(TempDestinationFile, ActualDestinationFile);
}
catch (Exception Ex)
{
// If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already.
// Either way, it's fine to just continue on.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName);
continue;
}
// If it doesn't exist, we actually failed to copy it entirely.
else
{
LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString());
}
}
// Delete the temp one no matter what, don't want them hanging around in the symstore
finally
{
FileReference.Delete(TempDestinationFile);
}
}
LogInformation("Took {0}s to copy the symbol files to the store", (DateTime.Now - CompressDone).TotalSeconds);
}
return(true);
}
/// <summary>
/// Reads the contents of the given token
/// </summary>
/// <returns>Contents of the token, or null if it does not exist</returns>
public string ReadTokenFile(FileReference Location)
{
return Location.Exists()? File.ReadAllText(Location.FullName) : null;
}
/// <summary>
/// Find all the build products created by compiling the given project file
/// </summary>
/// <param name="ProjectFiles">Initial project file to read. All referenced projects will also be read.</param>
/// <param name="InitialProperties">Mapping of property name to value</param>
/// <param name="OutBuildProducts">Receives a set of build products on success</param>
/// <param name="OutReferences">Receives a set of non-private references on success</param>
/// <returns>True if the build products were found, false otherwise.</returns>
static bool FindBuildProducts(HashSet <FileReference> ProjectFiles, Dictionary <string, string> InitialProperties, out HashSet <FileReference> OutBuildProducts, out HashSet <FileReference> OutReferences)
{
// Read all the project information into a dictionary
Dictionary <FileReference, CsProjectInfo> FileToProjectInfo = new Dictionary <FileReference, CsProjectInfo>();
foreach (FileReference ProjectFile in ProjectFiles)
{
if (!ReadProjectsRecursively(ProjectFile, InitialProperties, FileToProjectInfo))
{
OutBuildProducts = null;
OutReferences = null;
return(false);
}
}
// Find all the build products and references
HashSet <FileReference> BuildProducts = new HashSet <FileReference>();
HashSet <FileReference> References = new HashSet <FileReference>();
foreach (KeyValuePair <FileReference, CsProjectInfo> Pair in FileToProjectInfo)
{
CsProjectInfo ProjectInfo = Pair.Value;
// Add the standard build products
DirectoryReference OutputDir = ProjectInfo.GetOutputDir(Pair.Key.Directory);
ProjectInfo.AddBuildProducts(OutputDir, BuildProducts);
// Add the referenced assemblies
foreach (KeyValuePair <FileReference, bool> Reference in ProjectInfo.References)
{
FileReference OtherAssembly = Reference.Key;
if (Reference.Value)
{
// Add reference from the output dir
FileReference OutputFile = FileReference.Combine(OutputDir, OtherAssembly.GetFileName());
BuildProducts.Add(OutputFile);
FileReference OutputSymbolFile = OutputFile.ChangeExtension(".pdb");
if (OutputSymbolFile.Exists())
{
BuildProducts.Add(OutputSymbolFile);
}
}
else
{
// Add reference directly
References.Add(OtherAssembly);
FileReference SymbolFile = OtherAssembly.ChangeExtension(".pdb");
if (SymbolFile.Exists())
{
References.Add(SymbolFile);
}
}
}
// Add build products from all the referenced projects. MSBuild only copy the directly referenced build products, not recursive references or other assemblies.
foreach (CsProjectInfo OtherProjectInfo in ProjectInfo.ProjectReferences.Where(x => x.Value).Select(x => FileToProjectInfo[x.Key]))
{
OtherProjectInfo.AddBuildProducts(OutputDir, BuildProducts);
}
}
// Update the output set
OutBuildProducts = BuildProducts;
OutReferences = References;
return(true);
}
/// <summary>
/// Main entry point for the command
/// </summary>
public override void ExecuteBuild()
{
// Build a lookup of flags to set and clear for each identifier
Dictionary <string, int> IdentifierToIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < MacroPairs.GetLength(0); Idx++)
{
IdentifierToIndex[MacroPairs[Idx, 0]] = Idx;
IdentifierToIndex[MacroPairs[Idx, 1]] = ~Idx;
}
// Check if we want to just parse a single file
string FileParam = ParseParamValue("File");
if (FileParam != null)
{
// Check the file exists
FileReference File = new FileReference(FileParam);
if (!FileReference.Exists(File))
{
throw new AutomationException("File '{0}' does not exist", File);
}
CheckSourceFile(File, IdentifierToIndex, new object());
}
else
{
// Add the additional files to be ignored
foreach (string IgnoreFileName in ParseParamValues("Ignore"))
{
IgnoreFileNames.Add(IgnoreFileName);
}
// Create a list of all the root directories
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise directory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
string[] ProjectParams = ParseParamValues("Project");
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectFile = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Unable to find project '{0}'", ProjectFile);
}
RootDirs.Add(ProjectFile.Directory);
}
// Recurse through the tree
LogInformation("Finding source files...");
List <FileReference> SourceFiles = new List <FileReference>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (DirectoryReference RootDir in RootDirs)
{
DirectoryInfo PluginsDir = new DirectoryInfo(Path.Combine(RootDir.FullName, "Plugins"));
if (PluginsDir.Exists)
{
Queue.Enqueue(() => FindSourceFiles(PluginsDir, SourceFiles, Queue));
}
DirectoryInfo SourceDir = new DirectoryInfo(Path.Combine(RootDir.FullName, "Source"));
if (SourceDir.Exists)
{
Queue.Enqueue(() => FindSourceFiles(SourceDir, SourceFiles, Queue));
}
}
Queue.Wait();
}
// Loop through all the source files
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
object LogLock = new object();
foreach (FileReference SourceFile in SourceFiles)
{
Queue.Enqueue(() => CheckSourceFile(SourceFile, IdentifierToIndex, LogLock));
}
using (LogStatusScope Scope = new LogStatusScope("Checking source files..."))
{
while (!Queue.Wait(10 * 1000))
{
Scope.SetProgress("{0}/{1}", SourceFiles.Count - Queue.NumRemaining, SourceFiles.Count);
}
}
}
}
}
/// <summary>
/// Retrieve an output of the given node. Fetches and decompresses the files from shared storage if necessary, or validates the local files.
/// </summary>
/// <param name="NodeName">The node to retrieve build products for</param>
/// <param name="OutputName">The name of the node's output. May be null.</param>
/// <returns>Manifest of the files retrieved</returns>
public TempStorageManifest Retreive(string NodeName, string OutputName)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// Get the path to the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
bool bLocal = LocalManifestFile.Exists();
// Read the manifest, either from local storage or shared storage
TempStorageManifest Manifest;
if (bLocal)
{
CommandUtils.Log("Reading shared manifest from {0}", LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(LocalManifestFile);
}
else
{
// Check we have shared storage
if (SharedDir == null)
{
throw new AutomationException("Missing local manifest for node - {0}", LocalManifestFile.FullName);
}
// Get the shared directory for this node
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
// Make sure the manifest exists
if (!SharedManifestFile.Exists())
{
throw new AutomationException("Missing local or shared manifest for node - {0}", SharedManifestFile.FullName);
}
// Read the shared manifest
CommandUtils.Log("Copying shared manifest from {0} to {1}", SharedManifestFile.FullName, LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(SharedManifestFile);
// Unzip all the build products
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
FileInfo[] ZipFiles = Manifest.ZipFiles.Select(x => new FileInfo(FileReference.Combine(SharedNodeDir, x.Name).FullName)).ToArray();
ParallelUnzipFiles(ZipFiles, RootDir);
// Fix any Unix permissions/chmod issues, and update the timestamps to match the manifest. Zip files only use local time, and there's no guarantee it matches the local clock.
foreach (TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if (Utils.IsRunningOnMono)
{
CommandUtils.FixUnixFilePermissions(File.FullName);
}
System.IO.File.SetLastWriteTimeUtc(File.FullName, new DateTime(ManifestFile.LastWriteTimeUtcTicks, DateTimeKind.Utc));
}
// Save the manifest locally
LocalManifestFile.Directory.CreateDirectory();
Manifest.Save(LocalManifestFile);
}
// Check all the local files are as expected
bool bAllMatch = true;
foreach (TempStorageFile File in Manifest.Files)
{
bAllMatch &= File.Compare(RootDir);
}
if (!bAllMatch)
{
throw new AutomationException("Files have been modified");
}
// Update the stats and return
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Manifest.Files.Length, Manifest.Files.Sum(x => x.Length), bLocal? 0 : Manifest.ZipFiles.Sum(x => x.Length), bLocal? "Local" : "Remote", 0, 0, OutputName));
return(Manifest);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = ResolveFile(Parameters.Project);
if (!ProjectFile.Exists())
{
CommandUtils.LogError("Couldn't find project '{0}'", ProjectFile.FullName);
return(false);
}
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = UnrealBuildTool.UnrealBuildTool.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = ResolveDirectory(Parameters.ToDir);
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
string ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir.FullName, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, out Receipt))
{
CommandUtils.LogError("Couldn't read receipt '{0}'", ReceiptFileName);
return(false);
}
// Expand all the paths from the receipt
Receipt.ExpandPathVariables(SourceEngineDir, SourceProjectDir);
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts.Where(x => x.Type != BuildProductType.StaticLibrary && x.Type != BuildProductType.ImportLibrary))
{
SourceFiles.Add(new FileReference(BuildProduct.Path));
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.UnionWith(CommandUtils.ResolveFilespec(CommandUtils.RootDirectory, RuntimeDependency.Path, new string[] { ".../*.umap", ".../*.uasset" }));
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !TargetFile.Exists())
{
TargetFile.Directory.CreateDirectory();
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
return(true);
}
public override void ExecuteBuild()
{
string[] ProjectParams = ParseParamValues("Project");
string UpdateDirParam = ParseParamValue("UpdateDir", null);
if (UpdateDirParam == null)
{
throw new AutomationException("Missing -UpdateDir=... parameter");
}
DirectoryReference UpdateDir = new DirectoryReference(UpdateDirParam);
bool bWrite = ParseParam("Write");
// Get all the root dirs
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise edirectory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectLocation = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectLocation))
{
throw new AutomationException("Unable to find project '{0}'", ProjectLocation);
}
RootDirs.Add(ProjectLocation.Directory);
}
// Find all the modules
HashSet <DirectoryReference> ModuleDirs = new HashSet <DirectoryReference>();
foreach (DirectoryReference RootDir in RootDirs)
{
// Find all the modules from the source folder
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Source");
if (DirectoryReference.Exists(SourceDir))
{
foreach (FileReference ModuleFile in DirectoryReference.EnumerateFiles(SourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(ModuleFile.Directory);
}
}
// Find all the modules under the plugins folder
DirectoryReference PluginsDir = DirectoryReference.Combine(RootDir, "Plugins");
foreach (FileReference PluginFile in DirectoryReference.EnumerateFiles(PluginsDir, "*.uplugin", SearchOption.AllDirectories))
{
DirectoryReference PluginSourceDir = DirectoryReference.Combine(PluginFile.Directory, "Source");
if (DirectoryReference.Exists(PluginSourceDir))
{
foreach (FileReference PluginModuleFile in DirectoryReference.EnumerateFiles(PluginSourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(PluginModuleFile.Directory);
}
}
}
}
// Find a mapping from old to new include paths
Dictionary <string, Tuple <string, FileReference> > RemapIncludePaths = new Dictionary <string, Tuple <string, FileReference> >(StringComparer.InvariantCultureIgnoreCase);
foreach (DirectoryReference ModuleDir in ModuleDirs)
{
DirectoryReference ModulePublicDir = DirectoryReference.Combine(ModuleDir, "Public");
if (DirectoryReference.Exists(ModulePublicDir))
{
foreach (FileReference HeaderFile in DirectoryReference.EnumerateFiles(ModulePublicDir, "*.h", SearchOption.AllDirectories))
{
string BaseIncludeFile = HeaderFile.GetFileName();
Tuple <string, FileReference> ExistingIncludeName;
if (RemapIncludePaths.TryGetValue(BaseIncludeFile, out ExistingIncludeName))
{
LogWarning("Multiple include paths for {0}: {1}, {2}", BaseIncludeFile, ExistingIncludeName.Item2, HeaderFile);
}
else
{
RemapIncludePaths.Add(BaseIncludeFile, Tuple.Create(HeaderFile.MakeRelativeTo(ModulePublicDir).Replace('\\', '/'), HeaderFile));
}
}
}
}
// List of folders to exclude from updates
string[] ExcludeFoldersFromUpdate =
{
"Intermediate",
"ThirdParty"
};
// Enumerate all the files to update
HashSet <FileReference> UpdateFiles = new HashSet <FileReference>();
foreach (FileReference UpdateFile in DirectoryReference.EnumerateFiles(UpdateDir, "*", SearchOption.AllDirectories))
{
if (!UpdateFile.ContainsAnyNames(ExcludeFoldersFromUpdate, UpdateDir))
{
if (UpdateFile.HasExtension(".cpp") | UpdateFile.HasExtension(".h") || UpdateFile.HasExtension(".inl"))
{
UpdateFiles.Add(UpdateFile);
}
}
}
// Process all the source files
Dictionary <FileReference, string[]> ModifiedFiles = new Dictionary <FileReference, string[]>();
foreach (FileReference UpdateFile in UpdateFiles)
{
bool bModifiedFile = false;
string[] Lines = FileReference.ReadAllLines(UpdateFile);
for (int Idx = 0; Idx < Lines.Length; Idx++)
{
Match Match = Regex.Match(Lines[Idx], "^(\\s*#\\s*include\\s+\\\")([^\"]+)(\\\".*)$");
if (Match.Success)
{
string IncludePath = Match.Groups[2].Value;
Tuple <string, FileReference> NewIncludePath;
if (RemapIncludePaths.TryGetValue(IncludePath, out NewIncludePath))
{
if (IncludePath != NewIncludePath.Item1)
{
// Log("{0}: Changing {1} -> {2}", UpdateFile, IncludePath, NewIncludePath.Item1);
Lines[Idx] = String.Format("{0}{1}{2}", Match.Groups[1].Value, NewIncludePath.Item1, Match.Groups[3].Value);
bModifiedFile = true;
}
}
}
}
if (bModifiedFile)
{
ModifiedFiles.Add(UpdateFile, Lines);
}
}
// Output them all to disk
if (bWrite && ModifiedFiles.Count > 0)
{
LogInformation("Updating {0} files...", ModifiedFiles.Count);
List <FileReference> FilesToCheckOut = new List <FileReference>();
foreach (FileReference ModifiedFile in ModifiedFiles.Keys)
{
if ((FileReference.GetAttributes(ModifiedFile) & FileAttributes.ReadOnly) != 0)
{
FilesToCheckOut.Add(ModifiedFile);
}
}
if (FilesToCheckOut.Count > 0)
{
if (!P4Enabled)
{
throw new AutomationException("{0} files have been modified, but are read only. Run with -P4 to enable Perforce checkout.\n{1}", FilesToCheckOut.Count, String.Join("\n", FilesToCheckOut.Select(x => " " + x)));
}
LogInformation("Checking out files from Perforce");
int ChangeNumber = P4.CreateChange(Description: "Updating source files");
P4.Edit(ChangeNumber, FilesToCheckOut.Select(x => x.FullName).ToList(), false);
}
foreach (KeyValuePair <FileReference, string[]> FileToWrite in ModifiedFiles)
{
LogInformation("Writing {0}", FileToWrite.Key);
FileReference.WriteAllLines(FileToWrite.Key, FileToWrite.Value);
}
}
}
public override void ExecuteBuild()
{
int WorkingCL = -1;
if (P4Enabled && AllowSubmit)
{
string CmdLine = "";
foreach (var Arg in Params)
{
CmdLine += Arg.ToString() + " ";
}
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("MegaXGE build from changelist {0} - Params: {1}", P4Env.Changelist, CmdLine));
}
LogInformation("************************* MegaXGE");
bool Clean = ParseParam("Clean");
string CleanToolLocation = CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "Batchfiles", "Clean.bat");
bool ShowProgress = ParseParam("Progress");
var UE4Build = new UE4Build(this);
var Agenda = new UE4Build.BuildAgenda();
// we need to always build UHT when we use mega XGE
var ProgramTargets = new string[]
{
"UnrealHeaderTool",
};
Agenda.AddTargets(ProgramTargets, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development);
if (Clean)
{
LogSetProgress(ShowProgress, "Cleaning previous builds...");
foreach (var CurTarget in ProgramTargets)
{
string Args = String.Format("{0} {1} {2}", CurTarget, UnrealTargetPlatform.Win64.ToString(), UnrealTargetConfiguration.Development.ToString());
RunAndLog(CmdEnv, CleanToolLocation, Args);
}
}
LogInformation("*************************");
for (int Arg = 1; Arg < 100; Arg++)
{
string Parm = String.Format("Target{0}", Arg);
string Target = ParseParamValue(Parm, "");
if (String.IsNullOrEmpty(Target))
{
break;
}
FileReference ProjectFile = null;
string ProjectFileParam = ParseParamValue(String.Format("Project{0}", Arg), null);
if (ProjectFileParam != null)
{
ProjectFile = new FileReference(ProjectFileParam);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Project file '{0}' could not be found");
}
}
var Parts = Target.Split(' ');
string JustTarget = Parts[0];
if (String.IsNullOrEmpty(JustTarget))
{
throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target);
}
var Targets = JustTarget.Split('|');
if (Targets.Length < 1)
{
throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target);
}
var Platforms = new List <UnrealTargetPlatform>();
var Configurations = new List <UnrealTargetConfiguration>();
for (int Part = 1; Part < Parts.Length; Part++)
{
if (!String.IsNullOrEmpty(Parts[Part]))
{
var SubParts = Parts[Part].Split('|');
foreach (var SubPart in SubParts)
{
UnrealTargetPlatform Platform;
if (Enum.TryParse(SubPart, true, out Platform))
{
Platforms.Add(Platform);
}
else
{
switch (SubPart.ToUpperInvariant())
{
case "DEBUG":
Configurations.Add(UnrealTargetConfiguration.Debug);
break;
case "DEBUGGAME":
Configurations.Add(UnrealTargetConfiguration.DebugGame);
break;
case "DEVELOPMENT":
Configurations.Add(UnrealTargetConfiguration.Development);
break;
case "SHIPPING":
Configurations.Add(UnrealTargetConfiguration.Shipping);
break;
case "TEST":
Configurations.Add(UnrealTargetConfiguration.Test);
break;
default:
throw new AutomationException("BUILD FAILED target option {0} not recognized.", SubPart);
}
}
}
}
}
if (Platforms.Count < 1)
{
Platforms.Add(UnrealTargetPlatform.Win64);
}
if (Configurations.Count < 1)
{
Configurations.Add(UnrealTargetConfiguration.Development);
}
foreach (var Platform in Platforms)
{
foreach (var CurTarget in Targets)
{
foreach (var Configuration in Configurations)
{
Agenda.AddTargets(new string[] { CurTarget }, Platform, Configuration, ProjectFile);
LogInformation("Target {0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString());
if (Clean)
{
string Args = String.Format("{0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString());
RunAndLog(CmdEnv, CleanToolLocation, Args);
}
}
}
}
}
LogInformation("*************************");
UE4Build.Build(Agenda, InUpdateVersionFiles: IsBuildMachine, InUseParallelExecutor: ParseParam("useparallelexecutor"), InShowProgress: ShowProgress);
// if (WorkingCL > 0) // only move UAT files if we intend to check in some build products
// {
// UE4Build.CopyUATFilesAndAddToBuildProducts();
// }
UE4Build.CheckBuildProducts(UE4Build.BuildProductFiles);
if (WorkingCL > 0)
{
// Sign everything we built
CodeSign.SignMultipleIfEXEOrDLL(this, UE4Build.BuildProductFiles);
// Open files for add or edit
UE4Build.AddBuildProductsToChangelist(WorkingCL, UE4Build.BuildProductFiles);
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
PrintRunTime();
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// Engine non-ufs (binaries)
if (SC.bStageCrashReporter)
{
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(CommandUtils.EngineDirectory, "CrashReportClient", SC.StageTargetPlatform.PlatformType, UnrealTargetConfiguration.Shipping, null);
if (FileReference.Exists(ReceiptFileName))
{
DirectoryReference EngineDir = CommandUtils.EngineDirectory;
DirectoryReference ProjectDir = DirectoryReference.FromFile(Params.RawProjectPath);
TargetReceipt Receipt = TargetReceipt.Read(ReceiptFileName, EngineDir, ProjectDir);
SC.StageBuildProductsFromReceipt(Receipt, true, false);
}
}
// Stage all the build products
foreach (StageTarget Target in SC.StageTargets)
{
SC.StageBuildProductsFromReceipt(Target.Receipt, Target.RequireFilesExist, Params.bTreatNonShippingBinariesAsDebugFiles);
}
// Copy the splash screen, windows specific
FileReference SplashImage = FileReference.Combine(SC.ProjectRoot, "Content", "Splash", "Splash.bmp");
if (FileReference.Exists(SplashImage))
{
SC.StageFile(StagedFileType.NonUFS, SplashImage);
}
// Stage the bootstrap executable
if (!Params.NoBootstrapExe)
{
foreach (StageTarget Target in SC.StageTargets)
{
BuildProduct Executable = Target.Receipt.BuildProducts.FirstOrDefault(x => x.Type == BuildProductType.Executable);
if (Executable != null)
{
// only create bootstraps for executables
List <StagedFileReference> StagedFiles = SC.FilesToStage.NonUFSFiles.Where(x => x.Value == Executable.Path).Select(x => x.Key).ToList();
if (StagedFiles.Count > 0 && Executable.Path.HasExtension(".exe"))
{
string BootstrapArguments = "";
if (!ShouldStageCommandLine(Params, SC))
{
if (!SC.IsCodeBasedProject)
{
BootstrapArguments = String.Format("..\\..\\..\\{0}\\{0}.uproject", SC.ShortProjectName);
}
else
{
BootstrapArguments = SC.ShortProjectName;
}
}
string BootstrapExeName;
if (SC.StageTargetConfigurations.Count > 1)
{
BootstrapExeName = Executable.Path.GetFileName();
}
else if (Params.IsCodeBasedProject)
{
BootstrapExeName = Target.Receipt.TargetName + ".exe";
}
else
{
BootstrapExeName = SC.ShortProjectName + ".exe";
}
foreach (StagedFileReference StagePath in StagedFiles)
{
StageBootstrapExecutable(SC, BootstrapExeName, Executable.Path, StagePath, BootstrapArguments);
}
}
}
}
}
}
public override void ExecuteBuild()
{
var Params = new ProjectParams
(
Command: this,
// Shared
RawProjectPath: ProjectPath
);
LogInformation("********** CRYPTOKEYS COMMAND STARTED **********");
string UE4EditorExe = HostPlatform.Current.GetUE4ExePath(Params.UE4Exe);
if (!FileExists(UE4EditorExe))
{
throw new AutomationException("Missing " + UE4EditorExe + " executable. Needs to be built first.");
}
bool bCycleAllKeys = ParseParam("updateallkeys");
bool bCycleEncryptionKey = bCycleAllKeys || ParseParam("updateencryptionkey");
bool bCycleSigningKey = bCycleAllKeys || ParseParam("updatesigningkey");
if (!bCycleAllKeys && !bCycleEncryptionKey && !bCycleSigningKey)
{
throw new Exception("A target for key cycling must be specified when using the cryptokeys automation script\n\t-updateallkeys: Update all keys\n\t-updateencryptionkey: Update encryption key\n\t-updatesigningkey: Update signing key");
}
FileReference OutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "DefaultCrypto.ini");
FileReference NoRedistOutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "NoRedist", "DefaultCrypto.ini");
FileReference DestinationFile = OutputFile;
// If the project has a DefaultCrypto.ini in a NoRedist folder, we want to copy the newly generated file into that location
if (FileReference.Exists(NoRedistOutputFile))
{
DestinationFile = NoRedistOutputFile;
}
string ChangeDescription = "Automated update of ";
if (bCycleEncryptionKey)
{
ChangeDescription += "encryption";
}
if (bCycleSigningKey)
{
if (bCycleEncryptionKey)
{
ChangeDescription += " and ";
}
ChangeDescription += "signing";
}
ChangeDescription += " key";
if (bCycleEncryptionKey && bCycleSigningKey)
{
ChangeDescription += "s";
}
ChangeDescription += " for project " + Params.ShortProjectName;
P4Connection SubmitP4 = null;
int NewCL = 0;
if (CommandUtils.P4Enabled)
{
SubmitP4 = CommandUtils.P4;
NewCL = SubmitP4.CreateChange(Description: ChangeDescription);
SubmitP4.Revert(String.Format("-k \"{0}\"", DestinationFile.FullName));
SubmitP4.Sync(String.Format("-k \"{0}\"", DestinationFile.FullName), AllowSpew: false);
SubmitP4.Add(NewCL, String.Format("\"{0}\"", DestinationFile.FullName));
SubmitP4.Edit(NewCL, String.Format("\"{0}\"", DestinationFile.FullName));
}
else
{
LogInformation(ChangeDescription);
FileReference.MakeWriteable(OutputFile);
}
string CommandletParams = "";
if (bCycleAllKeys)
{
CommandletParams = "-updateallkeys";
}
else if (bCycleEncryptionKey)
{
CommandletParams = "-updateencryptionkey";
}
else if (bCycleSigningKey)
{
CommandletParams = "-updatesigningkey";
}
RunCommandlet(ProjectPath, UE4EditorExe, "CryptoKeys", CommandletParams);
if (DestinationFile != OutputFile)
{
File.Delete(DestinationFile.FullName);
FileReference.Move(OutputFile, DestinationFile);
}
if (SubmitP4 != null)
{
int ActualCL;
SubmitP4.Submit(NewCL, out ActualCL);
}
}
/// <summary>
/// Reads the contents of the given token
/// </summary>
/// <returns>Contents of the token, or null if it does not exist</returns>
public string ReadTokenFile(FileReference Location)
{
return(FileReference.Exists(Location)? File.ReadAllText(Location.FullName) : null);
}
/// <summary>
/// Entry point for the commandlet
/// </summary>
public override void ExecuteBuild()
{
string OutputDir = ParseParamValue("OutputDir");
string ContentOnlyPlatformsString = ParseParamValue("ContentOnlyPlatforms");
IEnumerable <UnrealTargetPlatform> ContentOnlyPlatforms = Enumerable.Empty <UnrealTargetPlatform>();
if (!String.IsNullOrWhiteSpace(ContentOnlyPlatformsString))
{
ContentOnlyPlatforms = ContentOnlyPlatformsString.Split(';').Where(x => !String.IsNullOrWhiteSpace(x)).Select(x => (UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), x));
}
string AnalyticsTypeOverride = ParseParamValue("AnalyticsTypeOverride");
// Write InstalledBuild.txt to indicate Engine is installed
string InstalledBuildFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/InstalledBuild.txt");
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(InstalledBuildFile));
CommandUtils.WriteAllText(InstalledBuildFile, "");
// Write InstalledBuild.txt to indicate Engine is installed
string Project = ParseParamValue("Project");
if (Project != null)
{
string InstalledProjectBuildFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/InstalledProjectBuild.txt");
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(InstalledProjectBuildFile));
CommandUtils.WriteAllText(InstalledProjectBuildFile, new FileReference(Project).MakeRelativeTo(new DirectoryReference(OutputDir)));
}
string OutputEnginePath = Path.Combine(OutputDir, "Engine");
string OutputBaseEnginePath = Path.Combine(OutputEnginePath, "Config", "BaseEngine.ini");
FileAttributes OutputAttributes = FileAttributes.ReadOnly;
List <String> IniLines = new List <String>();
// Should always exist but if not, we don't need extra line
if (File.Exists(OutputBaseEnginePath))
{
OutputAttributes = File.GetAttributes(OutputBaseEnginePath);
IniLines.Add("");
}
else
{
CommandUtils.CreateDirectory(CommandUtils.GetDirectoryName(OutputBaseEnginePath));
CommandUtils.WriteAllText(OutputBaseEnginePath, "");
OutputAttributes = File.GetAttributes(OutputBaseEnginePath) | OutputAttributes;
}
// Create list of platform configurations installed in a Rocket build
List <InstalledPlatformInfo.InstalledPlatformConfiguration> InstalledConfigs = new List <InstalledPlatformInfo.InstalledPlatformConfiguration>();
// Add the editor platform, otherwise we'll never be able to run UAT
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(UnrealTargetConfiguration.Development, HostPlatform.Current.HostEditorPlatform, TargetRules.TargetType.Editor, "", "", EProjectType.Unknown, false));
foreach (UnrealTargetPlatform CodeTargetPlatform in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (PlatformExports.IsPlatformAvailable(CodeTargetPlatform))
{
string Architecture = PlatformExports.GetDefaultArchitecture(CodeTargetPlatform, null);
// Try to parse additional Architectures from the command line
string Architectures = ParseParamValue(CodeTargetPlatform.ToString() + "Architectures");
string GPUArchitectures = ParseParamValue(CodeTargetPlatform.ToString() + "GPUArchitectures");
// Build a list of pre-compiled architecture combinations for this platform if any
List <string> AllArchNames;
if (!String.IsNullOrWhiteSpace(Architectures) && !String.IsNullOrWhiteSpace(GPUArchitectures))
{
AllArchNames = (from Arch in Architectures.Split('+')
from GPUArch in GPUArchitectures.Split('+')
select "-" + Arch + "-" + GPUArch).ToList();
}
else if (!String.IsNullOrWhiteSpace(Architectures))
{
AllArchNames = Architectures.Split('+').ToList();
}
else
{
AllArchNames = new List <string>();
}
// Check whether this platform should only be used for content based projects
EProjectType ProjectType = ContentOnlyPlatforms.Contains(CodeTargetPlatform) ? EProjectType.Content : EProjectType.Any;
// Allow Content only platforms to be shown as options in all projects
bool bCanBeDisplayed = ProjectType == EProjectType.Content;
foreach (UnrealTargetConfiguration CodeTargetConfiguration in Enum.GetValues(typeof(UnrealTargetConfiguration)))
{
// Need to check for development receipt as we use that for the Engine code in DebugGame
UnrealTargetConfiguration EngineConfiguration = (CodeTargetConfiguration == UnrealTargetConfiguration.DebugGame) ? UnrealTargetConfiguration.Development : CodeTargetConfiguration;
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(new DirectoryReference(OutputEnginePath), "UE4Game", CodeTargetPlatform, EngineConfiguration, Architecture);
if (FileReference.Exists(ReceiptFileName))
{
// Strip the output folder so that this can be used on any machine
string RelativeReceiptFileName = ReceiptFileName.MakeRelativeTo(new DirectoryReference(OutputDir));
// If we have pre-compiled architectures for this platform then add an entry for each of these -
// there isn't a receipt for each architecture like some other platforms
if (AllArchNames.Count > 0)
{
foreach (string Arch in AllArchNames)
{
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, TargetType.Game, Arch, RelativeReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
else
{
InstalledConfigs.Add(new InstalledPlatformInfo.InstalledPlatformConfiguration(CodeTargetConfiguration, CodeTargetPlatform, TargetType.Game, Architecture, RelativeReceiptFileName, ProjectType, bCanBeDisplayed));
}
}
}
}
}
UnrealBuildTool.InstalledPlatformInfo.WriteConfigFileEntries(InstalledConfigs, ref IniLines);
if (!String.IsNullOrEmpty(AnalyticsTypeOverride))
{
// Write Custom Analytics type setting
IniLines.Add("");
IniLines.Add("[Analytics]");
IniLines.Add(String.Format("UE4TypeOverride=\"{0}\"", AnalyticsTypeOverride));
}
// Make sure we can write to the the config file
File.SetAttributes(OutputBaseEnginePath, OutputAttributes & ~FileAttributes.ReadOnly);
File.AppendAllLines(OutputBaseEnginePath, IniLines);
File.SetAttributes(OutputBaseEnginePath, OutputAttributes);
}
/// <summary>
/// Adds a build product to the output list if it exists
/// </summary>
/// <param name="BuildProduct">The build product to add</param>
/// <param name="BuildProducts">List of output build products</param>
public static void AddOptionalBuildProduct(FileReference BuildProduct, HashSet<FileReference> BuildProducts)
{
if(BuildProduct.Exists())
{
BuildProducts.Add(BuildProduct);
}
}
public void StageRuntimeDependenciesFromReceipt(TargetReceipt Receipt, bool RequireDependenciesToExist, bool bUsingPakFile)
{
// Patterns to exclude from wildcard searches. Any maps and assets must be cooked.
List <string> ExcludePatterns = new List <string>();
ExcludePatterns.Add(".../*.umap");
ExcludePatterns.Add(".../*.uasset");
// Also stage any additional runtime dependencies, like ThirdParty DLLs
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
// allow missing files if needed
if ((RequireDependenciesToExist && RuntimeDependency.Type != StagedFileType.DebugNonUFS) || FileReference.Exists(RuntimeDependency.Path))
{
StageFile(RuntimeDependency.Type, RuntimeDependency.Path);
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference
> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Set the Engine directory
DirectoryReference EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.EngineDir))
{
EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.EngineDir);
}
// Set the Project directory
DirectoryReference ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.ProjectDir))
{
ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.ProjectDir);
}
// Resolve the input list
IEnumerable <FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
HashSet <FileReference> Files = new HashSet <FileReference>();
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
throw new AutomationException("Invalid file passed to TagReceipt task ({0})", TargetFile.FullName);
}
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile, EngineDir, ProjectDir, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
if (Parameters.BuildProducts)
{
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if ((String.IsNullOrEmpty(Parameters.BuildProductType) && TargetReceipt.GetStageTypeFromBuildProductType(BuildProduct) == StagedFileType) || BuildProduct.Type == BuildProductType)
{
Files.Add(BuildProduct.Path);
}
}
}
if (Parameters.RuntimeDependencies)
{
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
if (String.IsNullOrEmpty(Parameters.StagedFileType) || RuntimeDependency.Type == StagedFileType)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = RuntimeDependency.Path;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else
{
CommandUtils.LogWarning("File listed as RuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
}
if (Parameters.PrecompiledBuildDependencies)
{
foreach (FileReference PrecompiledBuildDependency in Receipt.PrecompiledBuildDependencies)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = PrecompiledBuildDependency;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else
{
CommandUtils.LogWarning("File listed as PrecompiledBuildDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
if (Parameters.PrecompiledRuntimeDependencies)
{
foreach (FileReference PrecompiledRuntimeDependency in Receipt.PrecompiledRuntimeDependencies)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = PrecompiledRuntimeDependency;
if (FileReference.Exists(DependencyPath))
{
Files.Add(DependencyPath);
}
else
{
CommandUtils.LogWarning("File listed as PrecompiledRuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
}
}
// Apply the tag to all the matching files
FindOrAddTagSet(TagNameToFileSet, Parameters.With).UnionWith(Files);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet<FileReference> BuildProducts, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Set the Engine directory
DirectoryReference EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.EngineDir))
{
EngineDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.EngineDir);
}
// Set the Project directory
DirectoryReference ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine");
if (!String.IsNullOrEmpty(Parameters.ProjectDir))
{
ProjectDir = DirectoryReference.Combine(CommandUtils.RootDirectory, Parameters.ProjectDir);
}
// Resolve the input list
IEnumerable<FileReference> TargetFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
HashSet<FileReference> Files = new HashSet<FileReference>();
HashSet<string> WildcardDependencies = new HashSet<string>();
foreach (FileReference TargetFile in TargetFiles)
{
// check all files are .target files
if (TargetFile.GetExtension() != ".target")
{
CommandUtils.LogError("Invalid file passed to TagReceipt task ({0})",TargetFile.FullName);
continue;
}
// Read the receipt
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(TargetFile.FullName, out Receipt))
{
CommandUtils.LogWarning("Unable to load file using TagReceipt task ({0})", TargetFile.FullName);
continue;
}
// Convert the paths to absolute
Receipt.ExpandPathVariables(EngineDir, ProjectDir);
if (Parameters.BuildProducts)
{
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
if (String.IsNullOrEmpty(Parameters.BuildProductType) || BuildProduct.Type == BuildProductType)
{
Files.Add(new FileReference(BuildProduct.Path));
}
}
}
if (Parameters.RuntimeDependencies)
{
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
if (String.IsNullOrEmpty(Parameters.StagedFileType) || RuntimeDependency.Type == StagedFileType)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(RuntimeDependency.Path) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(RuntimeDependency.Path);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as RuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(RuntimeDependency.Path);
}
}
}
}
if (Parameters.PrecompiledBuildDependencies)
{
foreach(string PrecompiledBuildDependency in Receipt.PrecompiledBuildDependencies)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(PrecompiledBuildDependency) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(PrecompiledBuildDependency);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as PrecompiledBuildDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(PrecompiledBuildDependency);
}
}
}
if (Parameters.PrecompiledRuntimeDependencies)
{
foreach (string PrecompiledRuntimeDependency in Receipt.PrecompiledRuntimeDependencies)
{
// If it doesn't contain any wildcards, just add the pattern directly
if (FileFilter.FindWildcardIndex(PrecompiledRuntimeDependency) == -1)
{
// Only add files that exist as dependencies are assumed to always exist
FileReference DependencyPath = new FileReference(PrecompiledRuntimeDependency);
if (DependencyPath.Exists())
{
Files.Add(DependencyPath);
}
else
{
// Give a warning about files that don't exist so that we can clean up build.cs files
CommandUtils.LogWarning("File listed as PrecompiledRuntimeDependency in {0} does not exist ({1})", TargetFile.FullName, DependencyPath.FullName);
}
}
else
{
WildcardDependencies.Add(PrecompiledRuntimeDependency);
}
}
}
}
// Turn any wildcards into a file list
Files.UnionWith(ResolveFilespecWithExcludePatterns(CommandUtils.RootDirectory, WildcardDependencies.ToList(), new List<string>(), TagNameToFileSet));
// Apply the tag to all the matching files
FindOrAddTagSet(TagNameToFileSet, Parameters.With).UnionWith(Files);
return true;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet<FileReference> BuildProducts, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Figure out the project that this target belongs to
FileReference ProjectFile = null;
if(Parameters.Project != null)
{
ProjectFile = new FileReference(Parameters.Project);
if(!ProjectFile.Exists())
{
CommandUtils.LogError("Missing project file - {0}", ProjectFile.FullName);
return false;
}
}
// Execute the cooker
using(TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", (ProjectFile == null)? "UE4" : ProjectFile.GetFileNameWithoutExtension(), Parameters.Platform))
{
string[] Maps = (Parameters.Maps == null)? null : Parameters.Maps.Split(new char[]{ '+' });
string Arguments = (Parameters.Versioned ? "" : "-Unversioned ") + "-LogCmds=\"LogSavePackage Warning\" " + Parameters.Arguments;
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, Parameters.Platform, Arguments);
}
// Find all the cooked files
List<FileReference> CookedFiles = new List<FileReference>();
foreach(string Platform in Parameters.Platform.Split('+'))
{
DirectoryReference PlatformCookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", Platform);
if(!PlatformCookedDirectory.Exists())
{
CommandUtils.LogError("Cook output directory not found ({0})", PlatformCookedDirectory.FullName);
return false;
}
List<FileReference> PlatformCookedFiles = PlatformCookedDirectory.EnumerateFileReferences("*", System.IO.SearchOption.AllDirectories).ToList();
if(PlatformCookedFiles.Count == 0)
{
CommandUtils.LogError("Cooking did not produce any files in {0}", PlatformCookedDirectory.FullName);
return false;
}
CookedFiles.AddRange(PlatformCookedFiles);
}
// Apply the optional tag to the build products
foreach(string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(CookedFiles);
}
// Add them to the set of build products
BuildProducts.UnionWith(CookedFiles);
return true;
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!PluginFile.Exists())
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (PackageDir.Exists())
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
PackageDir.CreateDirectory();
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!SourceFilterFile.Exists())
{
List <string> Lines = new List <string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
SourceFilterFile.Directory.CreateDirectory();
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.Log("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile, false);
// Compile the plugin for all the target platforms
List <UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List <UnrealTargetPlatform>() : new List <UnrealTargetPlatform> {
BuildHostPlatform.Current.Platform
};
List <UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform);
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, "");
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir);
// Remove the host project
if (!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
/// <summary>
/// Creates a text file with the given contents. If the contents of the text file aren't changed, it won't write the new contents to
/// the file to avoid causing an action to be considered outdated.
/// </summary>
public static FileItem CreateIntermediateTextFile(FileReference AbsolutePath, string Contents)
{
// Create the directory if it doesn't exist.
Directory.CreateDirectory(Path.GetDirectoryName(AbsolutePath.FullName));
// Only write the file if its contents have changed.
if (!AbsolutePath.Exists() || !String.Equals(Utils.ReadAllText(AbsolutePath.FullName), Contents, StringComparison.InvariantCultureIgnoreCase))
{
File.WriteAllText(AbsolutePath.FullName, Contents, GetEncodingForString(Contents));
}
return GetItemByFileReference(AbsolutePath);
}
/// <summary>
/// Generate HTML documentation for all the tasks
/// </summary>
/// <param name="NameToTask">Map of task name to implementation</param>
/// <param name="OutputFile">Output file</param>
static void GenerateDocumentation(Dictionary <string, ScriptTask> NameToTask, FileReference OutputFile)
{
// Find all the assemblies containing tasks
Assembly[] TaskAssemblies = NameToTask.Values.Select(x => x.ParametersClass.Assembly).Distinct().ToArray();
// Read documentation for each of them
Dictionary <string, XmlElement> MemberNameToElement = new Dictionary <string, XmlElement>();
foreach (Assembly TaskAssembly in TaskAssemblies)
{
string XmlFileName = Path.ChangeExtension(TaskAssembly.Location, ".xml");
if (File.Exists(XmlFileName))
{
// Read the document
XmlDocument Document = new XmlDocument();
Document.Load(XmlFileName);
// Parse all the members, and add them to the map
foreach (XmlElement Element in Document.SelectNodes("/doc/members/member"))
{
string Name = Element.GetAttribute("name");
MemberNameToElement.Add(Name, Element);
}
}
}
// Create the output directory
if (FileReference.Exists(OutputFile))
{
FileReference.MakeWriteable(OutputFile);
}
else
{
DirectoryReference.CreateDirectory(OutputFile.Directory);
}
// Write the output file
LogInformation("Writing {0}...", OutputFile);
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
Writer.WriteLine("<html>");
Writer.WriteLine(" <head>");
Writer.WriteLine(" <style>");
Writer.WriteLine(" table { border-collapse: collapse }");
Writer.WriteLine(" table, th, td { border: 1px solid black; }");
Writer.WriteLine(" </style>");
Writer.WriteLine(" </head>");
Writer.WriteLine(" <body>");
Writer.WriteLine(" <h1>BuildGraph Tasks</h1>");
foreach (string TaskName in NameToTask.Keys.OrderBy(x => x))
{
// Get the task object
ScriptTask Task = NameToTask[TaskName];
// Get the documentation for this task
XmlElement TaskElement;
if (MemberNameToElement.TryGetValue("T:" + Task.TaskClass.FullName, out TaskElement))
{
// Write the task heading
Writer.WriteLine(" <h2>{0}</h2>", TaskName);
Writer.WriteLine(" <p>{0}</p>", TaskElement.SelectSingleNode("summary").InnerXml.Trim());
// Start the parameter table
Writer.WriteLine(" <table>");
Writer.WriteLine(" <tr>");
Writer.WriteLine(" <th>Attribute</th>");
Writer.WriteLine(" <th>Type</th>");
Writer.WriteLine(" <th>Usage</th>");
Writer.WriteLine(" <th>Description</th>");
Writer.WriteLine(" </tr>");
// Document the parameters
foreach (string ParameterName in Task.NameToParameter.Keys)
{
// Get the parameter data
ScriptTaskParameter Parameter = Task.NameToParameter[ParameterName];
// Get the documentation for this parameter
XmlElement ParameterElement;
if (MemberNameToElement.TryGetValue("F:" + Parameter.FieldInfo.DeclaringType.FullName + "." + Parameter.Name, out ParameterElement))
{
string TypeName = Parameter.FieldInfo.FieldType.Name;
if (Parameter.ValidationType != TaskParameterValidationType.Default)
{
StringBuilder NewTypeName = new StringBuilder(Parameter.ValidationType.ToString());
for (int Idx = 1; Idx < NewTypeName.Length; Idx++)
{
if (Char.IsLower(NewTypeName[Idx - 1]) && Char.IsUpper(NewTypeName[Idx]))
{
NewTypeName.Insert(Idx, ' ');
}
}
TypeName = NewTypeName.ToString();
}
Writer.WriteLine(" <tr>");
Writer.WriteLine(" <td>{0}</td>", ParameterName);
Writer.WriteLine(" <td>{0}</td>", TypeName);
Writer.WriteLine(" <td>{0}</td>", Parameter.bOptional? "Optional" : "Required");
Writer.WriteLine(" <td>{0}</td>", ParameterElement.SelectSingleNode("summary").InnerXml.Trim());
Writer.WriteLine(" </tr>");
}
}
// Always include the "If" attribute
Writer.WriteLine(" <tr>");
Writer.WriteLine(" <td>If</td>");
Writer.WriteLine(" <td>Condition</td>");
Writer.WriteLine(" <td>Optional</td>");
Writer.WriteLine(" <td>Whether to execute this task. It is ignored if this condition evaluates to false.</td>");
Writer.WriteLine(" </tr>");
// Close the table
Writer.WriteLine(" <table>");
}
}
Writer.WriteLine(" </body>");
Writer.WriteLine("</html>");
}
}