/// <summary>
/// Detects root paths for the specified client.
/// </summary>
/// <param name="UATLocation">AutomationTool.exe location</param>
/// <param name="ThisClient">Client to detect the root paths for</param>
/// <param name="BuildRootPath">Build root</param>
/// <param name="LocalRootPath">Local root</param>
/// <param name="ClientRootPath">Client root</param>
private static void DetectRootPaths(P4Connection Connection, string LocalRootPath, P4ClientInfo ThisClient, out string BuildRootPath, out string ClientRootPath)
{
// Figure out the build root
string KnownFilePathFromRoot = CommandEnvironment.KnownFileRelativeToRoot;
string KnownLocalPath = CommandUtils.MakePathSafeToUseWithCommandLine(CommandUtils.CombinePaths(PathSeparator.Slash, LocalRootPath, KnownFilePathFromRoot));
ProcessResult P4Result = Connection.P4(String.Format("files -m 1 {0}", KnownLocalPath), AllowSpew: false);
string KnownFileDepotMapping = P4Result.Output;
// Get the build root
Log.TraceVerbose("Looking for {0} in {1}", KnownFilePathFromRoot, KnownFileDepotMapping);
int EndIdx = KnownFileDepotMapping.IndexOf(KnownFilePathFromRoot, StringComparison.CurrentCultureIgnoreCase);
if (EndIdx < 0)
{
EndIdx = KnownFileDepotMapping.IndexOf(CommandUtils.ConvertSeparators(PathSeparator.Slash, KnownFilePathFromRoot), StringComparison.CurrentCultureIgnoreCase);
}
// Get the root path without the trailing path separator
BuildRootPath = KnownFileDepotMapping.Substring(0, EndIdx - 1);
// Get the client root
if (LocalRootPath.StartsWith(CommandUtils.CombinePaths(PathSeparator.Slash, ThisClient.RootPath, "/"), StringComparison.InvariantCultureIgnoreCase) || LocalRootPath == ThisClient.RootPath)
{
ClientRootPath = CommandUtils.CombinePaths(PathSeparator.Depot, String.Format("//{0}/", ThisClient.Name), LocalRootPath.Substring(ThisClient.RootPath.Length));
}
else
{
throw new AutomationException("LocalRootPath ({0}) does not start with the client root path ({1})", LocalRootPath, ThisClient.RootPath);
}
}
/// <summary>
/// Finds the root path of the branch by looking in progressively higher folder locations until a file known to exist in the branch is found.
/// </summary>
/// <param name="UATLocation">Location of the currently executing assembly.</param>
/// <returns></returns>
private static string RootFromUATLocation(string UATLocation)
{
// pick a known file in the depot and try to build a relative path to it. This will tell us where the root path to the branch is.
var KnownFilePathFromRoot = CommandEnvironment.KnownFileRelativeToRoot;
var CurrentPath = CommandUtils.ConvertSeparators(PathSeparator.Slash, Path.GetDirectoryName(UATLocation));
var UATPathRoot = CommandUtils.CombinePaths(Path.GetPathRoot(CurrentPath), KnownFilePathFromRoot);
// walk parent dirs until we find the file or hit the path root, which means we aren't in a known location.
while (true)
{
var PathToCheck = Path.GetFullPath(CommandUtils.CombinePaths(CurrentPath, KnownFilePathFromRoot));
Log.TraceVerbose("Checking for {0}", PathToCheck);
if (!File.Exists(PathToCheck))
{
var LastSeparatorIndex = CurrentPath.LastIndexOf('/');
if (String.Compare(PathToCheck, UATPathRoot, true) == 0 || LastSeparatorIndex < 0)
{
throw new AutomationException("{0} does not appear to exist at a valid location with the branch, so the local root cannot be determined.", UATLocation);
}
// Step back one directory
CurrentPath = CurrentPath.Substring(0, LastSeparatorIndex);
}
else
{
return(CurrentPath);
}
}
}
/// <summary>
/// Gets the log folder used when running from installed build.
/// </summary>
/// <returns></returns>
private string GetInstalledLogFolder()
{
var LocalRootPath = CommandUtils.GetEnvVar(EnvVarNames.LocalRoot);
var LocalRootEscaped = CommandUtils.ConvertSeparators(PathSeparator.Slash, LocalRootPath.Replace(":", ""));
if (LocalRootEscaped[LocalRootEscaped.Length - 1] == '/')
{
LocalRootEscaped = LocalRootEscaped.Substring(0, LocalRootEscaped.Length - 1);
}
LocalRootEscaped = CommandUtils.EscapePath(LocalRootEscaped);
return(CommandUtils.CombinePaths(PathSeparator.Slash, HostPlatform.Current.LocalBuildsLogFolder, LocalRootEscaped));
}
/// <summary>
/// Gets project properties.
/// </summary>
/// <param name="RawProjectPath">Full project path.</param>
/// <returns>Properties of the project.</returns>
public static ProjectProperties GetProjectProperties(FileReference RawProjectPath, List<UnrealTargetPlatform> ClientTargetPlatforms = null, bool AssetNativizationRequested = false)
{
string ProjectKey = "UE4";
if (RawProjectPath != null)
{
ProjectKey = CommandUtils.ConvertSeparators(PathSeparator.Slash, RawProjectPath.FullName);
}
ProjectProperties Properties;
if (PropertiesCache.TryGetValue(ProjectKey, out Properties) == false)
{
Properties = DetectProjectProperties(RawProjectPath, ClientTargetPlatforms, AssetNativizationRequested);
PropertiesCache.Add(ProjectKey, Properties);
}
return Properties;
}
/// <summary>
/// Gets project properties.
/// </summary>
/// <param name="RawProjectPath">Full project path.</param>
/// <returns>Properties of the project.</returns>
public static ProjectProperties GetProjectProperties(string RawProjectPath, List <UnrealTargetPlatform> ClientTargetPlatforms = null)
{
string ProjectKey = "UE4";
if (!String.IsNullOrEmpty(RawProjectPath))
{
ProjectKey = CommandUtils.ConvertSeparators(PathSeparator.Slash, Path.GetFullPath(RawProjectPath));
}
ProjectProperties Properties;
if (PropertiesCache.TryGetValue(ProjectKey, out Properties) == false)
{
Properties = DetectProjectProperties(RawProjectPath, ClientTargetPlatforms);
PropertiesCache.Add(ProjectKey, Properties);
}
return(Properties);
}
/// <summary>
/// Initializes the environment.
/// </summary>
internal CommandEnvironment()
{
// Get the path to the UAT executable
UATExe = Assembly.GetEntryAssembly().GetOriginalLocation();
if (!CommandUtils.FileExists(UATExe))
{
throw new AutomationException("Could not find AutomationTool.exe. Reflection indicated it was here: {0}", UATExe);
}
// Find the root directory (containing the Engine folder)
LocalRoot = CommandUtils.GetEnvVar(EnvVarNames.LocalRoot);
if (String.IsNullOrEmpty(LocalRoot))
{
LocalRoot = CommandUtils.ConvertSeparators(PathSeparator.Slash, Path.GetFullPath(Path.Combine(Path.GetDirectoryName(UATExe), "..", "..", "..")));
CommandUtils.ConditionallySetEnvVar(EnvVarNames.LocalRoot, LocalRoot);
}
string SavedPath = CommandUtils.GetEnvVar(EnvVarNames.EngineSavedFolder);
if (String.IsNullOrEmpty(SavedPath))
{
SavedPath = CommandUtils.CombinePaths(PathSeparator.Slash, LocalRoot, "Engine", "Programs", "AutomationTool", "Saved");
CommandUtils.SetEnvVar(EnvVarNames.EngineSavedFolder, SavedPath);
}
EngineSavedFolder = CommandUtils.GetEnvVar(EnvVarNames.EngineSavedFolder);
CSVFile = CommandUtils.GetEnvVar(EnvVarNames.CSVFile);
LogFolder = CommandUtils.GetEnvVar(EnvVarNames.LogFolder);
if (String.IsNullOrEmpty(LogFolder))
{
if (GlobalCommandLine.Installed)
{
LogFolder = GetInstalledLogFolder();
}
else
{
LogFolder = CommandUtils.CombinePaths(PathSeparator.Slash, EngineSavedFolder, "Logs");
}
CommandUtils.SetEnvVar(EnvVarNames.LogFolder, LogFolder);
}
// clear the logfolder if we're the only running instance
if (InternalUtils.IsSoleInstance)
{
ClearLogFolder(LogFolder);
}
FinalLogFolder = CommandUtils.GetEnvVar(EnvVarNames.FinalLogFolder);
if (String.IsNullOrEmpty(FinalLogFolder))
{
FinalLogFolder = LogFolder;
CommandUtils.SetEnvVar(EnvVarNames.FinalLogFolder, FinalLogFolder);
}
RobocopyExe = GetSystemExePath("robocopy.exe");
MountExe = GetSystemExePath("mount.exe");
CmdExe = Utils.IsRunningOnMono ? "/bin/sh" : GetSystemExePath("cmd.exe");
MallocNanoZone = "0";
CommandUtils.SetEnvVar(EnvVarNames.MacMallocNanoZone, MallocNanoZone);
int IsChildInstanceInt;
int.TryParse(CommandUtils.GetEnvVar("uebp_UATChildInstance", "0"), out IsChildInstanceInt);
IsChildInstance = (IsChildInstanceInt != 0);
// Setup the timestamp string
DateTime LocalTime = DateTime.Now;
string TimeStamp = LocalTime.Year + "-"
+ LocalTime.Month.ToString("00") + "-"
+ LocalTime.Day.ToString("00") + "_"
+ LocalTime.Hour.ToString("00") + "."
+ LocalTime.Minute.ToString("00") + "."
+ LocalTime.Second.ToString("00");
TimestampAsString = TimeStamp;
SetupBuildEnvironment();
LogSettings();
}
private void SetLocalRootPath()
{
var LocalRootPath = CommandUtils.ConvertSeparators(PathSeparator.Slash, RootFromUATLocation(UATExe));
CommandUtils.ConditionallySetEnvVar(EnvVarNames.LocalRoot, LocalRootPath);
}
/// <summary>
/// Zips a set of files (that must be rooted at the given RootDir) to a set of zip files in the given OutputDir. The files will be prefixed with the given basename.
/// </summary>
/// <param name="Files">Fully qualified list of files to zip (must be rooted at RootDir).</param>
/// <param name="RootDir">Root Directory where all files will be extracted.</param>
/// <param name="OutputDir">Location to place the set of zip files created.</param>
/// <param name="StagingDir">Location to create zip files before copying them to the OutputDir. If the OutputDir is on a remote file share, staging may be more efficient. Use null to avoid using a staging copy.</param>
/// <param name="ZipBaseName">The basename of the set of zip files.</param>
/// <returns>Some metrics about the zip process.</returns>
/// <remarks>
/// This function tries to zip the files in parallel as fast as it can. It makes no guarantees about how many zip files will be created or which files will be in which zip,
/// but it does try to reasonably balance the file sizes.
/// </remarks>
private static FileInfo[] ParallelZipFiles(FileInfo[] InputFiles, DirectoryReference RootDir, DirectoryReference OutputDir, DirectoryReference StagingDir, string ZipBaseName)
{
// First get the sizes of all the files. We won't parallelize if there isn't enough data to keep the number of zips down.
var FilesInfo = InputFiles
.Select(InputFile => new { File = new FileReference(InputFile), FileSize = InputFile.Length })
.ToList();
// Profiling results show that we can zip 100MB quite fast and it is not worth parallelizing that case and creating a bunch of zips that are relatively small.
const long MinFileSizeToZipInParallel = 1024 * 1024 * 100L;
var bZipInParallel = FilesInfo.Sum(FileInfo => FileInfo.FileSize) >= MinFileSizeToZipInParallel;
// order the files in descending order so our threads pick up the biggest ones first.
// We want to end with the smaller files to more effectively fill in the gaps
var FilesToZip = new ConcurrentQueue <FileReference>(FilesInfo.OrderByDescending(FileInfo => FileInfo.FileSize).Select(FileInfo => FileInfo.File));
// We deliberately avoid Parallel.ForEach here because profiles have shown that dynamic partitioning creates
// too many zip files, and they can be of too varying size, creating uneven work when unzipping later,
// as ZipFile cannot unzip files in parallel from a single archive.
// We can safely assume the build system will not be doing more important things at the same time, so we simply use all our logical cores,
// which has shown to be optimal via profiling, and limits the number of resulting zip files to the number of logical cores.
//
// Sadly, mono implementation of System.IO.Compression is really poor (as of 2015/Aug), causing OOM when parallel zipping a large set of files.
// However, Ionic is MUCH slower than .NET's native implementation (2x+ slower in our build farm), so we stick to the faster solution on PC.
// The code duplication in the threadprocs is unfortunate here, and hopefully we can settle on .NET's implementation on both platforms eventually.
List <Thread> ZipThreads;
ConcurrentBag <FileInfo> ZipFiles = new ConcurrentBag <FileInfo>();
DirectoryReference ZipDir = StagingDir ?? OutputDir;
if (Utils.IsRunningOnMono)
{
ZipThreads = (
from CoreNum in Enumerable.Range(0, bZipInParallel ? Environment.ProcessorCount : 1)
let ZipFileName = FileReference.Combine(ZipDir, string.Format("{0}{1}.zip", ZipBaseName, bZipInParallel ? "-" + CoreNum.ToString("00") : ""))
select new Thread(() =>
{
// don't create the zip unless we have at least one file to add
FileReference File;
if (FilesToZip.TryDequeue(out File))
{
// Create one zip per thread using the given basename
using (var ZipArchive = new Ionic.Zip.ZipFile(ZipFileName.FullName)
{
CompressionLevel = Ionic.Zlib.CompressionLevel.BestSpeed
})
{
// pull from the queue until we are out of files.
do
{
// use fastest compression. In our best case we are CPU bound, so this is a good tradeoff,
// cutting overall time by 2/3 while only modestly increasing the compression ratio (22.7% -> 23.8% for RootEditor PDBs).
// This is in cases of a super hot cache, so the operation was largely CPU bound.
ZipArchive.AddFile(File.FullName, CommandUtils.ConvertSeparators(PathSeparator.Slash, File.Directory.MakeRelativeTo(RootDir)));
} while (FilesToZip.TryDequeue(out File));
ZipArchive.Save();
}
// if we are using a staging dir, copy to the final location and delete the staged copy.
FileInfo ZipFile = new FileInfo(ZipFileName.FullName);
if (StagingDir != null)
{
FileInfo NewZipFile = ZipFile.CopyTo(CommandUtils.MakeRerootedFilePath(ZipFile.FullName, StagingDir.FullName, OutputDir.FullName));
ZipFile.Delete();
ZipFile = NewZipFile;
}
ZipFiles.Add(ZipFile);
}
})).ToList();
}
else
{
ZipThreads = (
from CoreNum in Enumerable.Range(0, bZipInParallel ? Environment.ProcessorCount : 1)
let ZipFileName = FileReference.Combine(ZipDir, string.Format("{0}{1}.zip", ZipBaseName, bZipInParallel ? "-" + CoreNum.ToString("00") : ""))
select new Thread(() =>
{
// don't create the zip unless we have at least one file to add
FileReference File;
if (FilesToZip.TryDequeue(out File))
{
// Create one zip per thread using the given basename
using (var ZipArchive = System.IO.Compression.ZipFile.Open(ZipFileName.FullName, System.IO.Compression.ZipArchiveMode.Create))
{
// pull from the queue until we are out of files.
do
{
// use fastest compression. In our best case we are CPU bound, so this is a good tradeoff,
// cutting overall time by 2/3 while only modestly increasing the compression ratio (22.7% -> 23.8% for RootEditor PDBs).
// This is in cases of a super hot cache, so the operation was largely CPU bound.
// Also, sadly, mono appears to have a bug where nothing you can do will properly set the LastWriteTime on the created entry,
// so we have to ignore timestamps on files extracted from a zip, since it may have been created on a Mac.
ZipFileExtensions.CreateEntryFromFile(ZipArchive, File.FullName, CommandUtils.ConvertSeparators(PathSeparator.Slash, File.MakeRelativeTo(RootDir)), System.IO.Compression.CompressionLevel.Fastest);
} while (FilesToZip.TryDequeue(out File));
}
// if we are using a staging dir, copy to the final location and delete the staged copy.
FileInfo ZipFile = new FileInfo(ZipFileName.FullName);
if (StagingDir != null)
{
FileInfo NewZipFile = ZipFile.CopyTo(CommandUtils.MakeRerootedFilePath(ZipFile.FullName, StagingDir.FullName, OutputDir.FullName));
ZipFile.Delete();
ZipFile = NewZipFile;
}
ZipFiles.Add(ZipFile);
}
})).ToList();
}
ZipThreads.ForEach(thread => thread.Start());
ZipThreads.ForEach(thread => thread.Join());
return(ZipFiles.OrderBy(x => x.Name).ToArray());
}