int FindLastNonDuplicateFail(BuildNode NodeToDo, NodeHistory History, JobInfo JobInfo)
{
int Result = P4Env.Changelist;
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix);
List<int> BackwardsFails = new List<int>(History.AllFailed);
BackwardsFails.Add(P4Env.Changelist);
BackwardsFails.Sort();
BackwardsFails.Reverse();
HashSet<string> CurrentErrors = null;
foreach (int CL in BackwardsFails)
{
if (CL > P4Env.Changelist)
{
continue;
}
if (CL <= History.LastSucceeded)
{
break;
}
// Find any local temp storage manifest for this changelist and delete it.
var ThisTempStorageNodeInfo = new TempStorageNodeInfo(JobInfo.CreateWithNewChangelist(CL), TempStorageNodeInfo.NodeStorageName);
TempStorage.DeleteLocalTempStorageManifest(ThisTempStorageNodeInfo, true); // these all clash locally, which is fine we just retrieve them from shared
List<string> Files = null;
try
{
bool WasLocal;
Files = TempStorage.RetrieveFromTempStorage(ThisTempStorageNodeInfo, out WasLocal, GameNameIfAny, CmdEnv.LocalRoot); // this will fail on our CL if we didn't fail or we are just setting up the branch
}
catch (Exception)
{
}
if (Files == null)
{
continue;
}
if (Files.Count != 1)
{
throw new AutomationException("Unexpected number of files for fail record {0}", Files.Count);
}
string ErrorFile = Files[0];
HashSet<string> ThisErrors = ECJobPropsUtils.ErrorsFromProps(ErrorFile);
if (CurrentErrors == null)
{
CurrentErrors = ThisErrors;
}
else
{
if (CurrentErrors.Count == 0 || !HashSetEqual(CurrentErrors, ThisErrors))
{
break;
}
Result = CL;
}
}
return Result;
}
/// <summary>
/// Inverse of <see cref="StoreToTempStorage"/>.
/// Copies a block of files from a temp storage location given by a temp storage node and game to local storage rooted at the given root dir.
/// If the temp manifest for this block is found locally, the copy is skipped, as we assume this is the same machine that created the temp storage and the files are still there.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="WasLocal">upon return, this parameter is set to true if the temp manifest was found locally and the copy was avoided.</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="RootDir">Folder that all the retrieved files should be rooted from. If null or empty, CmdEnv.LocalRoot is used.</param>
/// <returns>List of fully qualified paths to all the files that were retrieved. This is returned even if we skip the copy (set WasLocal = true) .</returns>
public static List<string> RetrieveFromTempStorage(TempStorageNodeInfo TempStorageNodeInfo, out bool WasLocal, string GameName, string RootDir)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// use LocalRoot if one is not specified
if (String.IsNullOrEmpty(RootDir))
{
RootDir = CommandUtils.CmdEnv.LocalRoot;
}
// First see if the local manifest is there.
// If it is, then we must be on the same node as the one that originally created the temp storage.
// In that case, we just verify all the files exist as described in the manifest and use that.
// If there was any tampering, abort immediately because we never accept tampering, and it signifies a build problem.
var LocalManifest = LocalTempStorageManifestFilename(TempStorageNodeInfo);
if (CommandUtils.FileExists_NoExceptions(LocalManifest))
{
CommandUtils.LogVerbose("Found local manifest {0}", LocalManifest);
var Local = TempStorageManifest.Load(LocalManifest);
var Files = Local.GetFiles(RootDir);
var LocalTest = TempStorageManifest.Create(Files, RootDir);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.Local.{3}.{4}.{5}", Files.Count, Local.GetTotalSize(), 0L, 0L, 0L, TempStorageNodeInfo.NodeStorageName));
return Files;
}
WasLocal = false;
// We couldn't find the node storage locally, so get it from the shared location.
var BlockPath = SharedTempStorageDirectory(TempStorageNodeInfo, GameName);
CommandUtils.LogVerbose("Attempting to retrieve from {0}", BlockPath);
if (!CommandUtils.DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
InternalUtils.Robust_FileExists(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = TempStorageManifest.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath, !bZipTempStorage);
// We know the source files exist and are under RootDir because we created the manifest, which verifies it.
// Now create the list of target files
var DestFiles = SharedFiles.Select(Filename => CommandUtils.MakeRerootedFilePath(Filename, BlockPath, RootDir)).ToList();
long ZipFileSize = 0L;
long CopyTimeMS = 0L;
long UnzipTimeMS = 0L;
if (bZipTempStorage)
{
var LocalZipFilename = Path.ChangeExtension(LocalManifest, "zip");
var SharedZipFilename = Path.ChangeExtension(SharedManifest, "zip");
if (!CommandUtils.FileExists(SharedZipFilename))
{
throw new AutomationException("Storage block zip file does not exist! {0}", SharedZipFilename);
}
var CopyTimer = DateTimeStopwatch.Start();
InternalUtils.Robust_CopyFile(SharedZipFilename, LocalZipFilename);
CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
ZipFileSize = new FileInfo(LocalZipFilename).Length;
var UnzipTimer = DateTimeStopwatch.Start();
using (Ionic.Zip.ZipFile Zip = new Ionic.Zip.ZipFile(LocalZipFilename))
{
Zip.ExtractAll(RootDir, Ionic.Zip.ExtractExistingFileAction.OverwriteSilently);
}
CommandUtils.DeleteFile(LocalZipFilename);
UnzipTimeMS = (long)UnzipTimer.ElapsedTime.TotalMilliseconds;
}
else
{
var CopyTimer = DateTimeStopwatch.Start();
// If the local file already exists, it will be overwritten.
foreach (var DestFile in DestFiles)
{
if (CommandUtils.FileExists_NoExceptions(true, DestFile))
{
CommandUtils.LogVerbose("Dest file {0} already exists, deleting and overwriting", DestFile);
CommandUtils.DeleteFile(DestFile);
}
}
// Do the threaded copy to the local file system.
CommandUtils.ThreadedCopyFiles(SharedFiles, DestFiles, ThreadsToCopyWith());
CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
}
// Handle unix permissions/chmod issues.
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
foreach (string DestFile in DestFiles)
{
CommandUtils.FixUnixFilePermissions(DestFile);
}
}
var NewLocal = SaveLocalTempStorageManifest(RootDir, TempStorageNodeInfo, DestFiles);
// Now compare the created local files to ensure their attributes match the one we copied from the network.
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
CommandUtils.RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.Remote.{3}.{4}.{5}", DestFiles.Count, Shared.GetTotalSize(), ZipFileSize, CopyTimeMS, UnzipTimeMS, TempStorageNodeInfo.NodeStorageName));
return DestFiles;
}
}
public void SaveStatus(TempStorageNodeInfo TempStorageNodeInfo, bool bSaveSharedTempStorage, string GameNameIfAny, string JobStepIDForFailure = null)
{
string Contents = "Just a status record: " + TempStorageNodeInfo.NodeStorageName;
if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine)
{
try
{
Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true);
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Failed to get properties for jobstep to save them.");
CommandUtils.LogWarning(LogUtils.FormatException(Ex));
}
}
string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", TempStorageNodeInfo.NodeStorageName +".log");
CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess));
CommandUtils.WriteAllText(RecordOfSuccess, Contents);
TempStorage.StoreToTempStorage(TempStorageNodeInfo, new List<string> { RecordOfSuccess }, !bSaveSharedTempStorage, GameNameIfAny, CommandUtils.CmdEnv.LocalRoot);
}
/// <summary>
/// Checks if a temp storage manifest exists, either locally or in the shared folder (depending on the value of bLocalOnly) for the given temp storage node and game.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="bLocalOnly">If true, only ensures that the local temp storage manifest exists. Otherwise, checks if the manifest exists in either local or shared storage.</param>
/// <param name="bQuiet">True to suppress logging during the operation.</param>
/// <returns>true if the shared temp storage manifest file exists</returns>
public static bool TempStorageExists(TempStorageNodeInfo TempStorageNodeInfo, string GameName, bool bLocalOnly, bool bQuiet)
{
return LocalTempStorageManifestExists(TempStorageNodeInfo, bQuiet) || (!bLocalOnly && SharedTempStorageManifestExists(TempStorageNodeInfo, GameName, bQuiet));
}
/// <summary>
/// Saves the list of fully qualified files (that should be rooted at the shared temp storage location for the game) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="Files">Fully qualified names of files to reference in the manifest file.</param>
/// <param name="bLocalOnly">If true, only ensures that the local temp storage manifest exists. Otherwise, checks if the manifest exists in either local or shared storage.</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="RootDir">Folder that all the given files are rooted from. If null or empty, CmdEnv.LocalRoot is used.</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public static void StoreToTempStorage(TempStorageNodeInfo TempStorageNodeInfo, List<string> Files, bool bLocalOnly, string GameName, string RootDir)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// use LocalRoot if one is not specified
if (string.IsNullOrEmpty(RootDir))
{
RootDir = CommandUtils.CmdEnv.LocalRoot;
}
// save the manifest to local temp storage.
var Local = SaveLocalTempStorageManifest(RootDir, TempStorageNodeInfo, Files);
var LocalTotalSize = Local.GetTotalSize();
if (!bLocalOnly)
{
var BlockPath = SharedTempStorageDirectory(TempStorageNodeInfo, GameName);
CommandUtils.LogConsole("Storing to {0}", BlockPath);
if (CommandUtils.DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Already Exists! {0}", BlockPath);
}
CommandUtils.CreateDirectory(true, BlockPath);
if (!CommandUtils.DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Could Not Be Created! {0}", BlockPath);
}
// We know the source files exist and are under RootDir because we created the manifest, which verifies it.
// Now create the list of target files
var DestFiles = Files.Select(Filename => CommandUtils.MakeRerootedFilePath(Filename, RootDir, BlockPath)).ToList();
// make sure the target file doesn't exist, as we never expect this.
foreach (var DestFile in DestFiles)
{
if (CommandUtils.FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Dest file {0} already exists.", DestFile);
}
}
if (bZipTempStorage)
{
var ZipTimer = DateTimeStopwatch.Start();
var Zip = new Ionic.Zip.ZipFile
{
UseZip64WhenSaving = Ionic.Zip.Zip64Option.Always
};
foreach (string File in Files)
{
// strip the root dir off each file so we get a proper relative path.
Zip.AddFile(Path.Combine(RootDir, File), Path.GetDirectoryName(CommandUtils.StripBaseDirectory(File, RootDir)));
}
var LocalManifest = LocalTempStorageManifestFilename(TempStorageNodeInfo);
var SharedManifest = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
var ZipFilename = Path.ChangeExtension(LocalManifest, "zip");
Zip.Save(ZipFilename);
var ZipFileSize = new FileInfo(ZipFilename).Length;
var ZipTimeMS = (long)ZipTimer.ElapsedTime.TotalMilliseconds;
var CopyTimer = DateTimeStopwatch.Start();
InternalUtils.Robust_CopyFile(ZipFilename, CommandUtils.CombinePaths(BlockPath, Path.GetFileName(ZipFilename)));
var CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
CommandUtils.DeleteFile(ZipFilename);
// copy the local manifest to the shared location. We have to assume the zip is a good copy.
CommandUtils.CopyFile(LocalManifest, SharedManifest);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.Remote.{3}.{4}.{5}", Files.Count, LocalTotalSize, ZipFileSize, CopyTimeMS, ZipTimeMS, TempStorageNodeInfo.NodeStorageName));
}
else
{
var CopyTimer = DateTimeStopwatch.Start();
CommandUtils.ThreadedCopyFiles(Files, DestFiles, ThreadsToCopyWith());
var CopyTimeMS = (long)CopyTimer.ElapsedTime.TotalMilliseconds;
// save the shared temp storage manifest file.
var Shared = SaveSharedTempStorageManifest(TempStorageNodeInfo, GameName, DestFiles);
// These manifests better have the same files, timestamps, and sizes, else something failed in the copy.
if (!Local.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
var SharedTempManifestFilename = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
CommandUtils.RenameFile_NoExceptions(SharedTempManifestFilename, SharedTempManifestFilename + ".broken");
throw new AutomationException("Shared and Local manifest mismatch. Wrote out {0} for inspection.", SharedTempManifestFilename + ".broken");
}
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.Remote.{3}.{4}.{5}", Files.Count, LocalTotalSize, 0L, CopyTimeMS, 0L, TempStorageNodeInfo.NodeStorageName));
}
}
else
{
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.Local.{3}.{4}.{5}", Files.Count, LocalTotalSize, 0L, 0L, 0L, TempStorageNodeInfo.NodeStorageName));
}
}
}
/// <summary>
/// Runs a test of the temp storage system. This function is part of the class so it doesn't have to expose internals just to run the test.
/// </summary>
/// <param name="CmdEnv">The environment to use.</param>
static internal void TestTempStorage(CommandEnvironment CmdEnv)
{
// We are not a real GUBP job, so fake the values.
var TestTempStorageInfo = new TempStorageNodeInfo(new GUBP.JobInfo("Test", 0, 0, 0), "Test");
// Delete any local and shared temp storage that may exist.
TempStorage.DeleteLocalTempStorageManifests();
TempStorage.DeleteSharedTempStorage(TestTempStorageInfo, "UE4");
if (TempStorage.TempStorageExists(TestTempStorageInfo, "UE4", false, false))
{
throw new AutomationException("storage should not exist");
}
// Create some test files in various locations in the local root with unique content.
var TestFiles = new[]
{
CommandUtils.CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "Batchfiles", "TestFile.Txt"),
CommandUtils.CombinePaths(CmdEnv.LocalRoot, "TestFile2.Txt"),
CommandUtils.CombinePaths(CmdEnv.LocalRoot, "engine", "plugins", "TestFile3.Txt"),
}
.Select(TestFile => new
{
FileName = TestFile,
FileContents = string.Format("{0} - {1}", TestFile, DateTime.UtcNow)
})
.ToList();
foreach (var TestFile in TestFiles)
{
CommandUtils.Log("Test file {0}", TestFile.FileName);
File.WriteAllText(TestFile.FileName, TestFile.FileContents);
}
// wrap the operation so we are sure to clean up afterward.
try
{
// Store the test file to temp storage.
TempStorage.StoreToTempStorage(TestTempStorageInfo, TestFiles.Select(TestFile => TestFile.FileName).ToList(), false, "UE4", CmdEnv.LocalRoot);
// The manifest should exist locally.
if (!TempStorage.LocalTempStorageManifestExists(TestTempStorageInfo))
{
throw new AutomationException("local storage should exist");
}
// The manifest should exist on the shared drive.
if (!TempStorage.SharedTempStorageManifestExists(TestTempStorageInfo, "UE4", false))
{
throw new AutomationException("shared storage should exist");
}
// Now delete the local manifest
TempStorage.DeleteLocalTempStorageManifests();
// It should no longer be there.
if (TempStorage.LocalTempStorageManifestExists(TestTempStorageInfo))
{
throw new AutomationException("local storage should not exist");
}
// But the shared storage should still exist.
if (!TempStorage.TempStorageExists(TestTempStorageInfo, "UE4", false, false))
{
throw new AutomationException("some storage should exist");
}
// Ok, delete our test files locally.
foreach (var TestFile in TestFiles)
{
CommandUtils.DeleteFile(TestFile.FileName);
}
// Now we should be able to retrieve the test files from shared storage.
bool WasLocal;
TempStorage.RetrieveFromTempStorage(TestTempStorageInfo, out WasLocal, "UE4", CmdEnv.LocalRoot);
// the local manifest should be there, since we just retrieved from shared storage.
if (!TempStorage.LocalTempStorageManifestExists(TestTempStorageInfo))
{
throw new AutomationException("local storage should exist");
}
// The shared manifest should also still be there.
if (!TempStorage.SharedTempStorageManifestExists(TestTempStorageInfo, "UE4", false))
{
throw new AutomationException("shared storage should exist");
}
// Verify the retrieved files have the correct content.
foreach (var TestFile in TestFiles)
{
if (File.ReadAllText(TestFile.FileName) != TestFile.FileContents)
{
throw new AutomationException("Contents of the test file {0} was changed after restoring from shared temp storage.", TestFile.FileName);
}
}
// Now delete the shared temp storage
TempStorage.DeleteSharedTempStorage(TestTempStorageInfo, "UE4");
// Shared temp storage manifest should no longer exist.
if (TempStorage.SharedTempStorageManifestExists(TestTempStorageInfo, "UE4", false))
{
throw new AutomationException("shared storage should not exist");
}
// Retrieving temp storage should now just retrieve from local
TempStorage.RetrieveFromTempStorage(TestTempStorageInfo, out WasLocal, "UE4", CmdEnv.LocalRoot);
if (!WasLocal || !TempStorage.LocalTempStorageManifestExists(TestTempStorageInfo))
{
throw new AutomationException("local storage should exist");
}
// and now lets test tampering
TempStorage.DeleteLocalTempStorageManifests();
{
bool bFailedProperly = false;
var MissingFile = new List<string>(TestFiles.Select(TestFile => TestFile.FileName));
// add a file to the manifest that shouldn't be there.
MissingFile.Add(CommandUtils.CombinePaths(CmdEnv.LocalRoot, "Engine", "SomeFileThatDoesntExist.txt"));
try
{
TempStorage.StoreToTempStorage(TestTempStorageInfo, MissingFile, false, "UE4", CmdEnv.LocalRoot);
}
catch (AutomationException)
{
bFailedProperly = true;
}
if (!bFailedProperly)
{
throw new AutomationException("Missing file did not fail.");
}
}
// clear the shared temp storage again.
TempStorage.DeleteSharedTempStorage(TestTempStorageInfo, "UE4");
// store the test files to shared temp storage again.
TempStorage.StoreToTempStorage(TestTempStorageInfo, TestFiles.Select(TestFile => TestFile.FileName).ToList(), false, "UE4", CmdEnv.LocalRoot);
// force a load from shared by deleting the local manifest
TempStorage.DeleteLocalTempStorageManifests();
if (bZipTempStorage)
{
var ZipFile = Path.ChangeExtension(TempStorage.SharedTempStorageManifestFilename(TestTempStorageInfo, "UE4"), "zip");
// the shared storage file should exist.
if (!CommandUtils.FileExists_NoExceptions(ZipFile))
{
throw new AutomationException("Shared file {0} did not exist", ZipFile);
}
// delete the shared file.
CommandUtils.DeleteFile(ZipFile);
}
else
{
var SharedFile = CommandUtils.MakeRerootedFilePath(TestFiles[0].FileName, CmdEnv.LocalRoot, TempStorage.SharedTempStorageDirectory(TestTempStorageInfo, "UE4"));
// the shared storage file should exist.
if (!CommandUtils.FileExists_NoExceptions(SharedFile))
{
throw new AutomationException("Shared file {0} did not exist", SharedFile);
}
// delete the shared file.
CommandUtils.DeleteFile(SharedFile);
}
// now test that retrieving from shared temp storage properly balks that a file is missing.
{
bool bFailedProperly = false;
try
{
TempStorage.RetrieveFromTempStorage(TestTempStorageInfo, out WasLocal, "UE4", CmdEnv.LocalRoot);
}
catch (AutomationException)
{
bFailedProperly = true;
}
if (!bFailedProperly)
{
throw new AutomationException("Did not fail to load from missing file.");
}
}
// clear the shared temp storage again.
TempStorage.DeleteSharedTempStorage(TestTempStorageInfo, "UE4");
// Copy the files to temp storage.
TempStorage.StoreToTempStorage(TestTempStorageInfo, TestFiles.Select(TestFile => TestFile.FileName).ToList(), false, "UE4", CmdEnv.LocalRoot);
// Delete a local file.
CommandUtils.DeleteFile(TestFiles[0].FileName);
// retrieving from temp storage should use WasLocal, but should balk because a local file was deleted.
{
bool bFailedProperly = false;
try
{
TempStorage.RetrieveFromTempStorage(TestTempStorageInfo, out WasLocal, "UE4", CmdEnv.LocalRoot);
}
catch (AutomationException)
{
bFailedProperly = true;
}
if (!bFailedProperly)
{
throw new AutomationException("Did not fail to load from missing local file.");
}
}
}
finally
{
// Do a final cleanup.
TempStorage.DeleteSharedTempStorage(TestTempStorageInfo, "UE4");
TempStorage.DeleteLocalTempStorageManifests();
foreach (var TestFile in TestFiles)
{
CommandUtils.DeleteFile(TestFile.FileName);
}
}
}
/// <summary>
/// Checks if a shared temp storage manifest exists for the given temp storage node and game.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="bQuiet">True to suppress logging during the operation.</param>
/// <returns>true if the shared temp storage manifest file exists</returns>
private static bool SharedTempStorageManifestExists(TempStorageNodeInfo TempStorageNodeInfo, string GameName, bool bQuiet)
{
var SharedManifest = SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName);
if (CommandUtils.FileExists_NoExceptions(bQuiet, SharedManifest))
{
return true;
}
return false;
}
/// <summary>
/// Deletes all shared temp storage (file and manifest) for the given temp storage node and game.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
private static void DeleteSharedTempStorage(TempStorageNodeInfo TempStorageNodeInfo, string GameName)
{
CommandUtils.DeleteDirectory(true, SharedTempStorageDirectory(TempStorageNodeInfo, GameName));
}
/// <summary>
/// Checks if a local temp storage manifest exists for the given temp storage node.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="bQuiet">True to suppress logging during the operation.</param>
/// <returns>true if the local temp storage manifest file exists</returns>
public static bool LocalTempStorageManifestExists(TempStorageNodeInfo TempStorageNodeInfo, bool bQuiet = false)
{
var LocalManifest = LocalTempStorageManifestFilename(TempStorageNodeInfo);
return CommandUtils.FileExists_NoExceptions(bQuiet, LocalManifest);
}
/// <summary>
/// Gets the name of the temp storage manifest file for the given temp storage node and game (ie, P:\Builds\GameName\TmpStore\NodeInfoDirectory\NodeInfoFilename.TempManifest)
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <returns>The name of the temp storage manifest file for the given storage block name for the given game.</returns>
private static string SharedTempStorageManifestFilename(TempStorageNodeInfo TempStorageNodeInfo, string GameName)
{
return CommandUtils.CombinePaths(SharedTempStorageDirectory(TempStorageNodeInfo, GameName), TempStorageNodeInfo.GetManifestFilename());
}
/// <summary>
/// Saves the list of fully qualified files (that should be rooted at the shared temp storage location for the game) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <param name="Files">Fully qualified names of files to reference in the manifest file.</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
private static TempStorageManifest SaveSharedTempStorageManifest(TempStorageNodeInfo TempStorageNodeInfo, string GameName, List<string> Files)
{
return SaveTempStorageManifest(SharedTempStorageDirectory(TempStorageNodeInfo, GameName), SharedTempStorageManifestFilename(TempStorageNodeInfo, GameName), Files);
}
/// <summary>
/// Saves the list of fully qualified files rooted at RootDir to a local temp storage manifest with the given temp storage node.
/// </summary>
/// <param name="RootDir">Folder that all the given files are rooted from.</param>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="Files">Fully qualified names of files to reference in the manifest file.</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
private static TempStorageManifest SaveLocalTempStorageManifest(string RootDir, TempStorageNodeInfo TempStorageNodeInfo, List<string> Files)
{
return SaveTempStorageManifest(RootDir, LocalTempStorageManifestFilename(TempStorageNodeInfo), Files);
}
/// <summary>
/// Gets the name of the local temp storage manifest file for the given temp storage node (ie, Engine\Saved\TmpStore\NodeInfoDirectory\NodeInfoFilename.TempManifest)
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <returns>The name of the temp storage manifest file for the given storage block name for the given game.</returns>
private static string LocalTempStorageManifestFilename(TempStorageNodeInfo TempStorageNodeInfo)
{
return CommandUtils.CombinePaths(LocalTempStorageManifestDirectory(), TempStorageNodeInfo.GetManifestFilename());
}
/// <summary>
/// Returns the full path to the temp storage directory for the given temp storage node and game (ie, P:\Builds\GameName\TmpStore\NodeInfoDirectory)
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <returns>The full path to the temp storage directory for the given storage block name for the given game.</returns>
private static string SharedTempStorageDirectory(TempStorageNodeInfo TempStorageNodeInfo, string GameName)
{
return CommandUtils.CombinePaths(ResolveSharedTempStorageDirectory(GameName), TempStorageNodeInfo.GetRelativeDirectory());
}
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (NodeToDo.Node.BuildProducts != null)
{
throw new AutomationException("topological sort error");
}
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage;
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
if (string.IsNullOrEmpty(StorageRootIfAny))
{
StorageRootIfAny = CmdEnv.LocalRoot;
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.DoFakeBuild();
}
else
{
LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.DoBuild();
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
//@todo: This is a dup of non-exception code. Consolidate this!!
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
LogConsole("{0}", Ex);
if (History != null)
{
LogConsole("Changes since last green *********************************");
LogConsole("");
LogConsole("");
LogConsole("");
PrintDetailedChanges(History, P4Env.Changelist);
LogConsole("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
/// <summary>
/// Deletes a local temp storage manifest file for the given temp storage node if it exists.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="bQuiet">True to suppress logging during the operation.</param>
public static void DeleteLocalTempStorageManifest(TempStorageNodeInfo TempStorageNodeInfo, bool bQuiet = false)
{
var LocalManifest = LocalTempStorageManifestFilename(TempStorageNodeInfo);
CommandUtils.DeleteFile(bQuiet, LocalManifest);
}
static void FindCompletionState(IEnumerable<BuildNode> NodesToDo, JobInfo JobInfo, bool LocalOnly)
{
foreach(BuildNode NodeToDo in NodesToDo)
{
// Construct the full temp storage node info
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
if (LocalOnly)
{
NodeToDo.IsComplete = TempStorage.LocalTempStorageManifestExists(TempStorageNodeInfo, bQuiet: true);
}
else
{
NodeToDo.IsComplete = TempStorage.TempStorageExists(TempStorageNodeInfo, GameNameIfAny, bLocalOnly: false, bQuiet: true);
if(GameNameIfAny != "" && !NodeToDo.IsComplete)
{
NodeToDo.IsComplete = TempStorage.TempStorageExists(TempStorageNodeInfo, "UE4", bLocalOnly: false, bQuiet: true);
}
}
LogVerbose("** {0}", NodeToDo.Name);
if (!NodeToDo.IsComplete)
{
LogVerbose("***** GUBP Trigger Node was already triggered {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
}
else
{
LogVerbose("***** GUBP Trigger Node was NOT yet triggered {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
}
}
}
/// <summary>
/// Finds temp storage manifests that match the block name (where the block name has embedded wildcards to find all block names of a given pattern).
/// This is used by <see cref="GUBP.FindNodeHistory"/> to search for any temp storage manifests that match a certain GUBP node name.
/// This is used to construct a "history" of that node which is then used to log info and generate failure emails.
/// This method is used multiple times to find nodes matching a specific name with a suffix of _Started _Failed and _Succeeded.
/// The CL# is pulled from those names and used to generate a P4 history.
/// Main problem with this is that temp storage is ephemeral, so after a few days, this will typically not find any history for a node.
///
/// NOTE: This entire routine is dependent on how TempStorageNodeInfo lays out its directories!!!
/// If this layout is changed, this code needs to be changed as well.
/// @todo: Someday Make FindNodeHistory query the build database directly to get a true history of the node.
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <returns></returns>
public static List<int> FindMatchingSharedTempStorageNodeCLs(TempStorageNodeInfo TempStorageNodeInfo, string GameName)
{
// Find the shared temp storage folder for this game and branch
var SharedTempStorageDirectoryForGameAndBranch = CommandUtils.CombinePaths(ResolveSharedTempStorageDirectory(GameName), TempStorageNodeInfo.JobInfo.BranchNameForTempStorage);
int dummy;
return (
// Look for all folders underneath this, it should be a CL, or CL-PreflightInfo
from CLDir in Directory.EnumerateDirectories(SharedTempStorageDirectoryForGameAndBranch)
// only accept folders that are plain numbers. Any suffixes ('-') imply preflight builds or something else.
let PossibleCL = Path.GetFileName(CLDir)
where int.TryParse(PossibleCL, out dummy)
let CL = int.Parse(PossibleCL)
// if we have a real CL, try to find the node name we are looking for.
where File.Exists(CommandUtils.CombinePaths(CLDir, TempStorageNodeInfo.NodeStorageName, TempStorageNodeInfo.GetManifestFilename()))
select CL)
.OrderBy(CL => CL).ToList();
}
