/// <summary>
/// Checks the integrity of the give node's local build products.
/// </summary>
/// <param name="NodeName">The node to retrieve build products for</param>
/// <param name="OutputNames">List of output names from this node.</param>
/// <returns>True if the node is complete and valid, false if not (and typically followed by a call to CleanNode()).</returns>
public bool CheckLocalIntegrity(string NodeName, IEnumerable <string> OutputNames)
{
// If the node is not locally complete, fail immediately.
FileReference CompleteMarkerFile = GetCompleteMarkerFile(LocalDir, NodeName);
if (!CompleteMarkerFile.Exists())
{
return(false);
}
// Check that each of the outputs match
foreach (string OutputName in OutputNames)
{
// Check the local manifest exists
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
if (!LocalManifestFile.Exists())
{
return(false);
}
// Check the local manifest matches the shared manifest
if (SharedDir != null)
{
// Check the shared manifest exists
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
if (!SharedManifestFile.Exists())
{
return(false);
}
// Check the manifests are identical, byte by byte
byte[] LocalManifestBytes = File.ReadAllBytes(LocalManifestFile.FullName);
byte[] SharedManifestBytes = File.ReadAllBytes(SharedManifestFile.FullName);
if (!LocalManifestBytes.SequenceEqual(SharedManifestBytes))
{
return(false);
}
}
// Read the manifest and check the files
TempStorageManifest LocalManifest = TempStorageManifest.Load(LocalManifestFile);
if (LocalManifest.Files.Any(x => !x.Compare(RootDir)))
{
return(false);
}
}
return(true);
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List <string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return(Saver);
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return Files;
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List<string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return DestFiles;
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List<string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return Saver;
}
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List <string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return(Files);
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List <string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List <string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List <string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return(DestFiles);
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
return Files;
}
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder, false), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
if (!FileExists_NoExceptions(SharedManifest))
{
throw new AutomationException("Storage Block Manifest Does Not Exists! {0}", SharedManifest);
}
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
if (!FileExists_NoExceptions(true, Filename))
{
throw new AutomationException("Could not add {0} to manifest because it does not exist", Filename);
}
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
if (!FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Could not copy {0} to {1}", Filename, DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
return DestFiles;
}
/// <summary>
/// Retrieve an output of the given node. Fetches and decompresses the files from shared storage if necessary, or validates the local files.
/// </summary>
/// <param name="NodeName">The node to retrieve build products for</param>
/// <param name="OutputName">The name of the node's output. May be null.</param>
/// <returns>Manifest of the files retrieved</returns>
public TempStorageManifest Retreive(string NodeName, string OutputName)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// Get the path to the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
bool bLocal = LocalManifestFile.Exists();
// Read the manifest, either from local storage or shared storage
TempStorageManifest Manifest;
if (bLocal)
{
CommandUtils.Log("Reading shared manifest from {0}", LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(LocalManifestFile);
}
else
{
// Check we have shared storage
if (SharedDir == null)
{
throw new AutomationException("Missing local manifest for node - {0}", LocalManifestFile.FullName);
}
// Get the shared directory for this node
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
// Make sure the manifest exists
if (!SharedManifestFile.Exists())
{
throw new AutomationException("Missing local or shared manifest for node - {0}", SharedManifestFile.FullName);
}
// Read the shared manifest
CommandUtils.Log("Copying shared manifest from {0} to {1}", SharedManifestFile.FullName, LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(SharedManifestFile);
// Unzip all the build products
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
FileInfo[] ZipFiles = Manifest.ZipFiles.Select(x => new FileInfo(FileReference.Combine(SharedNodeDir, x.Name).FullName)).ToArray();
ParallelUnzipFiles(ZipFiles, RootDir);
// Fix any Unix permissions/chmod issues, and update the timestamps to match the manifest. Zip files only use local time, and there's no guarantee it matches the local clock.
foreach (TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if (Utils.IsRunningOnMono)
{
CommandUtils.FixUnixFilePermissions(File.FullName);
}
System.IO.File.SetLastWriteTimeUtc(File.FullName, new DateTime(ManifestFile.LastWriteTimeUtcTicks, DateTimeKind.Utc));
}
// Save the manifest locally
LocalManifestFile.Directory.CreateDirectory();
Manifest.Save(LocalManifestFile);
}
// Check all the local files are as expected
bool bAllMatch = true;
foreach (TempStorageFile File in Manifest.Files)
{
bAllMatch &= File.Compare(RootDir);
}
if (!bAllMatch)
{
throw new AutomationException("Files have been modified");
}
// Update the stats and return
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Manifest.Files.Length, Manifest.Files.Sum(x => x.Length), bLocal? 0 : Manifest.ZipFiles.Sum(x => x.Length), bLocal? "Local" : "Remote", 0, 0, OutputName));
return(Manifest);
}
}
