/// <summary>
/// Checks that a manifest matches the files on disk
/// </summary>
/// <param name="RootDir">Root directory for relative paths in the manifest</param>
/// <param name="Manifest">Manifest to check</param>
/// <param name="Files">Mapping of filename to timestamp as expected in the manifest</param>
static void CheckManifest(DirectoryReference RootDir, TempStorageManifest Manifest, Dictionary <FileReference, DateTime> Files)
{
if (Files.Count != Manifest.Files.Length)
{
throw new AutomationException("Number of files in manifest does not match");
}
foreach (TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if (!File.Exists())
{
throw new AutomationException("File in manifest does not exist");
}
DateTime OriginalTime;
if (!Files.TryGetValue(File, out OriginalTime))
{
throw new AutomationException("File in manifest did not exist previously");
}
double DiffSeconds = (new FileInfo(File.FullName).LastWriteTimeUtc - OriginalTime).TotalSeconds;
if (Math.Abs(DiffSeconds) > 2)
{
throw new AutomationException("Incorrect timestamp for {0}", ManifestFile.RelativePath);
}
}
}
/// <summary>
/// Checks the integrity of the give node's local build products.
/// </summary>
/// <param name="NodeName">The node to retrieve build products for</param>
/// <param name="OutputNames">List of output names from this node.</param>
/// <returns>True if the node is complete and valid, false if not (and typically followed by a call to CleanNode()).</returns>
public bool CheckLocalIntegrity(string NodeName, IEnumerable <string> OutputNames)
{
// If the node is not locally complete, fail immediately.
FileReference CompleteMarkerFile = GetCompleteMarkerFile(LocalDir, NodeName);
if (!CompleteMarkerFile.Exists())
{
return(false);
}
// Check that each of the outputs match
foreach (string OutputName in OutputNames)
{
// Check the local manifest exists
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
if (!LocalManifestFile.Exists())
{
return(false);
}
// Check the local manifest matches the shared manifest
if (SharedDir != null)
{
// Check the shared manifest exists
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
if (!SharedManifestFile.Exists())
{
return(false);
}
// Check the manifests are identical, byte by byte
byte[] LocalManifestBytes = File.ReadAllBytes(LocalManifestFile.FullName);
byte[] SharedManifestBytes = File.ReadAllBytes(SharedManifestFile.FullName);
if (!LocalManifestBytes.SequenceEqual(SharedManifestBytes))
{
return(false);
}
}
// Read the manifest and check the files
TempStorageManifest LocalManifest = TempStorageManifest.Load(LocalManifestFile);
if (LocalManifest.Files.Any(x => !x.Compare(RootDir)))
{
return(false);
}
}
return(true);
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List <string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return(Saver);
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which these build products belong to</param>
/// <param name="OutputName">The output name of the node.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string OutputName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using (TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if (bRemote)
{
// Create the shared directory for this node
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
SharedNodeDir.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedNodeDir, LocalNodeDir, OutputName);
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, OutputName));
return(Manifest);
}
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
// Create the mapping of tag names to file sets
Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();
// Read all the input tags for this node, and build a list of referenced input storage blocks
HashSet <TempStorageBlock> InputStorageBlocks = new HashSet <TempStorageBlock>();
foreach (NodeOutput Input in Node.Inputs)
{
TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
InputStorageBlocks.UnionWith(FileList.Blocks);
}
// Read the manifests for all the input storage blocks
Dictionary <TempStorageBlock, TempStorageManifest> InputManifests = new Dictionary <TempStorageBlock, TempStorageManifest>();
foreach (TempStorageBlock InputStorageBlock in InputStorageBlocks)
{
TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
InputManifests[InputStorageBlock] = Manifest;
}
// Read all the input storage blocks, keeping track of which block each file came from
Dictionary <FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary <FileReference, TempStorageBlock>();
foreach (KeyValuePair <TempStorageBlock, TempStorageManifest> Pair in InputManifests)
{
TempStorageBlock InputStorageBlock = Pair.Key;
foreach (FileReference File in Pair.Value.Files.Select(x => x.ToFileReference(RootDir)))
{
TempStorageBlock CurrentStorageBlock;
if (FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock) && !TempStorage.IsDuplicateBuildProduct(File))
{
LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
}
FileToStorageBlock[File] = InputStorageBlock;
}
}
// Add placeholder outputs for the current node
foreach (NodeOutput Output in Node.Outputs)
{
TagNameToFileSet.Add(Output.TagName, new HashSet <FileReference>());
}
// Execute the node
if (bWithBanner)
{
Console.WriteLine();
CommandUtils.LogInformation("========== Starting: {0} ==========", Node.Name);
}
if (!Node.Build(Job, TagNameToFileSet))
{
return(false);
}
if (bWithBanner)
{
CommandUtils.LogInformation("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Check that none of the inputs have been clobbered
Dictionary <string, string> ModifiedFiles = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (TempStorageFile File in InputManifests.Values.SelectMany(x => x.Files))
{
string Message;
if (!ModifiedFiles.ContainsKey(File.RelativePath) && !File.Compare(CommandUtils.RootDirectory, out Message))
{
ModifiedFiles.Add(File.RelativePath, Message);
}
}
if (ModifiedFiles.Count > 0)
{
throw new AutomationException("Build {0} from a previous step have been modified:\n{1}", (ModifiedFiles.Count == 1)? "product" : "products", String.Join("\n", ModifiedFiles.Select(x => x.Value)));
}
// Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
HashSet <FileReference> ReferencedOutputFiles = new HashSet <FileReference>();
foreach (Agent Agent in Graph.Agents)
{
bool bSameAgent = Agent.Nodes.Contains(Node);
foreach (Node OtherNode in Agent.Nodes)
{
if (!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
foreach (NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
{
ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
}
}
}
}
// Find a block name for all new outputs
Dictionary <FileReference, string> FileToOutputName = new Dictionary <FileReference, string>();
foreach (NodeOutput Output in Node.Outputs)
{
HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
foreach (FileReference File in Files)
{
if (!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
{
if (Output == Node.DefaultOutput)
{
if (!FileToOutputName.ContainsKey(File))
{
FileToOutputName[File] = "";
}
}
else
{
string OutputName;
if (FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
{
FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
}
else
{
FileToOutputName[File] = Output.TagName.Substring(1);
}
}
}
}
}
// Invert the dictionary to make a mapping of storage block to the files each contains
Dictionary <string, HashSet <FileReference> > OutputStorageBlockToFiles = new Dictionary <string, HashSet <FileReference> >();
foreach (KeyValuePair <FileReference, string> Pair in FileToOutputName)
{
HashSet <FileReference> Files;
if (!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
{
Files = new HashSet <FileReference>();
OutputStorageBlockToFiles.Add(Pair.Value, Files);
}
Files.Add(Pair.Key);
}
// Write all the storage blocks, and update the mapping from file to storage block
foreach (KeyValuePair <string, HashSet <FileReference> > Pair in OutputStorageBlockToFiles)
{
TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
foreach (FileReference File in Pair.Value)
{
FileToStorageBlock.Add(File, OutputBlock);
}
Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
}
// Publish all the output tags
foreach (NodeOutput Output in Node.Outputs)
{
HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
HashSet <TempStorageBlock> StorageBlocks = new HashSet <TempStorageBlock>();
foreach (FileReference File in Files)
{
TempStorageBlock StorageBlock;
if (FileToStorageBlock.TryGetValue(File, out StorageBlock))
{
StorageBlocks.Add(StorageBlock);
}
}
Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return(true);
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return Files;
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List<string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return DestFiles;
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List<string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return Saver;
}
public bool Compare(TempStorageManifest Other)
{
if (Directories.Count != Other.Directories.Count)
{
Log(System.Diagnostics.TraceEventType.Error, "Directory count mismatch {0} {1}", Directories.Count, Other.Directories.Count);
foreach (KeyValuePair<string, List<TempStorageFileInfo>> Directory in Directories)
{
List<TempStorageFileInfo> OtherDirectory;
if (Other.Directories.TryGetValue(Directory.Key, out OtherDirectory) == false)
{
Log(System.Diagnostics.TraceEventType.Error, "Missing Directory {0}", Directory.Key);
return false;
}
}
foreach (KeyValuePair<string, List<TempStorageFileInfo>> Directory in Other.Directories)
{
List<TempStorageFileInfo> OtherDirectory;
if (Directories.TryGetValue(Directory.Key, out OtherDirectory) == false)
{
Log(System.Diagnostics.TraceEventType.Error, "Missing Other Directory {0}", Directory.Key);
return false;
}
}
return false;
}
foreach (KeyValuePair<string, List<TempStorageFileInfo>> Directory in Directories)
{
List<TempStorageFileInfo> OtherDirectory;
if (Other.Directories.TryGetValue(Directory.Key, out OtherDirectory) == false)
{
Log(System.Diagnostics.TraceEventType.Error, "Missing Directory {0}", Directory.Key);
return false;
}
if (OtherDirectory.Count != Directory.Value.Count)
{
Log(System.Diagnostics.TraceEventType.Error, "File count mismatch {0} {1} {2}", Directory.Key, OtherDirectory.Count, Directory.Value.Count);
for (int FileIndex = 0; FileIndex < Directory.Value.Count; ++FileIndex)
{
Log("Manifest1: {0}", Directory.Value[FileIndex].Name);
}
for (int FileIndex = 0; FileIndex < OtherDirectory.Count; ++FileIndex)
{
Log("Manifest2: {0}", OtherDirectory[FileIndex].Name);
}
return false;
}
bool bResult = true;
for (int FileIndex = 0; FileIndex < Directory.Value.Count; ++FileIndex)
{
TempStorageFileInfo File = Directory.Value[FileIndex];
TempStorageFileInfo OtherFile = OtherDirectory[FileIndex];
if (File.Compare(OtherFile) == false)
{
bResult = false;
}
}
return bResult;
}
return true;
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
// Create a mapping from tag name to the files it contains, and seed it with invalid entries for everything in the graph
Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();
foreach (NodeOutput Output in Graph.Groups.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs))
{
TagNameToFileSet[Output.Name] = null;
}
// Fetch all the input dependencies for this node, and fill in the tag names with those files
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
foreach (NodeOutput Input in Node.Inputs)
{
TempStorageManifest Manifest = Storage.Retreive(Input.ProducingNode.Name, Input.Name);
TagNameToFileSet[Input.Name] = new HashSet <FileReference>(Manifest.Files.Select(x => x.ToFileReference(RootDir)));
}
// Add placeholder outputs for the current node
foreach (NodeOutput Output in Node.Outputs)
{
TagNameToFileSet[Output.Name] = new HashSet <FileReference>();
}
// Execute the node
if (bWithBanner)
{
Console.WriteLine();
CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
}
if (!Node.Build(Job, TagNameToFileSet))
{
return(false);
}
if (bWithBanner)
{
CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Determine all the outputs which are required to be copied to temp storage (because they're referenced by nodes in another agent group)
HashSet <NodeOutput> ReferencedOutputs = new HashSet <NodeOutput>();
foreach (AgentGroup Group in Graph.Groups)
{
bool bSameGroup = Group.Nodes.Contains(Node);
foreach (Node OtherNode in Group.Nodes)
{
if (!bSameGroup || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
ReferencedOutputs.UnionWith(OtherNode.Inputs);
}
}
}
// Publish all the outputs
foreach (NodeOutput Output in Node.Outputs)
{
Storage.Archive(Node.Name, Output.Name, TagNameToFileSet[Output.Name].ToArray(), ReferencedOutputs.Contains(Output));
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return(true);
}
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List <string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return(Files);
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List <string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List <string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List <string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return(DestFiles);
}
public bool Compare(TempStorageManifest Other)
{
if (Directories.Count != Other.Directories.Count)
{
Log(System.Diagnostics.TraceEventType.Error, "Directory count mismatch {0} {1}", Directories.Count, Other.Directories.Count);
foreach (KeyValuePair <string, List <TempStorageFileInfo> > Directory in Directories)
{
List <TempStorageFileInfo> OtherDirectory;
if (Other.Directories.TryGetValue(Directory.Key, out OtherDirectory) == false)
{
Log(System.Diagnostics.TraceEventType.Error, "Missing Directory {0}", Directory.Key);
return(false);
}
}
foreach (KeyValuePair <string, List <TempStorageFileInfo> > Directory in Other.Directories)
{
List <TempStorageFileInfo> OtherDirectory;
if (Directories.TryGetValue(Directory.Key, out OtherDirectory) == false)
{
Log(System.Diagnostics.TraceEventType.Error, "Missing Other Directory {0}", Directory.Key);
return(false);
}
}
return(false);
}
foreach (KeyValuePair <string, List <TempStorageFileInfo> > Directory in Directories)
{
List <TempStorageFileInfo> OtherDirectory;
if (Other.Directories.TryGetValue(Directory.Key, out OtherDirectory) == false)
{
Log(System.Diagnostics.TraceEventType.Error, "Missing Directory {0}", Directory.Key);
return(false);
}
if (OtherDirectory.Count != Directory.Value.Count)
{
Log(System.Diagnostics.TraceEventType.Error, "File count mismatch {0} {1} {2}", Directory.Key, OtherDirectory.Count, Directory.Value.Count);
for (int FileIndex = 0; FileIndex < Directory.Value.Count; ++FileIndex)
{
Log("Manifest1: {0}", Directory.Value[FileIndex].Name);
}
for (int FileIndex = 0; FileIndex < OtherDirectory.Count; ++FileIndex)
{
Log("Manifest2: {0}", OtherDirectory[FileIndex].Name);
}
return(false);
}
bool bResult = true;
for (int FileIndex = 0; FileIndex < Directory.Value.Count; ++FileIndex)
{
TempStorageFileInfo File = Directory.Value[FileIndex];
TempStorageFileInfo OtherFile = OtherDirectory[FileIndex];
if (File.Compare(OtherFile) == false)
{
bResult = false;
}
}
return(bResult);
}
return(true);
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which created the storage block</param>
/// <param name="BlockName">Name of the block to retrieve. May be null or empty.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string BlockName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using(TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if(bRemote)
{
// Create the shared directory for this node
FileReference SharedManifestFile = GetManifestLocation(SharedDir, NodeName, BlockName);
SharedManifestFile.Directory.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedManifestFile.Directory, LocalNodeDir, SharedManifestFile.GetFileNameWithoutExtension());
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestLocation(LocalDir, NodeName, BlockName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, BlockName));
return Manifest;
}
}
/// <summary>
/// Checks that a manifest matches the files on disk
/// </summary>
/// <param name="RootDir">Root directory for relative paths in the manifest</param>
/// <param name="Manifest">Manifest to check</param>
/// <param name="Files">Mapping of filename to timestamp as expected in the manifest</param>
static void CheckManifest(DirectoryReference RootDir, TempStorageManifest Manifest, Dictionary<FileReference, DateTime> Files)
{
if(Files.Count != Manifest.Files.Length)
{
throw new AutomationException("Number of files in manifest does not match");
}
foreach(TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if(!File.Exists())
{
throw new AutomationException("File in manifest does not exist");
}
DateTime OriginalTime;
if(!Files.TryGetValue(File, out OriginalTime))
{
throw new AutomationException("File in manifest did not exist previously");
}
double DiffSeconds = (new FileInfo(File.FullName).LastWriteTimeUtc - OriginalTime).TotalSeconds;
if(Math.Abs(DiffSeconds) > 2)
{
throw new AutomationException("Incorrect timestamp for {0}", ManifestFile.RelativePath);
}
}
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
return Files;
}
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder, false), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
if (!FileExists_NoExceptions(SharedManifest))
{
throw new AutomationException("Storage Block Manifest Does Not Exists! {0}", SharedManifest);
}
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
if (!FileExists_NoExceptions(true, Filename))
{
throw new AutomationException("Could not add {0} to manifest because it does not exist", Filename);
}
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
if (!FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Could not copy {0} to {1}", Filename, DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
return DestFiles;
}
/// <summary>
/// Run the automated tests
/// </summary>
public override void ExecuteBuild()
{
// Get all the shared directories
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local");
CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared");
CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName);
CommandUtils.DeleteDirectoryContents(SharedDir.FullName);
DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working");
CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
// Create the temp storage object
TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true);
// Create a working directory, and copy some source files into it
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime");
if (!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true))
{
throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName);
}
// Save the default output
Dictionary <FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f');
TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false);
Dictionary <FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i');
TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true);
// Check both outputs are still ok
TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null);
CheckManifest(WorkingDir, DefaultManifest, DefaultOutput);
TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifest, NamedOutput);
// Delete local temp storage and the working directory and try again
CommandUtils.Log("Clearing local folders...");
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
// First output should fail
CommandUtils.Log("Checking default manifest is now unavailable...");
bool bGotManifest = false;
try
{
TempStore.Retreive("TestNode", null);
}
catch
{
bGotManifest = false;
}
if (bGotManifest)
{
throw new AutomationException("Did not expect shared temp storage manifest to exist");
}
// Second one should be fine
TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput);
}
/// <summary>
/// Retrieve an output of the given node. Fetches and decompresses the files from shared storage if necessary, or validates the local files.
/// </summary>
/// <param name="NodeName">The node to retrieve build products for</param>
/// <param name="OutputName">The name of the node's output. May be null.</param>
/// <returns>Manifest of the files retrieved</returns>
public TempStorageManifest Retreive(string NodeName, string OutputName)
{
using (var TelemetryStopwatch = new TelemetryStopwatch("RetrieveFromTempStorage"))
{
// Get the path to the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
bool bLocal = LocalManifestFile.Exists();
// Read the manifest, either from local storage or shared storage
TempStorageManifest Manifest;
if (bLocal)
{
CommandUtils.Log("Reading shared manifest from {0}", LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(LocalManifestFile);
}
else
{
// Check we have shared storage
if (SharedDir == null)
{
throw new AutomationException("Missing local manifest for node - {0}", LocalManifestFile.FullName);
}
// Get the shared directory for this node
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
// Make sure the manifest exists
if (!SharedManifestFile.Exists())
{
throw new AutomationException("Missing local or shared manifest for node - {0}", SharedManifestFile.FullName);
}
// Read the shared manifest
CommandUtils.Log("Copying shared manifest from {0} to {1}", SharedManifestFile.FullName, LocalManifestFile.FullName);
Manifest = TempStorageManifest.Load(SharedManifestFile);
// Unzip all the build products
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
FileInfo[] ZipFiles = Manifest.ZipFiles.Select(x => new FileInfo(FileReference.Combine(SharedNodeDir, x.Name).FullName)).ToArray();
ParallelUnzipFiles(ZipFiles, RootDir);
// Fix any Unix permissions/chmod issues, and update the timestamps to match the manifest. Zip files only use local time, and there's no guarantee it matches the local clock.
foreach (TempStorageFile ManifestFile in Manifest.Files)
{
FileReference File = ManifestFile.ToFileReference(RootDir);
if (Utils.IsRunningOnMono)
{
CommandUtils.FixUnixFilePermissions(File.FullName);
}
System.IO.File.SetLastWriteTimeUtc(File.FullName, new DateTime(ManifestFile.LastWriteTimeUtcTicks, DateTimeKind.Utc));
}
// Save the manifest locally
LocalManifestFile.Directory.CreateDirectory();
Manifest.Save(LocalManifestFile);
}
// Check all the local files are as expected
bool bAllMatch = true;
foreach (TempStorageFile File in Manifest.Files)
{
bAllMatch &= File.Compare(RootDir);
}
if (!bAllMatch)
{
throw new AutomationException("Files have been modified");
}
// Update the stats and return
TelemetryStopwatch.Finish(string.Format("RetrieveFromTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Manifest.Files.Length, Manifest.Files.Sum(x => x.Length), bLocal? 0 : Manifest.ZipFiles.Sum(x => x.Length), bLocal? "Local" : "Remote", 0, 0, OutputName));
return(Manifest);
}
}
