/// <summary> /// Build a node /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param> /// <param name="Node">The node to build</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> /// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param> /// <returns>True if the node built successfully, false otherwise.</returns> bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner) { DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); // Create the mapping of tag names to file sets Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >(); // Read all the input tags for this node, and build a list of referenced input storage blocks HashSet <TempStorageBlock> InputStorageBlocks = new HashSet <TempStorageBlock>(); foreach (NodeOutput Input in Node.Inputs) { TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName); TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir); InputStorageBlocks.UnionWith(FileList.Blocks); } // Read the manifests for all the input storage blocks Dictionary <TempStorageBlock, TempStorageManifest> InputManifests = new Dictionary <TempStorageBlock, TempStorageManifest>(); foreach (TempStorageBlock InputStorageBlock in InputStorageBlocks) { TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName); InputManifests[InputStorageBlock] = Manifest; } // Read all the input storage blocks, keeping track of which block each file came from Dictionary <FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary <FileReference, TempStorageBlock>(); foreach (KeyValuePair <TempStorageBlock, TempStorageManifest> Pair in InputManifests) { TempStorageBlock InputStorageBlock = Pair.Key; foreach (FileReference File in Pair.Value.Files.Select(x => x.ToFileReference(RootDir))) { TempStorageBlock CurrentStorageBlock; if (FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock) && !TempStorage.IsDuplicateBuildProduct(File)) { LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock); } FileToStorageBlock[File] = InputStorageBlock; } } // Add placeholder outputs for the current node foreach (NodeOutput Output in Node.Outputs) { TagNameToFileSet.Add(Output.TagName, new HashSet <FileReference>()); } // Execute the node if (bWithBanner) { Console.WriteLine(); CommandUtils.LogInformation("========== Starting: {0} ==========", Node.Name); } if (!Node.Build(Job, TagNameToFileSet)) { return(false); } if (bWithBanner) { CommandUtils.LogInformation("========== Finished: {0} ==========", Node.Name); Console.WriteLine(); } // Check that none of the inputs have been clobbered Dictionary <string, string> ModifiedFiles = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (TempStorageFile File in InputManifests.Values.SelectMany(x => x.Files)) { string Message; if (!ModifiedFiles.ContainsKey(File.RelativePath) && !File.Compare(CommandUtils.RootDirectory, out Message)) { ModifiedFiles.Add(File.RelativePath, Message); } } if (ModifiedFiles.Count > 0) { throw new AutomationException("Build {0} from a previous step have been modified:\n{1}", (ModifiedFiles.Count == 1)? "product" : "products", String.Join("\n", ModifiedFiles.Select(x => x.Value))); } // Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent) HashSet <FileReference> ReferencedOutputFiles = new HashSet <FileReference>(); foreach (Agent Agent in Graph.Agents) { bool bSameAgent = Agent.Nodes.Contains(Node); foreach (Node OtherNode in Agent.Nodes) { if (!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger) { foreach (NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node)) { ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]); } } } } // Find a block name for all new outputs Dictionary <FileReference, string> FileToOutputName = new Dictionary <FileReference, string>(); foreach (NodeOutput Output in Node.Outputs) { HashSet <FileReference> Files = TagNameToFileSet[Output.TagName]; foreach (FileReference File in Files) { if (!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir)) { if (Output == Node.DefaultOutput) { if (!FileToOutputName.ContainsKey(File)) { FileToOutputName[File] = ""; } } else { string OutputName; if (FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0) { FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1)); } else { FileToOutputName[File] = Output.TagName.Substring(1); } } } } } // Invert the dictionary to make a mapping of storage block to the files each contains Dictionary <string, HashSet <FileReference> > OutputStorageBlockToFiles = new Dictionary <string, HashSet <FileReference> >(); foreach (KeyValuePair <FileReference, string> Pair in FileToOutputName) { HashSet <FileReference> Files; if (!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files)) { Files = new HashSet <FileReference>(); OutputStorageBlockToFiles.Add(Pair.Value, Files); } Files.Add(Pair.Key); } // Write all the storage blocks, and update the mapping from file to storage block foreach (KeyValuePair <string, HashSet <FileReference> > Pair in OutputStorageBlockToFiles) { TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key); foreach (FileReference File in Pair.Value) { FileToStorageBlock.Add(File, OutputBlock); } Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x))); } // Publish all the output tags foreach (NodeOutput Output in Node.Outputs) { HashSet <FileReference> Files = TagNameToFileSet[Output.TagName]; HashSet <TempStorageBlock> StorageBlocks = new HashSet <TempStorageBlock>(); foreach (FileReference File in Files) { TempStorageBlock StorageBlock; if (FileToStorageBlock.TryGetValue(File, out StorageBlock)) { StorageBlocks.Add(StorageBlock); } } Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray()); } // Mark the node as succeeded Storage.MarkAsComplete(Node.Name); return(true); }
/// <summary> /// Build a node /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param> /// <param name="Node">The node to build</param> /// <returns>True if the node built successfully, false otherwise.</returns> bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner) { // Create a mapping from tag name to the files it contains, and seed it with invalid entries for everything in the graph Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >(); foreach (NodeOutput Output in Graph.Groups.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs)) { TagNameToFileSet[Output.Name] = null; } // Fetch all the input dependencies for this node, and fill in the tag names with those files DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); foreach (NodeOutput Input in Node.Inputs) { TempStorageManifest Manifest = Storage.Retreive(Input.ProducingNode.Name, Input.Name); TagNameToFileSet[Input.Name] = new HashSet <FileReference>(Manifest.Files.Select(x => x.ToFileReference(RootDir))); } // Add placeholder outputs for the current node foreach (NodeOutput Output in Node.Outputs) { TagNameToFileSet[Output.Name] = new HashSet <FileReference>(); } // Execute the node if (bWithBanner) { Console.WriteLine(); CommandUtils.Log("========== Starting: {0} ==========", Node.Name); } if (!Node.Build(Job, TagNameToFileSet)) { return(false); } if (bWithBanner) { CommandUtils.Log("========== Finished: {0} ==========", Node.Name); Console.WriteLine(); } // Determine all the outputs which are required to be copied to temp storage (because they're referenced by nodes in another agent group) HashSet <NodeOutput> ReferencedOutputs = new HashSet <NodeOutput>(); foreach (AgentGroup Group in Graph.Groups) { bool bSameGroup = Group.Nodes.Contains(Node); foreach (Node OtherNode in Group.Nodes) { if (!bSameGroup || Node.ControllingTrigger != OtherNode.ControllingTrigger) { ReferencedOutputs.UnionWith(OtherNode.Inputs); } } } // Publish all the outputs foreach (NodeOutput Output in Node.Outputs) { Storage.Archive(Node.Name, Output.Name, TagNameToFileSet[Output.Name].ToArray(), ReferencedOutputs.Contains(Output)); } // Mark the node as succeeded Storage.MarkAsComplete(Node.Name); return(true); }
/// <summary> /// Run the automated tests /// </summary> public override void ExecuteBuild() { // Get all the shared directories DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local"); CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared"); CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName); CommandUtils.DeleteDirectoryContents(SharedDir.FullName); DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working"); CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); // Create the temp storage object TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true); // Create a working directory, and copy some source files into it DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime"); if (!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true)) { throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName); } // Save the default output Dictionary <FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f'); TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false); Dictionary <FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i'); TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true); // Check both outputs are still ok TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null); CheckManifest(WorkingDir, DefaultManifest, DefaultOutput); TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifest, NamedOutput); // Delete local temp storage and the working directory and try again CommandUtils.Log("Clearing local folders..."); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); // First output should fail CommandUtils.Log("Checking default manifest is now unavailable..."); bool bGotManifest = false; try { TempStore.Retreive("TestNode", null); } catch { bGotManifest = false; } if (bGotManifest) { throw new AutomationException("Did not expect shared temp storage manifest to exist"); } // Second one should be fine TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput); }
/// <summary> /// Run the automated tests /// </summary> public override void ExecuteBuild() { // Get all the shared directories DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local"); CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared"); CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName); CommandUtils.DeleteDirectoryContents(SharedDir.FullName); DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working"); CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); // Create the temp storage object TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true); // Create a working directory, and copy some source files into it DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime"); if(!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true)) { throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName); } // Save the default output Dictionary<FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f'); TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false); Dictionary<FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i'); TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true); // Check both outputs are still ok TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null); CheckManifest(WorkingDir, DefaultManifest, DefaultOutput); TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifest, NamedOutput); // Delete local temp storage and the working directory and try again CommandUtils.Log("Clearing local folders..."); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); // First output should fail CommandUtils.Log("Checking default manifest is now unavailable..."); bool bGotManifest = false; try { TempStore.Retreive("TestNode", null); } catch { bGotManifest = false; } if(bGotManifest) { throw new AutomationException("Did not expect shared temp storage manifest to exist"); } // Second one should be fine TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput); }
/// <summary> /// Build a node /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param> /// <param name="Node">The node to build</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> /// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param> /// <returns>True if the node built successfully, false otherwise.</returns> bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner) { DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); // Create the mapping of tag names to file sets Dictionary<string, HashSet<FileReference>> TagNameToFileSet = new Dictionary<string,HashSet<FileReference>>(); // Read all the input tags for this node, and build a list of referenced input storage blocks HashSet<TempStorageBlock> InputStorageBlocks = new HashSet<TempStorageBlock>(); foreach(NodeOutput Input in Node.Inputs) { TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName); TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir); InputStorageBlocks.UnionWith(FileList.Blocks); } // Read all the input storage blocks, keeping track of which block each file came from Dictionary<FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary<FileReference, TempStorageBlock>(); foreach(TempStorageBlock InputStorageBlock in InputStorageBlocks) { TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName); foreach(FileReference File in Manifest.Files.Select(x => x.ToFileReference(RootDir))) { TempStorageBlock CurrentStorageBlock; if(FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock)) { LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock); } FileToStorageBlock[File] = InputStorageBlock; } } // Add placeholder outputs for the current node foreach(NodeOutput Output in Node.Outputs) { TagNameToFileSet.Add(Output.TagName, new HashSet<FileReference>()); } // Execute the node if(bWithBanner) { Console.WriteLine(); CommandUtils.Log("========== Starting: {0} ==========", Node.Name); } if(!Node.Build(Job, TagNameToFileSet)) { return false; } if(bWithBanner) { CommandUtils.Log("========== Finished: {0} ==========", Node.Name); Console.WriteLine(); } // Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent) HashSet<FileReference> ReferencedOutputFiles = new HashSet<FileReference>(); foreach(Agent Agent in Graph.Agents) { bool bSameAgent = Agent.Nodes.Contains(Node); foreach(Node OtherNode in Agent.Nodes) { if(!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger) { foreach(NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node)) { ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]); } } } } // Find a block name for all new outputs Dictionary<FileReference, string> FileToOutputName = new Dictionary<FileReference, string>(); foreach(NodeOutput Output in Node.Outputs) { HashSet<FileReference> Files = TagNameToFileSet[Output.TagName]; foreach(FileReference File in Files) { if(!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir)) { if(Output == Node.DefaultOutput) { if(!FileToOutputName.ContainsKey(File)) { FileToOutputName[File] = ""; } } else { string OutputName; if(FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0) { FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1)); } else { FileToOutputName[File] = Output.TagName.Substring(1); } } } } } // Invert the dictionary to make a mapping of storage block to the files each contains Dictionary<string, HashSet<FileReference>> OutputStorageBlockToFiles = new Dictionary<string, HashSet<FileReference>>(); foreach(KeyValuePair<FileReference, string> Pair in FileToOutputName) { HashSet<FileReference> Files; if(!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files)) { Files = new HashSet<FileReference>(); OutputStorageBlockToFiles.Add(Pair.Value, Files); } Files.Add(Pair.Key); } // Write all the storage blocks, and update the mapping from file to storage block foreach(KeyValuePair<string, HashSet<FileReference>> Pair in OutputStorageBlockToFiles) { TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key); foreach(FileReference File in Pair.Value) { FileToStorageBlock.Add(File, OutputBlock); } Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x))); } // Publish all the output tags foreach(NodeOutput Output in Node.Outputs) { HashSet<FileReference> Files = TagNameToFileSet[Output.TagName]; HashSet<TempStorageBlock> StorageBlocks = new HashSet<TempStorageBlock>(); foreach(FileReference File in Files) { TempStorageBlock StorageBlock; if(FileToStorageBlock.TryGetValue(File, out StorageBlock)) { StorageBlocks.Add(StorageBlock); } } Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray()); } // Mark the node as succeeded Storage.MarkAsComplete(Node.Name); return true; }