/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
// Create a mapping from tag name to the files it contains, and seed it with invalid entries for everything in the graph
Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();
foreach (NodeOutput Output in Graph.Groups.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs))
{
TagNameToFileSet[Output.Name] = null;
}
// Fetch all the input dependencies for this node, and fill in the tag names with those files
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
foreach (NodeOutput Input in Node.Inputs)
{
TempStorageManifest Manifest = Storage.Retreive(Input.ProducingNode.Name, Input.Name);
TagNameToFileSet[Input.Name] = new HashSet <FileReference>(Manifest.Files.Select(x => x.ToFileReference(RootDir)));
}
// Add placeholder outputs for the current node
foreach (NodeOutput Output in Node.Outputs)
{
TagNameToFileSet[Output.Name] = new HashSet <FileReference>();
}
// Execute the node
if (bWithBanner)
{
Console.WriteLine();
CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
}
if (!Node.Build(Job, TagNameToFileSet))
{
return(false);
}
if (bWithBanner)
{
CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Determine all the outputs which are required to be copied to temp storage (because they're referenced by nodes in another agent group)
HashSet <NodeOutput> ReferencedOutputs = new HashSet <NodeOutput>();
foreach (AgentGroup Group in Graph.Groups)
{
bool bSameGroup = Group.Nodes.Contains(Node);
foreach (Node OtherNode in Group.Nodes)
{
if (!bSameGroup || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
ReferencedOutputs.UnionWith(OtherNode.Inputs);
}
}
}
// Publish all the outputs
foreach (NodeOutput Output in Node.Outputs)
{
Storage.Archive(Node.Name, Output.Name, TagNameToFileSet[Output.Name].ToArray(), ReferencedOutputs.Contains(Output));
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return(true);
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
// Create the mapping of tag names to file sets
Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();
// Read all the input tags for this node, and build a list of referenced input storage blocks
HashSet <TempStorageBlock> InputStorageBlocks = new HashSet <TempStorageBlock>();
foreach (NodeOutput Input in Node.Inputs)
{
TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
InputStorageBlocks.UnionWith(FileList.Blocks);
}
// Read the manifests for all the input storage blocks
Dictionary <TempStorageBlock, TempStorageManifest> InputManifests = new Dictionary <TempStorageBlock, TempStorageManifest>();
foreach (TempStorageBlock InputStorageBlock in InputStorageBlocks)
{
TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
InputManifests[InputStorageBlock] = Manifest;
}
// Read all the input storage blocks, keeping track of which block each file came from
Dictionary <FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary <FileReference, TempStorageBlock>();
foreach (KeyValuePair <TempStorageBlock, TempStorageManifest> Pair in InputManifests)
{
TempStorageBlock InputStorageBlock = Pair.Key;
foreach (FileReference File in Pair.Value.Files.Select(x => x.ToFileReference(RootDir)))
{
TempStorageBlock CurrentStorageBlock;
if (FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock) && !TempStorage.IsDuplicateBuildProduct(File))
{
LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
}
FileToStorageBlock[File] = InputStorageBlock;
}
}
// Add placeholder outputs for the current node
foreach (NodeOutput Output in Node.Outputs)
{
TagNameToFileSet.Add(Output.TagName, new HashSet <FileReference>());
}
// Execute the node
if (bWithBanner)
{
Console.WriteLine();
CommandUtils.LogInformation("========== Starting: {0} ==========", Node.Name);
}
if (!Node.Build(Job, TagNameToFileSet))
{
return(false);
}
if (bWithBanner)
{
CommandUtils.LogInformation("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Check that none of the inputs have been clobbered
Dictionary <string, string> ModifiedFiles = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (TempStorageFile File in InputManifests.Values.SelectMany(x => x.Files))
{
string Message;
if (!ModifiedFiles.ContainsKey(File.RelativePath) && !File.Compare(CommandUtils.RootDirectory, out Message))
{
ModifiedFiles.Add(File.RelativePath, Message);
}
}
if (ModifiedFiles.Count > 0)
{
throw new AutomationException("Build {0} from a previous step have been modified:\n{1}", (ModifiedFiles.Count == 1)? "product" : "products", String.Join("\n", ModifiedFiles.Select(x => x.Value)));
}
// Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
HashSet <FileReference> ReferencedOutputFiles = new HashSet <FileReference>();
foreach (Agent Agent in Graph.Agents)
{
bool bSameAgent = Agent.Nodes.Contains(Node);
foreach (Node OtherNode in Agent.Nodes)
{
if (!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
foreach (NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
{
ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
}
}
}
}
// Find a block name for all new outputs
Dictionary <FileReference, string> FileToOutputName = new Dictionary <FileReference, string>();
foreach (NodeOutput Output in Node.Outputs)
{
HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
foreach (FileReference File in Files)
{
if (!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
{
if (Output == Node.DefaultOutput)
{
if (!FileToOutputName.ContainsKey(File))
{
FileToOutputName[File] = "";
}
}
else
{
string OutputName;
if (FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
{
FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
}
else
{
FileToOutputName[File] = Output.TagName.Substring(1);
}
}
}
}
}
// Invert the dictionary to make a mapping of storage block to the files each contains
Dictionary <string, HashSet <FileReference> > OutputStorageBlockToFiles = new Dictionary <string, HashSet <FileReference> >();
foreach (KeyValuePair <FileReference, string> Pair in FileToOutputName)
{
HashSet <FileReference> Files;
if (!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
{
Files = new HashSet <FileReference>();
OutputStorageBlockToFiles.Add(Pair.Value, Files);
}
Files.Add(Pair.Key);
}
// Write all the storage blocks, and update the mapping from file to storage block
foreach (KeyValuePair <string, HashSet <FileReference> > Pair in OutputStorageBlockToFiles)
{
TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
foreach (FileReference File in Pair.Value)
{
FileToStorageBlock.Add(File, OutputBlock);
}
Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
}
// Publish all the output tags
foreach (NodeOutput Output in Node.Outputs)
{
HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
HashSet <TempStorageBlock> StorageBlocks = new HashSet <TempStorageBlock>();
foreach (FileReference File in Files)
{
TempStorageBlock StorageBlock;
if (FileToStorageBlock.TryGetValue(File, out StorageBlock))
{
StorageBlocks.Add(StorageBlock);
}
}
Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return(true);
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
// Create the mapping of tag names to file sets
Dictionary<string, HashSet<FileReference>> TagNameToFileSet = new Dictionary<string,HashSet<FileReference>>();
// Read all the input tags for this node, and build a list of referenced input storage blocks
HashSet<TempStorageBlock> InputStorageBlocks = new HashSet<TempStorageBlock>();
foreach(NodeOutput Input in Node.Inputs)
{
TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
InputStorageBlocks.UnionWith(FileList.Blocks);
}
// Read all the input storage blocks, keeping track of which block each file came from
Dictionary<FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary<FileReference, TempStorageBlock>();
foreach(TempStorageBlock InputStorageBlock in InputStorageBlocks)
{
TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
foreach(FileReference File in Manifest.Files.Select(x => x.ToFileReference(RootDir)))
{
TempStorageBlock CurrentStorageBlock;
if(FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock))
{
LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
}
FileToStorageBlock[File] = InputStorageBlock;
}
}
// Add placeholder outputs for the current node
foreach(NodeOutput Output in Node.Outputs)
{
TagNameToFileSet.Add(Output.TagName, new HashSet<FileReference>());
}
// Execute the node
if(bWithBanner)
{
Console.WriteLine();
CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
}
if(!Node.Build(Job, TagNameToFileSet))
{
return false;
}
if(bWithBanner)
{
CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
HashSet<FileReference> ReferencedOutputFiles = new HashSet<FileReference>();
foreach(Agent Agent in Graph.Agents)
{
bool bSameAgent = Agent.Nodes.Contains(Node);
foreach(Node OtherNode in Agent.Nodes)
{
if(!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
foreach(NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
{
ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
}
}
}
}
// Find a block name for all new outputs
Dictionary<FileReference, string> FileToOutputName = new Dictionary<FileReference, string>();
foreach(NodeOutput Output in Node.Outputs)
{
HashSet<FileReference> Files = TagNameToFileSet[Output.TagName];
foreach(FileReference File in Files)
{
if(!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
{
if(Output == Node.DefaultOutput)
{
if(!FileToOutputName.ContainsKey(File))
{
FileToOutputName[File] = "";
}
}
else
{
string OutputName;
if(FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
{
FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
}
else
{
FileToOutputName[File] = Output.TagName.Substring(1);
}
}
}
}
}
// Invert the dictionary to make a mapping of storage block to the files each contains
Dictionary<string, HashSet<FileReference>> OutputStorageBlockToFiles = new Dictionary<string, HashSet<FileReference>>();
foreach(KeyValuePair<FileReference, string> Pair in FileToOutputName)
{
HashSet<FileReference> Files;
if(!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
{
Files = new HashSet<FileReference>();
OutputStorageBlockToFiles.Add(Pair.Value, Files);
}
Files.Add(Pair.Key);
}
// Write all the storage blocks, and update the mapping from file to storage block
foreach(KeyValuePair<string, HashSet<FileReference>> Pair in OutputStorageBlockToFiles)
{
TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
foreach(FileReference File in Pair.Value)
{
FileToStorageBlock.Add(File, OutputBlock);
}
Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
}
// Publish all the output tags
foreach(NodeOutput Output in Node.Outputs)
{
HashSet<FileReference> Files = TagNameToFileSet[Output.TagName];
HashSet<TempStorageBlock> StorageBlocks = new HashSet<TempStorageBlock>();
foreach(FileReference File in Files)
{
TempStorageBlock StorageBlock;
if(FileToStorageBlock.TryGetValue(File, out StorageBlock))
{
StorageBlocks.Add(StorageBlock);
}
}
Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return true;
}
