public virtual void RetrieveBuildProducts(string SharedStorageDir)
 {
     CommandUtils.Log("***** Retrieving GUBP Node {0}", Name);
     try
     {
         BuildProducts = TempStorage.RetrieveFromTempStorage(SharedStorageDir, Name);
     }
     catch (Exception Ex)
     {
         throw new AutomationException(Ex, "Build Products cannot be found for node {0}", Name);
     }
 }
Пример #2
0
 public virtual void RetrieveBuildProducts(TempStorageNodeInfo TempStorageNodeInfo)
 {
     CommandUtils.Log("***** Retrieving GUBP Node {0} from {1}", Name, TempStorageNodeInfo.GetRelativeDirectory());
     try
     {
         BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, CommandUtils.CmdEnv.LocalRoot);
     }
     catch (Exception Ex)
     {
         throw new AutomationException(Ex, "Build Products cannot be found for node {0}", Name);
     }
 }
Пример #3
0
        /// <summary>
        /// Find all the nodes in the graph which are already completed
        /// </summary>
        /// <param name="Graph">The graph instance</param>
        /// <param name="Storage">The temp storage backend which stores the shared state</param>
        HashSet <Node> FindCompletedNodes(Graph Graph, TempStorage Storage)
        {
            HashSet <Node> CompletedNodes = new HashSet <Node>();

            foreach (Node Node in Graph.Agents.SelectMany(x => x.Nodes))
            {
                if (Storage.IsComplete(Node.Name))
                {
                    CompletedNodes.Add(Node);
                }
            }
            return(CompletedNodes);
        }
Пример #4
0
        /// <summary>
        /// Builds all the nodes in the graph
        /// </summary>
        /// <param name="Job">Information about the current job</param>
        /// <param name="Graph">The graph instance</param>
        /// <param name="Storage">The temp storage backend which stores the shared state</param>
        /// <returns>True if everything built successfully</returns>
        bool BuildAllNodes(JobContext Job, Graph Graph, TempStorage Storage)
        {
            // Build a flat list of nodes to execute, in order
            Node[] NodesToExecute = Graph.Agents.SelectMany(x => x.Nodes).ToArray();

            // Check the integrity of any local nodes that have been completed. It's common to run formal builds locally between regular development builds, so we may have
            // stale local state. Rather than failing later, detect and clean them up now.
            HashSet <Node> CleanedNodes = new HashSet <Node>();

            foreach (Node NodeToExecute in NodesToExecute)
            {
                if (NodeToExecute.InputDependencies.Any(x => CleanedNodes.Contains(x)) || !Storage.CheckLocalIntegrity(NodeToExecute.Name, NodeToExecute.Outputs.Select(x => x.TagName)))
                {
                    Storage.CleanLocalNode(NodeToExecute.Name);
                    CleanedNodes.Add(NodeToExecute);
                }
            }

            // Execute them in order
            int NodeIdx = 0;

            foreach (Node NodeToExecute in NodesToExecute)
            {
                LogInformation("****** [{0}/{1}] {2}", ++NodeIdx, NodesToExecute.Length, NodeToExecute.Name);
                if (!Storage.IsComplete(NodeToExecute.Name))
                {
                    LogInformation("");
                    if (!BuildNode(Job, Graph, NodeToExecute, Storage, false))
                    {
                        return(false);
                    }
                    LogInformation("");
                }
            }
            return(true);
        }
Пример #5
0
        public void SaveStatus(BuildNode NodeToDo, string Suffix, string NodeStoreName, bool bSaveSharedTempStorage, string GameNameIfAny, string JobStepIDForFailure = null)
        {
            string Contents = "Just a status record: " + Suffix;

            if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine)
            {
                try
                {
                    Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true);
                }
                catch (Exception Ex)
                {
                    CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to get properties for jobstep to save them.");
                    CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex));
                }
            }
            string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", NodeToDo.Name + Suffix + ".log");

            CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess));
            CommandUtils.WriteAllText(RecordOfSuccess, Contents);
            TempStorage.StoreToTempStorage(CommandUtils.CmdEnv, NodeStoreName + Suffix, new List <string> {
                RecordOfSuccess
            }, !bSaveSharedTempStorage, GameNameIfAny);
        }
Пример #6
0
        public void SaveStatus(TempStorageNodeInfo TempStorageNodeInfo, bool bSaveSharedTempStorage, string JobStepIDForFailure = null)
        {
            string Contents = "Just a status record: " + TempStorageNodeInfo.NodeStorageName;

            if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine)
            {
                try
                {
                    Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true);
                }
                catch (Exception Ex)
                {
                    CommandUtils.LogWarning("Failed to get properties for jobstep to save them.");
                    CommandUtils.LogWarning(LogUtils.FormatException(Ex));
                }
            }
            string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", TempStorageNodeInfo.NodeStorageName + ".log");

            CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess));
            CommandUtils.WriteAllText(RecordOfSuccess, Contents);
            TempStorage.StoreToTempStorage(TempStorageNodeInfo, new List <string> {
                RecordOfSuccess
            }, !bSaveSharedTempStorage, CommandUtils.CmdEnv.LocalRoot);
        }
Пример #7
0
        /// <summary>
        /// Main entry point for the BuildGraph command
        /// </summary>
        public override ExitCode Execute()
        {
            // Parse the command line parameters
            string ScriptFileName = ParseParamValue("Script", null);

            if (ScriptFileName == null)
            {
                LogError("Missing -Script= parameter for BuildGraph");
                return(ExitCode.Error_Unknown);
            }

            string TargetNames = ParseParamValue("Target", null);

            if (TargetNames == null)
            {
                LogError("Missing -Target= parameter for BuildGraph");
                return(ExitCode.Error_Unknown);
            }

            string SchemaFileName = ParseParamValue("Schema", null);
            string ExportFileName = ParseParamValue("Export", null);

            string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
            string SingleNodeName   = ParseParamValue("SingleNode", null);

            string[] TriggerNames          = ParseParamValue("Trigger", "").Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
            bool     bSkipTriggers         = ParseParam("SkipTriggers");
            bool     bClean                = ParseParam("Clean");
            bool     bListOnly             = ParseParam("ListOnly");
            bool     bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
            bool     bPublicTasksOnly      = ParseParam("PublicTasksOnly");

            GraphPrintOptions PrintOptions = 0;

            if (ParseParam("ShowDeps"))
            {
                PrintOptions |= GraphPrintOptions.ShowDependencies;
            }
            if (ParseParam("ShowNotifications"))
            {
                PrintOptions |= GraphPrintOptions.ShowNotifications;
            }

            // Parse any specific nodes to clean
            List <string> CleanNodes = new List <string>();

            foreach (string NodeList in ParseParamValues("CleanNode"))
            {
                foreach (string NodeName in NodeList.Split('+'))
                {
                    CleanNodes.Add(NodeName);
                }
            }

            // Read any environment variables
            Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            foreach (DictionaryEntry Entry in Environment.GetEnvironmentVariables())
            {
                DefaultProperties[Entry.Key.ToString()] = Entry.Value.ToString();
            }

            // Add any additional custom parameters from the command line (of the form -Set:X=Y)
            foreach (string Param in Params)
            {
                const string Prefix = "set:";
                if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
                {
                    int EqualsIdx = Param.IndexOf('=');
                    if (EqualsIdx >= 0)
                    {
                        DefaultProperties[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
                    }
                    else
                    {
                        LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
                    }
                }
            }

            // Set up the standard properties which build scripts might need
            DefaultProperties["Branch"]         = P4Enabled? P4Env.BuildRootP4 : "Unknown";
            DefaultProperties["EscapedBranch"]  = P4Enabled? P4Env.BuildRootEscaped : "Unknown";
            DefaultProperties["Change"]         = P4Enabled? P4Env.Changelist.ToString() : "0";
            DefaultProperties["RootDir"]        = CommandUtils.RootDirectory.FullName;
            DefaultProperties["IsBuildMachine"] = IsBuildMachine? "true" : "false";
            DefaultProperties["HostPlatform"]   = HostPlatform.Current.HostEditorPlatform.ToString();

            // Find all the tasks from the loaded assemblies
            Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();

            if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
            {
                return(ExitCode.Error_Unknown);
            }

            // Create a schema for the given tasks
            ScriptSchema Schema = new ScriptSchema(NameToTask);

            if (SchemaFileName != null)
            {
                Schema.Export(new FileReference(SchemaFileName));
            }

            // Read the script from disk
            Graph Graph;

            if (!ScriptReader.TryRead(new FileReference(ScriptFileName), DefaultProperties, Schema, out Graph))
            {
                return(ExitCode.Error_Unknown);
            }

            // Create the temp storage handler
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
            TempStorage        Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);

            if (bClean)
            {
                Storage.CleanLocal();
            }
            foreach (string CleanNode in CleanNodes)
            {
                Storage.CleanLocalNode(CleanNode);
            }

            // Convert the supplied target references into nodes
            HashSet <Node> TargetNodes = new HashSet <Node>();

            foreach (string TargetName in TargetNames.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
            {
                Node[] Nodes;
                if (!Graph.TryResolveReference(TargetName, out Nodes))
                {
                    LogError("Target '{0}' is not in graph", TargetName);
                    return(ExitCode.Error_Unknown);
                }
                TargetNodes.UnionWith(Nodes);
            }

            // Cull the graph to include only those nodes
            Graph.Select(TargetNodes);

            // Find the triggers which are explicitly activated, and all of its upstream triggers.
            HashSet <ManualTrigger> Triggers = new HashSet <ManualTrigger>();

            foreach (string TriggerName in TriggerNames)
            {
                ManualTrigger Trigger;
                if (!Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
                {
                    LogError("Couldn't find trigger '{0}'", TriggerName);
                    return(ExitCode.Error_Unknown);
                }
                while (Trigger != null)
                {
                    Triggers.Add(Trigger);
                    Trigger = Trigger.Parent;
                }
            }
            if (bSkipTriggers)
            {
                Triggers.UnionWith(Graph.NameToTrigger.Values);
            }

            // If we're just building a single node, find it
            Node SingleNode = null;

            if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
            {
                LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
                return(ExitCode.Error_Unknown);
            }

            // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system.
            if (SingleNode == null)
            {
                IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == null || Triggers.Contains(x.EnclosingTrigger));
                foreach (GraphDiagnostic Diagnostic in Diagnostics)
                {
                    if (Diagnostic.EventType == LogEventType.Warning)
                    {
                        CommandUtils.LogWarning(Diagnostic.Message);
                    }
                    else
                    {
                        CommandUtils.LogError(Diagnostic.Message);
                    }
                }
                if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
                {
                    return(ExitCode.Error_Unknown);
                }
            }

            // Execute the command
            if (bListOnly)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
            }
            else if (ExportFileName != null)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
                Graph.Export(new FileReference(ExportFileName), Triggers, CompletedNodes);
            }
            else if (SingleNode != null)
            {
                if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
                {
                    return(ExitCode.Error_Unknown);
                }
            }
            else
            {
                if (!BuildAllNodes(new JobContext(this), Graph, Storage))
                {
                    return(ExitCode.Error_Unknown);
                }
            }
            return(ExitCode.Success);
        }
Пример #8
0
        /// <summary>
        /// Build a node
        /// </summary>
        /// <param name="Job">Information about the current job</param>
        /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
        /// <param name="Node">The node to build</param>
        /// <returns>True if the node built successfully, false otherwise.</returns>
        bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
        {
            // Create a mapping from tag name to the files it contains, and seed it with invalid entries for everything in the graph
            Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();

            foreach (NodeOutput Output in Graph.Groups.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs))
            {
                TagNameToFileSet[Output.Name] = null;
            }

            // Fetch all the input dependencies for this node, and fill in the tag names with those files
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);

            foreach (NodeOutput Input in Node.Inputs)
            {
                TempStorageManifest Manifest = Storage.Retreive(Input.ProducingNode.Name, Input.Name);
                TagNameToFileSet[Input.Name] = new HashSet <FileReference>(Manifest.Files.Select(x => x.ToFileReference(RootDir)));
            }

            // Add placeholder outputs for the current node
            foreach (NodeOutput Output in Node.Outputs)
            {
                TagNameToFileSet[Output.Name] = new HashSet <FileReference>();
            }

            // Execute the node
            if (bWithBanner)
            {
                Console.WriteLine();
                CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
            }
            if (!Node.Build(Job, TagNameToFileSet))
            {
                return(false);
            }
            if (bWithBanner)
            {
                CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
                Console.WriteLine();
            }

            // Determine all the outputs which are required to be copied to temp storage (because they're referenced by nodes in another agent group)
            HashSet <NodeOutput> ReferencedOutputs = new HashSet <NodeOutput>();

            foreach (AgentGroup Group in Graph.Groups)
            {
                bool bSameGroup = Group.Nodes.Contains(Node);
                foreach (Node OtherNode in Group.Nodes)
                {
                    if (!bSameGroup || Node.ControllingTrigger != OtherNode.ControllingTrigger)
                    {
                        ReferencedOutputs.UnionWith(OtherNode.Inputs);
                    }
                }
            }

            // Publish all the outputs
            foreach (NodeOutput Output in Node.Outputs)
            {
                Storage.Archive(Node.Name, Output.Name, TagNameToFileSet[Output.Name].ToArray(), ReferencedOutputs.Contains(Output));
            }

            // Mark the node as succeeded
            Storage.MarkAsComplete(Node.Name);
            return(true);
        }
Пример #9
0
		/// <summary>
		/// Builds all the nodes in the graph
		/// </summary>
		/// <param name="Job">Information about the current job</param>
		/// <param name="Graph">The graph instance</param>
		/// <param name="Storage">The temp storage backend which stores the shared state</param>
		/// <returns>True if everything built successfully</returns>
		bool BuildAllNodes(JobContext Job, Graph Graph, TempStorage Storage)
		{
			// Build a flat list of nodes to execute, in order
			Node[] NodesToExecute = Graph.Agents.SelectMany(x => x.Nodes).ToArray();

			// Check the integrity of any local nodes that have been completed. It's common to run formal builds locally between regular development builds, so we may have 
			// stale local state. Rather than failing later, detect and clean them up now.
			HashSet<Node> CleanedNodes = new HashSet<Node>();
			foreach(Node NodeToExecute in NodesToExecute)
			{
				if(NodeToExecute.InputDependencies.Any(x => CleanedNodes.Contains(x)) || !Storage.CheckLocalIntegrity(NodeToExecute.Name, NodeToExecute.Outputs.Select(x => x.TagName)))
				{
					Storage.CleanLocalNode(NodeToExecute.Name);
					CleanedNodes.Add(NodeToExecute);
				}
			}

			// Execute them in order
			int NodeIdx = 0;
			foreach(Node NodeToExecute in NodesToExecute)
			{
				Log("****** [{0}/{1}] {2}", ++NodeIdx, NodesToExecute.Length, NodeToExecute.Name);
				if(!Storage.IsComplete(NodeToExecute.Name))
				{
					Log("");
					if(!BuildNode(Job, Graph, NodeToExecute, Storage, false))
					{
						return false;
					} 
					Log("");
				}
			}
			return true;
		}
Пример #10
0
		/// <summary>
		/// Run the automated tests
		/// </summary>
		public override void ExecuteBuild()
		{
			// Get all the shared directories
			DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);

			DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local");
			CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName);
			CommandUtils.DeleteDirectoryContents(LocalDir.FullName);

			DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared");
			CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName);
			CommandUtils.DeleteDirectoryContents(SharedDir.FullName);

			DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working");
			CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName);
			CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);

			// Create the temp storage object
			TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true);

			// Create a working directory, and copy some source files into it
			DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime");
			if(!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true))
			{
				throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName);
			}

			// Save the default output
			Dictionary<FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f');
			TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false);

			Dictionary<FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i');
			TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true);
			
			// Check both outputs are still ok
			TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null);
			CheckManifest(WorkingDir, DefaultManifest, DefaultOutput);

			TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput");
			CheckManifest(WorkingDir, NamedManifest, NamedOutput);

			// Delete local temp storage and the working directory and try again
			CommandUtils.Log("Clearing local folders...");
			CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
			CommandUtils.DeleteDirectoryContents(LocalDir.FullName);

			// First output should fail
			CommandUtils.Log("Checking default manifest is now unavailable...");
			bool bGotManifest = false;
			try
			{
				TempStore.Retreive("TestNode", null);
			}
			catch
			{
				bGotManifest = false;
			}
			if(bGotManifest)
			{
				throw new AutomationException("Did not expect shared temp storage manifest to exist");
			}

			// Second one should be fine
			TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput");
			CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput);
		}
 public virtual void ArchiveBuildProducts(string SharedStorageDir, bool bWriteToSharedStorage)
 {
     TempStorage.StoreToTempStorage(Name, BuildProducts, SharedStorageDir, bWriteToSharedStorage);
 }
        /// <summary>
        /// Run the automated tests
        /// </summary>
        public override void ExecuteBuild()
        {
            // Get all the shared directories
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);

            DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local");

            CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName);
            CommandUtils.DeleteDirectoryContents(LocalDir.FullName);

            DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared");

            CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName);
            CommandUtils.DeleteDirectoryContents(SharedDir.FullName);

            DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working");

            CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName);
            CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);

            // Create the temp storage object
            TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true);

            // Create a working directory, and copy some source files into it
            DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime");

            if (!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true))
            {
                throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName);
            }

            // Save the default output
            Dictionary <FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f');

            TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false);

            Dictionary <FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i');

            TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true);

            // Check both outputs are still ok
            TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null);

            CheckManifest(WorkingDir, DefaultManifest, DefaultOutput);

            TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput");

            CheckManifest(WorkingDir, NamedManifest, NamedOutput);

            // Delete local temp storage and the working directory and try again
            CommandUtils.Log("Clearing local folders...");
            CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
            CommandUtils.DeleteDirectoryContents(LocalDir.FullName);

            // First output should fail
            CommandUtils.Log("Checking default manifest is now unavailable...");
            bool bGotManifest = false;

            try
            {
                TempStore.Retreive("TestNode", null);
            }
            catch
            {
                bGotManifest = false;
            }
            if (bGotManifest)
            {
                throw new AutomationException("Did not expect shared temp storage manifest to exist");
            }

            // Second one should be fine
            TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput");

            CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput);
        }
Пример #13
0
		/// <summary>
		/// Main entry point for the BuildGraph command
		/// </summary>
		public override ExitCode Execute()
		{
			// Parse the command line parameters
			string ScriptFileName = ParseParamValue("Script", null);
			string TargetNames = ParseParamValue("Target", null);
			string DocumentationFileName = ParseParamValue("Documentation", null);
			string SchemaFileName = ParseParamValue("Schema", null);
			string ExportFileName = ParseParamValue("Export", null);
			string PreprocessedFileName = ParseParamValue("Preprocess", null);
			string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
			string SingleNodeName = ParseParamValue("SingleNode", null);
			string TriggerName = ParseParamValue("Trigger", null);
			string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
			bool bSkipTriggers = ParseParam("SkipTriggers");
			string TokenSignature = ParseParamValue("TokenSignature", null);
			bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
			bool bClearHistory = ParseParam("Clean") || ParseParam("ClearHistory");
			bool bListOnly = ParseParam("ListOnly");
			bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
			bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
			string ReportName = ParseParamValue("ReportName", null); 

			GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
			if(ParseParam("ShowDeps"))
			{
				PrintOptions |= GraphPrintOptions.ShowDependencies;
			}
			if(ParseParam("ShowNotifications"))
			{
				PrintOptions |= GraphPrintOptions.ShowNotifications;
			}

			// Parse any specific nodes to clean
			List<string> CleanNodes = new List<string>();
			foreach(string NodeList in ParseParamValues("CleanNode"))
			{
				foreach(string NodeName in NodeList.Split('+', ';'))
				{
					CleanNodes.Add(NodeName);
				}
			}

			// Set up the standard properties which build scripts might need
			Dictionary<string, string> DefaultProperties = new Dictionary<string,string>(StringComparer.InvariantCultureIgnoreCase);
			DefaultProperties["Branch"] = P4Enabled ? P4Env.BuildRootP4 : "Unknown";
			DefaultProperties["EscapedBranch"] = P4Enabled ? P4Env.BuildRootEscaped : "Unknown";
			DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
			DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
			DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
			DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
			DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();

			// Attempt to read existing Build Version information
			BuildVersion Version;
			if (BuildVersion.TryRead(FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Build", "Build.version").FullName, out Version))
			{
				DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
				DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
				DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
			}

			// Add any additional custom arguments from the command line (of the form -Set:X=Y)
			Dictionary<string, string> Arguments = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
			foreach (string Param in Params)
			{
				const string Prefix = "set:";
				if(Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
				{
					int EqualsIdx = Param.IndexOf('=');
					if(EqualsIdx >= 0)
					{
						Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
					}
					else
					{
						LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
					}
				}
			}

			// Find all the tasks from the loaded assemblies
			Dictionary<string, ScriptTask> NameToTask = new Dictionary<string,ScriptTask>();
			if(!FindAvailableTasks(NameToTask, bPublicTasksOnly))
			{
				return ExitCode.Error_Unknown;
			}

			// Generate documentation
			if(DocumentationFileName != null)
			{
				GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
				return ExitCode.Success;
			}

			// Create a schema for the given tasks
			ScriptSchema Schema = new ScriptSchema(NameToTask);
			if(SchemaFileName != null)
			{
				FileReference FullSchemaFileName = new FileReference(SchemaFileName);
				Log("Writing schema to {0}...", FullSchemaFileName.FullName);
				Schema.Export(FullSchemaFileName);
				if(ScriptFileName == null)
				{
					return ExitCode.Success;
				}
			}

			// Check there was a script specified
			if(ScriptFileName == null)
			{
				LogError("Missing -Script= parameter for BuildGraph");
				return ExitCode.Error_Unknown;
			}

			// Read the script from disk
			Graph Graph;
			if(!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
			{
				return ExitCode.Error_Unknown;
			}

			// Create the temp storage handler
			DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
			TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
			if(bClearHistory)
			{
				Storage.CleanLocal();
			}
			foreach(string CleanNode in CleanNodes)
			{
				Storage.CleanLocalNode(CleanNode);
			}

			// Convert the supplied target references into nodes 
			HashSet<Node> TargetNodes = new HashSet<Node>();
			if(TargetNames == null)
			{
				if(!bListOnly)
				{
					LogError("Missing -Target= parameter for BuildGraph");
					return ExitCode.Error_Unknown;
				}
				TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
			}
			else
			{
				foreach(string TargetName in TargetNames.Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
				{
					Node[] Nodes;
					if(!Graph.TryResolveReference(TargetName, out Nodes))
					{
						LogError("Target '{0}' is not in graph", TargetName);
						return ExitCode.Error_Unknown;
					}
					TargetNodes.UnionWith(Nodes);
				}
			}

			// Try to acquire tokens for all the target nodes we want to build
			if(TokenSignature != null)
			{
				// Find all the lock files
				HashSet<FileReference> RequiredTokens = new HashSet<FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));

				// List out all the required tokens
				if(SingleNodeName == null)
				{
					CommandUtils.Log("Required tokens:");
					foreach(Node Node in TargetNodes)
					{
						foreach(FileReference RequiredToken in Node.RequiredTokens)
						{
							CommandUtils.Log("  '{0}' requires {1}", Node, RequiredToken);
						}
					}
				}

				// Try to create all the lock files
				List<FileReference> CreatedTokens = new List<FileReference>();
				if(!bListOnly)
				{
					CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
				}

				// Find all the tokens that we don't have
				Dictionary<FileReference, string> MissingTokens = new Dictionary<FileReference, string>();
				foreach(FileReference RequiredToken in RequiredTokens)
				{
					string CurrentOwner = ReadTokenFile(RequiredToken);
					if(CurrentOwner != null && CurrentOwner != TokenSignature)
					{
						MissingTokens.Add(RequiredToken, CurrentOwner);
					}
				}

				// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
				if(MissingTokens.Count > 0)
				{
					if(bSkipTargetsWithoutTokens)
					{
						// Find all the nodes we're going to skip
						HashSet<Node> SkipNodes = new HashSet<Node>();
						foreach(IGrouping<string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
						{
							Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
							foreach(FileReference MissingToken in MissingTokensForBuild)
							{
								foreach(Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
								{
									Log("    {0}", SkipNode);
								}
							}
						}

						// Write a list of everything left over
						if(SkipNodes.Count > 0)
						{
							TargetNodes.ExceptWith(SkipNodes);
							Log("Remaining target nodes:");
							foreach(Node TargetNode in TargetNodes)
							{
								Log("    {0}", TargetNode);
							}
							if(TargetNodes.Count == 0)
							{
								Log("    None.");
							}
						}
					}
					else
					{
						foreach(KeyValuePair<FileReference, string> Pair in MissingTokens)
						{
							List<Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
							LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
						}
						foreach(FileReference CreatedToken in CreatedTokens)
						{
							CreatedToken.Delete();
						}
						return ExitCode.Error_Unknown;
					}
				}
			}

			// Cull the graph to include only those nodes
			Graph.Select(TargetNodes);

			// Collapse any triggers in the graph which are marked to be skipped
			HashSet<ManualTrigger> SkipTriggers = new HashSet<ManualTrigger>();
			if(bSkipTriggers)
			{
				SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
			}
			else
			{
				foreach(string SkipTriggerName in SkipTriggerNames)
				{
					ManualTrigger SkipTrigger;
					if(!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
					{
						LogError("Couldn't find trigger '{0}'", TriggerName);
						return ExitCode.Error_Unknown;
					}
					SkipTriggers.Add(SkipTrigger);
				}
			}
			Graph.SkipTriggers(SkipTriggers);

			// If a report for the whole build was requested, insert it into the graph
			if (ReportName != null)
			{
				Report NewReport = new Report(ReportName);
				NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
				Graph.NameToReport.Add(ReportName, NewReport);
			}

			// Write out the preprocessed script
			if (PreprocessedFileName != null)
			{
				Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
			}

			// Find the triggers which we are explicitly running.
			ManualTrigger Trigger = null;
			if(TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
				{
					LogError("Couldn't find trigger '{0}'", TriggerName);
					return ExitCode.Error_Unknown;
				}

			// If we're just building a single node, find it 
			Node SingleNode = null;
			if(SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
			{
				LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
				return ExitCode.Error_Unknown;
			}

			// If we just want to show the contents of the graph, do so and exit.
			if(bListOnly)
			{ 
				HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
				Graph.Print(CompletedNodes, PrintOptions);
				return ExitCode.Success;
			}

			// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file. 
			if(SingleNode == null)
			{
				IEnumerable<GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
				foreach(GraphDiagnostic Diagnostic in Diagnostics)
				{
					if(Diagnostic.EventType == LogEventType.Warning)
					{
						CommandUtils.LogWarning(Diagnostic.Message);
					}
					else
					{
						CommandUtils.LogError(Diagnostic.Message);
					}
				}
				if(Diagnostics.Any(x => x.EventType == LogEventType.Error))
				{
					return ExitCode.Error_Unknown;
				}
			}

			// Execute the command
			if(ExportFileName != null)
			{
				HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
				Graph.Print(CompletedNodes, PrintOptions);
				Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
			}
			else if(SingleNode != null)
			{
				if(!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
				{
					return ExitCode.Error_Unknown;
				}
			}
			else
			{
				if(!BuildAllNodes(new JobContext(this), Graph, Storage))
				{
					return ExitCode.Error_Unknown;
				}
			}
			return ExitCode.Success;
		}
Пример #14
0
		/// <summary>
		/// Build a node
		/// </summary>
		/// <param name="Job">Information about the current job</param>
		/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
		/// <param name="Node">The node to build</param>
		/// <param name="Storage">The temp storage backend which stores the shared state</param>
		/// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
		/// <returns>True if the node built successfully, false otherwise.</returns>
		bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
		{
			DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);

			// Create the mapping of tag names to file sets
			Dictionary<string, HashSet<FileReference>> TagNameToFileSet = new Dictionary<string,HashSet<FileReference>>();

			// Read all the input tags for this node, and build a list of referenced input storage blocks
			HashSet<TempStorageBlock> InputStorageBlocks = new HashSet<TempStorageBlock>();
			foreach(NodeOutput Input in Node.Inputs)
			{
				TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
				TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
				InputStorageBlocks.UnionWith(FileList.Blocks);
			}

			// Read all the input storage blocks, keeping track of which block each file came from
			Dictionary<FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary<FileReference, TempStorageBlock>();
			foreach(TempStorageBlock InputStorageBlock in InputStorageBlocks)
			{
				TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
				foreach(FileReference File in Manifest.Files.Select(x => x.ToFileReference(RootDir)))
				{
					TempStorageBlock CurrentStorageBlock;
					if(FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock))
					{
						LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
					}
					FileToStorageBlock[File] = InputStorageBlock;
				}
			}

			// Add placeholder outputs for the current node
			foreach(NodeOutput Output in Node.Outputs)
			{
				TagNameToFileSet.Add(Output.TagName, new HashSet<FileReference>());
			}

			// Execute the node
			if(bWithBanner)
			{
				Console.WriteLine();
				CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
			}
			if(!Node.Build(Job, TagNameToFileSet))
			{
				return false;
			}
			if(bWithBanner)
			{
				CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
				Console.WriteLine();
			}

			// Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
			HashSet<FileReference> ReferencedOutputFiles = new HashSet<FileReference>();
			foreach(Agent Agent in Graph.Agents)
			{
				bool bSameAgent = Agent.Nodes.Contains(Node);
				foreach(Node OtherNode in Agent.Nodes)
				{
					if(!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
					{
						foreach(NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
						{
							ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
						}
					}
				}
			}

			// Find a block name for all new outputs
			Dictionary<FileReference, string> FileToOutputName = new Dictionary<FileReference, string>();
			foreach(NodeOutput Output in Node.Outputs)
			{
				HashSet<FileReference> Files = TagNameToFileSet[Output.TagName]; 
				foreach(FileReference File in Files)
				{
					if(!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
					{
						if(Output == Node.DefaultOutput)
						{
							if(!FileToOutputName.ContainsKey(File))
							{
								FileToOutputName[File] = "";
							}
						}
						else
						{
							string OutputName;
							if(FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
							{
								FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
							}
							else
							{
								FileToOutputName[File] = Output.TagName.Substring(1);
							}
						}
					}
				}
			}

			// Invert the dictionary to make a mapping of storage block to the files each contains
			Dictionary<string, HashSet<FileReference>> OutputStorageBlockToFiles = new Dictionary<string, HashSet<FileReference>>();
			foreach(KeyValuePair<FileReference, string> Pair in FileToOutputName)
			{
				HashSet<FileReference> Files;
				if(!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
				{
					Files = new HashSet<FileReference>();
					OutputStorageBlockToFiles.Add(Pair.Value, Files);
				}
				Files.Add(Pair.Key);
			}

			// Write all the storage blocks, and update the mapping from file to storage block
			foreach(KeyValuePair<string, HashSet<FileReference>> Pair in OutputStorageBlockToFiles)
			{
				TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
				foreach(FileReference File in Pair.Value)
				{
					FileToStorageBlock.Add(File, OutputBlock);
				}
				Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
			}

			// Publish all the output tags
			foreach(NodeOutput Output in Node.Outputs)
			{
				HashSet<FileReference> Files = TagNameToFileSet[Output.TagName];

				HashSet<TempStorageBlock> StorageBlocks = new HashSet<TempStorageBlock>();
				foreach(FileReference File in Files)
				{
					TempStorageBlock StorageBlock;
					if(FileToStorageBlock.TryGetValue(File, out StorageBlock))
					{
						StorageBlocks.Add(StorageBlock);
					}
				}

				Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
			}

			// Mark the node as succeeded
			Storage.MarkAsComplete(Node.Name);
			return true;
		}
Пример #15
0
        /// <summary>
        /// Build a node
        /// </summary>
        /// <param name="Job">Information about the current job</param>
        /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
        /// <param name="Node">The node to build</param>
        /// <param name="Storage">The temp storage backend which stores the shared state</param>
        /// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
        /// <returns>True if the node built successfully, false otherwise.</returns>
        bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
        {
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);

            // Create the mapping of tag names to file sets
            Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();

            // Read all the input tags for this node, and build a list of referenced input storage blocks
            HashSet <TempStorageBlock> InputStorageBlocks = new HashSet <TempStorageBlock>();

            foreach (NodeOutput Input in Node.Inputs)
            {
                TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
                TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
                InputStorageBlocks.UnionWith(FileList.Blocks);
            }

            // Read the manifests for all the input storage blocks
            Dictionary <TempStorageBlock, TempStorageManifest> InputManifests = new Dictionary <TempStorageBlock, TempStorageManifest>();

            foreach (TempStorageBlock InputStorageBlock in InputStorageBlocks)
            {
                TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
                InputManifests[InputStorageBlock] = Manifest;
            }

            // Read all the input storage blocks, keeping track of which block each file came from
            Dictionary <FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary <FileReference, TempStorageBlock>();

            foreach (KeyValuePair <TempStorageBlock, TempStorageManifest> Pair in InputManifests)
            {
                TempStorageBlock InputStorageBlock = Pair.Key;
                foreach (FileReference File in Pair.Value.Files.Select(x => x.ToFileReference(RootDir)))
                {
                    TempStorageBlock CurrentStorageBlock;
                    if (FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock) && !TempStorage.IsDuplicateBuildProduct(File))
                    {
                        LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
                    }
                    FileToStorageBlock[File] = InputStorageBlock;
                }
            }

            // Add placeholder outputs for the current node
            foreach (NodeOutput Output in Node.Outputs)
            {
                TagNameToFileSet.Add(Output.TagName, new HashSet <FileReference>());
            }

            // Execute the node
            if (bWithBanner)
            {
                Console.WriteLine();
                CommandUtils.LogInformation("========== Starting: {0} ==========", Node.Name);
            }
            if (!Node.Build(Job, TagNameToFileSet))
            {
                return(false);
            }
            if (bWithBanner)
            {
                CommandUtils.LogInformation("========== Finished: {0} ==========", Node.Name);
                Console.WriteLine();
            }

            // Check that none of the inputs have been clobbered
            Dictionary <string, string> ModifiedFiles = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            foreach (TempStorageFile File in InputManifests.Values.SelectMany(x => x.Files))
            {
                string Message;
                if (!ModifiedFiles.ContainsKey(File.RelativePath) && !File.Compare(CommandUtils.RootDirectory, out Message))
                {
                    ModifiedFiles.Add(File.RelativePath, Message);
                }
            }
            if (ModifiedFiles.Count > 0)
            {
                throw new AutomationException("Build {0} from a previous step have been modified:\n{1}", (ModifiedFiles.Count == 1)? "product" : "products", String.Join("\n", ModifiedFiles.Select(x => x.Value)));
            }

            // Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
            HashSet <FileReference> ReferencedOutputFiles = new HashSet <FileReference>();

            foreach (Agent Agent in Graph.Agents)
            {
                bool bSameAgent = Agent.Nodes.Contains(Node);
                foreach (Node OtherNode in Agent.Nodes)
                {
                    if (!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
                    {
                        foreach (NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
                        {
                            ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
                        }
                    }
                }
            }

            // Find a block name for all new outputs
            Dictionary <FileReference, string> FileToOutputName = new Dictionary <FileReference, string>();

            foreach (NodeOutput Output in Node.Outputs)
            {
                HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
                foreach (FileReference File in Files)
                {
                    if (!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
                    {
                        if (Output == Node.DefaultOutput)
                        {
                            if (!FileToOutputName.ContainsKey(File))
                            {
                                FileToOutputName[File] = "";
                            }
                        }
                        else
                        {
                            string OutputName;
                            if (FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
                            {
                                FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
                            }
                            else
                            {
                                FileToOutputName[File] = Output.TagName.Substring(1);
                            }
                        }
                    }
                }
            }

            // Invert the dictionary to make a mapping of storage block to the files each contains
            Dictionary <string, HashSet <FileReference> > OutputStorageBlockToFiles = new Dictionary <string, HashSet <FileReference> >();

            foreach (KeyValuePair <FileReference, string> Pair in FileToOutputName)
            {
                HashSet <FileReference> Files;
                if (!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
                {
                    Files = new HashSet <FileReference>();
                    OutputStorageBlockToFiles.Add(Pair.Value, Files);
                }
                Files.Add(Pair.Key);
            }

            // Write all the storage blocks, and update the mapping from file to storage block
            foreach (KeyValuePair <string, HashSet <FileReference> > Pair in OutputStorageBlockToFiles)
            {
                TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
                foreach (FileReference File in Pair.Value)
                {
                    FileToStorageBlock.Add(File, OutputBlock);
                }
                Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
            }

            // Publish all the output tags
            foreach (NodeOutput Output in Node.Outputs)
            {
                HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];

                HashSet <TempStorageBlock> StorageBlocks = new HashSet <TempStorageBlock>();
                foreach (FileReference File in Files)
                {
                    TempStorageBlock StorageBlock;
                    if (FileToStorageBlock.TryGetValue(File, out StorageBlock))
                    {
                        StorageBlocks.Add(StorageBlock);
                    }
                }

                Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
            }

            // Mark the node as succeeded
            Storage.MarkAsComplete(Node.Name);
            return(true);
        }
Пример #16
0
 public virtual void ArchiveBuildProducts(TempStorageNodeInfo TempStorageNodeInfo, bool bLocalOnly)
 {
     TempStorage.StoreToTempStorage(TempStorageNodeInfo, BuildProducts, bLocalOnly, CommandUtils.CmdEnv.LocalRoot);
 }
Пример #17
0
        /// <summary>
        /// Main entry point for the BuildGraph command
        /// </summary>
        public override ExitCode Execute()
        {
            // Parse the command line parameters
            string ScriptFileName        = ParseParamValue("Script", null);
            string TargetNames           = ParseParamValue("Target", null);
            string DocumentationFileName = ParseParamValue("Documentation", null);
            string SchemaFileName        = ParseParamValue("Schema", null);
            string ExportFileName        = ParseParamValue("Export", null);
            string PreprocessedFileName  = ParseParamValue("Preprocess", null);
            string SharedStorageDir      = ParseParamValue("SharedStorageDir", null);
            string SingleNodeName        = ParseParamValue("SingleNode", null);
            string TriggerName           = ParseParamValue("Trigger", null);

            string[] SkipTriggerNames          = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
            bool     bSkipTriggers             = ParseParam("SkipTriggers");
            string   TokenSignature            = ParseParamValue("TokenSignature", null);
            bool     bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
            bool     bResume               = SingleNodeName != null || ParseParam("Resume");
            bool     bListOnly             = ParseParam("ListOnly");
            bool     bShowDiagnostics      = ParseParam("ShowDiagnostics");
            bool     bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
            bool     bPublicTasksOnly      = ParseParam("PublicTasksOnly");
            string   ReportName            = ParseParamValue("ReportName", null);

            GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;

            if (ParseParam("ShowDeps"))
            {
                PrintOptions |= GraphPrintOptions.ShowDependencies;
            }
            if (ParseParam("ShowNotifications"))
            {
                PrintOptions |= GraphPrintOptions.ShowNotifications;
            }

            // Parse any specific nodes to clean
            List <string> CleanNodes = new List <string>();

            foreach (string NodeList in ParseParamValues("CleanNode"))
            {
                foreach (string NodeName in NodeList.Split('+', ';'))
                {
                    CleanNodes.Add(NodeName);
                }
            }

            // Set up the standard properties which build scripts might need
            Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            DefaultProperties["Branch"]                 = P4Enabled ? P4Env.Branch : "Unknown";
            DefaultProperties["Depot"]                  = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown";
            DefaultProperties["EscapedBranch"]          = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown";
            DefaultProperties["Change"]                 = P4Enabled ? P4Env.Changelist.ToString() : "0";
            DefaultProperties["CodeChange"]             = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
            DefaultProperties["RootDir"]                = CommandUtils.RootDirectory.FullName;
            DefaultProperties["IsBuildMachine"]         = IsBuildMachine ? "true" : "false";
            DefaultProperties["HostPlatform"]           = HostPlatform.Current.HostEditorPlatform.ToString();
            DefaultProperties["RestrictedFolderNames"]  = String.Join(";", RestrictedFolders.Names);
            DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x)));

            // Attempt to read existing Build Version information
            BuildVersion Version;

            if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
            {
                DefaultProperties["EngineMajorVersion"]     = Version.MajorVersion.ToString();
                DefaultProperties["EngineMinorVersion"]     = Version.MinorVersion.ToString();
                DefaultProperties["EnginePatchVersion"]     = Version.PatchVersion.ToString();
                DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString();
            }

            // Add any additional custom arguments from the command line (of the form -Set:X=Y)
            Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            foreach (string Param in Params)
            {
                const string Prefix = "set:";
                if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
                {
                    int EqualsIdx = Param.IndexOf('=');
                    if (EqualsIdx >= 0)
                    {
                        Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
                    }
                    else
                    {
                        LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
                    }
                }
            }

            // Find all the tasks from the loaded assemblies
            Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();

            if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
            {
                return(ExitCode.Error_Unknown);
            }

            // Generate documentation
            if (DocumentationFileName != null)
            {
                GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
                return(ExitCode.Success);
            }

            // Create a schema for the given tasks
            ScriptSchema Schema = new ScriptSchema(NameToTask);

            if (SchemaFileName != null)
            {
                FileReference FullSchemaFileName = new FileReference(SchemaFileName);
                LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName);
                Schema.Export(FullSchemaFileName);
                if (ScriptFileName == null)
                {
                    return(ExitCode.Success);
                }
            }

            // Check there was a script specified
            if (ScriptFileName == null)
            {
                LogError("Missing -Script= parameter for BuildGraph");
                return(ExitCode.Error_Unknown);
            }

            // Read the script from disk
            Graph Graph;

            if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
            {
                return(ExitCode.Error_Unknown);
            }

            // Create the temp storage handler
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
            TempStorage        Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);

            if (!bResume)
            {
                Storage.CleanLocal();
            }
            foreach (string CleanNode in CleanNodes)
            {
                Storage.CleanLocalNode(CleanNode);
            }

            // Convert the supplied target references into nodes
            HashSet <Node> TargetNodes = new HashSet <Node>();

            if (TargetNames == null)
            {
                if (!bListOnly)
                {
                    LogError("Missing -Target= parameter for BuildGraph");
                    return(ExitCode.Error_Unknown);
                }
                TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
            }
            else
            {
                foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
                {
                    Node[] Nodes;
                    if (!Graph.TryResolveReference(TargetName, out Nodes))
                    {
                        LogError("Target '{0}' is not in graph", TargetName);
                        return(ExitCode.Error_Unknown);
                    }
                    TargetNodes.UnionWith(Nodes);
                }
            }

            // Try to acquire tokens for all the target nodes we want to build
            if (TokenSignature != null)
            {
                // Find all the lock files
                HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));

                // List out all the required tokens
                if (SingleNodeName == null)
                {
                    CommandUtils.LogInformation("Required tokens:");
                    foreach (Node Node in TargetNodes)
                    {
                        foreach (FileReference RequiredToken in Node.RequiredTokens)
                        {
                            CommandUtils.LogInformation("  '{0}' requires {1}", Node, RequiredToken);
                        }
                    }
                }

                // Try to create all the lock files
                List <FileReference> CreatedTokens = new List <FileReference>();
                if (!bListOnly)
                {
                    CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
                }

                // Find all the tokens that we don't have
                Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
                foreach (FileReference RequiredToken in RequiredTokens)
                {
                    string CurrentOwner = ReadTokenFile(RequiredToken);
                    if (CurrentOwner != null && CurrentOwner != TokenSignature)
                    {
                        MissingTokens.Add(RequiredToken, CurrentOwner);
                    }
                }

                // If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
                if (MissingTokens.Count > 0)
                {
                    if (bSkipTargetsWithoutTokens)
                    {
                        // Find all the nodes we're going to skip
                        HashSet <Node> SkipNodes = new HashSet <Node>();
                        foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
                        {
                            LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
                            foreach (FileReference MissingToken in MissingTokensForBuild)
                            {
                                foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
                                {
                                    LogInformation("    {0}", SkipNode);
                                }
                            }
                        }

                        // Write a list of everything left over
                        if (SkipNodes.Count > 0)
                        {
                            TargetNodes.ExceptWith(SkipNodes);
                            LogInformation("Remaining target nodes:");
                            foreach (Node TargetNode in TargetNodes)
                            {
                                LogInformation("    {0}", TargetNode);
                            }
                            if (TargetNodes.Count == 0)
                            {
                                LogInformation("    None.");
                            }
                        }
                    }
                    else
                    {
                        foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
                        {
                            List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
                            LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
                        }
                        foreach (FileReference CreatedToken in CreatedTokens)
                        {
                            FileReference.Delete(CreatedToken);
                        }
                        return(ExitCode.Error_Unknown);
                    }
                }
            }

            // Cull the graph to include only those nodes
            Graph.Select(TargetNodes);

            // Collapse any triggers in the graph which are marked to be skipped
            HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();

            if (bSkipTriggers)
            {
                SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
            }
            else
            {
                foreach (string SkipTriggerName in SkipTriggerNames)
                {
                    ManualTrigger SkipTrigger;
                    if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
                    {
                        LogError("Couldn't find trigger '{0}'", TriggerName);
                        return(ExitCode.Error_Unknown);
                    }
                    SkipTriggers.Add(SkipTrigger);
                }
            }
            Graph.SkipTriggers(SkipTriggers);

            // If a report for the whole build was requested, insert it into the graph
            if (ReportName != null)
            {
                Report NewReport = new Report(ReportName);
                NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
                Graph.NameToReport.Add(ReportName, NewReport);
            }

            // Write out the preprocessed script
            if (PreprocessedFileName != null)
            {
                FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName);
                LogInformation("Writing {0}...", PreprocessedFileLocation);
                Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
                return(ExitCode.Success);
            }

            // Find the triggers which we are explicitly running.
            ManualTrigger Trigger = null;

            if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
            {
                LogError("Couldn't find trigger '{0}'", TriggerName);
                return(ExitCode.Error_Unknown);
            }

            // If we're just building a single node, find it
            Node SingleNode = null;

            if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
            {
                LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
                return(ExitCode.Error_Unknown);
            }

            // If we just want to show the contents of the graph, do so and exit.
            if (bListOnly)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
            }

            // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
            if (SingleNode == null && (!bListOnly || bShowDiagnostics))
            {
                IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
                foreach (GraphDiagnostic Diagnostic in Diagnostics)
                {
                    if (Diagnostic.EventType == LogEventType.Console)
                    {
                        CommandUtils.LogInformation(Diagnostic.Message);
                    }
                    else if (Diagnostic.EventType == LogEventType.Warning)
                    {
                        CommandUtils.LogWarning(Diagnostic.Message);
                    }
                    else
                    {
                        CommandUtils.LogError(Diagnostic.Message);
                    }
                }
                if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
                {
                    return(ExitCode.Error_Unknown);
                }
            }

            // Export the graph to a file
            if (ExportFileName != null)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
                Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
                return(ExitCode.Success);
            }

            // Execute the command
            if (!bListOnly)
            {
                if (SingleNode != null)
                {
                    if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
                    {
                        return(ExitCode.Error_Unknown);
                    }
                }
                else
                {
                    if (!BuildAllNodes(new JobContext(this), Graph, Storage))
                    {
                        return(ExitCode.Error_Unknown);
                    }
                }
            }
            return(ExitCode.Success);
        }
Пример #18
0
		/// <summary>
		/// Find all the nodes in the graph which are already completed
		/// </summary>
		/// <param name="Graph">The graph instance</param>
		/// <param name="Storage">The temp storage backend which stores the shared state</param>
		HashSet<Node> FindCompletedNodes(Graph Graph, TempStorage Storage)
		{
			HashSet<Node> CompletedNodes = new HashSet<Node>();
			foreach(Node Node in Graph.Agents.SelectMany(x => x.Nodes))
			{
				if(Storage.IsComplete(Node.Name))
				{
					CompletedNodes.Add(Node);
				}
			}
			return CompletedNodes;
		}