Exemplo n.º 1
0
        /// <summary>
        /// Main entry point for the BuildGraph command
        /// </summary>
        public override ExitCode Execute()
        {
            // Parse the command line parameters
            string ScriptFileName        = ParseParamValue("Script", null);
            string TargetNames           = ParseParamValue("Target", null);
            string DocumentationFileName = ParseParamValue("Documentation", null);
            string SchemaFileName        = ParseParamValue("Schema", null);
            string ExportFileName        = ParseParamValue("Export", null);
            string PreprocessedFileName  = ParseParamValue("Preprocess", null);
            string SharedStorageDir      = ParseParamValue("SharedStorageDir", null);
            string SingleNodeName        = ParseParamValue("SingleNode", null);
            string TriggerName           = ParseParamValue("Trigger", null);

            string[] SkipTriggerNames          = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
            bool     bSkipTriggers             = ParseParam("SkipTriggers");
            string   TokenSignature            = ParseParamValue("TokenSignature", null);
            bool     bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
            bool     bResume               = SingleNodeName != null || ParseParam("Resume");
            bool     bListOnly             = ParseParam("ListOnly");
            bool     bShowDiagnostics      = ParseParam("ShowDiagnostics");
            bool     bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
            bool     bPublicTasksOnly      = ParseParam("PublicTasksOnly");
            string   ReportName            = ParseParamValue("ReportName", null);

            GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;

            if (ParseParam("ShowDeps"))
            {
                PrintOptions |= GraphPrintOptions.ShowDependencies;
            }
            if (ParseParam("ShowNotifications"))
            {
                PrintOptions |= GraphPrintOptions.ShowNotifications;
            }

            // Parse any specific nodes to clean
            List <string> CleanNodes = new List <string>();

            foreach (string NodeList in ParseParamValues("CleanNode"))
            {
                foreach (string NodeName in NodeList.Split('+', ';'))
                {
                    CleanNodes.Add(NodeName);
                }
            }

            // Set up the standard properties which build scripts might need
            Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            DefaultProperties["Branch"]                 = P4Enabled ? P4Env.Branch : "Unknown";
            DefaultProperties["Depot"]                  = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown";
            DefaultProperties["EscapedBranch"]          = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown";
            DefaultProperties["Change"]                 = P4Enabled ? P4Env.Changelist.ToString() : "0";
            DefaultProperties["CodeChange"]             = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
            DefaultProperties["RootDir"]                = CommandUtils.RootDirectory.FullName;
            DefaultProperties["IsBuildMachine"]         = IsBuildMachine ? "true" : "false";
            DefaultProperties["HostPlatform"]           = HostPlatform.Current.HostEditorPlatform.ToString();
            DefaultProperties["RestrictedFolderNames"]  = String.Join(";", RestrictedFolders.Names);
            DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x)));

            // Attempt to read existing Build Version information
            BuildVersion Version;

            if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
            {
                DefaultProperties["EngineMajorVersion"]     = Version.MajorVersion.ToString();
                DefaultProperties["EngineMinorVersion"]     = Version.MinorVersion.ToString();
                DefaultProperties["EnginePatchVersion"]     = Version.PatchVersion.ToString();
                DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString();
            }

            // Add any additional custom arguments from the command line (of the form -Set:X=Y)
            Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            foreach (string Param in Params)
            {
                const string Prefix = "set:";
                if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
                {
                    int EqualsIdx = Param.IndexOf('=');
                    if (EqualsIdx >= 0)
                    {
                        Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
                    }
                    else
                    {
                        LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
                    }
                }
            }

            // Find all the tasks from the loaded assemblies
            Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();

            if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
            {
                return(ExitCode.Error_Unknown);
            }

            // Generate documentation
            if (DocumentationFileName != null)
            {
                GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
                return(ExitCode.Success);
            }

            // Create a schema for the given tasks
            ScriptSchema Schema = new ScriptSchema(NameToTask);

            if (SchemaFileName != null)
            {
                FileReference FullSchemaFileName = new FileReference(SchemaFileName);
                LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName);
                Schema.Export(FullSchemaFileName);
                if (ScriptFileName == null)
                {
                    return(ExitCode.Success);
                }
            }

            // Check there was a script specified
            if (ScriptFileName == null)
            {
                LogError("Missing -Script= parameter for BuildGraph");
                return(ExitCode.Error_Unknown);
            }

            // Read the script from disk
            Graph Graph;

            if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
            {
                return(ExitCode.Error_Unknown);
            }

            // Create the temp storage handler
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
            TempStorage        Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);

            if (!bResume)
            {
                Storage.CleanLocal();
            }
            foreach (string CleanNode in CleanNodes)
            {
                Storage.CleanLocalNode(CleanNode);
            }

            // Convert the supplied target references into nodes
            HashSet <Node> TargetNodes = new HashSet <Node>();

            if (TargetNames == null)
            {
                if (!bListOnly)
                {
                    LogError("Missing -Target= parameter for BuildGraph");
                    return(ExitCode.Error_Unknown);
                }
                TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
            }
            else
            {
                foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
                {
                    Node[] Nodes;
                    if (!Graph.TryResolveReference(TargetName, out Nodes))
                    {
                        LogError("Target '{0}' is not in graph", TargetName);
                        return(ExitCode.Error_Unknown);
                    }
                    TargetNodes.UnionWith(Nodes);
                }
            }

            // Try to acquire tokens for all the target nodes we want to build
            if (TokenSignature != null)
            {
                // Find all the lock files
                HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));

                // List out all the required tokens
                if (SingleNodeName == null)
                {
                    CommandUtils.LogInformation("Required tokens:");
                    foreach (Node Node in TargetNodes)
                    {
                        foreach (FileReference RequiredToken in Node.RequiredTokens)
                        {
                            CommandUtils.LogInformation("  '{0}' requires {1}", Node, RequiredToken);
                        }
                    }
                }

                // Try to create all the lock files
                List <FileReference> CreatedTokens = new List <FileReference>();
                if (!bListOnly)
                {
                    CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
                }

                // Find all the tokens that we don't have
                Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
                foreach (FileReference RequiredToken in RequiredTokens)
                {
                    string CurrentOwner = ReadTokenFile(RequiredToken);
                    if (CurrentOwner != null && CurrentOwner != TokenSignature)
                    {
                        MissingTokens.Add(RequiredToken, CurrentOwner);
                    }
                }

                // If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
                if (MissingTokens.Count > 0)
                {
                    if (bSkipTargetsWithoutTokens)
                    {
                        // Find all the nodes we're going to skip
                        HashSet <Node> SkipNodes = new HashSet <Node>();
                        foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
                        {
                            LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
                            foreach (FileReference MissingToken in MissingTokensForBuild)
                            {
                                foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
                                {
                                    LogInformation("    {0}", SkipNode);
                                }
                            }
                        }

                        // Write a list of everything left over
                        if (SkipNodes.Count > 0)
                        {
                            TargetNodes.ExceptWith(SkipNodes);
                            LogInformation("Remaining target nodes:");
                            foreach (Node TargetNode in TargetNodes)
                            {
                                LogInformation("    {0}", TargetNode);
                            }
                            if (TargetNodes.Count == 0)
                            {
                                LogInformation("    None.");
                            }
                        }
                    }
                    else
                    {
                        foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
                        {
                            List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
                            LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
                        }
                        foreach (FileReference CreatedToken in CreatedTokens)
                        {
                            FileReference.Delete(CreatedToken);
                        }
                        return(ExitCode.Error_Unknown);
                    }
                }
            }

            // Cull the graph to include only those nodes
            Graph.Select(TargetNodes);

            // Collapse any triggers in the graph which are marked to be skipped
            HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();

            if (bSkipTriggers)
            {
                SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
            }
            else
            {
                foreach (string SkipTriggerName in SkipTriggerNames)
                {
                    ManualTrigger SkipTrigger;
                    if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
                    {
                        LogError("Couldn't find trigger '{0}'", TriggerName);
                        return(ExitCode.Error_Unknown);
                    }
                    SkipTriggers.Add(SkipTrigger);
                }
            }
            Graph.SkipTriggers(SkipTriggers);

            // If a report for the whole build was requested, insert it into the graph
            if (ReportName != null)
            {
                Report NewReport = new Report(ReportName);
                NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
                Graph.NameToReport.Add(ReportName, NewReport);
            }

            // Write out the preprocessed script
            if (PreprocessedFileName != null)
            {
                FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName);
                LogInformation("Writing {0}...", PreprocessedFileLocation);
                Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
                return(ExitCode.Success);
            }

            // Find the triggers which we are explicitly running.
            ManualTrigger Trigger = null;

            if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
            {
                LogError("Couldn't find trigger '{0}'", TriggerName);
                return(ExitCode.Error_Unknown);
            }

            // If we're just building a single node, find it
            Node SingleNode = null;

            if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
            {
                LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
                return(ExitCode.Error_Unknown);
            }

            // If we just want to show the contents of the graph, do so and exit.
            if (bListOnly)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
            }

            // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
            if (SingleNode == null && (!bListOnly || bShowDiagnostics))
            {
                IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
                foreach (GraphDiagnostic Diagnostic in Diagnostics)
                {
                    if (Diagnostic.EventType == LogEventType.Console)
                    {
                        CommandUtils.LogInformation(Diagnostic.Message);
                    }
                    else if (Diagnostic.EventType == LogEventType.Warning)
                    {
                        CommandUtils.LogWarning(Diagnostic.Message);
                    }
                    else
                    {
                        CommandUtils.LogError(Diagnostic.Message);
                    }
                }
                if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
                {
                    return(ExitCode.Error_Unknown);
                }
            }

            // Export the graph to a file
            if (ExportFileName != null)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
                Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
                return(ExitCode.Success);
            }

            // Execute the command
            if (!bListOnly)
            {
                if (SingleNode != null)
                {
                    if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
                    {
                        return(ExitCode.Error_Unknown);
                    }
                }
                else
                {
                    if (!BuildAllNodes(new JobContext(this), Graph, Storage))
                    {
                        return(ExitCode.Error_Unknown);
                    }
                }
            }
            return(ExitCode.Success);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Main entry point for the BuildGraph command
        /// </summary>
        public override ExitCode Execute()
        {
            // Parse the command line parameters
            string ScriptFileName = ParseParamValue("Script", null);

            if (ScriptFileName == null)
            {
                LogError("Missing -Script= parameter for BuildGraph");
                return(ExitCode.Error_Unknown);
            }

            string TargetNames = ParseParamValue("Target", null);

            if (TargetNames == null)
            {
                LogError("Missing -Target= parameter for BuildGraph");
                return(ExitCode.Error_Unknown);
            }

            string SchemaFileName = ParseParamValue("Schema", null);
            string ExportFileName = ParseParamValue("Export", null);

            string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
            string SingleNodeName   = ParseParamValue("SingleNode", null);

            string[] TriggerNames          = ParseParamValue("Trigger", "").Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
            bool     bSkipTriggers         = ParseParam("SkipTriggers");
            bool     bClean                = ParseParam("Clean");
            bool     bListOnly             = ParseParam("ListOnly");
            bool     bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
            bool     bPublicTasksOnly      = ParseParam("PublicTasksOnly");

            GraphPrintOptions PrintOptions = 0;

            if (ParseParam("ShowDeps"))
            {
                PrintOptions |= GraphPrintOptions.ShowDependencies;
            }
            if (ParseParam("ShowNotifications"))
            {
                PrintOptions |= GraphPrintOptions.ShowNotifications;
            }

            // Parse any specific nodes to clean
            List <string> CleanNodes = new List <string>();

            foreach (string NodeList in ParseParamValues("CleanNode"))
            {
                foreach (string NodeName in NodeList.Split('+'))
                {
                    CleanNodes.Add(NodeName);
                }
            }

            // Read any environment variables
            Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);

            foreach (DictionaryEntry Entry in Environment.GetEnvironmentVariables())
            {
                DefaultProperties[Entry.Key.ToString()] = Entry.Value.ToString();
            }

            // Add any additional custom parameters from the command line (of the form -Set:X=Y)
            foreach (string Param in Params)
            {
                const string Prefix = "set:";
                if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
                {
                    int EqualsIdx = Param.IndexOf('=');
                    if (EqualsIdx >= 0)
                    {
                        DefaultProperties[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
                    }
                    else
                    {
                        LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
                    }
                }
            }

            // Set up the standard properties which build scripts might need
            DefaultProperties["Branch"]         = P4Enabled? P4Env.BuildRootP4 : "Unknown";
            DefaultProperties["EscapedBranch"]  = P4Enabled? P4Env.BuildRootEscaped : "Unknown";
            DefaultProperties["Change"]         = P4Enabled? P4Env.Changelist.ToString() : "0";
            DefaultProperties["RootDir"]        = CommandUtils.RootDirectory.FullName;
            DefaultProperties["IsBuildMachine"] = IsBuildMachine? "true" : "false";
            DefaultProperties["HostPlatform"]   = HostPlatform.Current.HostEditorPlatform.ToString();

            // Find all the tasks from the loaded assemblies
            Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();

            if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
            {
                return(ExitCode.Error_Unknown);
            }

            // Create a schema for the given tasks
            ScriptSchema Schema = new ScriptSchema(NameToTask);

            if (SchemaFileName != null)
            {
                Schema.Export(new FileReference(SchemaFileName));
            }

            // Read the script from disk
            Graph Graph;

            if (!ScriptReader.TryRead(new FileReference(ScriptFileName), DefaultProperties, Schema, out Graph))
            {
                return(ExitCode.Error_Unknown);
            }

            // Create the temp storage handler
            DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
            TempStorage        Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);

            if (bClean)
            {
                Storage.CleanLocal();
            }
            foreach (string CleanNode in CleanNodes)
            {
                Storage.CleanLocalNode(CleanNode);
            }

            // Convert the supplied target references into nodes
            HashSet <Node> TargetNodes = new HashSet <Node>();

            foreach (string TargetName in TargetNames.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
            {
                Node[] Nodes;
                if (!Graph.TryResolveReference(TargetName, out Nodes))
                {
                    LogError("Target '{0}' is not in graph", TargetName);
                    return(ExitCode.Error_Unknown);
                }
                TargetNodes.UnionWith(Nodes);
            }

            // Cull the graph to include only those nodes
            Graph.Select(TargetNodes);

            // Find the triggers which are explicitly activated, and all of its upstream triggers.
            HashSet <ManualTrigger> Triggers = new HashSet <ManualTrigger>();

            foreach (string TriggerName in TriggerNames)
            {
                ManualTrigger Trigger;
                if (!Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
                {
                    LogError("Couldn't find trigger '{0}'", TriggerName);
                    return(ExitCode.Error_Unknown);
                }
                while (Trigger != null)
                {
                    Triggers.Add(Trigger);
                    Trigger = Trigger.Parent;
                }
            }
            if (bSkipTriggers)
            {
                Triggers.UnionWith(Graph.NameToTrigger.Values);
            }

            // If we're just building a single node, find it
            Node SingleNode = null;

            if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
            {
                LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
                return(ExitCode.Error_Unknown);
            }

            // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system.
            if (SingleNode == null)
            {
                IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == null || Triggers.Contains(x.EnclosingTrigger));
                foreach (GraphDiagnostic Diagnostic in Diagnostics)
                {
                    if (Diagnostic.EventType == LogEventType.Warning)
                    {
                        CommandUtils.LogWarning(Diagnostic.Message);
                    }
                    else
                    {
                        CommandUtils.LogError(Diagnostic.Message);
                    }
                }
                if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
                {
                    return(ExitCode.Error_Unknown);
                }
            }

            // Execute the command
            if (bListOnly)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
            }
            else if (ExportFileName != null)
            {
                HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
                Graph.Print(CompletedNodes, PrintOptions);
                Graph.Export(new FileReference(ExportFileName), Triggers, CompletedNodes);
            }
            else if (SingleNode != null)
            {
                if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
                {
                    return(ExitCode.Error_Unknown);
                }
            }
            else
            {
                if (!BuildAllNodes(new JobContext(this), Graph, Storage))
                {
                    return(ExitCode.Error_Unknown);
                }
            }
            return(ExitCode.Success);
        }