List<ChangedArtifact> LoadChanges(DateTime fromTime, DateTime toTime, CancellationToken ct) { var toS = toTime.ToString("yyyy-MM-ddTHH:mm:ss"); var fromS = fromTime.ToString("yyyy-MM-ddTHH:mm:ss"); var client = new RallyRestApi(Settings.Default.RallyUser, Settings.Default.RallyPassword, cancellationToken: ct); if (_currentUser == null) { backgroundOperation.SetProgress("Get current user ..."); if (string.IsNullOrWhiteSpace(textBoxUser.Text) || textBoxUser.Tag != null) { _currentUser = client.GetCurrentUser(); } else { var query = new Query("(UserName = "******")"); var resp = client.Query(new Request("user") { Query = query }); _currentUser = resp.Results.First(); } } // retrieve revisions var requestRevs = new Request("revision"); requestRevs.Limit = Int32.MaxValue; requestRevs.Fetch = new List<string> { "Description", "RevisionHistory", "CreationDate" }; requestRevs.Query = new Query("User", Query.Operator.Equals, _currentUser["UserName"]) .And(new Query("CreationDate", Query.Operator.GreaterThanOrEqualTo, fromS)) .And(new Query("CreationDate", Query.Operator.LessThanOrEqualTo, toS)) ; backgroundOperation.SetProgress("Load revisions ..."); var resultRevs = client.Query(requestRevs); var revHists = resultRevs.Results.Cast<DynamicJsonObject>() .GroupBy(r => (string)r["RevisionHistory"]["_refObjectUUID"]) .ToDictionary(g => g.Key, g => g.Select(r => (DynamicJsonObject)r).ToList()) ; // check cancellation ct.ThrowIfCancellationRequested(); // query artifacts var requestArtifacts = new Request("artifact"); requestArtifacts.Limit = Int32.MaxValue; requestArtifacts.Fetch = new List<string> { "FormattedID", "Name", "RevisionHistory", "LastUpdateDate", "Requirement", "WorkProduct", "ObjectID", "Owner", "Project" }; requestArtifacts.Query = new Query("LastUpdateDate", Query.Operator.GreaterThanOrEqualTo, fromS) .And(new Query("LastUpdateDate", Query.Operator.LessThanOrEqualTo, toS)) ; var artUuid2Revs = new Dictionary<string, List<DynamicJsonObject>>(); var artUuid2Art = new Dictionary<string, DynamicJsonObject>(); backgroundOperation.SetProgress("Load artifacts ..."); var resultArts = client.Query(requestArtifacts); foreach (var art in resultArts.Results) { // can't inject in Query .... var type = (string)art["_type"]; if (type != "Task" && type != "Defect") continue; var histUuid = (string)art["RevisionHistory"]["_refObjectUUID"]; if (!revHists.ContainsKey(histUuid)) continue; List<DynamicJsonObject> artifactRevs; if (!artUuid2Revs.TryGetValue((string)art["_refObjectUUID"], out artifactRevs)) { artifactRevs = new List<DynamicJsonObject>(); artUuid2Revs[(string)art["_refObjectUUID"]] = artifactRevs; } artifactRevs.AddRange(revHists[histUuid]); artUuid2Art[(string)art["_refObjectUUID"]] = art; } foreach (var group in artUuid2Art.GroupBy(kvp => new Artifact(kvp.Value).Type).ToArray()) { ct.ThrowIfCancellationRequested(); backgroundOperation.SetProgress(string.Format("Loading group of '{0}' ...", group.Key)); var allIds = group.Select(a => new Artifact(a.Value).ObjectID.ToString(CultureInfo.InvariantCulture)).ToArray(); var query = allIds .Skip(1) .Aggregate(new Query("ObjectId", Query.Operator.Equals, allIds.First()), (current, id) => current.Or(new Query("ObjectId", Query.Operator.Equals, id))) ; var r = new Request(group.First().Value["_type"]); r.Limit = Int32.MaxValue; r.Query = query; var qr = client.Query(r); foreach (var result in qr.Results) { if (result["_type"] == "Task") { artUuid2Art[result["_refObjectUUID"]]["Estimate"] = (double)result["Estimate"]; artUuid2Art[result["_refObjectUUID"]]["ToDo"] = (double)result["ToDo"]; artUuid2Art[result["_refObjectUUID"]]["State"] = (string)result["State"]; } else if (result["_type"] == "Defect") { //var additionalInfo = client.GetByReference(art.Reference); //artUuid2Art[kvp.Key]["Estimate"] = additionalInfo["Estimate"]; //artUuid2Art["ToDo"] = additionalInfo["ToDo"]; } } } var changes = artUuid2Revs .Select(kvp => new ChangedArtifact { Artifact = artUuid2Art[kvp.Key], Revisions = kvp.Value, WorkItem = new Artifact(artUuid2Art[kvp.Key]) }) .ToList() ; UpdateCache(changes); return changes; }
/// <summary> /// Main entry point for the BuildGraph command /// </summary> public override ExitCode Execute() { // Parse the command line parameters string ScriptFileName = ParseParamValue("Script", null); string TargetNames = ParseParamValue("Target", null); string DocumentationFileName = ParseParamValue("Documentation", null); string SchemaFileName = ParseParamValue("Schema", null); string ExportFileName = ParseParamValue("Export", null); string PreprocessedFileName = ParseParamValue("Preprocess", null); string SharedStorageDir = ParseParamValue("SharedStorageDir", null); string SingleNodeName = ParseParamValue("SingleNode", null); string TriggerName = ParseParamValue("Trigger", null); string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); bool bSkipTriggers = ParseParam("SkipTriggers"); string TokenSignature = ParseParamValue("TokenSignature", null); bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens"); bool bClearHistory = ParseParam("Clean") || ParseParam("ClearHistory"); bool bListOnly = ParseParam("ListOnly"); bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine; bool bPublicTasksOnly = ParseParam("PublicTasksOnly"); string ReportName = ParseParamValue("ReportName", null); GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions; if(ParseParam("ShowDeps")) { PrintOptions |= GraphPrintOptions.ShowDependencies; } if(ParseParam("ShowNotifications")) { PrintOptions |= GraphPrintOptions.ShowNotifications; } // Parse any specific nodes to clean List<string> CleanNodes = new List<string>(); foreach(string NodeList in ParseParamValues("CleanNode")) { foreach(string NodeName in NodeList.Split('+', ';')) { CleanNodes.Add(NodeName); } } // Set up the standard properties which build scripts might need Dictionary<string, string> DefaultProperties = new Dictionary<string,string>(StringComparer.InvariantCultureIgnoreCase); DefaultProperties["Branch"] = P4Enabled ? P4Env.BuildRootP4 : "Unknown"; DefaultProperties["EscapedBranch"] = P4Enabled ? P4Env.BuildRootEscaped : "Unknown"; DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0"; DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0"; DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName; DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false"; DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString(); // Attempt to read existing Build Version information BuildVersion Version; if (BuildVersion.TryRead(FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Build", "Build.version").FullName, out Version)) { DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString(); DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString(); DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString(); } // Add any additional custom arguments from the command line (of the form -Set:X=Y) Dictionary<string, string> Arguments = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (string Param in Params) { const string Prefix = "set:"; if(Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase)) { int EqualsIdx = Param.IndexOf('='); if(EqualsIdx >= 0) { Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1); } else { LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length)); } } } // Find all the tasks from the loaded assemblies Dictionary<string, ScriptTask> NameToTask = new Dictionary<string,ScriptTask>(); if(!FindAvailableTasks(NameToTask, bPublicTasksOnly)) { return ExitCode.Error_Unknown; } // Generate documentation if(DocumentationFileName != null) { GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName)); return ExitCode.Success; } // Create a schema for the given tasks ScriptSchema Schema = new ScriptSchema(NameToTask); if(SchemaFileName != null) { FileReference FullSchemaFileName = new FileReference(SchemaFileName); Log("Writing schema to {0}...", FullSchemaFileName.FullName); Schema.Export(FullSchemaFileName); if(ScriptFileName == null) { return ExitCode.Success; } } // Check there was a script specified if(ScriptFileName == null) { LogError("Missing -Script= parameter for BuildGraph"); return ExitCode.Error_Unknown; } // Read the script from disk Graph Graph; if(!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph)) { return ExitCode.Error_Unknown; } // Create the temp storage handler DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage); if(bClearHistory) { Storage.CleanLocal(); } foreach(string CleanNode in CleanNodes) { Storage.CleanLocalNode(CleanNode); } // Convert the supplied target references into nodes HashSet<Node> TargetNodes = new HashSet<Node>(); if(TargetNames == null) { if(!bListOnly) { LogError("Missing -Target= parameter for BuildGraph"); return ExitCode.Error_Unknown; } TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); } else { foreach(string TargetName in TargetNames.Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())) { Node[] Nodes; if(!Graph.TryResolveReference(TargetName, out Nodes)) { LogError("Target '{0}' is not in graph", TargetName); return ExitCode.Error_Unknown; } TargetNodes.UnionWith(Nodes); } } // Try to acquire tokens for all the target nodes we want to build if(TokenSignature != null) { // Find all the lock files HashSet<FileReference> RequiredTokens = new HashSet<FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens)); // List out all the required tokens if(SingleNodeName == null) { CommandUtils.Log("Required tokens:"); foreach(Node Node in TargetNodes) { foreach(FileReference RequiredToken in Node.RequiredTokens) { CommandUtils.Log(" '{0}' requires {1}", Node, RequiredToken); } } } // Try to create all the lock files List<FileReference> CreatedTokens = new List<FileReference>(); if(!bListOnly) { CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature))); } // Find all the tokens that we don't have Dictionary<FileReference, string> MissingTokens = new Dictionary<FileReference, string>(); foreach(FileReference RequiredToken in RequiredTokens) { string CurrentOwner = ReadTokenFile(RequiredToken); if(CurrentOwner != null && CurrentOwner != TokenSignature) { MissingTokens.Add(RequiredToken, CurrentOwner); } } // If we want to skip all the nodes with missing locks, adjust the target nodes to account for it if(MissingTokens.Count > 0) { if(bSkipTargetsWithoutTokens) { // Find all the nodes we're going to skip HashSet<Node> SkipNodes = new HashSet<Node>(); foreach(IGrouping<string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key)) { Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key); foreach(FileReference MissingToken in MissingTokensForBuild) { foreach(Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x))) { Log(" {0}", SkipNode); } } } // Write a list of everything left over if(SkipNodes.Count > 0) { TargetNodes.ExceptWith(SkipNodes); Log("Remaining target nodes:"); foreach(Node TargetNode in TargetNodes) { Log(" {0}", TargetNode); } if(TargetNodes.Count == 0) { Log(" None."); } } } else { foreach(KeyValuePair<FileReference, string> Pair in MissingTokens) { List<Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList(); LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value); } foreach(FileReference CreatedToken in CreatedTokens) { CreatedToken.Delete(); } return ExitCode.Error_Unknown; } } } // Cull the graph to include only those nodes Graph.Select(TargetNodes); // Collapse any triggers in the graph which are marked to be skipped HashSet<ManualTrigger> SkipTriggers = new HashSet<ManualTrigger>(); if(bSkipTriggers) { SkipTriggers.UnionWith(Graph.NameToTrigger.Values); } else { foreach(string SkipTriggerName in SkipTriggerNames) { ManualTrigger SkipTrigger; if(!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return ExitCode.Error_Unknown; } SkipTriggers.Add(SkipTrigger); } } Graph.SkipTriggers(SkipTriggers); // If a report for the whole build was requested, insert it into the graph if (ReportName != null) { Report NewReport = new Report(ReportName); NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); Graph.NameToReport.Add(ReportName, NewReport); } // Write out the preprocessed script if (PreprocessedFileName != null) { Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null); } // Find the triggers which we are explicitly running. ManualTrigger Trigger = null; if(TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return ExitCode.Error_Unknown; } // If we're just building a single node, find it Node SingleNode = null; if(SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode)) { LogError("Node '{0}' is not in the trimmed graph", SingleNodeName); return ExitCode.Error_Unknown; } // If we just want to show the contents of the graph, do so and exit. if(bListOnly) { HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); return ExitCode.Success; } // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file. if(SingleNode == null) { IEnumerable<GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger); foreach(GraphDiagnostic Diagnostic in Diagnostics) { if(Diagnostic.EventType == LogEventType.Warning) { CommandUtils.LogWarning(Diagnostic.Message); } else { CommandUtils.LogError(Diagnostic.Message); } } if(Diagnostics.Any(x => x.EventType == LogEventType.Error)) { return ExitCode.Error_Unknown; } } // Execute the command if(ExportFileName != null) { HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes); } else if(SingleNode != null) { if(!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true)) { return ExitCode.Error_Unknown; } } else { if(!BuildAllNodes(new JobContext(this), Graph, Storage)) { return ExitCode.Error_Unknown; } } return ExitCode.Success; }