List<ChangedArtifact> LoadChanges(DateTime fromTime, DateTime toTime, CancellationToken ct)
{
var toS = toTime.ToString("yyyy-MM-ddTHH:mm:ss");
var fromS = fromTime.ToString("yyyy-MM-ddTHH:mm:ss");
var client = new RallyRestApi(Settings.Default.RallyUser, Settings.Default.RallyPassword, cancellationToken: ct);
if (_currentUser == null)
{
backgroundOperation.SetProgress("Get current user ...");
if (string.IsNullOrWhiteSpace(textBoxUser.Text) || textBoxUser.Tag != null)
{
_currentUser = client.GetCurrentUser();
}
else
{
var query = new Query("(UserName = "******")");
var resp = client.Query(new Request("user") { Query = query });
_currentUser = resp.Results.First();
}
}
// retrieve revisions
var requestRevs = new Request("revision");
requestRevs.Limit = Int32.MaxValue;
requestRevs.Fetch = new List<string> { "Description", "RevisionHistory", "CreationDate" };
requestRevs.Query = new Query("User", Query.Operator.Equals, _currentUser["UserName"])
.And(new Query("CreationDate", Query.Operator.GreaterThanOrEqualTo, fromS))
.And(new Query("CreationDate", Query.Operator.LessThanOrEqualTo, toS))
;
backgroundOperation.SetProgress("Load revisions ...");
var resultRevs = client.Query(requestRevs);
var revHists = resultRevs.Results.Cast<DynamicJsonObject>()
.GroupBy(r => (string)r["RevisionHistory"]["_refObjectUUID"])
.ToDictionary(g => g.Key, g => g.Select(r => (DynamicJsonObject)r).ToList())
;
// check cancellation
ct.ThrowIfCancellationRequested();
// query artifacts
var requestArtifacts = new Request("artifact");
requestArtifacts.Limit = Int32.MaxValue;
requestArtifacts.Fetch = new List<string> { "FormattedID", "Name", "RevisionHistory", "LastUpdateDate", "Requirement", "WorkProduct", "ObjectID", "Owner", "Project" };
requestArtifacts.Query = new Query("LastUpdateDate", Query.Operator.GreaterThanOrEqualTo, fromS)
.And(new Query("LastUpdateDate", Query.Operator.LessThanOrEqualTo, toS))
;
var artUuid2Revs = new Dictionary<string, List<DynamicJsonObject>>();
var artUuid2Art = new Dictionary<string, DynamicJsonObject>();
backgroundOperation.SetProgress("Load artifacts ...");
var resultArts = client.Query(requestArtifacts);
foreach (var art in resultArts.Results)
{
// can't inject in Query ....
var type = (string)art["_type"];
if (type != "Task" && type != "Defect")
continue;
var histUuid = (string)art["RevisionHistory"]["_refObjectUUID"];
if (!revHists.ContainsKey(histUuid))
continue;
List<DynamicJsonObject> artifactRevs;
if (!artUuid2Revs.TryGetValue((string)art["_refObjectUUID"], out artifactRevs))
{
artifactRevs = new List<DynamicJsonObject>();
artUuid2Revs[(string)art["_refObjectUUID"]] = artifactRevs;
}
artifactRevs.AddRange(revHists[histUuid]);
artUuid2Art[(string)art["_refObjectUUID"]] = art;
}
foreach (var group in artUuid2Art.GroupBy(kvp => new Artifact(kvp.Value).Type).ToArray())
{
ct.ThrowIfCancellationRequested();
backgroundOperation.SetProgress(string.Format("Loading group of '{0}' ...", group.Key));
var allIds = group.Select(a => new Artifact(a.Value).ObjectID.ToString(CultureInfo.InvariantCulture)).ToArray();
var query = allIds
.Skip(1)
.Aggregate(new Query("ObjectId", Query.Operator.Equals, allIds.First()), (current, id) => current.Or(new Query("ObjectId", Query.Operator.Equals, id)))
;
var r = new Request(group.First().Value["_type"]);
r.Limit = Int32.MaxValue;
r.Query = query;
var qr = client.Query(r);
foreach (var result in qr.Results)
{
if (result["_type"] == "Task")
{
artUuid2Art[result["_refObjectUUID"]]["Estimate"] = (double)result["Estimate"];
artUuid2Art[result["_refObjectUUID"]]["ToDo"] = (double)result["ToDo"];
artUuid2Art[result["_refObjectUUID"]]["State"] = (string)result["State"];
}
else if (result["_type"] == "Defect")
{
//var additionalInfo = client.GetByReference(art.Reference);
//artUuid2Art[kvp.Key]["Estimate"] = additionalInfo["Estimate"];
//artUuid2Art["ToDo"] = additionalInfo["ToDo"];
}
}
}
var changes = artUuid2Revs
.Select(kvp => new ChangedArtifact { Artifact = artUuid2Art[kvp.Key], Revisions = kvp.Value, WorkItem = new Artifact(artUuid2Art[kvp.Key]) })
.ToList()
;
UpdateCache(changes);
return changes;
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bClearHistory = ParseParam("Clean") || ParseParam("ClearHistory");
bool bListOnly = ParseParam("ListOnly");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if(ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if(ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List<string> CleanNodes = new List<string>();
foreach(string NodeList in ParseParamValues("CleanNode"))
{
foreach(string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary<string, string> DefaultProperties = new Dictionary<string,string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.BuildRootP4 : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? P4Env.BuildRootEscaped : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Build", "Build.version").FullName, out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary<string, string> Arguments = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if(Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if(EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary<string, ScriptTask> NameToTask = new Dictionary<string,ScriptTask>();
if(!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return ExitCode.Error_Unknown;
}
// Generate documentation
if(DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return ExitCode.Success;
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if(SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
Log("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if(ScriptFileName == null)
{
return ExitCode.Success;
}
}
// Check there was a script specified
if(ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return ExitCode.Error_Unknown;
}
// Read the script from disk
Graph Graph;
if(!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return ExitCode.Error_Unknown;
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if(bClearHistory)
{
Storage.CleanLocal();
}
foreach(string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet<Node> TargetNodes = new HashSet<Node>();
if(TargetNames == null)
{
if(!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return ExitCode.Error_Unknown;
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach(string TargetName in TargetNames.Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if(!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return ExitCode.Error_Unknown;
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if(TokenSignature != null)
{
// Find all the lock files
HashSet<FileReference> RequiredTokens = new HashSet<FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if(SingleNodeName == null)
{
CommandUtils.Log("Required tokens:");
foreach(Node Node in TargetNodes)
{
foreach(FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.Log(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List<FileReference> CreatedTokens = new List<FileReference>();
if(!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary<FileReference, string> MissingTokens = new Dictionary<FileReference, string>();
foreach(FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if(CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if(MissingTokens.Count > 0)
{
if(bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet<Node> SkipNodes = new HashSet<Node>();
foreach(IGrouping<string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach(FileReference MissingToken in MissingTokensForBuild)
{
foreach(Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
Log(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if(SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
Log("Remaining target nodes:");
foreach(Node TargetNode in TargetNodes)
{
Log(" {0}", TargetNode);
}
if(TargetNodes.Count == 0)
{
Log(" None.");
}
}
}
else
{
foreach(KeyValuePair<FileReference, string> Pair in MissingTokens)
{
List<Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach(FileReference CreatedToken in CreatedTokens)
{
CreatedToken.Delete();
}
return ExitCode.Error_Unknown;
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet<ManualTrigger> SkipTriggers = new HashSet<ManualTrigger>();
if(bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach(string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if(!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return ExitCode.Error_Unknown;
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if(TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return ExitCode.Error_Unknown;
}
// If we're just building a single node, find it
Node SingleNode = null;
if(SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return ExitCode.Error_Unknown;
}
// If we just want to show the contents of the graph, do so and exit.
if(bListOnly)
{
HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
return ExitCode.Success;
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if(SingleNode == null)
{
IEnumerable<GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach(GraphDiagnostic Diagnostic in Diagnostics)
{
if(Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if(Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return ExitCode.Error_Unknown;
}
}
// Execute the command
if(ExportFileName != null)
{
HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
}
else if(SingleNode != null)
{
if(!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return ExitCode.Error_Unknown;
}
}
else
{
if(!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return ExitCode.Error_Unknown;
}
}
return ExitCode.Success;
}
