public void BeginSave(string fileName, AsyncCallback callback, object state)
{
if (Format == DicomFileFormat.ACRNEMA1 || Format == DicomFileFormat.ACRNEMA2)
{
throw new DicomFileException(this, "Unable to save ACR-NEMA file");
}
if (Format == DicomFileFormat.DICOM3NoFileMetaInfo)
{
// create file meta information from dataset
FileMetaInfo = new DicomFileMetaInformation(Dataset);
}
File = new FileReference(fileName);
File.Delete();
OnSave();
FileByteTarget target = new FileByteTarget(File);
EventAsyncResult result = new EventAsyncResult(callback, state);
DicomFileWriter writer = new DicomFileWriter(DicomWriteOptions.Default);
writer.BeginWrite(target, FileMetaInfo, Dataset, OnWriteComplete, new Tuple <DicomFileWriter, EventAsyncResult>(writer, result));
}
public override void StripSymbols(FileReference SourceFile, FileReference TargetFile)
{
bool bStripInPlace = false;
if (SourceFile == TargetFile)
{
// PDBCopy only supports creation of a brand new stripped file so we have to create a temporary filename
TargetFile = new FileReference(Path.Combine(TargetFile.Directory.FullName, Guid.NewGuid().ToString() + TargetFile.GetExtension()));
bStripInPlace = true;
}
ProcessStartInfo StartInfo = new ProcessStartInfo();
string PDBCopyPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "MSBuild", "Microsoft", "VisualStudio", "v14.0", "AppxPackage", "PDBCopy.exe");
if (!File.Exists(PDBCopyPath))
{
// Fall back on VS2013 version
PDBCopyPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "MSBuild", "Microsoft", "VisualStudio", "v12.0", "AppxPackage", "PDBCopy.exe");
}
StartInfo.FileName = PDBCopyPath;
StartInfo.Arguments = String.Format("\"{0}\" \"{1}\" -p", SourceFile.FullName, TargetFile.FullName);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
Utils.RunLocalProcessAndLogOutput(StartInfo);
if (bStripInPlace)
{
// Copy stripped file to original location and delete the temporary file
File.Copy(TargetFile.FullName, SourceFile.FullName, true);
FileReference.Delete(TargetFile);
}
}
public override void StripSymbols(FileReference SourceFile, FileReference TargetFile)
{
bool bStripInPlace = false;
if (SourceFile == TargetFile)
{
// PDBCopy only supports creation of a brand new stripped file so we have to create a temporary filename
TargetFile = new FileReference(Path.Combine(TargetFile.Directory.FullName, Guid.NewGuid().ToString() + TargetFile.GetExtension()));
bStripInPlace = true;
}
FileReference PdbCopyLocation;
if (!TryGetPdbCopyLocation(out PdbCopyLocation))
{
throw new AutomationException("Unable to find installation of PDBCOPY.EXE, which is required to strip symbols. This tool is included as part of the 'Windows Debugging Tools' component of the Windows 10 SDK (https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk).");
}
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = PdbCopyLocation.FullName;
StartInfo.Arguments = String.Format("\"{0}\" \"{1}\" -p", SourceFile.FullName, TargetFile.FullName);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
Utils.RunLocalProcessAndLogOutput(StartInfo);
if (bStripInPlace)
{
// Copy stripped file to original location and delete the temporary file
File.Copy(TargetFile.FullName, SourceFile.FullName, true);
FileReference.Delete(TargetFile);
}
}
public void BeginSave(string fileName, AsyncCallback callback, object state)
{
File = new FileReference(fileName);
File.Delete();
FileByteTarget target = new FileByteTarget(File);
EventAsyncResult async = new EventAsyncResult(callback, state);
DicomFileWriter writer = new DicomFileWriter(DicomWriteOptions.Default);
writer.BeginWrite(target, FileMetaInfo, Dataset, OnWriteComplete, new Tuple <DicomFileWriter, EventAsyncResult>(writer, async));
}
public void Save(string fileName)
{
File = new FileReference(fileName);
File.Delete();
FileByteTarget target = new FileByteTarget(File);
DicomFileWriter writer = new DicomFileWriter(DicomWriteOptions.Default);
writer.Write(target, FileMetaInfo, Dataset);
target.Close();
}
/// <summary>
/// Writes the state to disk in a way that can be recovered if the operation is interrupted
/// </summary>
/// <param name="StateFile">The file to write to</param>
/// <param name="State">The state object</param>
static void WriteState(FileReference StateFile, BuildHealthState State)
{
Stopwatch Timer = Stopwatch.StartNew();
// Write out the state to the transaction file
FileReference StateTransactionFile = GetStateTransactionFile(StateFile);
SerializeJson(StateTransactionFile, State);
// Remove the original file, then move the transaction file into place
FileReference.Delete(StateFile);
FileReference.Move(StateTransactionFile, StateFile);
Log.TraceInformation("Took {0}s to write state", Timer.Elapsed.TotalSeconds);
}
/// <summary>
/// Saves the contents of this object to disk
/// </summary>
/// <param name="OutputFile">The output file to write to</param>
public void Save(FileReference OutputFile, bool Overwrite = false)
{
if (Overwrite && FileReference.Exists(OutputFile))
{
FileReference.Delete(OutputFile);
}
using (FileStream Stream = File.Open(OutputFile.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.Read))
{
Stream.Write(CurrentSignature, 0, CurrentSignature.Length);
using (GZipStream CompressedStream = new GZipStream(Stream, CompressionMode.Compress))
{
using (BinaryWriter Writer = new BinaryWriter(CompressedStream, Encoding.UTF8, true))
{
Write(Writer);
}
}
}
}
public void Dispose()
{
// Try to write the output file in a transactional way; write it to a temporary file and rename it.
DirectoryReference.CreateDirectory(OutputFile.Directory);
InputData Data = new InputData();
Data.Jobs.Add(Job);
FileReference TempOutputFile = new FileReference(OutputFile.FullName + ".incoming");
using (MemoryStream Stream = new MemoryStream())
{
DataContractJsonSerializer InputFileDataSerializer = new DataContractJsonSerializer(typeof(InputData));
InputFileDataSerializer.WriteObject(Stream, Data);
FileReference.WriteAllBytes(TempOutputFile, Stream.ToArray());
}
FileReference.Delete(OutputFile);
FileReference.Move(TempOutputFile, OutputFile);
}
public void Save(string fileName)
{
if (Format == DicomFileFormat.ACRNEMA1 || Format == DicomFileFormat.ACRNEMA2)
{
throw new DicomFileException(this, "Unable to save ACR-NEMA file");
}
if (Format == DicomFileFormat.DICOM3NoFileMetaInfo)
{
// create file meta information from dataset
FileMetaInfo = new DicomFileMetaInformation(Dataset);
}
File = new FileReference(fileName);
File.Delete();
using (var target = new FileByteTarget(File)) {
DicomFileWriter writer = new DicomFileWriter(DicomWriteOptions.Default);
writer.Write(target, FileMetaInfo, Dataset);
}
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bResume = SingleNodeName != null || ParseParam("Resume");
bool bListOnly = ParseParam("ListOnly");
bool bShowDiagnostics = ParseParam("ShowDiagnostics");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if (ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if (ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List <string> CleanNodes = new List <string>();
foreach (string NodeList in ParseParamValues("CleanNode"))
{
foreach (string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.Branch : "Unknown";
DefaultProperties["Depot"] = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
DefaultProperties["RestrictedFolderNames"] = String.Join(";", RestrictedFolders.Names);
DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x)));
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if (EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();
if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return(ExitCode.Error_Unknown);
}
// Generate documentation
if (DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return(ExitCode.Success);
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if (SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if (ScriptFileName == null)
{
return(ExitCode.Success);
}
}
// Check there was a script specified
if (ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
// Read the script from disk
Graph Graph;
if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return(ExitCode.Error_Unknown);
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if (!bResume)
{
Storage.CleanLocal();
}
foreach (string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet <Node> TargetNodes = new HashSet <Node>();
if (TargetNames == null)
{
if (!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if (!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if (TokenSignature != null)
{
// Find all the lock files
HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if (SingleNodeName == null)
{
CommandUtils.LogInformation("Required tokens:");
foreach (Node Node in TargetNodes)
{
foreach (FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.LogInformation(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List <FileReference> CreatedTokens = new List <FileReference>();
if (!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
foreach (FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if (CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if (MissingTokens.Count > 0)
{
if (bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet <Node> SkipNodes = new HashSet <Node>();
foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach (FileReference MissingToken in MissingTokensForBuild)
{
foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
LogInformation(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if (SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
LogInformation("Remaining target nodes:");
foreach (Node TargetNode in TargetNodes)
{
LogInformation(" {0}", TargetNode);
}
if (TargetNodes.Count == 0)
{
LogInformation(" None.");
}
}
}
else
{
foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
{
List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach (FileReference CreatedToken in CreatedTokens)
{
FileReference.Delete(CreatedToken);
}
return(ExitCode.Error_Unknown);
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();
if (bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach (string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName);
LogInformation("Writing {0}...", PreprocessedFileLocation);
Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
return(ExitCode.Success);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
// If we're just building a single node, find it
Node SingleNode = null;
if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return(ExitCode.Error_Unknown);
}
// If we just want to show the contents of the graph, do so and exit.
if (bListOnly)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if (SingleNode == null && (!bListOnly || bShowDiagnostics))
{
IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach (GraphDiagnostic Diagnostic in Diagnostics)
{
if (Diagnostic.EventType == LogEventType.Console)
{
CommandUtils.LogInformation(Diagnostic.Message);
}
else if (Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return(ExitCode.Error_Unknown);
}
}
// Export the graph to a file
if (ExportFileName != null)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
return(ExitCode.Success);
}
// Execute the command
if (!bListOnly)
{
if (SingleNode != null)
{
if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return(ExitCode.Error_Unknown);
}
}
else
{
if (!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return(ExitCode.Error_Unknown);
}
}
}
return(ExitCode.Success);
}
/// <summary>
/// Wait for this worker to complete
/// </summary>
/// <param name="Writer">Writer for log output</param>
/// <returns>Return code from the thread</returns>
public SequenceProbeResult Join(LineBasedTextWriter Writer)
{
// Finish the task instance
BufferedTextWriter BufferedWriter = new BufferedTextWriter();
int ExitCode = ActiveInstance.Join(BufferedWriter);
ActiveInstance.Dispose();
ActiveInstance = null;
// Read the new state
FileReference OutputFile = new FileReference(TaskStateFile.FullName + ".out");
SequenceWorker NewWorker = SequenceWorker.Deserialize(OutputFile.FullName);
OutputFile.Delete();
// Make sure the exit code reflects the failure state. XGE can sometimes fail transferring back.
if (ExitCode == 0 && !NewWorker.bResult)
{
ExitCode = -1;
}
// If it's a hard failure, print the compile log to the regular log
if (ExitCode == -1)
{
Writer.WriteLine("Failed to compile {0}, exit code {1}:", UniqueName, ExitCode);
foreach (string Line in NewWorker.CompileLog)
{
Writer.WriteLine(" > {0}", Line.Replace("error:", "err:"));
}
}
// Annotate the log data if this is from a failed attempt. It still may be useful for debugging purposes.
if (ExitCode != 0)
{
NewWorker.SummaryLog.Insert(0, String.Format("ExitCode={0}", ExitCode));
NewWorker.SummaryLog = NewWorker.SummaryLog.Select(x => "FAIL > " + x).ToList();
NewWorker.CompileLog.Insert(0, String.Format("ExitCode={0}", ExitCode));
NewWorker.CompileLog = NewWorker.CompileLog.Select(x => "FAIL > " + x).ToList();
}
// Append the log data back to the local output
File.AppendAllLines(SummaryLogFile.FullName, NewWorker.SummaryLog);
NewWorker.SummaryLog.Clear();
File.AppendAllLines(CompileLogFile.FullName, NewWorker.CompileLog);
NewWorker.CompileLog.Clear();
// If we failed, return the
if (ExitCode != 0)
{
if (ExitCode == -1)
{
Writer.WriteLine("Warning: Failed to compile {0}, exit code {1}. Aborting.", UniqueName, ExitCode);
return(SequenceProbeResult.Failed);
}
else
{
Writer.WriteLine("Warning: Failed to compile {0}; exit code {1}. Will retry.", UniqueName, ExitCode);
return(SequenceProbeResult.FailedAllowRetry);
}
}
// Update the task
Worker = NewWorker;
// Check if this is just an incremental update
if (Type == SequenceProbeType.Verify)
{
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Return that we're done
return(SequenceProbeResult.Completed);
}
else if (Type == SequenceProbeType.Optimize)
{
if (Worker.RemainingFragmentCount > 0)
{
// Get the top-most fragment - the one we've just established is a dependency for this leaf node - and add it to the list of known dependencies
SourceFragment NextFragment = Fragments[Worker.RemainingFragmentCount - 1];
AddDependency(Worker, Fragments, Worker.RemainingFragmentCount - 1);
Worker.SummaryLog.Add(String.Format(" [Added {0}: {1}]", Worker.RemainingFragmentCount - 1, Fragments[Worker.RemainingFragmentCount - 1].Location));
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Otherwise, return that we've just updated
return(SequenceProbeResult.Updated);
}
else
{
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Return that we're done
SetCompletedDependencies();
return(SequenceProbeResult.Completed);
}
}
else
{
throw new NotImplementedException();
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}
/// <summary>
/// Loads the cache from disk
/// </summary>
/// <param name="Cache">The file to load</param>
/// <returns>The loaded instance</returns>
public static FlatCPPIncludeDependencyCache Load(FileReference BackingFile)
{
FlatCPPIncludeDependencyCache Result = null;
try
{
string CacheBuildMutexPath = BackingFile.FullName + ".buildmutex";
// If the .buildmutex file for the cache is present, it means that something went wrong between loading
// and saving the cache last time (most likely the UBT process being terminated), so we don't want to load
// it.
if (!File.Exists(CacheBuildMutexPath))
{
using (File.Create(CacheBuildMutexPath))
{
}
using (BinaryReader Reader = new BinaryReader(new FileStream(BackingFile.FullName, FileMode.Open, FileAccess.Read)))
{
// @todo ubtmake: We can store the cache in a cheaper/smaller way using hash file names and indices into included headers, but it might actually slow down load times
// @todo ubtmake: If we can index PCHs here, we can avoid storing all of the PCH's included headers (PCH's action should have been invalidated, so we shouldn't even have to report the PCH's includes as our indirect includes)
if (Reader.ReadInt32() == FileSignature)
{
Result = Deserialize(Reader);
}
}
}
}
catch (Exception Ex)
{
Console.Error.WriteLine("Failed to read FlatCPPIncludeDependencyCache: {0}", Ex.Message);
BackingFile.Delete();
}
return Result;
}
/// <summary>
/// Loads the cache from the passed in file.
/// </summary>
/// <param name="Cache">File to deserialize from</param>
public static DependencyCache Load(FileReference CacheFile)
{
DependencyCache Result = null;
try
{
string CacheBuildMutexPath = CacheFile.FullName + ".buildmutex";
// If the .buildmutex file for the cache is present, it means that something went wrong between loading
// and saving the cache last time (most likely the UBT process being terminated), so we don't want to load
// it.
if (!File.Exists(CacheBuildMutexPath))
{
using (File.Create(CacheBuildMutexPath))
{
}
using (BinaryReader Reader = new BinaryReader(new FileStream(CacheFile.FullName, FileMode.Open, FileAccess.Read)))
{
if (Reader.ReadInt32() == FileSignature)
{
Result = DependencyCache.Deserialize(Reader);
}
}
}
}
catch (Exception Ex)
{
Console.Error.WriteLine("Failed to read dependency cache: {0}", Ex.Message);
CacheFile.Delete();
}
return Result;
}
public override void ExecuteBuild()
{
int WorkingCL = -1;
FileReference PluginFile = null;
string ProjectFileName = ParseParamValue("Project");
if (ProjectFileName == null)
{
ProjectFileName = CombinePaths(CmdEnv.LocalRoot, "SimpleGame", "SimpleGame.uproject");
}
LogInformation(ProjectFileName);
ProjectParams Params = GetParams(this, ProjectFileName, out PluginFile);
// Check whether folder already exists so we know if we can delete it later
string PlatformStageDir = Path.Combine(Params.StageDirectoryParam, "WindowsNoEditor");
bool bPreExistingStageDir = Directory.Exists(PlatformStageDir);
PluginDescriptor Plugin = PluginDescriptor.FromFile(PluginFile);
FileReference ProjectFile = new FileReference(ProjectFileName);
// Add Plugin to folders excluded for nativization in config file
FileReference UserEditorIni = new FileReference(Path.Combine(Path.GetDirectoryName(ProjectFileName), "Config", "UserEditor.ini"));
bool bPreExistingUserEditorIni = FileReference.Exists(UserEditorIni);
if (!bPreExistingUserEditorIni)
{
// Expect this most of the time so we will create and clean up afterwards
DirectoryReference.CreateDirectory(UserEditorIni.Directory);
CommandUtils.WriteAllText(UserEditorIni.FullName, "");
}
const string ConfigSection = "BlueprintNativizationSettings";
const string ConfigKey = "ExcludedFolderPaths";
string ConfigValue = "/" + PluginFile.GetFileNameWithoutAnyExtensions() + "/";
ConfigFile UserEditorConfig = new ConfigFile(UserEditorIni);
ConfigFileSection BPNSection = UserEditorConfig.FindOrAddSection(ConfigSection);
bool bUpdateConfigFile = !BPNSection.Lines.Exists(x => String.Equals(x.Key, ConfigKey, StringComparison.OrdinalIgnoreCase) && String.Equals(x.Value, ConfigValue, StringComparison.OrdinalIgnoreCase));
if (bUpdateConfigFile)
{
BPNSection.Lines.Add(new ConfigLine(ConfigLineAction.Add, ConfigKey, ConfigValue));
UserEditorConfig.Write(UserEditorIni);
}
Project.Cook(Params);
if (!bPreExistingUserEditorIni)
{
FileReference.Delete(UserEditorIni);
}
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
// Get path to where the plugin was staged
string StagedPluginDir = Path.Combine(PlatformStageDir, Path.GetFileNameWithoutExtension(ProjectFileName), PluginFile.Directory.MakeRelativeTo(ProjectFile.Directory));
string ZipFile = Path.Combine(Params.StageDirectoryParam, PluginFile.GetFileNameWithoutAnyExtensions());
CommandUtils.DeleteFile(ZipFile);
System.IO.Compression.ZipFile.CreateFromDirectory(StagedPluginDir, ZipFile + ".zip");
if (!bPreExistingStageDir)
{
CommandUtils.DeleteDirectory(PlatformStageDir);
}
}
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null)
{
// Get the SYMSTORE.EXE path, using the latest SDK version we can find.
FileReference SymStoreExe = GetSymStoreExe();
List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList();
if (FilesToAdd.Count > 0)
{
DateTime Start = DateTime.Now;
DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore");
DirectoryReference.CreateDirectory(TempSymStoreDir);
DeleteDirectoryContents(TempSymStoreDir);
string TempFileName = Path.GetTempFileName();
try
{
File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII);
// Copy everything to the temp symstore
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = SymStoreExe.FullName;
StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\" /compress", TempFileName, TempSymStoreDir, Product);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0)
{
return(false);
}
}
finally
{
File.Delete(TempFileName);
}
DateTime CompressDone = DateTime.Now;
LogInformation("Took {0}s to compress the symbol files", (CompressDone - Start).TotalSeconds);
// Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files
foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => File.HasExtension(".dl_") || File.HasExtension(".ex_") || File.HasExtension(".pd_")))
{
string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir));
FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath);
// Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example)
if (!string.IsNullOrWhiteSpace(BuildVersion))
{
FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion));
// Attempt to create the file. Just continue if it fails.
try
{
DirectoryReference.CreateDirectory(BuildVersionFile.Directory);
FileReference.WriteAllText(BuildVersionFile, string.Empty);
}
catch (Exception Ex)
{
LogWarning("Failed to write the version file, reason {0}", Ex.ToString());
}
}
// Don't bother copying the temp file if the destination file is there already.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName);
continue;
}
FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString());
try
{
CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName);
}
catch (Exception Ex)
{
throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString());
}
// Move the file in the temp store over.
try
{
FileReference.Move(TempDestinationFile, ActualDestinationFile);
}
catch (Exception Ex)
{
// If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already.
// Either way, it's fine to just continue on.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName);
continue;
}
// If it doesn't exist, we actually failed to copy it entirely.
else
{
LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString());
}
}
// Delete the temp one no matter what, don't want them hanging around in the symstore
finally
{
FileReference.Delete(TempDestinationFile);
}
}
LogInformation("Took {0}s to copy the symbol files to the store", (DateTime.Now - CompressDone).TotalSeconds);
}
return(true);
}