/// <summary>
/// Returns the build root path (P:\Builds on build machines usually)
/// </summary>
/// <returns></returns>
static public string GetBuildRootPath()
{
return(CommandUtils.P4Enabled && CommandUtils.AllowSubmit
? CommandUtils.RootSharedTempStorageDirectory()
: CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds"));
}
public override void ExecuteBuild()
{
// Command parameters - not all required if using existing manifest
string ProjectPath = ParseParamValue("ProjectPath");
string ManifestFile = ParseParamValue("ManifestFile");
string UE4Exe = ParseParamValue("UE4Exe", "UE4Editor-Cmd.exe");
string ReplacedPaths = ParseParamValue("ReplacedPaths", "");
string ReplacedClasses = ParseParamValue("ReplacedClasses", "");
string ExcludedPaths = ParseParamValue("ExcludedPaths", "");
string ExcludedClasses = ParseParamValue("ExcludedClasses", "");
DirectoryReference BaseDir = new DirectoryReference(ParseParamValue("BaseDir"));
DirectoryReference AssetSourcePath = new DirectoryReference(ParseParamValue("AssetSourcePath"));
bool UseExistingManifest = ParseParam("UseExistingManifest");
if (!UseExistingManifest || !FileExists_NoExceptions(ManifestFile))
{
// Run commandlet to generate list of assets to replace
FileReference Project = new FileReference(ProjectPath);
var Dir = Path.GetDirectoryName(ManifestFile);
var Filename = Path.GetFileName(ManifestFile);
if (String.IsNullOrEmpty(Dir) || String.IsNullOrEmpty(Filename))
{
throw new AutomationException("GenerateDistillFileSets should have a full path and file for {0}.", ManifestFile);
}
CreateDirectory_NoExceptions(Dir);
if (FileExists_NoExceptions(ManifestFile))
{
DeleteFile(ManifestFile);
}
RunCommandlet(Project, UE4Exe, "GenerateAssetManifest", String.Format("-ManifestFile={0} -IncludedPaths={1} -IncludedClasses={2} -ExcludedPaths={3} -ExcludedClasses={4}", CommandUtils.MakePathSafeToUseWithCommandLine(ManifestFile), ReplacedPaths, ReplacedClasses, ExcludedPaths, ExcludedClasses));
if (!FileExists_NoExceptions(ManifestFile))
{
throw new AutomationException("GenerateAssetManifest did not produce a manifest for {0}.", Project);
}
}
// Read Source Files from Manifest
List <string> Lines = new List <string>(ReadAllLines(ManifestFile));
if (Lines.Count < 1)
{
throw new AutomationException("Manifest file {0} does not list any files.", ManifestFile);
}
List <string> SourceFiles = new List <string>();
foreach (var ThisFile in Lines)
{
var TestFile = CombinePaths(ThisFile);
if (!FileExists_NoExceptions(TestFile))
{
throw new AutomationException("GenerateAssetManifest produced {0}, but {1} doesn't exist.", ThisFile, TestFile);
}
// we correct the case here
var TestFileInfo = new FileInfo(TestFile);
var FinalFile = CombinePaths(TestFileInfo.FullName);
if (!FileExists_NoExceptions(FinalFile))
{
throw new AutomationException("GenerateDistillFileSets produced {0}, but {1} doesn't exist.", ThisFile, FinalFile);
}
SourceFiles.Add(FinalFile);
}
// Delete all original files - might not always be an asset to replace them with
foreach (string SourceFile in SourceFiles)
{
CommandUtils.DeleteFile(SourceFile);
}
// Convert Source file paths to Asset Source Paths
foreach (string ReplacedFile in SourceFiles)
{
FileReference ReplacementFile = FileReference.Combine(AssetSourcePath, new FileReference(ReplacedFile).MakeRelativeTo(BaseDir));
if (FileReference.Exists(ReplacementFile))
{
CommandUtils.CopyFile(ReplacementFile.FullName, ReplacedFile);
}
}
}
public async Task <int> Collect(
CancellationToken ct,
List <string> counter_list,
string counters,
IConsole console,
int processId,
int refreshInterval,
CountersExportFormat format,
string output,
string name,
string diagnosticPort,
bool resumeRuntime,
int maxHistograms,
int maxTimeSeries)
{
try
{
// System.CommandLine does have an option to specify arguments as uint and it would validate they are non-negative. However the error
// message is "Cannot parse argument '-1' for option '--maxTimeSeries' as expected type System.UInt32" which is not as user friendly.
// If there was another option to leverage System.CommandLine that provides a little more user friendly error message we could switch
// to it.
ValidateNonNegative(maxHistograms, nameof(maxHistograms));
ValidateNonNegative(maxTimeSeries, nameof(maxTimeSeries));
if (!ProcessLauncher.Launcher.HasChildProc && !CommandUtils.ValidateArgumentsForAttach(processId, name, diagnosticPort, out _processId))
{
return(ReturnCode.ArgumentError);
}
ct.Register(() => _shouldExit.TrySetResult(ReturnCode.Ok));
DiagnosticsClientBuilder builder = new DiagnosticsClientBuilder("dotnet-counters", 10);
using (DiagnosticsClientHolder holder = await builder.Build(ct, _processId, diagnosticPort, showChildIO: false, printLaunchCommand: false))
{
if (holder == null)
{
return(ReturnCode.Ok);
}
try
{
_console = console;
// the launch command may misinterpret app arguments as the old space separated
// provider list so we need to ignore it in that case
_counterList = ConfigureCounters(counters, _processId != 0 ? counter_list : null);
_ct = ct;
_interval = refreshInterval;
_maxHistograms = maxHistograms;
_maxTimeSeries = maxTimeSeries;
_output = output;
_diagnosticsClient = holder.Client;
if (_output.Length == 0)
{
_console.Error.WriteLine("Output cannot be an empty string");
return(ReturnCode.ArgumentError);
}
if (format == CountersExportFormat.csv)
{
_renderer = new CSVExporter(output);
}
else if (format == CountersExportFormat.json)
{
// Try getting the process name.
string processName = "";
try
{
if (ProcessLauncher.Launcher.HasChildProc)
{
_processId = ProcessLauncher.Launcher.ChildProc.Id;
}
processName = Process.GetProcessById(_processId).ProcessName;
}
catch (Exception) { }
_renderer = new JSONExporter(output, processName);
}
else
{
_console.Error.WriteLine($"The output format {format} is not a valid output format.");
return(ReturnCode.ArgumentError);
}
_resumeRuntime = resumeRuntime;
int ret = await Start();
return(ret);
}
catch (OperationCanceledException)
{
try
{
_session.Stop();
}
catch (Exception) { } // session.Stop() can throw if target application already stopped before we send the stop command.
return(ReturnCode.Ok);
}
}
}
catch (CommandLineErrorException e)
{
console.Error.WriteLine(e.Message);
return(ReturnCode.ArgumentError);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
HashSet <FileReference> Files = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
if (Files.Count == 0)
{
Log.TraceInformation("No files to submit.");
}
else if (!CommandUtils.AllowSubmit)
{
Log.TraceWarning("Submitting to Perforce is disabled by default. Run with the -submit argument to allow.");
}
else
{
// Get the connection that we're going to submit with
P4Connection SubmitP4 = CommandUtils.P4;
if (Parameters.Workspace != null)
{
// Create a brand new workspace
P4ClientInfo Client = new P4ClientInfo();
Client.Owner = CommandUtils.P4Env.User;
Client.Host = Environment.MachineName;
Client.Stream = Parameters.Stream ?? CommandUtils.P4Env.Branch;
Client.RootPath = Parameters.RootDir ?? CommandUtils.RootDirectory.FullName;
Client.Name = Parameters.Workspace;
Client.Options = P4ClientOption.NoAllWrite | P4ClientOption.Clobber | P4ClientOption.NoCompress | P4ClientOption.Unlocked | P4ClientOption.NoModTime | P4ClientOption.RmDir;
Client.LineEnd = P4LineEnd.Local;
CommandUtils.P4.CreateClient(Client, AllowSpew: false);
// Create a new connection for it
SubmitP4 = new P4Connection(Client.Owner, Client.Name);
}
// Get the latest version of it
int NewCL = SubmitP4.CreateChange(Description: Parameters.Description.Replace("\\n", "\n"));
foreach (FileReference File in Files)
{
SubmitP4.Revert(String.Format("-k \"{0}\"", File.FullName));
SubmitP4.Sync(String.Format("-k \"{0}\"", File.FullName), AllowSpew: false);
SubmitP4.Add(NewCL, String.Format("\"{0}\"", File.FullName));
SubmitP4.Edit(NewCL, String.Format("\"{0}\"", File.FullName));
if (Parameters.FileType != null)
{
SubmitP4.P4(String.Format("reopen -t \"{0}\" \"{1}\"", Parameters.FileType, File.FullName), AllowSpew: false);
}
}
// Revert any unchanged files
if (Parameters.RevertUnchanged)
{
SubmitP4.RevertUnchanged(NewCL);
if (SubmitP4.TryDeleteEmptyChange(NewCL))
{
CommandUtils.Log("No files to submit; ignored.");
return;
}
}
// Submit it
int SubmittedCL;
SubmitP4.Submit(NewCL, out SubmittedCL);
if (SubmittedCL <= 0)
{
throw new AutomationException("Submit failed.");
}
CommandUtils.Log("Submitted in changelist {0}", SubmittedCL);
}
}
public override void ExecuteBuild()
{
// Get the list of platform names
string[] FeaturePacks = ParseParamValue("FeaturePacks").Split(';');
string TempDir = ParseParamValue("TempDir");
UnrealTargetPlatform HostPlatform = (UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), ParseParamValue("HostPlatform"));
string TargetPlatforms = ParseParamValue("TargetPlatforms");
string SavedDir = ParseParamValue("SavedDir");
// Get paths to everything within the temporary directory
string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform);
string RelativePakPath = "Engine/DerivedDataCache/Compressed.ddp";
string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath);
string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv");
List <string> ProjectPakFiles = new List <string>();
List <string> FeaturePackPaths = new List <string>();
// loop through all the projects first and bail out if one of them doesn't exist.
foreach (string FeaturePack in FeaturePacks)
{
if (!String.IsNullOrWhiteSpace(FeaturePack))
{
string FeaturePackPath = CommandUtils.CombinePaths(CommandUtils.RootDirectory.FullName, FeaturePack);
if (!CommandUtils.FileExists(FeaturePackPath))
{
throw new AutomationException("Could not find project: " + FeaturePack);
}
FeaturePackPaths.Add(FeaturePackPath);
}
}
// loop through all the paths and generate ddc data for them
foreach (string FeaturePackPath in FeaturePackPaths)
{
string ProjectSpecificPlatforms = TargetPlatforms;
FileReference FileRef = new FileReference(FeaturePackPath);
string GameName = FileRef.GetFileNameWithoutAnyExtensions();
ProjectDescriptor Project = ProjectDescriptor.FromFile(FeaturePackPath);
if (Project.TargetPlatforms != null && Project.TargetPlatforms.Length > 0)
{
// Restrict target platforms used to those specified in project file
List <string> FilteredPlatforms = new List <string>();
// Always include the editor platform for cooking
string EditorCookPlatform = Platform.GetPlatform(HostPlatform).GetEditorCookPlatform();
if (TargetPlatforms.Contains(EditorCookPlatform))
{
FilteredPlatforms.Add(EditorCookPlatform);
}
foreach (string TargetPlatform in Project.TargetPlatforms)
{
if (TargetPlatforms.Contains(TargetPlatform))
{
FilteredPlatforms.Add(TargetPlatform);
}
}
ProjectSpecificPlatforms = CommandUtils.CombineCommandletParams(FilteredPlatforms.Distinct().ToArray());
}
CommandUtils.Log("Generating DDC data for {0} on {1}", GameName, ProjectSpecificPlatforms);
CommandUtils.DDCCommandlet(FileRef, EditorExe, null, ProjectSpecificPlatforms, "-fill -DDC=CreateInstalledEnginePak -ProjectOnly");
string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", GameName));
CommandUtils.DeleteFile(ProjectPakFile);
CommandUtils.RenameFile(OutputPakFile, ProjectPakFile);
string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv");
CommandUtils.DeleteFile(ProjectCsvFile);
CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile);
ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile));
}
// Generate DDC for the editor, and merge all the other PAK files in
CommandUtils.Log("Generating DDC data for engine content on {0}", TargetPlatforms);
CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak " + CommandUtils.MakePathSafeToUseWithCommandLine("-MergePaks=" + String.Join("+", ProjectPakFiles)));
string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath);
CommandUtils.CopyFile(OutputPakFile, SavedPakFile);
}
public List <ISharedCookedBuild> FindBestBuilds()
{
// Attempt manifest searching first
ConfigHierarchy Hierarchy = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, DirectoryReference.FromFile(ProjectFile), UnrealTargetPlatform.Win64);
IEnumerable <string> RawSharedCookedSources = null;
Hierarchy.TryGetValues("SharedCookedBuildSettings", "SharedCookedSources", out RawSharedCookedSources);
if (RawSharedCookedSources == null)
{
throw new AutomationException("Unable to locate shared cooked builds. SharedCookedSources not set in Engine.ini [SharedCookedBuildSettings]");
}
List <Dictionary <string, string> > ParsedSharedCookSources = new List <Dictionary <string, string> >();
foreach (string RawConfig in RawSharedCookedSources)
{
Dictionary <string, string> ParsedSource = null;
if (ConfigHierarchy.TryParse(RawConfig, out ParsedSource))
{
ParsedSharedCookSources.Add(ParsedSource);
}
}
List <ISharedCookedBuild> CandidateBuilds = new List <ISharedCookedBuild>();
// If existing sync is present, stick to it. Read version out of sync file
foreach (string Platform in TargetPlatforms)
{
FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(InstallPath.FullName, Platform, SyncedBuildFileName));
if (FileReference.Exists(SyncedBuildFile))
{
string[] SyncedBuildInfo = FileReference.ReadAllLines(SyncedBuildFile);
int SyncedCL = int.Parse(SyncedBuildInfo[0]);
if (IsValidCL(SyncedCL, BuildType, LocalSync))
{
CandidateBuilds.Add(new ExistingSharedCookedBuild()
{
CL = SyncedCL, Platform = Platform
});
}
}
}
foreach (Dictionary <string, string> Source in ParsedSharedCookSources)
{
SharedCookSource SourceType = (SharedCookSource)Enum.Parse(typeof(SharedCookSource), Source["Type"], true);
foreach (string Platform in TargetPlatforms)
{
if (SourceType == SharedCookSource.Manifest)
{
CandidateBuilds.AddRange(FindValidManifestBuilds(Source["Path"], Platform));
}
else if (SourceType == SharedCookSource.LooseFiles)
{
CandidateBuilds.AddRange(FindValidLooseBuilds(Source["Path"], Platform));
}
}
}
// Strip all failed searches
CandidateBuilds.RemoveAll(x => x == null);
// Make sure we have a matching CL for all target platforms, regardless of source
List <int> OrderedDistinctCLs = CandidateBuilds.Select(x => x.CL).Distinct().OrderByDescending(i => i).ToList();
int BestCL = -1;
foreach (int CL in OrderedDistinctCLs)
{
// Ensure we have a platform for each
HashSet <string> CLPlatforms = new HashSet <string>(CandidateBuilds.Where(x => x.CL == CL).Select(x => x.Platform).ToList());
if (CLPlatforms.SetEquals(TargetPlatforms))
{
BestCL = CL;
break;
}
}
if (BestCL < 0)
{
CommandUtils.LogError("Could not locate valid shared cooked build for all target platforms");
CommandUtils.LogError("Current CL: {0}, Current Code CL: {1}", LocalSync.Changelist, LocalSync.CompatibleChangelist);
}
return(CandidateBuilds.Where(x => x.CL == BestCL).ToList());
}
/// <summary>
/// Collects a diagnostic trace from a currently running process.
/// </summary>
/// <param name="ct">The cancellation token</param>
/// <param name="console"></param>
/// <param name="processId">The process to collect the trace from.</param>
/// <param name="name">The name of process to collect the trace from.</param>
/// <param name="output">The output path for the collected trace data.</param>
/// <param name="buffersize">Sets the size of the in-memory circular buffer in megabytes.</param>
/// <param name="providers">A list of EventPipe providers to be enabled. This is in the form 'Provider[,Provider]', where Provider is in the form: 'KnownProviderName[:Flags[:Level][:KeyValueArgs]]', and KeyValueArgs is in the form: '[key1=value1][;key2=value2]'</param>
/// <param name="profile">A named pre-defined set of provider configurations that allows common tracing scenarios to be specified succinctly.</param>
/// <param name="format">The desired format of the created trace file.</param>
/// <param name="duration">The duration of trace to be taken. </param>
/// <param name="clrevents">A list of CLR events to be emitted.</param>
/// <param name="clreventlevel">The verbosity level of CLR events</param>
/// <param name="port">Path to the diagnostic port to be created.</param>
/// <returns></returns>
private static async Task <int> Collect(CancellationToken ct, IConsole console, int processId, FileInfo output, uint buffersize, string providers, string profile, TraceFileFormat format, TimeSpan duration, string clrevents, string clreventlevel, string name, string diagnosticPort)
{
int ret = 0;
try
{
Debug.Assert(output != null);
Debug.Assert(profile != null);
if (!ProcessLauncher.Launcher.HasChildProc)
{
if (CommandUtils.ValidateArguments(processId, name, diagnosticPort, out int resolvedProcessId))
{
processId = resolvedProcessId;
}
else
{
return(ErrorCodes.ArgumentError);
}
}
if (profile.Length == 0 && providers.Length == 0 && clrevents.Length == 0)
{
Console.Out.WriteLine("No profile or providers specified, defaulting to trace profile 'cpu-sampling'");
profile = "cpu-sampling";
}
Dictionary <string, string> enabledBy = new Dictionary <string, string>();
var providerCollection = Extensions.ToProviders(providers);
foreach (EventPipeProvider providerCollectionProvider in providerCollection)
{
enabledBy[providerCollectionProvider.Name] = "--providers ";
}
if (profile.Length != 0)
{
var selectedProfile = ListProfilesCommandHandler.DotNETRuntimeProfiles
.FirstOrDefault(p => p.Name.Equals(profile, StringComparison.OrdinalIgnoreCase));
if (selectedProfile == null)
{
Console.Error.WriteLine($"Invalid profile name: {profile}");
return(ErrorCodes.ArgumentError);
}
Profile.MergeProfileAndProviders(selectedProfile, providerCollection, enabledBy);
}
// Parse --clrevents parameter
if (clrevents.Length != 0)
{
// Ignore --clrevents if CLR event provider was already specified via --profile or --providers command.
if (enabledBy.ContainsKey(Extensions.CLREventProviderName))
{
Console.WriteLine($"The argument --clrevents {clrevents} will be ignored because the CLR provider was configured via either --profile or --providers command.");
}
else
{
var clrProvider = Extensions.ToCLREventPipeProvider(clrevents, clreventlevel);
providerCollection.Add(clrProvider);
enabledBy[Extensions.CLREventProviderName] = "--clrevents";
}
}
if (providerCollection.Count <= 0)
{
Console.Error.WriteLine("No providers were specified to start a trace.");
return(ErrorCodes.ArgumentError);
}
PrintProviders(providerCollection, enabledBy);
DiagnosticsClient diagnosticsClient;
Process process;
DiagnosticsClientBuilder builder = new DiagnosticsClientBuilder("dotnet-trace", 10);
bool shouldResumeRuntime = ProcessLauncher.Launcher.HasChildProc || !string.IsNullOrEmpty(diagnosticPort);
using (DiagnosticsClientHolder holder = await builder.Build(ct, processId, diagnosticPort))
{
diagnosticsClient = holder.Client;
if (shouldResumeRuntime)
{
process = Process.GetProcessById(holder.EndpointInfo.ProcessId);
}
else
{
process = Process.GetProcessById(processId);
}
var shouldExit = new ManualResetEvent(false);
var shouldStopAfterDuration = duration != default(TimeSpan);
var rundownRequested = false;
System.Timers.Timer durationTimer = null;
ct.Register(() => shouldExit.Set());
using (VirtualTerminalMode vTermMode = VirtualTerminalMode.TryEnable())
{
EventPipeSession session = null;
try
{
session = diagnosticsClient.StartEventPipeSession(providerCollection, true, (int)buffersize);
if (shouldResumeRuntime)
{
diagnosticsClient.ResumeRuntime();
}
}
catch (DiagnosticsClientException e)
{
Console.Error.WriteLine($"Unable to start a tracing session: {e.ToString()}");
}
if (session == null)
{
Console.Error.WriteLine("Unable to create session.");
return(ErrorCodes.SessionCreationError);
}
if (shouldStopAfterDuration)
{
durationTimer = new System.Timers.Timer(duration.TotalMilliseconds);
durationTimer.Elapsed += (s, e) => shouldExit.Set();
durationTimer.AutoReset = false;
}
var stopwatch = new Stopwatch();
durationTimer?.Start();
stopwatch.Start();
LineRewriter rewriter = null;
using (var fs = new FileStream(output.FullName, FileMode.Create, FileAccess.Write))
{
Console.Out.WriteLine($"Process : {process.MainModule.FileName}");
Console.Out.WriteLine($"Output File : {fs.Name}");
if (shouldStopAfterDuration)
{
Console.Out.WriteLine($"Trace Duration : {duration.ToString(@"dd\:hh\:mm\:ss")}");
}
Console.Out.WriteLine("\n\n");
var fileInfo = new FileInfo(output.FullName);
Task copyTask = session.EventStream.CopyToAsync(fs).ContinueWith((task) => shouldExit.Set());
if (!Console.IsOutputRedirected)
{
rewriter = new LineRewriter {
LineToClear = Console.CursorTop - 1
};
Console.CursorVisible = false;
}
Action printStatus = () =>
{
if (!Console.IsOutputRedirected)
{
rewriter?.RewriteConsoleLine();
fileInfo.Refresh();
Console.Out.WriteLine($"[{stopwatch.Elapsed.ToString(@"dd\:hh\:mm\:ss")}]\tRecording trace {GetSize(fileInfo.Length)}");
Console.Out.WriteLine("Press <Enter> or <Ctrl+C> to exit...");
}
if (rundownRequested)
{
Console.Out.WriteLine("Stopping the trace. This may take up to minutes depending on the application being traced.");
}
};
while (!shouldExit.WaitOne(100) && !(!Console.IsInputRedirected && Console.KeyAvailable && Console.ReadKey(true).Key == ConsoleKey.Enter))
{
printStatus();
}
// if the CopyToAsync ended early (target program exited, etc.), the we don't need to stop the session.
if (!copyTask.Wait(0))
{
// Behavior concerning Enter moving text in the terminal buffer when at the bottom of the buffer
// is different between Console/Terminals on Windows and Mac/Linux
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
!Console.IsOutputRedirected &&
rewriter != null &&
Math.Abs(Console.CursorTop - Console.BufferHeight) == 1)
{
rewriter.LineToClear--;
}
durationTimer?.Stop();
rundownRequested = true;
session.Stop();
do
{
printStatus();
} while (!copyTask.Wait(100));
}
}
Console.Out.WriteLine("\nTrace completed.");
if (format != TraceFileFormat.NetTrace)
{
TraceFileFormatConverter.ConvertToFormat(format, output.FullName);
}
}
}
}
catch (Exception ex)
{
Console.Error.WriteLine($"[ERROR] {ex.ToString()}");
ret = ErrorCodes.TracingError;
}
finally
{
if (console.GetTerminal() != null)
{
Console.CursorVisible = true;
}
// If we launched a child proc that hasn't exited yet, terminate it before we exit.
if (ProcessLauncher.Launcher.HasChildProc && !ProcessLauncher.Launcher.ChildProc.HasExited)
{
ProcessLauncher.Launcher.ChildProc.Kill();
}
}
return(await Task.FromResult(ret));
}
public int ArchiveFiles(string InPath, string Wildcard = "*", bool bRecursive = true, string[] ExcludeWildcard = null)
{
int FilesAdded = 0;
if (CommandUtils.DirectoryExists(InPath))
{
var All = CommandUtils.FindFiles(Wildcard, bRecursive, InPath);
var Exclude = new HashSet <string>();
if (ExcludeWildcard != null)
{
foreach (var Excl in ExcludeWildcard)
{
var Remove = CommandUtils.FindFiles(Excl, bRecursive, InPath);
foreach (var File in Remove)
{
Exclude.Add(CommandUtils.CombinePaths(File));
}
}
}
foreach (var AllFile in All)
{
var FileToCopy = CommandUtils.CombinePaths(AllFile);
if (Exclude.Contains(FileToCopy))
{
continue;
}
bool OtherPlatform = false;
foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (Plat != StageTargetPlatform.PlatformType && Plat != UnrealTargetPlatform.Unknown && FileToCopy.IndexOf(CommandUtils.CombinePaths("/" + Plat.ToString() + "/"), 0, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
OtherPlatform = true;
break;
}
}
if (OtherPlatform)
{
continue;
}
string Dest;
if (!FileToCopy.StartsWith(InPath))
{
throw new AutomationException("Can't archive {0}; it was supposed to start with {1}", FileToCopy, InPath);
}
Dest = FileToCopy.Substring(InPath.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
ArchivedFiles.Add(FileToCopy, Dest);
FilesAdded++;
}
}
return(FilesAdded);
}
public DeploymentContext(
string RawProjectPathOrName,
string InLocalRoot,
string BaseStageDirectory,
string BaseArchiveDirectory,
string CookFlavor,
Platform InSourcePlatform,
Platform InTargetPlatform,
List <UnrealTargetConfiguration> InTargetConfigurations,
IEnumerable <BuildReceipt> InStageTargetReceipts,
List <String> InStageExecutables,
bool InServer,
bool InCooked,
bool InStageCrashReporter,
bool InStage,
bool InCookOnTheFly,
bool InArchive,
bool InProgram,
bool bHasDedicatedServerAndClient,
bool bInUseWebsocketNetDriver = false
)
{
bStageCrashReporter = InStageCrashReporter;
RawProjectPath = RawProjectPathOrName;
DedicatedServer = InServer;
LocalRoot = CommandUtils.CombinePaths(InLocalRoot);
CookSourcePlatform = InSourcePlatform;
StageTargetPlatform = InTargetPlatform;
StageTargetConfigurations = new List <UnrealTargetConfiguration>(InTargetConfigurations);
StageTargetReceipts = new List <BuildReceipt>(InStageTargetReceipts);
StageExecutables = InStageExecutables;
IsCodeBasedProject = ProjectUtils.IsCodeBasedUProjectFile(RawProjectPath);
ShortProjectName = ProjectUtils.GetShortProjectName(RawProjectPath);
Stage = InStage;
Archive = InArchive;
bUseWebsocketNetDriver = bInUseWebsocketNetDriver;
if (CookSourcePlatform != null && InCooked)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(DedicatedServer, bHasDedicatedServerAndClient, CookFlavor);
}
else if (CookSourcePlatform != null && InProgram)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(false, false, "");
}
else
{
CookPlatform = "";
}
if (StageTargetPlatform != null && InCooked)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(DedicatedServer, bHasDedicatedServerAndClient, CookFlavor);
}
else if (StageTargetPlatform != null && InProgram)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(false, false, "");
}
else
{
FinalCookPlatform = "";
}
PlatformDir = StageTargetPlatform.PlatformType.ToString();
StageDirectory = CommandUtils.CombinePaths(BaseStageDirectory, FinalCookPlatform);
ArchiveDirectory = CommandUtils.CombinePaths(BaseArchiveDirectory, FinalCookPlatform);
if (!CommandUtils.FileExists(RawProjectPath))
{
throw new AutomationException("Can't find uproject file {0}.", RawProjectPathOrName);
}
ProjectRoot = CommandUtils.CombinePaths(CommandUtils.GetDirectoryName(Path.GetFullPath(RawProjectPath)));
if (!CommandUtils.DirectoryExists(ProjectRoot))
{
throw new AutomationException("Project Directory {0} doesn't exist.", ProjectRoot);
}
RelativeProjectRootForStage = ShortProjectName;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(RawProjectPath);
CookSourceRuntimeRootDir = RuntimeRootDir = LocalRoot;
RuntimeProjectRootDir = ProjectRoot;
RelativeProjectRootForUnrealPak = CommandUtils.CombinePaths(RelativeProjectRootForStage).Replace("\\", "/");
if (RelativeProjectRootForUnrealPak.StartsWith("/"))
{
RelativeProjectRootForUnrealPak = RelativeProjectRootForUnrealPak.Substring(1);
RelativeProjectRootForStage = RelativeProjectRootForStage.Substring(1);
}
SourceRelativeProjectRoot = RelativeProjectRootForStage; // for foreign projects this doesn't make much sense, but it turns into a noop on staging files
if (ProjectRoot.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
SourceRelativeProjectRoot = ProjectRoot.Substring(LocalRoot.Length);
}
if (SourceRelativeProjectRoot.StartsWith("/") || SourceRelativeProjectRoot.StartsWith("\\"))
{
SourceRelativeProjectRoot = SourceRelativeProjectRoot.Substring(1);
}
if (Stage)
{
CommandUtils.CreateDirectory(StageDirectory);
StageProjectRoot = CommandUtils.CombinePaths(StageDirectory, RelativeProjectRootForStage);
RuntimeRootDir = StageDirectory;
CookSourceRuntimeRootDir = CommandUtils.CombinePaths(BaseStageDirectory, CookPlatform);
RuntimeProjectRootDir = StageProjectRoot;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(UProjectCommandLineArgInternalRoot + RelativeProjectRootForStage + "/" + ShortProjectName + ".uproject");
}
if (Archive)
{
CommandUtils.CreateDirectory(ArchiveDirectory);
}
ProjectArgForCommandLines = ProjectArgForCommandLines.Replace("\\", "/");
ProjectBinariesFolder = CommandUtils.CombinePaths(ProjectUtils.GetClientProjectBinariesRootPath(RawProjectPath, TargetRules.TargetType.Game, IsCodeBasedProject), PlatformDir);
}
public override void ExecuteBuild()
{
CommandUtils.LogInformation("************************* List Third Party Software");
string ProjectPath = ParseParamValue("Project", String.Empty);
//Add quotes to avoid issues with spaces in project path
if (ProjectPath != String.Empty)
{
ProjectPath = "\"" + ProjectPath + "\"";
}
// Parse the list of targets to list TPS for. Each target is specified by -Target="Name|Configuration|Platform" on the command line.
HashSet <FileReference> TpsFiles = new HashSet <FileReference>();
foreach (string Target in ParseParamValues(Params, "Target"))
{
// Get the path to store the exported JSON target data
FileReference OutputFile = FileReference.Combine(CommandUtils.EngineDirectory, "Intermediate", "Build", "ThirdParty.json");
IProcessResult Result;
Result = Run(UE4Build.GetUBTExecutable(), String.Format("{0} {1} -Mode=JsonExport -OutputFile=\"{2}\"", Target.Replace('|', ' '), ProjectPath, OutputFile.FullName), Options: ERunOptions.Default);
if (Result.ExitCode != 0)
{
throw new AutomationException("Failed to run UBT");
}
// Read the exported target info back in
JsonObject Object = JsonObject.Read(OutputFile);
// Get the project file if there is one
FileReference ProjectFile = null;
string ProjectFileName;
if (Object.TryGetStringField("ProjectFile", out ProjectFileName))
{
ProjectFile = new FileReference(ProjectFileName);
}
// Get the default paths to search
HashSet <DirectoryReference> DirectoriesToScan = new HashSet <DirectoryReference>();
DirectoriesToScan.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Shaders"));
DirectoriesToScan.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Content"));
if (ProjectFile != null)
{
DirectoriesToScan.Add(DirectoryReference.Combine(ProjectFile.Directory, "Content"));
}
// Add all the paths for each module, and its runtime dependencies
JsonObject Modules = Object.GetObjectField("Modules");
foreach (string ModuleName in Modules.KeyNames)
{
JsonObject Module = Modules.GetObjectField(ModuleName);
DirectoriesToScan.Add(new DirectoryReference(Module.GetStringField("Directory")));
foreach (JsonObject RuntimeDependency in Module.GetObjectArrayField("RuntimeDependencies"))
{
string RuntimeDependencyPath;
if (RuntimeDependency.TryGetStringField("SourcePath", out RuntimeDependencyPath) || RuntimeDependency.TryGetStringField("Path", out RuntimeDependencyPath))
{
List <FileReference> Files = FileFilter.ResolveWildcard(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Source"), RuntimeDependencyPath);
DirectoriesToScan.UnionWith(Files.Select(x => x.Directory));
}
}
}
// Remove any directories that are under other directories, and sort the output list
List <DirectoryReference> SortedDirectoriesToScan = new List <DirectoryReference>();
foreach (DirectoryReference DirectoryToScan in DirectoriesToScan.OrderBy(x => x.FullName))
{
if (SortedDirectoriesToScan.Count == 0 || !DirectoryToScan.IsUnderDirectory(SortedDirectoriesToScan[SortedDirectoriesToScan.Count - 1]))
{
SortedDirectoriesToScan.Add(DirectoryToScan);
}
}
// Get the platforms to exclude
List <UnrealTargetPlatform> SupportedPlatforms = new List <UnrealTargetPlatform> {
UnrealTargetPlatform.Parse(Object.GetStringField("Platform"))
};
string[] ExcludePlatformNames = Utils.MakeListOfUnsupportedPlatforms(SupportedPlatforms, bIncludeUnbuildablePlatforms: true).ToArray();
// Find all the TPS files under the engine directory which match
foreach (DirectoryReference DirectoryToScan in SortedDirectoriesToScan)
{
foreach (FileReference TpsFile in DirectoryReference.EnumerateFiles(DirectoryToScan, "*.tps", SearchOption.AllDirectories))
{
if (!TpsFile.ContainsAnyNames(ExcludePlatformNames, DirectoryToScan))
{
TpsFiles.Add(TpsFile);
}
}
}
}
// Also add any redirects
List <string> OutputMessages = new List <string>();
foreach (FileReference TpsFile in TpsFiles)
{
string Message = TpsFile.FullName;
try
{
string[] Lines = FileReference.ReadAllLines(TpsFile);
foreach (string Line in Lines)
{
const string RedirectPrefix = "Redirect:";
int Idx = Line.IndexOf(RedirectPrefix, StringComparison.InvariantCultureIgnoreCase);
if (Idx >= 0)
{
FileReference RedirectTpsFile = FileReference.Combine(TpsFile.Directory, Line.Substring(Idx + RedirectPrefix.Length).Trim());
Message = String.Format("{0} (redirect from {1})", RedirectTpsFile.FullName, TpsFile.FullName);
break;
}
}
}
catch (Exception Ex)
{
ExceptionUtils.AddContext(Ex, "while processing {0}", TpsFile);
throw;
}
OutputMessages.Add(Message);
}
OutputMessages.Sort();
// Print them all out
foreach (string OutputMessage in OutputMessages)
{
CommandUtils.LogInformation(OutputMessage);
}
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Plugin file name was not specified via the -plugin argument");
}
// Read the plugin
FileReference PluginFile = new FileReference(PluginParam);
DirectoryReference PluginDirectory = PluginFile.Directory;
PluginDescriptor Plugin = PluginDescriptor.FromFile(PluginFile);
// Clean the intermediate build directory
DirectoryReference IntermediateBuildDirectory = DirectoryReference.Combine(PluginDirectory, "Intermediate", "Build");
if (CommandUtils.DirectoryExists(IntermediateBuildDirectory.FullName))
{
CommandUtils.DeleteDirectory(IntermediateBuildDirectory.FullName);
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it - and that can only happen if it's enabled in a project.
DirectoryReference HostProjectDirectory = DirectoryReference.Combine(new DirectoryReference(CommandUtils.CmdEnv.LocalRoot), "HostProject");
if (CommandUtils.DirectoryExists(HostProjectDirectory.FullName))
{
CommandUtils.DeleteDirectory(HostProjectDirectory.FullName);
}
DirectoryReference HostProjectPluginDirectory = DirectoryReference.Combine(HostProjectDirectory, "Plugins", PluginFile.GetFileNameWithoutExtension());
string[] CopyPluginFiles = Directory.EnumerateFiles(PluginDirectory.FullName, "*", SearchOption.AllDirectories).ToArray();
foreach (string CopyPluginFile in CopyPluginFiles)
{
CommandUtils.CopyFile(CopyPluginFile, CommandUtils.MakeRerootedFilePath(CopyPluginFile, PluginDirectory.FullName, HostProjectPluginDirectory.FullName));
}
FileReference HostProjectPluginFile = FileReference.Combine(HostProjectPluginDirectory, PluginFile.GetFileName());
FileReference HostProjectFile = FileReference.Combine(HostProjectDirectory, "HostProject.uproject");
File.WriteAllText(HostProjectFile.FullName, "{ \"FileVersion\": 3, \"Plugins\": [ { \"Name\": \"" + PluginFile.GetFileNameWithoutExtension() + "\", \"Enabled\": true } ] }");
// Get any additional arguments from the commandline
string AdditionalArgs = "";
// Build the host platforms
List <string> ReceiptFileNames = new List <string>();
UnrealTargetPlatform HostPlatform = BuildHostPlatform.Current.Platform;
if (!ParseParam("NoHostPlatform"))
{
if (Plugin.bCanBeUsedWithUnrealHeaderTool)
{
BuildPluginWithUBT(PluginFile, Plugin, null, "UnrealHeaderTool", TargetRules.TargetType.Program, HostPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, String.Format("{0} -plugin {1}", AdditionalArgs, CommandUtils.MakePathSafeToUseWithCommandLine(HostProjectPluginFile.FullName)));
}
BuildPluginWithUBT(PluginFile, Plugin, HostProjectFile, "UE4Editor", TargetRules.TargetType.Editor, HostPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, AdditionalArgs);
}
// Add the game targets
List <UnrealTargetPlatform> TargetPlatforms = Rocket.RocketBuild.GetTargetPlatforms(this, HostPlatform);
foreach (UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if (Rocket.RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform))
{
BuildPluginWithUBT(PluginFile, Plugin, HostProjectFile, "UE4Game", TargetRules.TargetType.Game, TargetPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, AdditionalArgs);
BuildPluginWithUBT(PluginFile, Plugin, HostProjectFile, "UE4Game", TargetRules.TargetType.Game, TargetPlatform, UnrealTargetConfiguration.Shipping, ReceiptFileNames, AdditionalArgs);
}
}
// Package the plugin to the output folder
string PackageDirectory = ParseParamValue("Package");
if (PackageDirectory != null)
{
List <BuildProduct> BuildProducts = GetBuildProductsFromReceipts(UnrealBuildTool.UnrealBuildTool.EngineDirectory, HostProjectDirectory, ReceiptFileNames);
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDirectory);
}
}
public async Task Stats()
{
long temp;
var owner = (await Context.Client.GetApplicationInfoAsync()).Owner;
var cpuUsage = (int)await CommandUtils.GetCpuUsageForProcessAsync();
string cpu = null;
long? totalRamUsage = null;
long processRamUsage = 0;
long? totalRam = null;
string os = RuntimeInformation.OSDescription;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
// CPU Name
if (File.Exists("/proc/cpuinfo"))
{
var cpuinfo = File.ReadAllLines("/proc/cpuinfo");
cpu = cpuinfo.ElementAtOrDefault(4)?.Split(':').ElementAtOrDefault(1);
}
// OS Name
if (File.Exists("/etc/lsb-release"))
{
var distroInfo = File.ReadAllLines("/etc/lsb-release");
os = distroInfo.ElementAtOrDefault(3)?.Split('=').ElementAtOrDefault(1)?.Trim('\"');
}
// Total RAM & total RAM usage
var output = CommandUtils.RunCommand("free -m")?.Split(Environment.NewLine);
var memory = output?.ElementAtOrDefault(1)?.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (long.TryParse(memory?.ElementAtOrDefault(1), out temp))
{
totalRam = temp;
}
if (long.TryParse(memory?.ElementAtOrDefault(2), out temp))
{
totalRamUsage = temp;
}
// Process RAM usage
processRamUsage = Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024;
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// CPU Name
cpu = CommandUtils.RunCommand("wmic cpu get name")
?.Trim()
.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries)
.ElementAtOrDefault(1);
// Total RAM & total RAM usage
var output = CommandUtils.RunCommand("wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value")
?.Trim()
.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
if (output?.Length > 1)
{
long freeRam = 0;
var split = output[0].Split('=', StringSplitOptions.RemoveEmptyEntries);
if (split.Length > 1 && long.TryParse(split[1], out temp))
{
freeRam = temp / 1024;
}
split = output[1].Split('=', StringSplitOptions.RemoveEmptyEntries);
if (split.Length > 1 && long.TryParse(split[1], out temp))
{
totalRam = temp / 1024;
}
if (totalRam != null && freeRam != 0)
{
totalRamUsage = totalRam - freeRam;
}
}
// Process RAM usage
processRamUsage = Process.GetCurrentProcess().PrivateMemorySize64 / 1024 / 1024;
}
else
{
// TODO: Get system info from the remaining platforms
}
int totalUsers = 0;
foreach (var guild in Context.Client.Guilds)
{
totalUsers += guild.MemberCount;
}
string version = $"v{Constants.Version}";
if (FergunClient.IsDebugMode)
{
version += "-dev";
}
var elapsed = DateTimeOffset.UtcNow - FergunClient.Uptime;
var builder = new EmbedBuilder()
.WithTitle("Fergun Stats")
.AddField(Locate("OperatingSystem"), os, true)
.AddField("\u200b", "\u200b", true)
.AddField("CPU", cpu ?? "?", true)
.AddField(Locate("CPUUsage"), cpuUsage + "%", true)
.AddField("\u200b", "\u200b", true)
.AddField(Locate("RAMUsage"),
$"{processRamUsage}MB ({(totalRam == null ? 0 : Math.Round((double)processRamUsage / totalRam.Value * 100, 2))}%) " +
$"/ {(totalRamUsage == null || totalRam == null ? "?MB" : $"{totalRamUsage}MB ({Math.Round((double)totalRamUsage.Value / totalRam.Value * 100, 2)}%)")} " +
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null)
{
// Get the SYMSTORE.EXE path, using the latest SDK version we can find.
FileReference SymStoreExe = GetSymStoreExe();
List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList();
if (FilesToAdd.Count > 0)
{
DateTime Start = DateTime.Now;
DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore");
if (DirectoryReference.Exists(TempSymStoreDir))
{
CommandUtils.DeleteDirectory(TempSymStoreDir);
DirectoryReference.CreateDirectory(TempSymStoreDir);
}
string TempFileName = Path.GetTempFileName();
try
{
File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII);
// Copy everything to the temp symstore
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = SymStoreExe.FullName;
StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\"", TempFileName, TempSymStoreDir, Product);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0)
{
return(false);
}
}
finally
{
File.Delete(TempFileName);
}
DateTime CompressDone = DateTime.Now;
LogInformation("Took {0}s to compress the symbol files to temp path {1}", (CompressDone - Start).TotalSeconds, TempSymStoreDir);
int CopiedCount = 0;
// Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files
foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => IsSymbolFile(File)))
{
string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir));
FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath);
// Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example)
if (!string.IsNullOrWhiteSpace(BuildVersion))
{
FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion));
// Attempt to create the file. Just continue if it fails.
try
{
DirectoryReference.CreateDirectory(BuildVersionFile.Directory);
FileReference.WriteAllText(BuildVersionFile, string.Empty);
}
catch (Exception Ex)
{
LogWarning("Failed to write the version file, reason {0}", Ex.ToString());
}
}
// Don't bother copying the temp file if the destination file is there already.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName);
continue;
}
FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString());
try
{
CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName);
}
catch (Exception Ex)
{
throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString());
}
// Move the file in the temp store over.
try
{
FileReference.Move(TempDestinationFile, ActualDestinationFile);
//LogVerbose("Moved {0} to {1}", TempDestinationFile, ActualDestinationFile);
CopiedCount++;
}
catch (Exception Ex)
{
// If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already.
// Either way, it's fine to just continue on.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName);
continue;
}
// If it doesn't exist, we actually failed to copy it entirely.
else
{
LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString());
}
}
// Delete the temp one no matter what, don't want them hanging around in the symstore
finally
{
FileReference.Delete(TempDestinationFile);
}
}
LogInformation("Took {0}s to copy {1} symbol files to the store at {2}", (DateTime.Now - CompressDone).TotalSeconds, CopiedCount, SymbolStoreDirectory);
FileReference PingmeFile = FileReference.Combine(SymbolStoreDirectory, "pingme.txt");
if (!FileReference.Exists(PingmeFile))
{
LogInformation("Creating {0} to mark path as three-tiered symbol location", PingmeFile);
File.WriteAllText(PingmeFile.FullName, "Exists to mark this as a three-tiered symbol location");
}
}
return(true);
}
public static UE4Build.BuildAgenda MakeAgenda(UnrealBuildTool.UnrealTargetPlatform[] Platforms, List <string> ExtraBuildProducts)
{
// Create the build agenda
UE4Build.BuildAgenda Agenda = new UE4Build.BuildAgenda();
// C# binaries
Agenda.SwarmProject = @"Engine\Source\Programs\UnrealSwarm\UnrealSwarm.sln";
Agenda.DotNetProjects.Add(@"Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.csproj");
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/DotNETCommon/DotNETUtilities/DotNETUtilities.csproj");
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/RPCUtility/RPCUtility.csproj");
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/UnrealControls/UnrealControls.csproj");
// Windows binaries
if (Platforms.Contains(UnrealBuildTool.UnrealTargetPlatform.Win64))
{
Agenda.AddTarget("CrashReportClient", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("CrashReportClient", UnrealBuildTool.UnrealTargetPlatform.Win32, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("UnrealHeaderTool", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("UnrealPak", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("UnrealLightmass", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("ShaderCompileWorker", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("UnrealVersionSelector", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Shipping);
Agenda.AddTarget("BootstrapPackagedGame", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Shipping);
Agenda.AddTarget("BootstrapPackagedGame", UnrealBuildTool.UnrealTargetPlatform.Win32, UnrealBuildTool.UnrealTargetConfiguration.Shipping);
Agenda.AddTarget("UnrealCEFSubProcess", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.AddTarget("UnrealCEFSubProcess", UnrealBuildTool.UnrealTargetPlatform.Win32, UnrealBuildTool.UnrealTargetConfiguration.Development);
}
// Mac binaries
if (Platforms.Contains(UnrealBuildTool.UnrealTargetPlatform.Mac))
{
Agenda.AddTarget("CrashReportClient", UnrealBuildTool.UnrealTargetPlatform.Mac, UnrealBuildTool.UnrealTargetConfiguration.Development, InAddArgs: "-CopyAppBundleBackToDevice");
Agenda.AddTarget("UnrealPak", UnrealBuildTool.UnrealTargetPlatform.Mac, UnrealBuildTool.UnrealTargetConfiguration.Development, InAddArgs: "-CopyAppBundleBackToDevice");
Agenda.AddTarget("UnrealLightmass", UnrealBuildTool.UnrealTargetPlatform.Mac, UnrealBuildTool.UnrealTargetConfiguration.Development, InAddArgs: "-CopyAppBundleBackToDevice");
Agenda.AddTarget("ShaderCompileWorker", UnrealBuildTool.UnrealTargetPlatform.Mac, UnrealBuildTool.UnrealTargetConfiguration.Development, InAddArgs: "-CopyAppBundleBackToDevice");
Agenda.AddTarget("UE4EditorServices", UnrealBuildTool.UnrealTargetPlatform.Mac, UnrealBuildTool.UnrealTargetConfiguration.Development, InAddArgs: "-CopyAppBundleBackToDevice");
Agenda.AddTarget("UnrealCEFSubProcess", UnrealBuildTool.UnrealTargetPlatform.Mac, UnrealBuildTool.UnrealTargetConfiguration.Development, InAddArgs: "-CopyAppBundleBackToDevice");
}
// iOS binaries
if (Platforms.Contains(UnrealBuildTool.UnrealTargetPlatform.IOS))
{
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/iOS/iPhonePackager/iPhonePackager.csproj");
ExtraBuildProducts.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/iOS/iPhonePackager.exe"));
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/iOS/DeploymentServer/DeploymentServer.csproj");
ExtraBuildProducts.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/iOS/DeploymentServer.exe"));
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/iOS/DeploymentInterface/DeploymentInterface.csproj");
ExtraBuildProducts.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/iOS/DeploymentInterface.dll"));
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/iOS/MobileDeviceInterface/MobileDeviceInterface.csproj");
ExtraBuildProducts.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/iOS/MobileDeviceInterface.dll"));
}
// PS4 binaries
if (Platforms.Contains(UnrealBuildTool.UnrealTargetPlatform.PS4))
{
Agenda.AddTarget("PS4MapFileUtil", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
Agenda.DotNetProjects.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Source/Programs/PS4/PS4DevKitUtil/PS4DevKitUtil.csproj"));
ExtraBuildProducts.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/PS4/PS4DevKitUtil.exe"));
}
// Xbox One binaries
if (Platforms.Contains(UnrealBuildTool.UnrealTargetPlatform.XboxOne))
{
Agenda.AddTarget("XboxOnePDBFileUtil", UnrealBuildTool.UnrealTargetPlatform.Win64, UnrealBuildTool.UnrealTargetConfiguration.Development);
}
// HTML5 binaries
if (Platforms.Contains(UnrealBuildTool.UnrealTargetPlatform.HTML5))
{
Agenda.DotNetProjects.Add(@"Engine/Source/Programs/HTML5/HTML5LaunchHelper/HTML5LaunchHelper.csproj");
ExtraBuildProducts.Add(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/HTML5LaunchHelper.exe"));
}
return(Agenda);
}
FileReference[] CompilePlugin(FileReference HostProjectFile, FileReference HostProjectPluginFile, PluginDescriptor Plugin, List <UnrealTargetPlatform> HostPlatforms, List <UnrealTargetPlatform> TargetPlatforms, string AdditionalArgs)
{
List <string> ReceiptFileNames = new List <string>();
// Build the host platforms
if (HostPlatforms.Count > 0)
{
CommandUtils.Log("Building plugin for host platforms: {0}", String.Join(", ", HostPlatforms));
foreach (UnrealTargetPlatform HostPlatform in HostPlatforms)
{
if (Plugin.bCanBeUsedWithUnrealHeaderTool)
{
CompilePluginWithUBT(null, HostProjectPluginFile, Plugin, "UnrealHeaderTool", TargetType.Program, HostPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, String.Format("{0} -plugin {1}", AdditionalArgs, CommandUtils.MakePathSafeToUseWithCommandLine(HostProjectPluginFile.FullName)));
}
CompilePluginWithUBT(HostProjectFile, HostProjectPluginFile, Plugin, "UE4Editor", TargetType.Editor, HostPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, AdditionalArgs);
}
}
// Add the game targets
if (TargetPlatforms.Count > 0)
{
CommandUtils.Log("Building plugin for target platforms: {0}", String.Join(", ", TargetPlatforms));
foreach (UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
CompilePluginWithUBT(HostProjectFile, HostProjectPluginFile, Plugin, "UE4Game", TargetType.Game, TargetPlatform, UnrealTargetConfiguration.Development, ReceiptFileNames, null);
CompilePluginWithUBT(HostProjectFile, HostProjectPluginFile, Plugin, "UE4Game", TargetType.Game, TargetPlatform, UnrealTargetConfiguration.Shipping, ReceiptFileNames, null);
}
}
// Package the plugin to the output folder
List <BuildProduct> BuildProducts = GetBuildProductsFromReceipts(CommandUtils.EngineDirectory, HostProjectFile.Directory, ReceiptFileNames);
return(BuildProducts.Select(x => new FileReference(x.Path)).ToArray());
}
public int StageFiles(StagedFileType FileType, string InPath, string Wildcard = "*", bool bRecursive = true, string[] ExcludeWildcard = null, string NewPath = null, bool bAllowNone = false, bool bRemap = true, string NewName = null, bool bAllowNotForLicenseesFiles = true, bool bStripFilesForOtherPlatforms = true)
{
int FilesAdded = 0;
// make sure any ..'s are removed
Utils.CollapseRelativeDirectories(ref InPath);
if (CommandUtils.DirectoryExists(InPath))
{
var All = CommandUtils.FindFiles(Wildcard, bRecursive, InPath);
var Exclude = new HashSet <string>();
if (ExcludeWildcard != null)
{
foreach (var Excl in ExcludeWildcard)
{
var Remove = CommandUtils.FindFiles(Excl, bRecursive, InPath);
foreach (var File in Remove)
{
Exclude.Add(CommandUtils.CombinePaths(File));
}
}
}
foreach (var AllFile in All)
{
var FileToCopy = CommandUtils.CombinePaths(AllFile);
if (Exclude.Contains(FileToCopy))
{
continue;
}
if (!bAllowNotForLicenseesFiles && (FileToCopy.Contains("NotForLicensees") || FileToCopy.Contains("NoRedist")))
{
continue;
}
if (bStripFilesForOtherPlatforms && !bIsCombiningMultiplePlatforms)
{
bool OtherPlatform = false;
foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (Plat != StageTargetPlatform.PlatformType && Plat != UnrealTargetPlatform.Unknown)
{
var Search = FileToCopy;
if (Search.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
if (LocalRoot.EndsWith("\\") || LocalRoot.EndsWith("/"))
{
Search = Search.Substring(LocalRoot.Length - 1);
}
else
{
Search = Search.Substring(LocalRoot.Length);
}
}
if (Search.StartsWith(ProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
if (ProjectRoot.EndsWith("\\") || ProjectRoot.EndsWith("/"))
{
Search = Search.Substring(ProjectRoot.Length - 1);
}
else
{
Search = Search.Substring(ProjectRoot.Length);
}
}
if (Search.IndexOf(CommandUtils.CombinePaths("/" + Plat.ToString() + "/"), 0, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
OtherPlatform = true;
break;
}
}
}
if (OtherPlatform)
{
continue;
}
}
string Dest;
if (!FileToCopy.StartsWith(InPath))
{
throw new AutomationException("Can't deploy {0}; it was supposed to start with {1}", FileToCopy, InPath);
}
string FileToRemap = FileToCopy;
// If the specified a new directory, first we deal with that, then apply the other things
// this is used to collapse the sandbox, among other things
if (NewPath != null)
{
Dest = FileToRemap.Substring(InPath.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
Dest = CommandUtils.CombinePaths(NewPath, Dest);
#if false // if the cooker rebases, I don't think we need to ever rebase while staging to a new path
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
// project relative stuff in a collapsed sandbox
if (Dest.StartsWith(SourceRelativeProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
Dest = Dest.Substring(SourceRelativeProjectRoot.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
Dest = CommandUtils.CombinePaths(RelativeProjectRootForStage, Dest);
}
#endif
}
// project relative file
else if (FileToRemap.StartsWith(ProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
Dest = FileToRemap.Substring(ProjectRoot.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
Dest = CommandUtils.CombinePaths(RelativeProjectRootForStage, Dest);
}
// engine relative file
else if (FileToRemap.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
Dest = CommandUtils.CombinePaths(FileToRemap.Substring(LocalRoot.Length));
}
else
{
throw new AutomationException("Can't deploy {0} because it doesn't start with {1} or {2}", FileToRemap, ProjectRoot, LocalRoot);
}
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
if (NewName != null)
{
Dest = CommandUtils.CombinePaths(Path.GetDirectoryName(Dest), NewName);
}
if (bRemap)
{
Dest = StageTargetPlatform.Remap(Dest);
}
if (FileType == StagedFileType.UFS)
{
AddUniqueStagingFile(UFSStagingFiles, FileToCopy, Dest);
}
else if (FileType == StagedFileType.NonUFS)
{
AddUniqueStagingFile(NonUFSStagingFiles, FileToCopy, Dest);
}
else if (FileType == StagedFileType.DebugNonUFS)
{
AddUniqueStagingFile(NonUFSStagingFilesDebug, FileToCopy, Dest);
}
FilesAdded++;
}
}
if (FilesAdded == 0 && !bAllowNone && !bIsCombiningMultiplePlatforms)
{
throw new AutomationException(ErrorCodes.Error_StageMissingFile, "No files found to deploy for {0} with wildcard {1} and exclusions {2}", InPath, Wildcard, ExcludeWildcard);
}
return(FilesAdded);
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!FileReference.Exists(PluginFile))
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (DirectoryReference.Exists(PackageDir))
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
DirectoryReference.CreateDirectory(PackageDir);
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!FileReference.Exists(SourceFilterFile))
{
List <string> Lines = new List <string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
DirectoryReference.CreateDirectory(SourceFilterFile.Directory);
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.Log("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile, false);
// Compile the plugin for all the target platforms
List <UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List <UnrealTargetPlatform>() : new List <UnrealTargetPlatform> {
BuildHostPlatform.Current.Platform
};
List <UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform);
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, "");
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir);
// Remove the host project
if (!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
public int ArchiveFiles(string InPath, string Wildcard = "*", bool bRecursive = true, string[] ExcludeWildcard = null)
{
int FilesAdded = 0;
if (CommandUtils.DirectoryExists(InPath))
{
var All = CommandUtils.FindFiles(Wildcard, bRecursive, InPath);
var Exclude = new HashSet <string>();
if (ExcludeWildcard != null)
{
foreach (var Excl in ExcludeWildcard)
{
var Remove = CommandUtils.FindFiles(Excl, bRecursive, InPath);
foreach (var File in Remove)
{
Exclude.Add(CommandUtils.CombinePaths(File));
}
}
}
foreach (var AllFile in All)
{
var FileToCopy = CommandUtils.CombinePaths(AllFile);
if (Exclude.Contains(FileToCopy))
{
continue;
}
if (!bIsCombiningMultiplePlatforms)
{
bool OtherPlatform = false;
foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (Plat != StageTargetPlatform.PlatformType && Plat != UnrealTargetPlatform.Unknown)
{
var Search = FileToCopy;
if (Search.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
if (LocalRoot.EndsWith("\\") || LocalRoot.EndsWith("/"))
{
Search = Search.Substring(LocalRoot.Length - 1);
}
else
{
Search = Search.Substring(LocalRoot.Length);
}
}
if (Search.StartsWith(ProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
if (ProjectRoot.EndsWith("\\") || ProjectRoot.EndsWith("/"))
{
Search = Search.Substring(ProjectRoot.Length - 1);
}
else
{
Search = Search.Substring(ProjectRoot.Length);
}
}
if (Search.IndexOf(CommandUtils.CombinePaths("/" + Plat.ToString() + "/"), 0, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
OtherPlatform = true;
break;
}
}
}
if (OtherPlatform)
{
continue;
}
}
string Dest;
if (!FileToCopy.StartsWith(InPath))
{
throw new AutomationException("Can't archive {0}; it was supposed to start with {1}", FileToCopy, InPath);
}
Dest = FileToCopy.Substring(InPath.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
if (ArchivedFiles.ContainsKey(FileToCopy))
{
if (ArchivedFiles[FileToCopy] != Dest)
{
throw new AutomationException("Can't archive {0}: it was already in the files to archive with a different destination '{1}'", FileToCopy, Dest);
}
}
else
{
ArchivedFiles.Add(FileToCopy, Dest);
}
FilesAdded++;
}
}
return(FilesAdded);
}
public bool CopyBuild(DirectoryReference InstallPath)
{
CommandUtils.LogInformation("Using previously synced shared cooked build");
return(true);
}
public void SpewValues()
{
CommandUtils.Log("Name : {0}", Name);
CommandUtils.Log("ApiKey : {0}", ApiKey);
//CommandUtils.LogConsole("ApiSecret : {0}", ApiSecret); // This should probably never be revealed.
}
public override void ExecuteBuild()
{
LogInformation("************************* Build Third Party Libs");
// figure out what batch/script to run
string CompileScriptName;
if (UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64)
{
CompileScriptName = WindowsCompileScript;
}
else if (UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac)
{
CompileScriptName = MacCompileScript;
}
else if (UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux)
{
CompileScriptName = LinuxCompileScript;
}
else
{
throw new AutomationException("Unknown runtime platform!");
}
// look for changelist on the command line
int WorkingCL = Int32.Parse(ParseParamValue("Changelist", "-1"));
// if not specified, make one
if (WorkingCL == -1)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("Third party libs built from changelist {0}", P4Env.Changelist));
}
LogInformation("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL);
// go to the third party lib dir
string SearchLibraryDir = ParseParamValue("SearchDir", DefaultLibraryDir);
CommandUtils.PushDir(SearchLibraryDir);
// figure out what libraries to compile
string LibsToCompileString = ParseParamValue("Libs");
// hunt down build batch files if the caller didn't specify a list to compile
List <string> LibsToCompile = new List <string>();
if (string.IsNullOrEmpty(LibsToCompileString))
{
// loop over third party directories looking for the right batch files
foreach (string Dir in Directory.EnumerateDirectories("."))
{
if (File.Exists(Path.Combine(Dir, CompileScriptName)))
{
LibsToCompile.Add(Path.GetFileName(Dir));
}
}
}
else
{
// just split up the param and make sure the batch file exists
string[] Libs = LibsToCompileString.Split('+');
bool bHadError = false;
foreach (string Dir in Libs)
{
if (File.Exists(Path.Combine(Dir, CompileScriptName)))
{
LibsToCompile.Add(Path.GetFileName(Dir));
}
else
{
LogError("Error: Requested lib {0} does not have a {1}", Dir, CompileScriptName);
bHadError = true;
}
}
if (bHadError)
{
// error out so that we don't fail to build some and have it lost in the noise
throw new AutomationException("One or more libs were not set up to compile.");
}
}
// set an envvar so that the inner batch files can check out files into a shared changelist
Environment.SetEnvironmentVariable("THIRD_PARTY_CHANGELIST", string.Format("-c {0}", WorkingCL));
// now go through and run each batch file,
foreach (string Lib in LibsToCompile)
{
LogInformation("Building {0}", Lib);
// go into the lib dir
CommandUtils.PushDir(Lib);
// run the builder batch file
CommandUtils.RunAndLog(CmdEnv, CompileScriptName, "", "ThirdPartyLib_" + Lib);
// go back to ThirdParty dir
CommandUtils.PopDir();
}
// undo the SearchLibraryDir push
CommandUtils.PopDir();
PrintRunTime();
// revert any unchanged files
P4.RevertUnchanged(WorkingCL);
if (AllowSubmit)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
LogInformation("Submitted changelist {0}", SubmittedCL);
}
}
public override void GetFilesToDeployOrStage(ProjectParams Params, DeploymentContext SC)
{
// FIXME: use build architecture
string BuildArchitecture = "x86_64-unknown-linux-gnu";
if (SC.bStageCrashReporter)
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir), "CrashReportClient", false, null, null, true);
}
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries/ThirdParty/ICU/icu4c-53_1/", SC.PlatformDir, BuildArchitecture), Params.bDebugBuildsActuallyUseDebugCRT ? "*d.so*" : "*.so*", false, new[] { Params.bDebugBuildsActuallyUseDebugCRT ? "*.so*" : "*d.so*" }, CombinePaths("Engine/Binaries", SC.PlatformDir));
}
// assume that we always have to deploy Steam (FIXME: should be automatic - UEPLAT-807)
{
string SteamVersion = "Steamv132";
// Check if the Steam directory exists. We need it for Steam controller support, so we include it whenever we can.
if (Directory.Exists(CommandUtils.CombinePaths(SC.LocalRoot, "Engine/Binaries/ThirdParty/Steamworks/" + SteamVersion)))
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries/ThirdParty/Steamworks/" + SteamVersion, SC.PlatformDir), "libsteam_api.so", false, null, CombinePaths("Engine/Binaries", SC.PlatformDir));
}
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Config"), "controller.vdf", false, null, CommandUtils.CombinePaths(SC.RelativeProjectRootForStage, "Saved/Config"));
// copy optional perfcounters definition file
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.ProjectRoot, "Config"), "PerfCounters.json", false, null, CommandUtils.CombinePaths(SC.RelativeProjectRootForStage, "Saved/Config"), true);
}
// stage libLND (omit it for dedservers and Rocket - proper resolution is to use build receipts, see UEPLAT-807)
if (!SC.DedicatedServer && !Automation.RunningRocket())
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries/ThirdParty/LinuxNativeDialogs/", SC.PlatformDir, BuildArchitecture), "libLND*.so");
}
// assume that we always have to deploy OpenAL (FIXME: should be automatic - UEPLAT-807)
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries/ThirdParty/OpenAL/", SC.PlatformDir), "libopenal.so.1", false, null, CombinePaths("Engine/Binaries", SC.PlatformDir));
}
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.ProjectRoot, "Content/Splash"), "Splash.bmp", false, null, null, true);
List <string> Exes = GetExecutableNames(SC);
foreach (var Exe in Exes)
{
if (Exe.StartsWith(CombinePaths(SC.RuntimeProjectRootDir, "Binaries", SC.PlatformDir)))
{
// remap the project root. For content-only projects, rename the executable to the project name.
if (!Params.IsCodeBasedProject && Exe == Exes[0])
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.ProjectRoot, "Binaries", SC.PlatformDir), Path.GetFileNameWithoutExtension(Exe), true, null, CommandUtils.CombinePaths(SC.RelativeProjectRootForStage, "Binaries", SC.PlatformDir), false, true, SC.ShortProjectName);
}
else
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.ProjectRoot, "Binaries", SC.PlatformDir), Path.GetFileNameWithoutExtension(Exe), true, null, CommandUtils.CombinePaths(SC.RelativeProjectRootForStage, "Binaries", SC.PlatformDir), false);
}
}
else if (Exe.StartsWith(CombinePaths(SC.RuntimeRootDir, "Engine/Binaries", SC.PlatformDir)))
{
// Move the executable for content-only projects into the project directory, using the project name, so it can figure out the UProject to look for and is consistent with code projects.
if (!Params.IsCodeBasedProject && Exe == Exes[0])
{
// ensure the ue4game binary exists, if applicable
if (!SC.IsCodeBasedProject && !FileExists_NoExceptions(Params.ProjectGameExeFilename) && !SC.bIsCombiningMultiplePlatforms)
{
LogError("Failed to find game binary " + Params.ProjectGameExeFilename);
throw new AutomationException(ExitCode.Error_MissingExecutable, "Stage Failed. Could not find game binary {0}. You may need to build the UE4 project with your target configuration and platform.", Params.ProjectGameExeFilename);
}
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir), Path.GetFileNameWithoutExtension(Exe), true, null, CommandUtils.CombinePaths(SC.RelativeProjectRootForStage, "Binaries", SC.PlatformDir), false, true, SC.ShortProjectName);
}
else
{
SC.StageFiles(StagedFileType.NonUFS, CombinePaths(SC.LocalRoot, "Engine/Binaries", SC.PlatformDir), Path.GetFileNameWithoutExtension(Exe), true, null, null, false);
}
}
else
{
throw new AutomationException("Can't stage the exe {0} because it doesn't start with {1} or {2}", Exe, CombinePaths(SC.RuntimeProjectRootDir, "Binaries", SC.PlatformDir), CombinePaths(SC.RuntimeRootDir, "Engine/Binaries", SC.PlatformDir));
}
}
}
public override void GetTargetFile(string RemoteFilePath, string LocalFile, ProjectParams Params)
{
var SourceFile = FileReference.Combine(new DirectoryReference(Params.BaseStageDirectory), GetCookPlatform(Params.HasServerCookedTargets, Params.HasClientTargetDetected), RemoteFilePath);
CommandUtils.CopyFile(SourceFile.FullName, LocalFile);
}
public override void Package(ProjectParams Params, DeploymentContext SC, int WorkingCL)
{
Log("Package {0}", Params.RawProjectPath);
Log("Setting Emscripten SDK for packaging..");
HTML5SDKInfo.SetupEmscriptenTemp();
HTML5SDKInfo.SetUpEmscriptenConfigFile();
string PackagePath = Path.Combine(Path.GetDirectoryName(Params.RawProjectPath), "Binaries", "HTML5");
if (!Directory.Exists(PackagePath))
{
Directory.CreateDirectory(PackagePath);
}
string FinalDataLocation = Path.Combine(PackagePath, Params.ShortProjectName) + ".data";
var ConfigCache = new UnrealBuildTool.ConfigCacheIni(UnrealTargetPlatform.HTML5, "Engine", Path.GetDirectoryName(Params.RawProjectPath), CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine"));
if (HTMLPakAutomation.CanCreateMapPaks(Params))
{
HTMLPakAutomation PakAutomation = new HTMLPakAutomation(Params, SC);
// Create Necessary Paks.
PakAutomation.CreateEnginePak();
PakAutomation.CreateGamePak();
PakAutomation.CreateContentDirectoryPak();
// Create Emscripten Package from Necessary Paks. - This will be the VFS.
PakAutomation.CreateEmscriptenDataPackage(PackagePath, FinalDataLocation);
// Create All Map Paks which will be downloaded on the fly.
PakAutomation.CreateMapPak();
// Create Delta Paks if setup.
List <string> Paks = new List <string>();
ConfigCache.GetArray("/Script/HTML5PlatformEditor.HTML5TargetSettings", "LevelTransitions", out Paks);
if (Paks != null)
{
foreach (var Pak in Paks)
{
var Matched = Regex.Matches(Pak, "\"[^\"]+\"", RegexOptions.IgnoreCase);
string MapFrom = Path.GetFileNameWithoutExtension(Matched[0].ToString().Replace("\"", ""));
string MapTo = Path.GetFileNameWithoutExtension(Matched[1].ToString().Replace("\"", ""));
PakAutomation.CreateDeltaMapPaks(MapFrom, MapTo);
}
}
}
else
{
// we need to operate in the root
string PythonPath = HTML5SDKInfo.Python();
string PackagerPath = HTML5SDKInfo.EmscriptenPackager();
using (new ScopedEnvVar("EM_CONFIG", HTML5SDKInfo.DOT_EMSCRIPTEN))
{
using (new PushedDirectory(Path.Combine(Params.BaseStageDirectory, "HTML5")))
{
string CmdLine = string.Format("\"{0}\" \"{1}\" --preload . --js-output=\"{1}.js\"", PackagerPath, FinalDataLocation);
RunAndLog(CmdEnv, PythonPath, CmdLine);
}
}
}
// copy the "Executable" to the package directory
string GameExe = Path.GetFileNameWithoutExtension(Params.ProjectGameExeFilename);
if (Params.ClientConfigsToBuild[0].ToString() != "Development")
{
GameExe += "-HTML5-" + Params.ClientConfigsToBuild[0].ToString();
}
GameExe += ".js";
// ensure the ue4game binary exists, if applicable
string FullGameExePath = Path.Combine(Path.GetDirectoryName(Params.ProjectGameExeFilename), GameExe);
if (!SC.IsCodeBasedProject && !FileExists_NoExceptions(FullGameExePath))
{
Log("Failed to find game application " + FullGameExePath);
throw new AutomationException(ErrorCodes.Error_MissingExecutable, "Stage Failed. Could not find application {0}. You may need to build the UE4 project with your target configuration and platform.", FullGameExePath);
}
if (Path.Combine(Path.GetDirectoryName(Params.ProjectGameExeFilename), GameExe) != Path.Combine(PackagePath, GameExe))
{
File.Copy(Path.Combine(Path.GetDirectoryName(Params.ProjectGameExeFilename), GameExe), Path.Combine(PackagePath, GameExe), true);
File.Copy(Path.Combine(Path.GetDirectoryName(Params.ProjectGameExeFilename), GameExe) + ".mem", Path.Combine(PackagePath, GameExe) + ".mem", true);
File.Copy(Path.Combine(Path.GetDirectoryName(Params.ProjectGameExeFilename), GameExe) + ".symbols", Path.Combine(PackagePath, GameExe) + ".symbols", true);
}
File.SetAttributes(Path.Combine(PackagePath, GameExe), FileAttributes.Normal);
File.SetAttributes(Path.Combine(PackagePath, GameExe) + ".mem", FileAttributes.Normal);
File.SetAttributes(Path.Combine(PackagePath, GameExe) + ".symbols", FileAttributes.Normal);
// put the HTML file to the package directory
string TemplateFileName = "GameX.html.template";
string TemplateFile = Path.Combine(CombinePaths(CmdEnv.LocalRoot, "Engine"), "Build", "HTML5", TemplateFileName);
string OutputFile = Path.Combine(PackagePath, (Params.ClientConfigsToBuild[0].ToString() != "Development" ? (Params.ShortProjectName + "-HTML5-" + Params.ClientConfigsToBuild[0].ToString()) : Params.ShortProjectName)) + ".html";
// find Heap Size.
ulong HeapSize;
int ConfigHeapSize = 0;
// Valuer set by Editor UI
var bGotHeapSize = ConfigCache.GetInt32("/Script/HTML5PlatformEditor.HTML5TargetSettings", "HeapSize" + Params.ClientConfigsToBuild[0].ToString(), out ConfigHeapSize);
// Fallback if the previous method failed
if (!bGotHeapSize && !ConfigCache.GetInt32("BuildSettings", "HeapSize" + Params.ClientConfigsToBuild[0].ToString(), out ConfigHeapSize)) // in Megs.
{
// we couldn't find a per config heap size, look for a common one.
if (!ConfigCache.GetInt32("BuildSettings", "HeapSize", out ConfigHeapSize))
{
ConfigHeapSize = Params.IsCodeBasedProject ? 1024 : 512;
Log("Could not find Heap Size setting in .ini for Client config {0}", Params.ClientConfigsToBuild[0].ToString());
}
}
HeapSize = (ulong)ConfigHeapSize * 1024L * 1024L; // convert to bytes.
Log("Setting Heap size to {0} Mb ", ConfigHeapSize);
GenerateFileFromTemplate(TemplateFile, OutputFile, Params.ShortProjectName, Params.ClientConfigsToBuild[0].ToString(), Params.StageCommandline, !Params.IsCodeBasedProject, HeapSize);
string MacBashTemplateFile = Path.Combine(CombinePaths(CmdEnv.LocalRoot, "Engine"), "Build", "HTML5", "RunMacHTML5LaunchHelper.command.template");
string MacBashOutputFile = Path.Combine(PackagePath, "RunMacHTML5LaunchHelper.command");
string MonoPath = Path.Combine(CombinePaths(CmdEnv.LocalRoot, "Engine"), "Build", "BatchFiles", "Mac", "SetupMono.sh");
GenerateMacCommandFromTemplate(MacBashTemplateFile, MacBashOutputFile, MonoPath);
string JSDir = Path.Combine(CombinePaths(CmdEnv.LocalRoot, "Engine"), "Build", "HTML5");
string OutDir = PackagePath;
// Gather utlity .js files and combine into one file
string[] UtilityJavaScriptFiles = Directory.GetFiles(JSDir, "*.js");
string DestinationFile = OutDir + "/Utility.js";
foreach (var UtilityFile in UtilityJavaScriptFiles)
{
string Data = File.ReadAllText(UtilityFile);
File.AppendAllText(DestinationFile, Data);
}
// Compress all files. These are independent tasks which can be threaded.
Task[] CompressionTasks = new Task[6];
//data file.
CompressionTasks[0] = Task.Factory.StartNew(() => CompressFile(FinalDataLocation, FinalDataLocation + ".gz"));
// data file .js driver.
CompressionTasks[1] = Task.Factory.StartNew(() => CompressFile(FinalDataLocation + ".js", FinalDataLocation + ".js.gz"));
// main js.
CompressionTasks[2] = Task.Factory.StartNew(() => CompressFile(Path.Combine(PackagePath, GameExe), Path.Combine(PackagePath, GameExe) + ".gz"));
// mem init file.
CompressionTasks[3] = Task.Factory.StartNew(() => CompressFile(Path.Combine(PackagePath, GameExe) + ".mem", Path.Combine(PackagePath, GameExe) + ".mem.gz"));
// symbols file.
CompressionTasks[4] = Task.Factory.StartNew(() => CompressFile(Path.Combine(PackagePath, GameExe) + ".symbols", Path.Combine(PackagePath, GameExe) + ".symbols.gz"));
// Utility
CompressionTasks[5] = Task.Factory.StartNew(() => CompressFile(OutDir + "/Utility.js", OutDir + "/Utility.js.gz"));
File.Copy(CombinePaths(CmdEnv.LocalRoot, "Engine/Binaries/DotNET/HTML5LaunchHelper.exe"), CombinePaths(OutDir, "HTML5LaunchHelper.exe"), true);
Task.WaitAll(CompressionTasks);
PrintRunTime();
}
public async Task <int> Collect(CancellationToken ct, List <string> counter_list, string counters, IConsole console, int processId, int refreshInterval, CountersExportFormat format, string output, string name, string diagnosticPort)
{
if (!ProcessLauncher.Launcher.HasChildProc && !CommandUtils.ValidateArgumentsForAttach(processId, name, diagnosticPort, out _processId))
{
return(0);
}
shouldExit = new ManualResetEvent(false);
_ct.Register(() => shouldExit.Set());
DiagnosticsClientBuilder builder = new DiagnosticsClientBuilder("dotnet-counters", 10);
using (DiagnosticsClientHolder holder = await builder.Build(ct, _processId, diagnosticPort, showChildIO: false, printLaunchCommand: false))
{
if (holder == null)
{
return(1);
}
try
{
InitializeCounterList(counters, counter_list);
_ct = ct;
_console = console;
_interval = refreshInterval;
_output = output;
_diagnosticsClient = holder.Client;
if (_output.Length == 0)
{
_console.Error.WriteLine("Output cannot be an empty string");
return(0);
}
if (format == CountersExportFormat.csv)
{
_renderer = new CSVExporter(output);
}
else if (format == CountersExportFormat.json)
{
// Try getting the process name.
string processName = "";
try
{
if (ProcessLauncher.Launcher.HasChildProc)
{
_processId = ProcessLauncher.Launcher.ChildProc.Id;
}
processName = Process.GetProcessById(_processId).ProcessName;
}
catch (Exception) { }
_renderer = new JSONExporter(output, processName);
}
else
{
_console.Error.WriteLine($"The output format {format} is not a valid output format.");
return(0);
}
shouldResumeRuntime = ProcessLauncher.Launcher.HasChildProc || !string.IsNullOrEmpty(diagnosticPort);
int ret = await Start();
return(ret);
}
catch (OperationCanceledException)
{
try
{
_session.Stop();
}
catch (Exception) { } // session.Stop() can throw if target application already stopped before we send the stop command.
return(1);
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = Parameters.Project;
if (Parameters.Project != null && !FileReference.Exists(ProjectFile))
{
throw new AutomationException("Couldn't find project '{0}'", ProjectFile.FullName);
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = CommandUtils.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = Parameters.ToDir;
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, out Receipt))
{
throw new AutomationException("Couldn't read receipt '{0}'", ReceiptFileName);
}
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts)
{
SourceFiles.Add(BuildProduct.Path);
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.Add(RuntimeDependency.Path);
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !FileReference.Exists(TargetFile))
{
DirectoryReference.CreateDirectory(TargetFile.Directory);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
// Force all destination files to not readonly.
CommandUtils.SetFileAttributes(TargetFile.FullName, ReadOnly: false);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
}
public async Task <int> Monitor(
CancellationToken ct,
List <string> counter_list,
string counters,
IConsole console,
int processId,
int refreshInterval,
string name,
string diagnosticPort,
bool resumeRuntime,
int maxHistograms,
int maxTimeSeries)
{
try
{
// System.CommandLine does have an option to specify arguments as uint and it would validate they are non-negative. However the error
// message is "Cannot parse argument '-1' for option '--maxTimeSeries' as expected type System.UInt32" which is not as user friendly.
// If there was another option to leverage System.CommandLine that provides a little more user friendly error message we could switch
// to it.
ValidateNonNegative(maxHistograms, nameof(maxHistograms));
ValidateNonNegative(maxTimeSeries, nameof(maxTimeSeries));
if (!ProcessLauncher.Launcher.HasChildProc && !CommandUtils.ValidateArgumentsForAttach(processId, name, diagnosticPort, out _processId))
{
return(ReturnCode.ArgumentError);
}
ct.Register(() => _shouldExit.TrySetResult(ReturnCode.Ok));
DiagnosticsClientBuilder builder = new DiagnosticsClientBuilder("dotnet-counters", 10);
using (DiagnosticsClientHolder holder = await builder.Build(ct, _processId, diagnosticPort, showChildIO: false, printLaunchCommand: false))
using (VirtualTerminalMode vTerm = VirtualTerminalMode.TryEnable())
{
bool useAnsi = vTerm.IsEnabled;
if (holder == null)
{
return(ReturnCode.Ok);
}
try
{
_console = console;
// the launch command may misinterpret app arguments as the old space separated
// provider list so we need to ignore it in that case
_counterList = ConfigureCounters(counters, _processId != 0 ? counter_list : null);
_ct = ct;
_interval = refreshInterval;
_maxHistograms = maxHistograms;
_maxTimeSeries = maxTimeSeries;
_renderer = new ConsoleWriter(useAnsi);
_diagnosticsClient = holder.Client;
_resumeRuntime = resumeRuntime;
int ret = await Start();
ProcessLauncher.Launcher.Cleanup();
return(ret);
}
catch (OperationCanceledException)
{
try
{
_session.Stop();
}
catch (Exception) { } // Swallow all exceptions for now.
console.Out.WriteLine($"Complete");
return(ReturnCode.Ok);
}
}
}
catch (CommandLineErrorException e)
{
console.Error.WriteLine(e.Message);
return(ReturnCode.ArgumentError);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the pattern to match against. If it's a simple pattern (eg. *.cpp, Engine/Build/...), automatically infer the source wildcard
string FromPattern = Parameters.From;
if (FromPattern == null)
{
List <string> Patterns = SplitDelimitedList(Parameters.Files);
if (Patterns.Count != 1 || Patterns[0].StartsWith("#"))
{
CommandUtils.LogError("Missing 'From' attribute specifying pattern to match source files against");
return(false);
}
FromPattern = Patterns[0];
int SlashIdx = FromPattern.LastIndexOfAny(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar });
if (SlashIdx != -1)
{
FromPattern = FromPattern.Substring(SlashIdx + 1);
}
if (FromPattern.StartsWith("..."))
{
FromPattern = "*" + FromPattern.Substring(3);
}
}
// Convert the source pattern into a regex
string EscapedFromPattern = "^" + Regex.Escape(FromPattern) + "$";
EscapedFromPattern = EscapedFromPattern.Replace("\\*", "(.*)");
EscapedFromPattern = EscapedFromPattern.Replace("\\?", "(.)");
Regex FromRegex = new Regex(EscapedFromPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
// Split the output pattern into fragments that we can insert captures between
string[] FromFragments = FromPattern.Split('*', '?');
string[] ToFragments = Parameters.To.Split('*', '?');
if (FromFragments.Length < ToFragments.Length)
{
CommandUtils.LogError("Too few capture groups in source pattern '{0}' to rename to '{1}'", FromPattern, Parameters.To);
return(false);
}
// Find the input files
HashSet <FileReference> InputFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
// Find all the corresponding output files
Dictionary <FileReference, FileReference> RenameFiles = new Dictionary <FileReference, FileReference>();
foreach (FileReference InputFile in InputFiles)
{
Match Match = FromRegex.Match(InputFile.GetFileName());
if (Match.Success)
{
StringBuilder OutputName = new StringBuilder(ToFragments[0]);
for (int Idx = 1; Idx < ToFragments.Length; Idx++)
{
OutputName.Append(Match.Groups[Idx].Value);
OutputName.Append(ToFragments[Idx]);
}
RenameFiles[InputFile] = FileReference.Combine(InputFile.Directory, OutputName.ToString());
}
}
// Print out everything we're going to do
foreach (KeyValuePair <FileReference, FileReference> Pair in RenameFiles)
{
CommandUtils.RenameFile(Pair.Key.FullName, Pair.Value.FullName, true);
}
// Add the build product
BuildProducts.UnionWith(RenameFiles.Values);
// Apply the optional output tag to them
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(RenameFiles.Values);
}
return(true);
}
// based on ExampleMod
public override CommandReply Action(CommandCaller caller, string[] args)
{
int npcType = 0;
int x = 0;
int y = 0;
int amount = 1;
bool xRelative = false;
bool yRelative = false;
if (!int.TryParse(args[0], out npcType))
{
npcType = CommandUtils.GetNPCType(args[0]);
}
if (npcType == 0 || npcType >= NPCLoader.NPCCount)
{
throw new UsageException("Unknown NPC type: " + npcType);
}
if (!CommandUtils.IsValidNPC(npcType))
{
throw new UsageException("Invalid NPC type: " + npcType);
}
if (args.Length > 1)
{
if (args[1].StartsWith("~"))
{
xRelative = true;
args[1] = args[1].Remove(0, 1);
}
if (!int.TryParse(args[1], out x))
{
xRelative = true;
}
}
else
{
xRelative = true;
}
if (args.Length > 2)
{
if (args[2].StartsWith("~"))
{
yRelative = true;
args[2] = args[2].Remove(0, 1);
}
if (!int.TryParse(args[2], out x))
{
yRelative = true;
}
}
else
{
yRelative = true;
}
if (args.Length > 3)
{
if (!int.TryParse(args[3], out amount))
{
amount = 1;
}
}
if (xRelative)
{
x += (int)caller.Player.Bottom.X;
}
if (yRelative)
{
y += (int)caller.Player.Bottom.Y;
}
for (int i = 0; i < amount; i++)
{
int newNPC = NPC.NewNPC(x, y, npcType);
if (newNPC < 200)
{
NetMessage.SendData(MessageID.SyncNPC, number: newNPC);
}
else
{
break;
}
}
return(new CommandReply(caller.Player.name + " spawned " + amount + " NPCs!"));
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Parse all the source patterns
FilePattern SourcePattern = new FilePattern(CommandUtils.RootDirectory, Parameters.From);
// Parse the target pattern
FilePattern TargetPattern = new FilePattern(CommandUtils.RootDirectory, Parameters.To);
// Apply the filter to the source files
HashSet <FileReference> Files = null;
if (!String.IsNullOrEmpty(Parameters.Files))
{
SourcePattern = SourcePattern.AsDirectoryPattern();
Files = ResolveFilespec(SourcePattern.BaseDirectory, Parameters.Files, TagNameToFileSet);
}
// Build the file mapping
Dictionary <FileReference, FileReference> TargetFileToSourceFile = FilePattern.CreateMapping(Files, ref SourcePattern, ref TargetPattern);
// If we're not overwriting, remove any files where the destination file already exists.
if (!Parameters.Overwrite)
{
TargetFileToSourceFile = TargetFileToSourceFile.Where(File =>
{
if (FileReference.Exists(File.Key))
{
CommandUtils.Log("Not copying existing file {0}", File.Key);
return(false);
}
return(true);
}).ToDictionary(Pair => Pair.Key, Pair => Pair.Value);
}
// Check we got some files
if (TargetFileToSourceFile.Count == 0)
{
CommandUtils.Log("No files found matching '{0}'", SourcePattern);
return;
}
// If the target is on a network share, retry creating the first directory until it succeeds
DirectoryReference FirstTargetDirectory = TargetFileToSourceFile.First().Key.Directory;
if (!DirectoryReference.Exists(FirstTargetDirectory))
{
const int MaxNumRetries = 15;
for (int NumRetries = 0;; NumRetries++)
{
try
{
DirectoryReference.CreateDirectory(FirstTargetDirectory);
if (NumRetries == 1)
{
Log.TraceInformation("Created target directory {0} after 1 retry.", FirstTargetDirectory);
}
else if (NumRetries > 1)
{
Log.TraceInformation("Created target directory {0} after {1} retries.", FirstTargetDirectory, NumRetries);
}
break;
}
catch (Exception Ex)
{
if (NumRetries == 0)
{
Log.TraceInformation("Unable to create directory '{0}' on first attempt. Retrying {1} times...", FirstTargetDirectory, MaxNumRetries);
}
Log.TraceLog(" {0}", Ex);
if (NumRetries >= 15)
{
throw new AutomationException(Ex, "Unable to create target directory '{0}' after {1} retries.", FirstTargetDirectory, NumRetries);
}
Thread.Sleep(2000);
}
}
}
// Copy them all
KeyValuePair <FileReference, FileReference>[] FilePairs = TargetFileToSourceFile.ToArray();
CommandUtils.Log("Copying {0} file{1} from {2} to {3}...", FilePairs.Length, (FilePairs.Length == 1)? "" : "s", SourcePattern.BaseDirectory, TargetPattern.BaseDirectory);
CommandUtils.ThreadedCopyFiles(FilePairs.Select(x => x.Value.FullName).ToList(), FilePairs.Select(x => x.Key.FullName).ToList(), bQuiet: true);
// Update the list of build products
BuildProducts.UnionWith(TargetFileToSourceFile.Keys);
// Apply the optional output tag to them
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFileToSourceFile.Keys);
}
}
