void ProcessInternal(string Line, ref bool bSkipLine)
{
List <string> Tokens = ParseTokens(Line);
for (int TokenIdx = 0; TokenIdx < Tokens.Count;)
{
float Fraction;
if (ReadFraction(Tokens, ref TokenIdx, out Fraction))
{
Value.Set(Fraction);
}
else if (Tokens[TokenIdx] == "push")
{
TokenIdx++;
if (ReadFraction(Tokens, ref TokenIdx, out Fraction))
{
Value.Push(Fraction);
}
}
else if (Tokens[TokenIdx] == "pop")
{
TokenIdx++;
Value.Pop();
}
else if (Tokens[TokenIdx] == "increment")
{
TokenIdx++;
if (ReadFraction(Tokens, ref TokenIdx, out Fraction))
{
Value.Increment(Fraction);
}
}
else if (Tokens[TokenIdx] == "skipline")
{
TokenIdx++;
bSkipLine = true;
}
else if (Tokens[TokenIdx].Length >= 2 && (Tokens[TokenIdx][0] == '\'' || Tokens[TokenIdx][0] == '\"') && Tokens[TokenIdx].Last() == Tokens[TokenIdx].First())
{
string Message = Tokens[TokenIdx++];
Value.Set(Message.Substring(1, Message.Length - 2));
}
else
{
TokenIdx++;
}
}
}
WorkspaceUpdateResult UpdateWorkspaceInternal(WorkspaceUpdateContext Context, out string StatusMessage)
{
string CmdExe = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "cmd.exe");
if (!File.Exists(CmdExe))
{
StatusMessage = String.Format("Missing {0}.", CmdExe);
return(WorkspaceUpdateResult.FailedToSync);
}
List <Tuple <string, TimeSpan> > Times = new List <Tuple <string, TimeSpan> >();
int NumFilesSynced = 0;
if (Context.Options.HasFlag(WorkspaceUpdateOptions.Sync) || Context.Options.HasFlag(WorkspaceUpdateOptions.SyncSingleChange))
{
using (TelemetryStopwatch Stopwatch = new TelemetryStopwatch("Sync", TelemetryProjectPath))
{
Log.WriteLine("Syncing to {0}...", PendingChangeNumber);
// Find all the files that are out of date
Progress.Set("Finding files to sync...");
// Get the user's sync filter
FileFilter UserFilter = null;
if (Context.SyncFilter != null)
{
UserFilter = new FileFilter(FileFilterType.Include);
UserFilter.AddRules(Context.SyncFilter.Select(x => x.Trim()).Where(x => x.Length > 0 && !x.StartsWith(";") && !x.StartsWith("#")));
}
// Find all the server changes, and anything that's opened for edit locally. We need to sync files we have open to schedule a resolve.
List <string> SyncFiles = new List <string>();
foreach (string SyncPath in SyncPaths)
{
List <PerforceFileRecord> SyncRecords;
if (!Perforce.SyncPreview(SyncPath, PendingChangeNumber, !Context.Options.HasFlag(WorkspaceUpdateOptions.Sync), out SyncRecords, Log))
{
StatusMessage = String.Format("Couldn't enumerate changes matching {0}.", SyncPath);
return(WorkspaceUpdateResult.FailedToSync);
}
if (UserFilter != null)
{
SyncRecords.RemoveAll(x => !String.IsNullOrEmpty(x.ClientPath) && !MatchFilter(x.ClientPath, UserFilter));
}
SyncFiles.AddRange(SyncRecords.Select(x => x.DepotPath));
List <PerforceFileRecord> OpenRecords;
if (!Perforce.GetOpenFiles(SyncPath, out OpenRecords, Log))
{
StatusMessage = String.Format("Couldn't find open files matching {0}.", SyncPath);
return(WorkspaceUpdateResult.FailedToSync);
}
// don't force a sync on added files
SyncFiles.AddRange(OpenRecords.Where(x => x.Action != "add" && x.Action != "branch" && x.Action != "move/add").Select(x => x.DepotPath));
}
// Filter out all the binaries that we don't want
FileFilter Filter = new FileFilter(FileFilterType.Include);
Filter.Exclude("..." + BuildVersionFileName);
Filter.Exclude("..." + VersionHeaderFileName);
Filter.Exclude("..." + ObjectVersionFileName);
if (Context.Options.HasFlag(WorkspaceUpdateOptions.ContentOnly))
{
Filter.Exclude("*.usf");
}
SyncFiles.RemoveAll(x => !Filter.Matches(x));
// Sync them all
List <string> TamperedFiles = new List <string>();
HashSet <string> RemainingFiles = new HashSet <string>(SyncFiles, StringComparer.InvariantCultureIgnoreCase);
if (!Perforce.Sync(SyncFiles, PendingChangeNumber, Record => UpdateSyncProgress(Record, RemainingFiles, SyncFiles.Count), TamperedFiles, Log))
{
StatusMessage = "Aborted sync due to errors.";
return(WorkspaceUpdateResult.FailedToSync);
}
// If any files need to be clobbered, defer to the main thread to figure out which ones
if (TamperedFiles.Count > 0)
{
int NumNewFilesToClobber = 0;
foreach (string TamperedFile in TamperedFiles)
{
if (!Context.ClobberFiles.ContainsKey(TamperedFile))
{
Context.ClobberFiles[TamperedFile] = true;
NumNewFilesToClobber++;
}
}
if (NumNewFilesToClobber > 0)
{
StatusMessage = String.Format("Cancelled sync after checking files to clobber ({0} new files).", NumNewFilesToClobber);
return(WorkspaceUpdateResult.FilesToClobber);
}
foreach (string TamperedFile in TamperedFiles)
{
if (Context.ClobberFiles[TamperedFile] && !Perforce.ForceSync(TamperedFile, PendingChangeNumber, Log))
{
StatusMessage = String.Format("Couldn't sync {0}.", TamperedFile);
return(WorkspaceUpdateResult.FailedToSync);
}
}
}
if (Context.Options.HasFlag(WorkspaceUpdateOptions.Sync))
{
// Read the new config file
ProjectConfigFile = ReadProjectConfigFile(LocalRootPath, SelectedLocalFileName, Log);
// Get the branch name
string BranchOrStreamName;
if (!Perforce.GetActiveStream(out BranchOrStreamName, Log))
{
string DepotFileName;
if (!Perforce.ConvertToDepotPath(ClientRootPath + "/GenerateProjectFiles.bat", out DepotFileName, Log))
{
StatusMessage = String.Format("Couldn't determine branch name for {0}.", SelectedClientFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
BranchOrStreamName = PerforceUtils.GetClientOrDepotDirectoryName(DepotFileName);
}
// Find the last code change before this changelist. For consistency in versioning between local builds and precompiled binaries, we need to use the last submitted code changelist as our version number.
List <PerforceChangeSummary> CodeChanges;
if (!Perforce.FindChanges(new string[] { ".cs", ".h", ".cpp", ".usf" }.SelectMany(x => SyncPaths.Select(y => String.Format("{0}{1}@<={2}", y, x, PendingChangeNumber))), 1, out CodeChanges, Log))
{
StatusMessage = String.Format("Couldn't determine last code changelist before CL {0}.", PendingChangeNumber);
return(WorkspaceUpdateResult.FailedToSync);
}
if (CodeChanges.Count == 0)
{
StatusMessage = String.Format("Could not find any code changes before CL {0}.", PendingChangeNumber);
return(WorkspaceUpdateResult.FailedToSync);
}
// Get the last code change
int VersionChangeNumber;
if (ProjectConfigFile.GetValue("Options.VersionToLastCodeChange", true))
{
VersionChangeNumber = CodeChanges.Max(x => x.Number);
}
else
{
VersionChangeNumber = PendingChangeNumber;
}
// Update the version files
if (ProjectConfigFile.GetValue("Options.UseFastModularVersioning", false))
{
Dictionary <string, string> BuildVersionStrings = new Dictionary <string, string>();
BuildVersionStrings["\"Changelist\":"] = String.Format(" {0},", VersionChangeNumber);
BuildVersionStrings["\"BranchName\":"] = String.Format(" \"{0}\"", BranchOrStreamName.Replace('/', '+'));
if (!UpdateVersionFile(ClientRootPath + BuildVersionFileName, BuildVersionStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}.", BuildVersionFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
Dictionary <string, string> VersionHeaderStrings = new Dictionary <string, string>();
VersionHeaderStrings["#define ENGINE_IS_PROMOTED_BUILD"] = " (0)";
VersionHeaderStrings["#define BUILT_FROM_CHANGELIST"] = " 0";
VersionHeaderStrings["#define BRANCH_NAME"] = " \"" + BranchOrStreamName.Replace('/', '+') + "\"";
if (!UpdateVersionFile(ClientRootPath + VersionHeaderFileName, VersionHeaderStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}.", VersionHeaderFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
if (!UpdateVersionFile(ClientRootPath + ObjectVersionFileName, new Dictionary <string, string>(), PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}.", ObjectVersionFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
}
else
{
if (!UpdateVersionFile(ClientRootPath + BuildVersionFileName, new Dictionary <string, string>(), PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}", BuildVersionFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
Dictionary <string, string> VersionStrings = new Dictionary <string, string>();
VersionStrings["#define ENGINE_VERSION"] = " " + VersionChangeNumber.ToString();
VersionStrings["#define ENGINE_IS_PROMOTED_BUILD"] = " (0)";
VersionStrings["#define BUILT_FROM_CHANGELIST"] = " " + VersionChangeNumber.ToString();
VersionStrings["#define BRANCH_NAME"] = " \"" + BranchOrStreamName.Replace('/', '+') + "\"";
if (!UpdateVersionFile(ClientRootPath + VersionHeaderFileName, VersionStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}", VersionHeaderFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
if (!UpdateVersionFile(ClientRootPath + ObjectVersionFileName, VersionStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}", ObjectVersionFileName);
return(WorkspaceUpdateResult.FailedToSync);
}
}
// Remove all the receipts for build targets in this branch
if (SelectedClientFileName.EndsWith(".uproject", StringComparison.InvariantCultureIgnoreCase))
{
Perforce.Sync(PerforceUtils.GetClientOrDepotDirectoryName(SelectedClientFileName) + "/Build/Receipts/...#0", Log);
}
}
// Check if there are any files which need resolving
List <PerforceFileRecord> UnresolvedFiles;
if (!FindUnresolvedFiles(SyncPaths, out UnresolvedFiles))
{
StatusMessage = "Couldn't get list of unresolved files.";
return(WorkspaceUpdateResult.FailedToSync);
}
if (UnresolvedFiles.Count > 0 && Context.Options.HasFlag(WorkspaceUpdateOptions.AutoResolveChanges))
{
foreach (PerforceFileRecord UnresolvedFile in UnresolvedFiles)
{
Perforce.AutoResolveFile(UnresolvedFile.DepotPath, Log);
}
if (!FindUnresolvedFiles(SyncPaths, out UnresolvedFiles))
{
StatusMessage = "Couldn't get list of unresolved files.";
return(WorkspaceUpdateResult.FailedToSync);
}
}
if (UnresolvedFiles.Count > 0)
{
Log.WriteLine("{0} files need resolving:", UnresolvedFiles.Count);
foreach (PerforceFileRecord UnresolvedFile in UnresolvedFiles)
{
Log.WriteLine(" {0}", UnresolvedFile.ClientPath);
}
StatusMessage = "Files need resolving.";
return(WorkspaceUpdateResult.FilesToResolve);
}
// Update the current change number. Everything else happens for the new change.
if (Context.Options.HasFlag(WorkspaceUpdateOptions.Sync))
{
CurrentChangeNumber = PendingChangeNumber;
}
// Update the timing info
Times.Add(new Tuple <string, TimeSpan>("Sync", Stopwatch.Stop("Success")));
// Save the number of files synced
NumFilesSynced = SyncFiles.Count;
Log.WriteLine();
}
}
// Extract an archive from the depot path
if (Context.Options.HasFlag(WorkspaceUpdateOptions.SyncArchives))
{
using (TelemetryStopwatch Stopwatch = new TelemetryStopwatch("Archives", TelemetryProjectPath))
{
// Create the directory for extracted archive manifests
string ManifestDirectoryName;
if (SelectedLocalFileName.EndsWith(".uproject", StringComparison.InvariantCultureIgnoreCase))
{
ManifestDirectoryName = Path.Combine(Path.GetDirectoryName(SelectedLocalFileName), "Saved", "UnrealGameSync");
}
else
{
ManifestDirectoryName = Path.Combine(Path.GetDirectoryName(SelectedLocalFileName), "Engine", "Saved", "UnrealGameSync");
}
Directory.CreateDirectory(ManifestDirectoryName);
// Sync and extract (or just remove) the given archives
foreach (KeyValuePair <string, string> ArchiveTypeAndDepotPath in Context.ArchiveTypeToDepotPath)
{
// Remove any existing binaries
string ManifestFileName = Path.Combine(ManifestDirectoryName, String.Format("{0}.zipmanifest", ArchiveTypeAndDepotPath.Key));
if (File.Exists(ManifestFileName))
{
Log.WriteLine("Removing {0} binaries...", ArchiveTypeAndDepotPath.Key);
Progress.Set(String.Format("Removing {0} binaries...", ArchiveTypeAndDepotPath.Key), 0.0f);
ArchiveUtils.RemoveExtractedFiles(LocalRootPath, ManifestFileName, Progress, Log);
Log.WriteLine();
}
// If we have a new depot path, sync it down and extract it
if (ArchiveTypeAndDepotPath.Value != null)
{
string TempZipFileName = Path.GetTempFileName();
try
{
Log.WriteLine("Syncing {0} binaries...", ArchiveTypeAndDepotPath.Key.ToLowerInvariant());
Progress.Set(String.Format("Syncing {0} binaries...", ArchiveTypeAndDepotPath.Key.ToLowerInvariant()), 0.0f);
if (!Perforce.PrintToFile(ArchiveTypeAndDepotPath.Value, TempZipFileName, Log) || new FileInfo(TempZipFileName).Length == 0)
{
StatusMessage = String.Format("Couldn't read {0}", ArchiveTypeAndDepotPath.Value);
return(WorkspaceUpdateResult.FailedToSync);
}
ArchiveUtils.ExtractFiles(TempZipFileName, LocalRootPath, ManifestFileName, Progress, Log);
Log.WriteLine();
}
finally
{
File.SetAttributes(TempZipFileName, FileAttributes.Normal);
File.Delete(TempZipFileName);
}
}
}
// Add the finish time
Times.Add(new Tuple <string, TimeSpan>("Archive", Stopwatch.Stop("Success")));
}
}
// Generate project files in the workspace
if (Context.Options.HasFlag(WorkspaceUpdateOptions.GenerateProjectFiles))
{
using (TelemetryStopwatch Stopwatch = new TelemetryStopwatch("Prj gen", TelemetryProjectPath))
{
Progress.Set("Generating project files...", 0.0f);
string ProjectFileArgument = "";
if (SelectedLocalFileName.EndsWith(".uproject", StringComparison.InvariantCultureIgnoreCase))
{
ProjectFileArgument = String.Format("\"{0}\" ", SelectedLocalFileName);
}
string CommandLine = String.Format("/C \"\"{0}\" {1}-progress\"", Path.Combine(LocalRootPath, "GenerateProjectFiles.bat"), ProjectFileArgument);
Log.WriteLine("Generating project files...");
Log.WriteLine("gpf> Running {0} {1}", CmdExe, CommandLine);
int GenerateProjectFilesResult = Utility.ExecuteProcess(CmdExe, CommandLine, null, new ProgressTextWriter(Progress, new PrefixedTextWriter("gpf> ", Log)));
if (GenerateProjectFilesResult != 0)
{
StatusMessage = String.Format("Failed to generate project files (exit code {0}).", GenerateProjectFilesResult);
return(WorkspaceUpdateResult.FailedToCompile);
}
Log.WriteLine();
Times.Add(new Tuple <string, TimeSpan>("Prj gen", Stopwatch.Stop("Success")));
}
}
// Build everything using MegaXGE
if (Context.Options.HasFlag(WorkspaceUpdateOptions.Build))
{
// Compile all the build steps together
Dictionary <Guid, ConfigObject> BuildStepObjects = Context.DefaultBuildSteps.ToDictionary(x => x.Key, x => new ConfigObject(x.Value));
BuildStep.MergeBuildStepObjects(BuildStepObjects, ProjectConfigFile.GetValues("Build.Step", new string[0]).Select(x => new ConfigObject(x)));
BuildStep.MergeBuildStepObjects(BuildStepObjects, Context.UserBuildStepObjects);
// Construct build steps from them
List <BuildStep> BuildSteps = BuildStepObjects.Values.Select(x => new BuildStep(x)).OrderBy(x => x.OrderIndex).ToList();
if (Context.CustomBuildSteps != null && Context.CustomBuildSteps.Count > 0)
{
BuildSteps.RemoveAll(x => !Context.CustomBuildSteps.Contains(x.UniqueId));
}
else if (Context.Options.HasFlag(WorkspaceUpdateOptions.ScheduledBuild))
{
BuildSteps.RemoveAll(x => !x.bScheduledSync);
}
else
{
BuildSteps.RemoveAll(x => !x.bNormalSync);
}
// Check if the last successful build was before a change that we need to force a clean for
bool bForceClean = false;
if (LastBuiltChangeNumber != 0)
{
foreach (string CleanBuildChange in ProjectConfigFile.GetValues("ForceClean.Changelist", new string[0]))
{
int ChangeNumber;
if (int.TryParse(CleanBuildChange, out ChangeNumber))
{
if ((LastBuiltChangeNumber >= ChangeNumber) != (CurrentChangeNumber >= ChangeNumber))
{
Log.WriteLine("Forcing clean build due to changelist {0}.", ChangeNumber);
Log.WriteLine();
bForceClean = true;
break;
}
}
}
}
// Execute them all
string TelemetryEventName = (Context.UserBuildStepObjects.Count > 0)? "CustomBuild" : Context.Options.HasFlag(WorkspaceUpdateOptions.UseIncrementalBuilds) ? "Compile" : "FullCompile";
using (TelemetryStopwatch Stopwatch = new TelemetryStopwatch(TelemetryEventName, TelemetryProjectPath))
{
Progress.Set("Starting build...", 0.0f);
// Check we've built UBT (it should have been compiled by generating project files)
string UnrealBuildToolPath = Path.Combine(LocalRootPath, "Engine", "Binaries", "DotNET", "UnrealBuildTool.exe");
if (!File.Exists(UnrealBuildToolPath))
{
StatusMessage = String.Format("Couldn't find {0}", UnrealBuildToolPath);
return(WorkspaceUpdateResult.FailedToCompile);
}
// Execute all the steps
float MaxProgressFraction = 0.0f;
foreach (BuildStep Step in BuildSteps)
{
MaxProgressFraction += (float)Step.EstimatedDuration / (float)Math.Max(BuildSteps.Sum(x => x.EstimatedDuration), 1);
Progress.Set(Step.StatusText);
Progress.Push(MaxProgressFraction);
Log.WriteLine(Step.StatusText);
switch (Step.Type)
{
case BuildStepType.Compile:
using (TelemetryStopwatch StepStopwatch = new TelemetryStopwatch("Compile:" + Step.Target, TelemetryProjectPath))
{
string CommandLine = String.Format("{0} {1} {2} {3} -NoHotReloadFromIDE", Step.Target, Step.Platform, Step.Configuration, Utility.ExpandVariables(Step.Arguments, Context.Variables));
if (!Context.Options.HasFlag(WorkspaceUpdateOptions.UseIncrementalBuilds) || bForceClean)
{
Log.WriteLine("ubt> Running {0} {1} -clean", UnrealBuildToolPath, CommandLine);
Utility.ExecuteProcess(UnrealBuildToolPath, CommandLine + " -clean", null, new ProgressTextWriter(Progress, new PrefixedTextWriter("ubt> ", Log)));
}
Log.WriteLine("ubt> Running {0} {1} -progress", UnrealBuildToolPath, CommandLine);
int ResultFromBuild = Utility.ExecuteProcess(UnrealBuildToolPath, CommandLine + " -progress", null, new ProgressTextWriter(Progress, new PrefixedTextWriter("ubt> ", Log)));
if (ResultFromBuild != 0)
{
StatusMessage = String.Format("Failed to compile {0}.", Step.Target);
return((HasModifiedSourceFiles() || Context.UserBuildStepObjects.Count > 0)? WorkspaceUpdateResult.FailedToCompile : WorkspaceUpdateResult.FailedToCompileWithCleanWorkspace);
}
}
break;
case BuildStepType.Cook:
using (TelemetryStopwatch StepStopwatch = new TelemetryStopwatch("Cook/Launch: " + Path.GetFileNameWithoutExtension(Step.FileName), TelemetryProjectPath))
{
string LocalRunUAT = Path.Combine(LocalRootPath, "Engine", "Build", "BatchFiles", "RunUAT.bat");
string Arguments = String.Format("/C \"\"{0}\" -profile=\"{1}\"\"", LocalRunUAT, Path.Combine(LocalRootPath, Step.FileName));
Log.WriteLine("uat> Running {0} {1}", LocalRunUAT, Arguments);
int ResultFromUAT = Utility.ExecuteProcess(CmdExe, Arguments, null, new ProgressTextWriter(Progress, new PrefixedTextWriter("uat> ", Log)));
if (ResultFromUAT != 0)
{
StatusMessage = String.Format("Cook failed. ({0})", ResultFromUAT);
return(WorkspaceUpdateResult.FailedToCompile);
}
}
break;
case BuildStepType.Other:
using (TelemetryStopwatch StepStopwatch = new TelemetryStopwatch("Custom: " + Path.GetFileNameWithoutExtension(Step.FileName), TelemetryProjectPath))
{
string ToolFileName = Path.Combine(LocalRootPath, Utility.ExpandVariables(Step.FileName, Context.Variables));
string ToolArguments = Utility.ExpandVariables(Step.Arguments, Context.Variables);
Log.WriteLine("tool> Running {0} {1}", ToolFileName, ToolArguments);
if (Step.bUseLogWindow)
{
int ResultFromTool = Utility.ExecuteProcess(ToolFileName, ToolArguments, null, new ProgressTextWriter(Progress, new PrefixedTextWriter("tool> ", Log)));
if (ResultFromTool != 0)
{
StatusMessage = String.Format("Tool terminated with exit code {0}.", ResultFromTool);
return(WorkspaceUpdateResult.FailedToCompile);
}
}
else
{
using (Process.Start(ToolFileName, ToolArguments))
{
}
}
}
break;
}
Log.WriteLine();
Progress.Pop();
}
Times.Add(new Tuple <string, TimeSpan>("Build", Stopwatch.Stop("Success")));
}
// Update the last successful build change number
if (Context.CustomBuildSteps == null || Context.CustomBuildSteps.Count == 0)
{
LastBuiltChangeNumber = CurrentChangeNumber;
}
}
// Write out all the timing information
Log.WriteLine("Total time : " + FormatTime(Times.Sum(x => (long)(x.Item2.TotalMilliseconds / 1000))));
foreach (Tuple <string, TimeSpan> Time in Times)
{
Log.WriteLine(" {0,-8}: {1}", Time.Item1, FormatTime((long)(Time.Item2.TotalMilliseconds / 1000)));
}
if (NumFilesSynced > 0)
{
Log.WriteLine("{0} files synced.", NumFilesSynced);
}
Log.WriteLine();
Log.WriteLine("UPDATE SUCCEEDED");
StatusMessage = "Update succeeded";
return(WorkspaceUpdateResult.Success);
}