/// <summary>
/// Copy constructor
/// </summary>
/// <param name="Other">Filter to copy from</param>
public FileFilter(FileFilter Other)
: this(Other.DefaultNode.Type)
{
foreach (FileFilterNode OtherRule in Other.Rules)
{
AddRule(OtherRule.ToString(), OtherRule.Type);
}
}
bool MatchFilter(string FileName, FileFilter Filter)
{
bool bMatch = true;
if(FileName.StartsWith(LocalRootPath, StringComparison.InvariantCultureIgnoreCase))
{
if(!Filter.Matches(FileName.Substring(LocalRootPath.Length)))
{
bMatch = false;
}
}
return bMatch;
}
WorkspaceUpdateResult UpdateWorkspaceInternal(WorkspaceUpdateContext Context, out string StatusMessage)
{
string CmdExe = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "cmd.exe");
if(!File.Exists(CmdExe))
{
StatusMessage = String.Format("Missing {0}.", CmdExe);
return WorkspaceUpdateResult.FailedToSync;
}
List<Tuple<string, TimeSpan>> Times = new List<Tuple<string,TimeSpan>>();
int NumFilesSynced = 0;
if(Context.Options.HasFlag(WorkspaceUpdateOptions.Sync) || Context.Options.HasFlag(WorkspaceUpdateOptions.SyncSingleChange))
{
using(TelemetryStopwatch Stopwatch = new TelemetryStopwatch("Sync", TelemetryProjectPath))
{
Log.WriteLine("Syncing to {0}...", PendingChangeNumber);
// Find all the files that are out of date
Progress.Set("Finding files to sync...");
// Get the user's sync filter
FileFilter UserFilter = null;
if(Context.SyncFilter != null)
{
UserFilter = new FileFilter(FileFilterType.Include);
UserFilter.AddRules(Context.SyncFilter.Select(x => x.Trim()).Where(x => x.Length > 0 && !x.StartsWith(";") && !x.StartsWith("#")));
}
// Find all the server changes, and anything that's opened for edit locally. We need to sync files we have open to schedule a resolve.
List<string> SyncFiles = new List<string>();
foreach(string SyncPath in SyncPaths)
{
List<PerforceFileRecord> SyncRecords;
if(!Perforce.SyncPreview(SyncPath, PendingChangeNumber, !Context.Options.HasFlag(WorkspaceUpdateOptions.Sync), out SyncRecords, Log))
{
StatusMessage = String.Format("Couldn't enumerate changes matching {0}.", SyncPath);
return WorkspaceUpdateResult.FailedToSync;
}
if(UserFilter != null)
{
SyncRecords.RemoveAll(x => !String.IsNullOrEmpty(x.ClientPath) && !MatchFilter(x.ClientPath, UserFilter));
}
SyncFiles.AddRange(SyncRecords.Select(x => x.DepotPath));
List<PerforceFileRecord> OpenRecords;
if(!Perforce.GetOpenFiles(SyncPath, out OpenRecords, Log))
{
StatusMessage = String.Format("Couldn't find open files matching {0}.", SyncPath);
return WorkspaceUpdateResult.FailedToSync;
}
// don't force a sync on added files
SyncFiles.AddRange(OpenRecords.Where(x => x.Action != "add" && x.Action != "branch" && x.Action != "move/add").Select(x => x.DepotPath));
}
// Filter out all the binaries that we don't want
FileFilter Filter = new FileFilter(FileFilterType.Include);
Filter.Exclude("..." + BuildVersionFileName);
Filter.Exclude("..." + VersionHeaderFileName);
Filter.Exclude("..." + ObjectVersionFileName);
if(Context.Options.HasFlag(WorkspaceUpdateOptions.SkipShaders))
{
Filter.Exclude("*.usf");
}
SyncFiles.RemoveAll(x => !Filter.Matches(x));
// Sync them all
List<string> TamperedFiles = new List<string>();
HashSet<string> RemainingFiles = new HashSet<string>(SyncFiles, StringComparer.InvariantCultureIgnoreCase);
if(!Perforce.Sync(SyncFiles, PendingChangeNumber, Record => UpdateSyncProgress(Record, RemainingFiles, SyncFiles.Count), TamperedFiles, Log))
{
StatusMessage = "Aborted sync due to errors.";
return WorkspaceUpdateResult.FailedToSync;
}
// If any files need to be clobbered, defer to the main thread to figure out which ones
if(TamperedFiles.Count > 0)
{
int NumNewFilesToClobber = 0;
foreach(string TamperedFile in TamperedFiles)
{
if(!Context.ClobberFiles.ContainsKey(TamperedFile))
{
Context.ClobberFiles[TamperedFile] = true;
NumNewFilesToClobber++;
}
}
if(NumNewFilesToClobber > 0)
{
StatusMessage = String.Format("Cancelled sync after checking files to clobber ({0} new files).", NumNewFilesToClobber);
return WorkspaceUpdateResult.FilesToClobber;
}
foreach(string TamperedFile in TamperedFiles)
{
if(Context.ClobberFiles[TamperedFile] && !Perforce.ForceSync(TamperedFile, PendingChangeNumber, Log))
{
StatusMessage = String.Format("Couldn't sync {0}.", TamperedFile);
return WorkspaceUpdateResult.FailedToSync;
}
}
}
ConfigFile NewProjectConfigFile = null;
if(Context.Options.HasFlag(WorkspaceUpdateOptions.Sync))
{
// Read the new config file
NewProjectConfigFile = ReadProjectConfigFile(LocalRootPath, SelectedLocalFileName, Log);
// Get the branch name
string BranchOrStreamName;
if(!Perforce.GetActiveStream(out BranchOrStreamName, Log))
{
string DepotFileName;
if(!Perforce.ConvertToDepotPath(ClientRootPath + "/GenerateProjectFiles.bat", out DepotFileName, Log))
{
StatusMessage = String.Format("Couldn't determine branch name for {0}.", SelectedClientFileName);
return WorkspaceUpdateResult.FailedToSync;
}
BranchOrStreamName = PerforceUtils.GetClientOrDepotDirectoryName(DepotFileName);
}
// Find the last code change before this changelist. For consistency in versioning between local builds and precompiled binaries, we need to use the last submitted code changelist as our version number.
List<PerforceChangeSummary> CodeChanges;
if(!Perforce.FindChanges(new string[]{ ".cs", ".h", ".cpp", ".usf" }.SelectMany(x => SyncPaths.Select(y => String.Format("{0}{1}@<={2}", y, x, PendingChangeNumber))), 1, out CodeChanges, Log))
{
StatusMessage = String.Format("Couldn't determine last code changelist before CL {0}.", PendingChangeNumber);
return WorkspaceUpdateResult.FailedToSync;
}
if(CodeChanges.Count == 0)
{
StatusMessage = String.Format("Could not find any code changes before CL {0}.", PendingChangeNumber);
return WorkspaceUpdateResult.FailedToSync;
}
// Get the last code change
int VersionChangeNumber;
if(NewProjectConfigFile.GetValue("Options.VersionToLastCodeChange", true))
{
VersionChangeNumber = CodeChanges.Max(x => x.Number);
}
else
{
VersionChangeNumber = PendingChangeNumber;
}
// Update the version files
if(NewProjectConfigFile.GetValue("Options.UseFastModularVersioning", false))
{
Dictionary<string, string> BuildVersionStrings = new Dictionary<string,string>();
BuildVersionStrings["\"Changelist\":"] = String.Format(" {0},", VersionChangeNumber);
BuildVersionStrings["\"BranchName\":"] = String.Format(" \"{0}\"", BranchOrStreamName.Replace('/', '+'));
if(!UpdateVersionFile(ClientRootPath + BuildVersionFileName, BuildVersionStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}.", BuildVersionFileName);
return WorkspaceUpdateResult.FailedToSync;
}
Dictionary<string, string> VersionHeaderStrings = new Dictionary<string,string>();
VersionHeaderStrings["#define ENGINE_IS_PROMOTED_BUILD"] = " (0)";
VersionHeaderStrings["#define BUILT_FROM_CHANGELIST"] = " 0";
VersionHeaderStrings["#define BRANCH_NAME"] = " \"" + BranchOrStreamName.Replace('/', '+') + "\"";
if(!UpdateVersionFile(ClientRootPath + VersionHeaderFileName, VersionHeaderStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}.", VersionHeaderFileName);
return WorkspaceUpdateResult.FailedToSync;
}
if(!UpdateVersionFile(ClientRootPath + ObjectVersionFileName, new Dictionary<string,string>(), PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}.", ObjectVersionFileName);
return WorkspaceUpdateResult.FailedToSync;
}
}
else
{
if(!UpdateVersionFile(ClientRootPath + BuildVersionFileName, new Dictionary<string, string>(), PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}", BuildVersionFileName);
return WorkspaceUpdateResult.FailedToSync;
}
Dictionary<string, string> VersionStrings = new Dictionary<string,string>();
VersionStrings["#define ENGINE_VERSION"] = " " + VersionChangeNumber.ToString();
VersionStrings["#define ENGINE_IS_PROMOTED_BUILD"] = " (0)";
VersionStrings["#define BUILT_FROM_CHANGELIST"] = " " + VersionChangeNumber.ToString();
VersionStrings["#define BRANCH_NAME"] = " \"" + BranchOrStreamName.Replace('/', '+') + "\"";
if(!UpdateVersionFile(ClientRootPath + VersionHeaderFileName, VersionStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}", VersionHeaderFileName);
return WorkspaceUpdateResult.FailedToSync;
}
if(!UpdateVersionFile(ClientRootPath + ObjectVersionFileName, VersionStrings, PendingChangeNumber))
{
StatusMessage = String.Format("Failed to update {0}", ObjectVersionFileName);
return WorkspaceUpdateResult.FailedToSync;
}
}
// Remove all the receipts for build targets in this branch
if(SelectedClientFileName.EndsWith(".uproject", StringComparison.InvariantCultureIgnoreCase))
{
Perforce.Sync(PerforceUtils.GetClientOrDepotDirectoryName(SelectedClientFileName) + "/Build/Receipts/...#0", Log);
}
}
// Check if there are any files which need resolving
List<PerforceFileRecord> UnresolvedFiles;
if(!FindUnresolvedFiles(SyncPaths, out UnresolvedFiles))
{
StatusMessage = "Couldn't get list of unresolved files.";
return WorkspaceUpdateResult.FailedToSync;
}
if(UnresolvedFiles.Count > 0 && Context.Options.HasFlag(WorkspaceUpdateOptions.AutoResolveChanges))
{
foreach (PerforceFileRecord UnresolvedFile in UnresolvedFiles)
{
Perforce.AutoResolveFile(UnresolvedFile.DepotPath, Log);
}
if(!FindUnresolvedFiles(SyncPaths, out UnresolvedFiles))
{
StatusMessage = "Couldn't get list of unresolved files.";
return WorkspaceUpdateResult.FailedToSync;
}
}
if(UnresolvedFiles.Count > 0)
{
Log.WriteLine("{0} files need resolving:", UnresolvedFiles.Count);
foreach(PerforceFileRecord UnresolvedFile in UnresolvedFiles)
{
Log.WriteLine(" {0}", UnresolvedFile.ClientPath);
}
StatusMessage = "Files need resolving.";
return WorkspaceUpdateResult.FilesToResolve;
}
// Update the current change number. Everything else happens for the new change.
if(Context.Options.HasFlag(WorkspaceUpdateOptions.Sync))
{
ProjectConfigFile = NewProjectConfigFile;
CurrentChangeNumber = PendingChangeNumber;
}
// Update the timing info
Times.Add(new Tuple<string,TimeSpan>("Sync", Stopwatch.Stop("Success")));
// Save the number of files synced
NumFilesSynced = SyncFiles.Count;
Log.WriteLine();
}
}
// Extract an archive from the depot path
if(Context.Options.HasFlag(WorkspaceUpdateOptions.SyncArchives))
{
using(TelemetryStopwatch Stopwatch = new TelemetryStopwatch("Archives", TelemetryProjectPath))
{
// Create the directory for extracted archive manifests
string ManifestDirectoryName;
if(SelectedLocalFileName.EndsWith(".uproject", StringComparison.InvariantCultureIgnoreCase))
{
ManifestDirectoryName = Path.Combine(Path.GetDirectoryName(SelectedLocalFileName), "Saved", "UnrealGameSync");
}
else
{
ManifestDirectoryName = Path.Combine(Path.GetDirectoryName(SelectedLocalFileName), "Engine", "Saved", "UnrealGameSync");
}
Directory.CreateDirectory(ManifestDirectoryName);
// Sync and extract (or just remove) the given archives
foreach(KeyValuePair<string, string> ArchiveTypeAndDepotPath in Context.ArchiveTypeToDepotPath)
{
// Remove any existing binaries
string ManifestFileName = Path.Combine(ManifestDirectoryName, String.Format("{0}.zipmanifest", ArchiveTypeAndDepotPath.Key));
if(File.Exists(ManifestFileName))
{
Log.WriteLine("Removing {0} binaries...", ArchiveTypeAndDepotPath.Key);
Progress.Set(String.Format("Removing {0} binaries...", ArchiveTypeAndDepotPath.Key), 0.0f);
ArchiveUtils.RemoveExtractedFiles(LocalRootPath, ManifestFileName, Progress, Log);
Log.WriteLine();
}
// If we have a new depot path, sync it down and extract it
if(ArchiveTypeAndDepotPath.Value != null)
{
string TempZipFileName = Path.GetTempFileName();
try
{
Log.WriteLine("Syncing {0} binaries...", ArchiveTypeAndDepotPath.Key.ToLowerInvariant());
Progress.Set(String.Format("Syncing {0} binaries...", ArchiveTypeAndDepotPath.Key.ToLowerInvariant()), 0.0f);
if(!Perforce.PrintToFile(ArchiveTypeAndDepotPath.Value, TempZipFileName, Log) || new FileInfo(TempZipFileName).Length == 0)
{
StatusMessage = String.Format("Couldn't read {0}", ArchiveTypeAndDepotPath.Value);
return WorkspaceUpdateResult.FailedToSync;
}
ArchiveUtils.ExtractFiles(TempZipFileName, LocalRootPath, ManifestFileName, Progress, Log);
Log.WriteLine();
}
finally
{
File.SetAttributes(TempZipFileName, FileAttributes.Normal);
File.Delete(TempZipFileName);
}
}
}
// Add the finish time
Times.Add(new Tuple<string,TimeSpan>("Archive", Stopwatch.Stop("Success")));
}
}
// Generate project files in the workspace
if(Context.Options.HasFlag(WorkspaceUpdateOptions.GenerateProjectFiles))
{
using(TelemetryStopwatch Stopwatch = new TelemetryStopwatch("Prj gen", TelemetryProjectPath))
{
Progress.Set("Generating project files...", 0.0f);
string ProjectFileArgument = "";
if(SelectedLocalFileName.EndsWith(".uproject", StringComparison.InvariantCultureIgnoreCase))
{
ProjectFileArgument = String.Format("\"{0}\" ", SelectedLocalFileName);
}
string CommandLine = String.Format("/C \"\"{0}\" {1}-progress\"", Path.Combine(LocalRootPath, "GenerateProjectFiles.bat"), ProjectFileArgument);
Log.WriteLine("Generating project files...");
Log.WriteLine("gpf> Running {0} {1}", CmdExe, CommandLine);
int GenerateProjectFilesResult = Utility.ExecuteProcess(CmdExe, CommandLine, null, new ProgressTextWriter(Progress, new PrefixedTextWriter("gpf> ", Log)));
if(GenerateProjectFilesResult != 0)
{
StatusMessage = String.Format("Failed to generate project files (exit code {0}).", GenerateProjectFilesResult);
return WorkspaceUpdateResult.FailedToCompile;
}
Log.WriteLine();
Times.Add(new Tuple<string,TimeSpan>("Prj gen", Stopwatch.Stop("Success")));
}
}
// Build everything using MegaXGE
if(Context.Options.HasFlag(WorkspaceUpdateOptions.Build))
{
// Compile all the build steps together
Dictionary<Guid, ConfigObject> BuildStepObjects = Context.DefaultBuildSteps.ToDictionary(x => x.Key, x => new ConfigObject(x.Value));
BuildStep.MergeBuildStepObjects(BuildStepObjects, ProjectConfigFile.GetValues("Build.Step", new string[0]).Select(x => new ConfigObject(x)));
BuildStep.MergeBuildStepObjects(BuildStepObjects, Context.UserBuildStepObjects);
// Construct build steps from them
List<BuildStep> BuildSteps = BuildStepObjects.Values.Select(x => new BuildStep(x)).OrderBy(x => x.OrderIndex).ToList();
if(Context.CustomBuildSteps != null && Context.CustomBuildSteps.Count > 0)
{
BuildSteps.RemoveAll(x => !Context.CustomBuildSteps.Contains(x.UniqueId));
}
else if(Context.Options.HasFlag(WorkspaceUpdateOptions.ScheduledBuild))
{
BuildSteps.RemoveAll(x => !x.bScheduledSync);
}
else
{
BuildSteps.RemoveAll(x => !x.bNormalSync);
}
// Check if the last successful build was before a change that we need to force a clean for
bool bForceClean = false;
if(LastBuiltChangeNumber != 0)
{
foreach(string CleanBuildChange in ProjectConfigFile.GetValues("ForceClean.Changelist", new string[0]))
{
int ChangeNumber;
if(int.TryParse(CleanBuildChange, out ChangeNumber))
{
if((LastBuiltChangeNumber >= ChangeNumber) != (CurrentChangeNumber >= ChangeNumber))
{
Log.WriteLine("Forcing clean build due to changelist {0}.", ChangeNumber);
Log.WriteLine();
bForceClean = true;
break;
}
}
}
}
// Execute them all
string TelemetryEventName = (Context.UserBuildStepObjects.Count > 0)? "CustomBuild" : Context.Options.HasFlag(WorkspaceUpdateOptions.UseIncrementalBuilds) ? "Compile" : "FullCompile";
using(TelemetryStopwatch Stopwatch = new TelemetryStopwatch(TelemetryEventName, TelemetryProjectPath))
{
Progress.Set("Starting build...", 0.0f);
// Check we've built UBT (it should have been compiled by generating project files)
string UnrealBuildToolPath = Path.Combine(LocalRootPath, "Engine", "Binaries", "DotNET", "UnrealBuildTool.exe");
if(!File.Exists(UnrealBuildToolPath))
{
StatusMessage = String.Format("Couldn't find {0}", UnrealBuildToolPath);
return WorkspaceUpdateResult.FailedToCompile;
}
// Execute all the steps
float MaxProgressFraction = 0.0f;
foreach (BuildStep Step in BuildSteps)
{
MaxProgressFraction += (float)Step.EstimatedDuration / (float)Math.Max(BuildSteps.Sum(x => x.EstimatedDuration), 1);
Progress.Set(Step.StatusText);
Progress.Push(MaxProgressFraction);
Log.WriteLine(Step.StatusText);
switch(Step.Type)
{
case BuildStepType.Compile:
using(TelemetryStopwatch StepStopwatch = new TelemetryStopwatch("Compile:" + Step.Target, TelemetryProjectPath))
{
string CommandLine = String.Format("{0} {1} {2} {3} -NoHotReloadFromIDE", Step.Target, Step.Platform, Step.Configuration, Utility.ExpandVariables(Step.Arguments, Context.Variables));
if(!Context.Options.HasFlag(WorkspaceUpdateOptions.UseIncrementalBuilds) || bForceClean)
{
Log.WriteLine("ubt> Running {0} {1} -clean", UnrealBuildToolPath, CommandLine);
Utility.ExecuteProcess(UnrealBuildToolPath, CommandLine + " -clean", null, new ProgressTextWriter(Progress, new PrefixedTextWriter("ubt> ", Log)));
}
Log.WriteLine("ubt> Running {0} {1} -progress", UnrealBuildToolPath, CommandLine);
int ResultFromBuild = Utility.ExecuteProcess(UnrealBuildToolPath, CommandLine + " -progress", null, new ProgressTextWriter(Progress, new PrefixedTextWriter("ubt> ", Log)));
if(ResultFromBuild != 0)
{
StatusMessage = String.Format("Failed to compile {0}.", Step.Target);
return (HasModifiedSourceFiles() || Context.UserBuildStepObjects.Count > 0)? WorkspaceUpdateResult.FailedToCompile : WorkspaceUpdateResult.FailedToCompileWithCleanWorkspace;
}
}
break;
case BuildStepType.Cook:
using(TelemetryStopwatch StepStopwatch = new TelemetryStopwatch("Cook/Launch: " + Path.GetFileNameWithoutExtension(Step.FileName), TelemetryProjectPath))
{
string LocalRunUAT = Path.Combine(LocalRootPath, "Engine", "Build", "BatchFiles", "RunUAT.bat");
string Arguments = String.Format("/C \"\"{0}\" -profile=\"{1}\"\"", LocalRunUAT, Path.Combine(LocalRootPath, Step.FileName));
Log.WriteLine("uat> Running {0} {1}", LocalRunUAT, Arguments);
int ResultFromUAT = Utility.ExecuteProcess(CmdExe, Arguments, null, new ProgressTextWriter(Progress, new PrefixedTextWriter("uat> ", Log)));
if(ResultFromUAT != 0)
{
StatusMessage = String.Format("Cook failed. ({0})", ResultFromUAT);
return WorkspaceUpdateResult.FailedToCompile;
}
}
break;
case BuildStepType.Other:
using(TelemetryStopwatch StepStopwatch = new TelemetryStopwatch("Custom: " + Path.GetFileNameWithoutExtension(Step.FileName), TelemetryProjectPath))
{
string ToolFileName = Path.Combine(LocalRootPath, Utility.ExpandVariables(Step.FileName, Context.Variables));
string ToolArguments = Utility.ExpandVariables(Step.Arguments, Context.Variables);
Log.WriteLine("tool> Running {0} {1}", ToolFileName, ToolArguments);
if(Step.bUseLogWindow)
{
int ResultFromTool = Utility.ExecuteProcess(ToolFileName, ToolArguments, null, new ProgressTextWriter(Progress, new PrefixedTextWriter("tool> ", Log)));
if(ResultFromTool != 0)
{
StatusMessage = String.Format("Tool terminated with exit code {0}.", ResultFromTool);
return WorkspaceUpdateResult.FailedToCompile;
}
}
else
{
using(Process.Start(ToolFileName, ToolArguments))
{
}
}
}
break;
}
Log.WriteLine();
Progress.Pop();
}
Times.Add(new Tuple<string,TimeSpan>("Build", Stopwatch.Stop("Success")));
}
// Update the last successful build change number
if(Context.CustomBuildSteps == null || Context.CustomBuildSteps.Count == 0)
{
LastBuiltChangeNumber = CurrentChangeNumber;
}
}
// Write out all the timing information
Log.WriteLine("Total time : " + FormatTime(Times.Sum(x => (long)(x.Item2.TotalMilliseconds / 1000))));
foreach(Tuple<string, TimeSpan> Time in Times)
{
Log.WriteLine(" {0,-8}: {1}", Time.Item1, FormatTime((long)(Time.Item2.TotalMilliseconds / 1000)));
}
if(NumFilesSynced > 0)
{
Log.WriteLine("{0} files synced.", NumFilesSynced);
}
Log.WriteLine();
Log.WriteLine("UPDATE SUCCEEDED");
StatusMessage = "Update succeeded";
return WorkspaceUpdateResult.Success;
}
