/** Builds a list of actions that need to be executed to produce the specified output items. */
public static List<Action> GetActionsToExecute(Action[] PrerequisiteActions, List<STBuildTarget> Targets, out Dictionary<STBuildTarget, List<FileItem>> TargetToOutdatedPrerequisitesMap)
{
var CheckOutdatednessStartTime = DateTime.UtcNow;
// Build a set of all actions needed for this target.
var IsActionOutdatedMap = new Dictionary<Action, bool>();
foreach (var Action in PrerequisiteActions)
{
IsActionOutdatedMap.Add(Action, true);
}
// For all targets, build a set of all actions that are outdated.
var OutdatedActionDictionary = new Dictionary<Action, bool>();
var HistoryList = new List<ActionHistory>();
var OpenHistoryFiles = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
TargetToOutdatedPrerequisitesMap = new Dictionary<STBuildTarget, List<FileItem>>();
foreach (var BuildTarget in Targets) // @todo ubtmake: Optimization: Ideally we don't even need to know about targets for ubtmake -- everything would come from the files
{
var HistoryFilename = ActionHistory.GeneratePathForTarget(BuildTarget);
if (!OpenHistoryFiles.Contains(HistoryFilename)) // @todo ubtmake: Optimization: We should be able to move the command-line outdatedness and build product deletion over to the 'gather' phase, as the command-lines won't change between assembler runs
{
var History = new ActionHistory(HistoryFilename);
HistoryList.Add(History);
OpenHistoryFiles.Add(HistoryFilename);
GatherAllOutdatedActions(BuildTarget, History, ref OutdatedActionDictionary, TargetToOutdatedPrerequisitesMap);
}
}
// Delete produced items that are outdated.
DeleteOutdatedProducedItems(OutdatedActionDictionary, BuildConfiguration.bShouldDeleteAllOutdatedProducedItems);
// Save the action history.
// This must happen after deleting outdated produced items to ensure that the action history on disk doesn't have
// command-lines that don't match the produced items on disk.
foreach (var TargetHistory in HistoryList)
{
TargetHistory.Save();
}
// Create directories for the outdated produced items.
CreateDirectoriesForProducedItems(OutdatedActionDictionary);
// Build a list of actions that are both needed for this target and outdated.
List<Action> ActionsToExecute = new List<Action>();
bool bHasOutdatedNonLinkActions = false;
foreach (Action Action in AllActions)
{
if (Action.CommandPath != null && IsActionOutdatedMap.ContainsKey(Action) && OutdatedActionDictionary[Action])
{
ActionsToExecute.Add(Action);
if (Action.ActionType != ActionType.Link)
{
bHasOutdatedNonLinkActions = true;
}
}
}
// Remove link actions if asked to
if (STBuildConfiguration.bSkipLinkingWhenNothingToCompile && !bHasOutdatedNonLinkActions)
{
ActionsToExecute.Clear();
}
if (BuildConfiguration.bPrintPerformanceInfo)
{
var CheckOutdatednessTime = (DateTime.UtcNow - CheckOutdatednessStartTime).TotalSeconds;
Log.TraceInformation("Checking outdatedness took " + CheckOutdatednessTime + "s");
}
return ActionsToExecute;
}
/**
* Determines whether an action is outdated based on the modification times for its prerequisite
* and produced items.
* @param RootAction - The action being considered.
* @param OutdatedActionDictionary -
* @return true if outdated
*/
public static bool IsActionOutdated(STBuildTarget Target, Action RootAction, ref Dictionary<Action, bool> OutdatedActionDictionary, ActionHistory ActionHistory, Dictionary<STBuildTarget, List<FileItem>> TargetToOutdatedPrerequisitesMap)
{
// Only compute the outdated-ness for actions that don't aren't cached in the outdated action dictionary.
bool bIsOutdated = false;
if (!OutdatedActionDictionary.TryGetValue(RootAction, out bIsOutdated))
{
// Determine the last time the action was run based on the write times of its produced files.
string LatestUpdatedProducedItemName = null;
DateTimeOffset LastExecutionTime = DateTimeOffset.MaxValue;
foreach (FileItem ProducedItem in RootAction.ProducedItems)
{
// Optionally skip the action history check, as this only works for local builds
if (BuildConfiguration.bUseActionHistory)
{
// Check if the command-line of the action previously used to produce the item is outdated.
string OldProducingCommandLine = "";
string NewProducingCommandLine = RootAction.CommandPath + " " + RootAction.CommandArguments;
if (!ActionHistory.GetProducingCommandLine(ProducedItem, out OldProducingCommandLine)
|| !String.Equals(OldProducingCommandLine, NewProducingCommandLine, StringComparison.InvariantCultureIgnoreCase))
{
Log.TraceVerbose(
"{0}: Produced item \"{1}\" was produced by outdated command-line.\nOld command-line: {2}\nNew command-line: {3}",
RootAction.StatusDescription,
Path.GetFileName(ProducedItem.AbsolutePath),
OldProducingCommandLine,
NewProducingCommandLine
);
bIsOutdated = true;
// Update the command-line used to produce this item in the action history.
ActionHistory.SetProducingCommandLine(ProducedItem, NewProducingCommandLine);
}
}
// If the produced file doesn't exist or has zero size, consider it outdated. The zero size check is to detect cases
// where aborting an earlier compile produced invalid zero-sized obj files, but that may cause actions where that's
// legitimate output to always be considered outdated.
if (ProducedItem.bExists && (ProducedItem.bIsRemoteFile || ProducedItem.Length > 0 || ProducedItem.IsDirectory))
{
// When linking incrementally, don't use LIB, EXP pr PDB files when checking for the oldest produced item,
// as those files aren't always touched.
if (BuildConfiguration.bUseIncrementalLinking)
{
String ProducedItemExtension = Path.GetExtension(ProducedItem.AbsolutePath).ToUpperInvariant();
if (ProducedItemExtension == ".LIB" || ProducedItemExtension == ".EXP" || ProducedItemExtension == ".PDB")
{
continue;
}
}
// Use the oldest produced item's time as the last execution time.
if (ProducedItem.LastWriteTime < LastExecutionTime)
{
LastExecutionTime = ProducedItem.LastWriteTime;
LatestUpdatedProducedItemName = ProducedItem.AbsolutePath;
}
}
else
{
// If any of the produced items doesn't exist, the action is outdated.
Log.TraceVerbose(
"{0}: Produced item \"{1}\" doesn't exist.",
RootAction.StatusDescription,
Path.GetFileName(ProducedItem.AbsolutePath)
);
bIsOutdated = true;
}
}
Log.WriteLineIf(BuildConfiguration.bLogDetailedActionStats && !String.IsNullOrEmpty(LatestUpdatedProducedItemName),
TraceEventType.Verbose, "{0}: Oldest produced item is {1}", RootAction.StatusDescription, LatestUpdatedProducedItemName);
bool bFindCPPIncludePrerequisites = false;
if (RootAction.ActionType == ActionType.Compile)
{
// Outdated targets don't need their headers scanned yet, because presumably they would already be out of dated based on already-cached
// includes before getting this far. However, if we find them to be outdated after processing includes, we'll do a deep scan later
// on and cache all of the includes so that we have them for a quick outdatedness check the next run.
if (!bIsOutdated &&
BuildConfiguration.bUseExperimentalFastBuildIteration &&
STBuildTool.IsAssemblingBuild &&
RootAction.ActionType == ActionType.Compile)
{
bFindCPPIncludePrerequisites = true;
}
// Were we asked to force an update of our cached includes BEFORE we try to build? This may be needed if our cache can no longer
// be trusted and we need to fill it with perfectly valid data (even if we're in assembler only mode)
if (BuildConfiguration.bUseExperimentalFastDependencyScan &&
STBuildTool.bNeedsFullCPPIncludeRescan)
{
bFindCPPIncludePrerequisites = true;
}
}
if (bFindCPPIncludePrerequisites)
{
// Scan this file for included headers that may be out of date. Note that it's OK if we break out early because we found
// the action to be outdated. For outdated actions, we kick off a separate include scan in a background thread later on to
// catch all of the other includes and form an exhaustive set.
foreach (FileItem PrerequisiteItem in RootAction.PrerequisiteItems)
{
// @todo ubtmake: Make sure we are catching RC files here too. Anything that the toolchain would have tried it on. Logic should match the CACHING stuff below
if (PrerequisiteItem.CachedCPPIncludeInfo != null)
{
var BuildPlatform = STBuildPlatform.GetBuildPlatform(Target.GetTargetInfo().Platform);
var IncludedFileList = CPPEnvironment.FindAndCacheAllIncludedFiles(Target, PrerequisiteItem, BuildPlatform, PrerequisiteItem.CachedCPPIncludeInfo, bOnlyCachedDependencies: BuildConfiguration.bUseExperimentalFastDependencyScan);
foreach (var IncludedFile in IncludedFileList) // @todo fastubt: @todo ubtmake: Optimization: This is "retesting" a lot of the same files over and over in a single run (common indirect includes)
{
if (IncludedFile.bExists)
{
// allow a 1 second slop for network copies
TimeSpan TimeDifference = IncludedFile.LastWriteTime - LastExecutionTime;
bool bPrerequisiteItemIsNewerThanLastExecution = TimeDifference.TotalSeconds > 1;
if (bPrerequisiteItemIsNewerThanLastExecution)
{
Log.TraceVerbose(
"{0}: Included file {1} is newer than the last execution of the action: {2} vs {3}",
RootAction.StatusDescription,
Path.GetFileName(IncludedFile.AbsolutePath),
IncludedFile.LastWriteTime.LocalDateTime,
LastExecutionTime.LocalDateTime
);
bIsOutdated = true;
// Don't bother checking every single include if we've found one that is out of date
break;
}
}
}
}
if (bIsOutdated)
{
break;
}
}
}
if (!bIsOutdated)
{
// Check if any of the prerequisite items are produced by outdated actions, or have changed more recently than
// the oldest produced item.
foreach (FileItem PrerequisiteItem in RootAction.PrerequisiteItems)
{
// Only check for outdated import libraries if we were configured to do so. Often, a changed import library
// won't affect a dependency unless a public header file was also changed, in which case we would be forced
// to recompile anyway. This just allows for faster iteration when working on a subsystem in a DLL, as we
// won't have to wait for dependent targets to be relinked after each change.
bool bIsImportLibraryFile = false;
if (PrerequisiteItem.ProducingAction != null && PrerequisiteItem.ProducingAction.bProducesImportLibrary)
{
bIsImportLibraryFile = PrerequisiteItem.AbsolutePath.EndsWith(".LIB", StringComparison.InvariantCultureIgnoreCase);
}
if (!bIsImportLibraryFile || !BuildConfiguration.bIgnoreOutdatedImportLibraries)
{
// If the prerequisite is produced by an outdated action, then this action is outdated too.
if (PrerequisiteItem.ProducingAction != null)
{
if (IsActionOutdated(Target, PrerequisiteItem.ProducingAction, ref OutdatedActionDictionary, ActionHistory, TargetToOutdatedPrerequisitesMap))
{
Log.TraceVerbose(
"{0}: Prerequisite {1} is produced by outdated action.",
RootAction.StatusDescription,
Path.GetFileName(PrerequisiteItem.AbsolutePath)
);
bIsOutdated = true;
}
}
if (PrerequisiteItem.bExists)
{
// allow a 1 second slop for network copies
TimeSpan TimeDifference = PrerequisiteItem.LastWriteTime - LastExecutionTime;
bool bPrerequisiteItemIsNewerThanLastExecution = TimeDifference.TotalSeconds > 1;
if (bPrerequisiteItemIsNewerThanLastExecution)
{
Log.TraceVerbose(
"{0}: Prerequisite {1} is newer than the last execution of the action: {2} vs {3}",
RootAction.StatusDescription,
Path.GetFileName(PrerequisiteItem.AbsolutePath),
PrerequisiteItem.LastWriteTime.LocalDateTime,
LastExecutionTime.LocalDateTime
);
bIsOutdated = true;
}
}
// GatherAllOutdatedActions will ensure all actions are checked for outdated-ness, so we don't need to recurse with
// all this action's prerequisites once we've determined it's outdated.
if (bIsOutdated)
{
break;
}
}
}
}
// For compile actions, we have C++ files that are actually dependent on header files that could have been changed. We only need to
// know about the set of header files that are included for files that are already determined to be out of date (such as if the file
// is missing or was modified.) In the case that the file is out of date, we'll perform a deep scan to update our cached set of
// includes for this file, so that we'll be able to determine whether it is out of date next time very quickly.
if (BuildConfiguration.bUseExperimentalFastDependencyScan)
{
var DeepIncludeScanStartTime = DateTime.UtcNow;
// @todo fastubt: we may be scanning more files than we need to here -- indirectly outdated files are bIsOutdated=true by this point (for example basemost includes when deeper includes are dirty)
if (bIsOutdated && RootAction.ActionType == ActionType.Compile) // @todo fastubt: Does this work with RC files? See above too.
{
Log.TraceVerbose("Outdated action: {0}", RootAction.StatusDescription);
foreach (FileItem PrerequisiteItem in RootAction.PrerequisiteItems)
{
if (PrerequisiteItem.CachedCPPIncludeInfo != null)
{
if (!IsCPPFile(PrerequisiteItem))
{
throw new BuildException("Was only expecting C++ files to have CachedCPPEnvironments!");
}
Log.TraceVerbose(" -> DEEP include scan: {0}", PrerequisiteItem.AbsolutePath);
List<FileItem> OutdatedPrerequisites;
if (!TargetToOutdatedPrerequisitesMap.TryGetValue(Target, out OutdatedPrerequisites))
{
OutdatedPrerequisites = new List<FileItem>();
TargetToOutdatedPrerequisitesMap.Add(Target, OutdatedPrerequisites);
}
OutdatedPrerequisites.Add(PrerequisiteItem);
}
else if (IsCPPImplementationFile(PrerequisiteItem) || IsCPPResourceFile(PrerequisiteItem))
{
if (PrerequisiteItem.CachedCPPIncludeInfo == null)
{
Log.TraceVerbose(" -> WARNING: No CachedCPPEnvironment: {0}", PrerequisiteItem.AbsolutePath);
}
}
}
}
if (BuildConfiguration.bPrintPerformanceInfo)
{
double DeepIncludeScanTime = (DateTime.UtcNow - DeepIncludeScanStartTime).TotalSeconds;
STBuildTool.TotalDeepIncludeScanTime += DeepIncludeScanTime;
}
}
// Cache the outdated-ness of this action.
OutdatedActionDictionary.Add(RootAction, bIsOutdated);
}
return bIsOutdated;
}
/**
* Builds a dictionary containing the actions from AllActions that are outdated by calling
* IsActionOutdated.
*/
static void GatherAllOutdatedActions(STBuildTarget Target, ActionHistory ActionHistory, ref Dictionary<Action, bool> OutdatedActions, Dictionary<STBuildTarget, List<FileItem>> TargetToOutdatedPrerequisitesMap)
{
var CheckOutdatednessStartTime = DateTime.UtcNow;
foreach (var Action in AllActions)
{
IsActionOutdated(Target, Action, ref OutdatedActions, ActionHistory, TargetToOutdatedPrerequisitesMap);
}
if (BuildConfiguration.bPrintPerformanceInfo)
{
var CheckOutdatednessTime = (DateTime.UtcNow - CheckOutdatednessStartTime).TotalSeconds;
Log.TraceInformation("Checking actions for " + Target.GetTargetName() + " took " + CheckOutdatednessTime + "s");
}
}