public BuildStepInfo(BuildStep buildStep, long executionId, string description, TimestampLocalLogger logger)
{
BuildStep = buildStep;
ExecutionId = executionId;
Description = description;
Logger = logger;
HasBeenSend = false;
}
public BuildStepLogger(BuildStep buildStep, ILogger mainLogger, DateTime startTime)
{
this.buildStep = buildStep;
this.mainLogger = mainLogger;
StepLogger = new TimestampLocalLogger(startTime);
// Let's receive all level messages, each logger will filter them itself
ActivateLog(LogMessageType.Debug);
// StepLogger messages will be forwarded to the monitor, which will also filter itself
StepLogger.ActivateLog(LogMessageType.Debug);
}
//private static PluginResolver pluginManager;
private static void BuildStepsRecursively(BuildEngine.Builder builder, ICollection<BuildStep> steps, int stepsPerLevel, int maxLevel, BuildStep curParent = null, int curLevel = 0)
{
if (curLevel == maxLevel)
return;
for (var i = 0; i < stepsPerLevel; ++i)
{
BuildStep step = builder.Root.Add(new DoNothingCommand());
if (curParent != null)
BuildStep.LinkBuildSteps(curParent, step);
BuildStepsRecursively(builder, steps, stepsPerLevel, maxLevel, step, curLevel + 1);
steps.Add(step);
}
}
internal void RegisterBuildStep(BuildStep buildStep, TimestampLocalLogger logger)
{
lock (buildStepInfosToSend)
{
buildStepInfosToSend.Add(new BuildStepInfo(buildStep, buildStep.ExecutionId, buildStep.Description, logger));
}
}
/// <summary>
/// Sets recursively the <see cref="BuildStep.Module"/>.
/// </summary>
/// <param name="buildStep">The build step.</param>
/// <param name="module">The module.</param>
private void SetModule(BuildStep buildStep, string module)
{
if (buildStep.Module == null)
buildStep.Module = module;
var enumerableBuildStep = buildStep as EnumerableBuildStep;
if (enumerableBuildStep != null && enumerableBuildStep.Steps != null)
{
foreach (var child in enumerableBuildStep.Steps)
{
SetModule(child, module);
}
}
}
private static IEnumerable<CommandBuildStep> CollectCommandSteps(BuildStep step)
{
if (step is CommandBuildStep)
{
yield return (CommandBuildStep)step;
}
// NOTE: We assume that only EnumerableBuildStep is the base class for sub-steps and that ContentReferencable BuildStep are accessible from them (not through dynamic build step)
var enumerateBuildStep = step as EnumerableBuildStep;
if (enumerateBuildStep?.Steps != null)
{
foreach (var subStep in enumerateBuildStep.Steps)
{
foreach (var command in CollectCommandSteps(subStep))
{
yield return command;
}
}
}
}
private static void CollectContentReferenceDependencies(BuildStep step, HashSet<string> locations)
{
// For each CommandStep for the current build step, collects all dependencies to ContenrReference-BuildStep
foreach (var commandStep in CollectCommandSteps(step))
{
foreach (var inputFile in commandStep.Command.GetInputFiles())
{
if (inputFile.Type == UrlType.Content)
{
locations.Add(inputFile.Path);
}
}
}
}
public void ScheduleBuildStep(BuildStep step)
{
builder.ScheduleBuildStep(builderContext, buildStep, step, Variables);
}
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary<string, string> variables)
{
if (buildStep.ExecutionId == 0)
{
if (buildStep.Parent != null && buildStep.Parent != instigator)
throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent");
if (buildStep.Parent == null)
{
buildStep.Parent = instigator;
}
// Compute content dependencies before scheduling the build
GenerateDependencies(buildStep);
var executeContext = new ExecuteContext(this, builderContext, buildStep) { Variables = new Dictionary<string, string>(variables) };
//buildStep.ExpandStrings(executeContext);
if (runMode == Mode.Build)
{
MicroThread microThread = scheduler.Create();
// Find priority from this build step, or one of its parent.
var buildStepPriority = buildStep;
while (buildStepPriority != null)
{
if (buildStepPriority.Priority.HasValue)
{
microThread.Priority = buildStepPriority.Priority.Value;
break;
}
buildStepPriority = buildStepPriority.Parent;
}
buildStep.ExecutionId = microThread.Id;
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.RegisterBuildStep(buildStep, ((BuildStepLogger)executeContext.Logger).StepLogger);
}
microThread.Name = buildStep.ToString();
// Default:
// Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished.
// Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep.
// This should also reduce memory usage, and might improve cache coherency as well.
microThread.ScheduleMode = ScheduleMode.First;
microThread.Start(async () =>
{
// Wait for prerequisites
await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray());
// Check for failed prerequisites
var status = ResultStatus.NotProcessed;
if (buildStep.ArePrerequisitesSuccessful)
{
try
{
IndexFileCommand.MountDatabase(executeContext.GetOutputObjectsGroups());
// Execute
status = await buildStep.Execute(executeContext, builderContext);
}
catch (TaskCanceledException e)
{
// Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again
executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully.");
executeContext.Logger.Warning(e.Message);
status = ResultStatus.Cancelled;
}
catch (Exception e)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + e);
status = ResultStatus.Failed;
}
finally
{
IndexFileCommand.UnmountDatabase();
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion.");
}
if (microThread.Exception != null)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception);
status = ResultStatus.Failed;
}
}
else
{
status = ResultStatus.NotTriggeredPrerequisiteFailed;
}
//if (completedTask.IsCanceled)
//{
// completedStep.Status = ResultStatus.Cancelled;
//}
var logType = LogMessageType.Info;
string logText = null;
switch (status)
{
case ResultStatus.Successful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Failed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredPrerequisiteFailed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed of previous failed prerequisites.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Cancelled:
logType = LogMessageType.Warning;
logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredWasSuccessful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotProcessed:
throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled");
}
if (logText != null)
{
var logMessage = new LogMessage(buildStep.Module, logType, logText);
Logger.Log(logMessage);
}
buildStep.RegisterResult(executeContext, status);
stepCounter.AddStepResult(status);
});
}
else
{
buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete);
}
}
}
public void ScheduleBuildStep(BuildStep step)
{
builder.ScheduleBuildStep(builderContext, buildStep, step, Variables);
}
public ExecuteContext(Builder builder, BuilderContext builderContext, BuildStep buildStep)
{
Logger = new BuildStepLogger(buildStep, builder.Logger, builder.startTime);
this.builderContext = builderContext;
this.builder = builder;
this.buildStep = buildStep;
buildTransaction = new BuildTransaction(null, buildStep.GetOutputObjectsGroups());
}
/// <summary>
/// Sets recursively the <see cref="BuildStep.Module"/>.
/// </summary>
/// <param name="buildStep">The build step.</param>
/// <param name="module">The module.</param>
private void SetAssetLogger(BuildStep buildStep, Package package, IReference assetReference, string assetFullPath)
{
if (buildStep.TransformExecuteContextLogger == null)
buildStep.TransformExecuteContextLogger = (ref Logger logger) => logger = new AssetLogger(package, assetReference, assetFullPath, logger);
var enumerableBuildStep = buildStep as EnumerableBuildStep;
if (enumerableBuildStep != null && enumerableBuildStep.Steps != null)
{
foreach (var child in enumerableBuildStep.Steps)
{
SetAssetLogger(child, package, assetReference, assetFullPath);
}
}
}
public ExecuteContext(Builder builder, BuilderContext builderContext, BuildStep buildStep, Logger logger)
{
Logger = logger;
this.builderContext = builderContext;
this.builder = builder;
this.buildStep = buildStep;
buildTransaction = new BuildTransaction(null, buildStep.GetOutputObjectsGroups());
}
public BuildStepEventArgs(BuildStep step, ILogger logger)
{
Step = step;
Logger = logger;
}
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary <string, string> variables)
{
if (buildStep.ExecutionId == 0)
{
if (buildStep.Parent != null && buildStep.Parent != instigator)
{
throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent");
}
if (buildStep.Parent == null)
{
buildStep.Parent = instigator;
}
var executeContext = new ExecuteContext(this, builderContext, buildStep)
{
Variables = new Dictionary <string, string>(variables)
};
//buildStep.ExpandStrings(executeContext);
if (runMode == Mode.Build)
{
MicroThread microThread = scheduler.Create();
// Find priority from this build step, or one of its parent.
var buildStepPriority = buildStep;
while (buildStepPriority != null)
{
if (buildStepPriority.Priority.HasValue)
{
microThread.Priority = buildStepPriority.Priority.Value;
break;
}
buildStepPriority = buildStepPriority.Parent;
}
buildStep.ExecutionId = microThread.Id;
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.RegisterBuildStep(buildStep, ((BuildStepLogger)executeContext.Logger).StepLogger);
}
microThread.Name = buildStep.ToString();
// Default:
// Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished.
// Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep.
// This should also reduce memory usage, and might improve cache coherency as well.
microThread.ScheduleMode = ScheduleMode.First;
microThread.Start(async() =>
{
// Wait for prerequisites
await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray());
// Check for failed prerequisites
var status = ResultStatus.NotProcessed;
if (buildStep.ArePrerequisitesSuccessful)
{
try
{
IndexFileCommand.MountDatabase(executeContext.GetOutputObjectsGroups());
// Execute
status = await buildStep.Execute(executeContext, builderContext);
}
catch (TaskCanceledException e)
{
// Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again
executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully.");
executeContext.Logger.Warning(e.Message);
status = ResultStatus.Cancelled;
}
catch (Exception e)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + e);
status = ResultStatus.Failed;
}
finally
{
IndexFileCommand.UnmountDatabase();
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
{
throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion.");
}
}
if (microThread.Exception != null)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception);
status = ResultStatus.Failed;
}
}
else
{
status = ResultStatus.NotTriggeredPrerequisiteFailed;
}
//if (completedTask.IsCanceled)
//{
// completedStep.Status = ResultStatus.Cancelled;
//}
var logType = LogMessageType.Info;
string logText = null;
switch (status)
{
case ResultStatus.Successful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Failed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Cancelled:
logType = LogMessageType.Warning;
logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredWasSuccessful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotProcessed:
throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled");
}
if (logText != null)
{
var logMessage = new LogMessage(buildStep.Module, logType, logText);
Logger.Log(logMessage);
}
buildStep.RegisterResult(executeContext, status);
stepCounter.AddStepResult(status);
});
}
else
{
buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete);
}
}
}
/// <summary>
/// Collects dependencies between <see cref="BuildStep.OutputLocation"/> and fill the <see cref="BuildStep.PrerequisiteSteps"/> accordingly.
/// </summary>
/// <param name="rootStep">The root BuildStep</param>
private void GenerateDependencies(BuildStep rootStep)
{
// TODO: Support proper incremental dependecies
if (rootStep.ProcessedDependencies)
return;
rootStep.ProcessedDependencies = true;
var contentBuildSteps = new Dictionary<string, KeyValuePair<BuildStep, HashSet<string>>>();
PrepareDependencyGraph(rootStep, contentBuildSteps);
ComputeDependencyGraph(contentBuildSteps);
}
private static IEnumerable<BuildStep> EnumerateSpawnedBuildSteps(BuildStep buildStep)
{
foreach (var spawnedStep in buildStep.SpawnedSteps)
{
yield return spawnedStep;
foreach (var childSpawnedStep in EnumerateSpawnedBuildSteps(spawnedStep))
{
yield return childSpawnedStep;
}
}
}
/// <summary>
/// Collects dependencies between <see cref="BuildStep.OutputLocation"/> BuildStep. See remarks.
/// </summary>
/// <param name="step">The step to compute the dependencies for</param>
/// <param name="contentBuildSteps">A cache of content reference location to buildsteps </param>
/// <remarks>
/// Each BuildStep inheriting from <see cref="BuildStep.OutputLocation"/> is considered as a top-level dependency step that can have depedencies
/// on other top-level dependency. We are collecting all of them here.
/// </remarks>
private static void PrepareDependencyGraph(BuildStep step, Dictionary<string, KeyValuePair<BuildStep, HashSet<string>>> contentBuildSteps)
{
step.ProcessedDependencies = true;
var outputLocation = step.OutputLocation;
if (outputLocation != null)
{
var dependencies = new HashSet<string>();
if (!contentBuildSteps.ContainsKey(outputLocation))
{
contentBuildSteps.Add(outputLocation, new KeyValuePair<BuildStep, HashSet<string>>(step, dependencies));
CollectContentReferenceDependencies(step, dependencies);
}
// If we have a reference, we don't need to iterate further
return;
}
// NOTE: We assume that only EnumerableBuildStep is the base class for sub-steps and that ContentReferencable BuildStep are accessible from them (not through dynamic build step)
var enumerateBuildStep = step as EnumerableBuildStep;
if (enumerateBuildStep?.Steps != null)
{
foreach (var subStep in enumerateBuildStep.Steps)
{
PrepareDependencyGraph(subStep, contentBuildSteps);
}
}
}
public static void LinkBuildSteps(BuildStep parent, BuildStep child)
{
lock (child.prerequisiteSteps)
{
child.prerequisiteSteps.Add(parent);
}
}
//private static PluginResolver pluginManager;
private static void BuildStepsRecursively(Builder builder, ICollection <BuildStep> steps, int stepsPerLevel, int maxLevel, BuildStep curParent = null, int curLevel = 0)
{
if (curLevel == maxLevel)
{
return;
}
for (var i = 0; i < stepsPerLevel; ++i)
{
BuildStep step = builder.Root.Add(new DoNothingCommand());
if (curParent != null)
{
BuildStep.LinkBuildSteps(curParent, step);
}
BuildStepsRecursively(builder, steps, stepsPerLevel, maxLevel, step, curLevel + 1);
steps.Add(step);
}
}
