public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List <BuildStep>();
// Process prerequisites build steps first
if (PrerequisiteSteps.Count > 0)
{
await CompleteCommands(executeContext, PrerequisiteSteps.ToList());
}
foreach (var child in Steps)
{
executeContext.ScheduleBuildStep(child);
buildStepsToWait.Add(child);
executedSteps.Add(child);
}
await CompleteCommands(executeContext, buildStepsToWait);
return(ComputeResultStatusFromExecutedSteps());
}
public RemoteCommandContext(IProcessBuilderRemote processBuilderRemote, Command command, BuilderContext builderContext, Logger logger) : base(command, builderContext)
{
this.processBuilderRemote = processBuilderRemote;
this.logger = logger;
}
/// <summary>
/// Runs this instance.
/// </summary>
public BuildResultCode Run(Mode mode, bool writeIndexFile = true, bool enableMonitor = false)
{
// When we setup the database ourself we have to take responsibility to close it after
var shouldCloseDatabase = ObjectDatabase == null;
OpenObjectDatabase(buildPath, indexName);
PreRun();
runMode = mode;
if (IsRunning)
{
throw new InvalidOperationException("An instance of this Builder is already running.");
}
// reset build cache from previous build run
cancellationTokenSource = new CancellationTokenSource();
Cancelled = false;
IsRunning = true;
DisableCompressionIds.Clear();
// Reseting result map
var inputHashes = FileVersionTracker.GetDefault();
{
var builderContext = new BuilderContext(inputHashes, TryExecuteRemote);
resultMap = ObjectDatabase;
scheduler = new Scheduler();
if (enableMonitor)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId));
foreach (var monitorPipeName in MonitorPipeNames)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId, monitorPipeName));
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Start();
}
}
// Schedule the build
ScheduleBuildStep(builderContext, null, Root, InitialVariables);
// Create threads
var threads = Enumerable.Range(0, ThreadCount).Select(x => new Thread(SafeAction.Wrap(RunUntilEnd))
{
IsBackground = true
}).ToArray();
// Start threads
int threadId = 0;
foreach (var thread in threads)
{
thread.Name = (BuilderName ?? "Builder") + " worker thread " + (++threadId);
thread.Start();
}
// Wait for all threads to finish
foreach (var thread in threads)
{
thread.Join();
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Finish();
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Join();
}
}
threadMonitors.Clear();
BuildResultCode result;
if (runMode == Mode.Build)
{
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error("Build cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure.");
Logger.Error("Build failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Info($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure.");
Logger.Info("Build is successful.");
result = BuildResultCode.Successful;
}
}
else
{
string modeName;
switch (runMode)
{
case Mode.Clean:
modeName = "Clean";
break;
case Mode.CleanAndDelete:
modeName = "Clean-and-delete";
break;
default:
throw new InvalidOperationException("Builder executed in unknown mode.");
}
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error(modeName + " has been cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error(modeName + " has failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Error(modeName + " has been successfully completed.");
result = BuildResultCode.Successful;
}
}
scheduler = null;
resultMap = null;
IsRunning = false;
if (shouldCloseDatabase)
{
CloseObjectDatabase();
}
return(result);
}
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary <string, string> variables)
{
if (buildStep.ExecutionId == 0)
{
if (buildStep.Parent != null && buildStep.Parent != instigator)
{
throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent");
}
if (buildStep.Parent == null)
{
buildStep.Parent = instigator;
}
// Compute content dependencies before scheduling the build
GenerateDependencies(buildStep);
// TODO: Big review of the log infrastructure of CompilerApp & BuildEngine!
// Create a logger that redirects to various places (BuildStep.Logger, timestampped log, global log, etc...)
var buildStepLogger = new BuildStepLogger(buildStep, Logger, startTime);
var logger = (Logger)buildStepLogger;
// Apply user-registered callbacks to the logger
buildStep.TransformExecuteContextLogger?.Invoke(ref logger);
// Create execute context
var executeContext = new ExecuteContext(this, builderContext, buildStep, logger)
{
Variables = new Dictionary <string, string>(variables)
};
//buildStep.ExpandStrings(executeContext);
if (runMode == Mode.Build)
{
MicroThread microThread = scheduler.Create();
// Set priority from this build step, if we have one.
if (buildStep.Priority.HasValue)
{
microThread.Priority = buildStep.Priority.Value;
}
buildStep.ExecutionId = microThread.Id;
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.RegisterBuildStep(buildStep, buildStepLogger.StepLogger);
}
microThread.Name = buildStep.ToString();
// Default:
// Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished.
// Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep.
// This should also reduce memory usage, and might improve cache coherency as well.
microThread.ScheduleMode = ScheduleMode.First;
microThread.Start(async() =>
{
// Wait for prerequisites
await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray());
// Check for failed prerequisites
var status = ResultStatus.NotProcessed;
if (buildStep.ArePrerequisitesSuccessful)
{
try
{
var outputObjectsGroups = executeContext.GetOutputObjectsGroups();
MicrothreadLocalDatabases.MountDatabase(outputObjectsGroups);
// Execute
status = await buildStep.Execute(executeContext, builderContext);
}
catch (TaskCanceledException e)
{
// Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again
executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully.");
executeContext.Logger.Warning(e.Message);
status = ResultStatus.Cancelled;
}
catch (Exception e)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + e);
status = ResultStatus.Failed;
}
finally
{
MicrothreadLocalDatabases.UnmountDatabase();
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
{
throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion.");
}
}
if (microThread.Exception != null)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception);
status = ResultStatus.Failed;
}
}
else
{
status = ResultStatus.NotTriggeredPrerequisiteFailed;
}
//if (completedTask.IsCanceled)
//{
// completedStep.Status = ResultStatus.Cancelled;
//}
var logType = LogMessageType.Info;
string logText = null;
switch (status)
{
case ResultStatus.Successful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Failed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredPrerequisiteFailed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed of previous failed prerequisites.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Cancelled:
logType = LogMessageType.Warning;
logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredWasSuccessful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotProcessed:
throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled");
}
if (logText != null)
{
var logMessage = new LogMessage(null, logType, logText);
executeContext.Logger.Log(logMessage);
}
buildStep.RegisterResult(executeContext, status);
stepCounter.AddStepResult(status);
});
}
else
{
buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete);
}
}
}
protected CommandContextBase(Command command, BuilderContext builderContext)
{
CurrentCommand = command;
ResultEntry = new CommandResultEntry();
}
public static BuildResultCode BuildSlave(BuilderOptions options)
{
// Mount build path
((FileSystemProvider)VirtualFileSystem.ApplicationData).ChangeBasePath(options.BuildDirectory);
PrepareDatabases(options);
try
{
VirtualFileSystem.CreateDirectory("/data/");
VirtualFileSystem.CreateDirectory("/data/db/");
}
catch (Exception)
{
throw new OptionException("Invalid Build database path", "database");
}
// Open WCF channel with master builder
var namedPipeBinding = new NetNamedPipeBinding(NetNamedPipeSecurityMode.None)
{
SendTimeout = TimeSpan.FromSeconds(300.0)
};
var processBuilderRemote = ChannelFactory <IProcessBuilderRemote> .CreateChannel(namedPipeBinding, new EndpointAddress(options.SlavePipe));
try
{
RegisterRemoteLogger(processBuilderRemote);
// Create scheduler
var scheduler = new Scheduler();
var status = ResultStatus.NotProcessed;
// Schedule command
string buildPath = options.BuildDirectory;
Logger logger = options.Logger;
MicroThread microthread = scheduler.Add(async() =>
{
// Deserialize command and parameters
Command command = processBuilderRemote.GetCommandToExecute();
BuildParameterCollection parameters = processBuilderRemote.GetBuildParameters();
// Run command
var inputHashes = new DictionaryStore <InputVersionKey, ObjectId>(VirtualFileSystem.OpenStream("/data/db/InputHashes", VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite));
var builderContext = new BuilderContext(buildPath, inputHashes, parameters, 0, null);
var commandContext = new RemoteCommandContext(processBuilderRemote, command, builderContext, logger);
command.PreCommand(commandContext);
status = await command.DoCommand(commandContext);
command.PostCommand(commandContext, status);
// Returns result to master builder
processBuilderRemote.RegisterResult(commandContext.ResultEntry);
});
while (true)
{
scheduler.Run();
// Exit loop if no more micro threads
lock (scheduler.MicroThreads)
{
if (!scheduler.MicroThreads.Any())
{
break;
}
}
Thread.Sleep(0);
}
// Rethrow any exception that happened in microthread
if (microthread.Exception != null)
{
options.Logger.Fatal(microthread.Exception.ToString());
return(BuildResultCode.BuildError);
}
if (status == ResultStatus.Successful || status == ResultStatus.NotTriggeredWasSuccessful)
{
return(BuildResultCode.Successful);
}
return(BuildResultCode.BuildError);
}
finally
{
// Close WCF channel
// ReSharper disable SuspiciousTypeConversion.Global
((IClientChannel)processBuilderRemote).Close();
// ReSharper restore SuspiciousTypeConversion.Global
}
}
