public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = new ForwardingLoggerResult(executeContext.Logger);
Step = step;
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List <BuildStep>();
while (true)
{
// interrupt the build if cancellation is required.
if (executeContext.CancellationTokenSource.Token.IsCancellationRequested)
{
return(ResultStatus.Cancelled);
}
// Clean completed build steps
for (int index = buildStepsToWait.Count - 1; index >= 0; index--)
{
var buildStepToWait = buildStepsToWait[index];
if (buildStepToWait.Processed)
{
buildStepsToWait.RemoveAt(index);
}
}
// wait for a task to complete
if (buildStepsToWait.Count >= MaxParallelSteps)
{
await CompleteOneBuildStep(executeContext, buildStepsToWait);
}
// Should we check for all tasks or only high priority tasks? (priority < 0)
bool checkOnlyForHighPriorityTasks = buildStepsToWait.Count >= MaxParallelSteps;
// Transform item into build step
var buildStep = buildStepProvider.GetNextBuildStep(checkOnlyForHighPriorityTasks ? -1 : int.MaxValue);
// No job => passively wait
if (buildStep == null)
{
newWorkAvailable.WaitOne();
continue;
}
// Safeguard if the provided build step is already processed
if (buildStep.Processed)
{
continue;
}
// Schedule build step
executeContext.ScheduleBuildStep(buildStep);
buildStepsToWait.Add(buildStep);
}
}
private async Task <ResultStatus> StartCommand(IExecuteContext executeContext, ListStore <CommandResultEntry> commandResultEntries, BuilderContext builderContext)
{
var logger = executeContext.Logger;
//await Scheduler.Yield();
ResultStatus status;
using (commandResultEntries)
{
logger.Debug($"Starting command {Command}...");
// Creating the CommandResult object
var commandContext = new LocalCommandContext(executeContext, this, builderContext);
// Try to execute remotely
if (!(Command.ShouldSpawnNewProcess() &&
builderContext.TryExecuteRemote != null &&
(status = await builderContext.TryExecuteRemote(Command, builderContext, executeContext, commandContext)) != ResultStatus.NotProcessed))
{
Command.PreCommand(commandContext);
if (!Command.BasePreCommandCalled)
{
throw new InvalidOperationException("base.PreCommand not called in command " + Command);
}
try
{
// Merge results from prerequisites
// TODO: This will prevent us from overwriting this asset with different content as it will result in a write conflict
// At some point we _might_ want to get rid of WaitBuildStep/ListBuildStep system and write a fully stateless input/output-based system; probably need further discussions
var fileProvider = MicrothreadLocalDatabases.DatabaseFileProvider;
if (fileProvider != null)
{
var assetIndexMap = fileProvider.ContentIndexMap;
foreach (var prerequisiteStep in PrerequisiteSteps)
{
foreach (var output in prerequisiteStep.OutputObjectIds)
{
assetIndexMap[output.Key.Path] = output.Value;
}
}
}
status = await Command.DoCommand(commandContext);
}
catch (Exception ex)
{
executeContext.Logger.Error("Exception in command " + this + ": " + ex);
status = ResultStatus.Failed;
}
Command.PostCommand(commandContext, status);
if (!Command.BasePostCommandCalled)
{
throw new InvalidOperationException("base.PostCommand not called in command " + Command);
}
}
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
{
throw new InvalidDataException("The command " + Command + " returned ResultStatus.NotProcessed after completion.");
}
// Registering the result to the build cache
RegisterCommandResult(commandResultEntries, commandContext.ResultEntry, status);
}
return(status);
}
public RemoteCommandContext(IProcessBuilderRemote processBuilderRemote, Command command, BuilderContext builderContext, Logger logger) : base(command, builderContext)
{
this.processBuilderRemote = processBuilderRemote;
this.logger = logger;
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore <CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
bool monitorExited = false;
var status = ResultStatus.NotProcessed;
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult;
try
{
// try to retrieve result from one of the object store
var commandHash = Command.ComputeCommandHash(executeContext);
// Early exit if the hash of the command failed
if (commandHash == ObjectId.Empty)
{
return(ResultStatus.Failed);
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore <CommandResultEntry>(commandResultsFileStream)
{
AutoLoadNewValues = false
};
commandResultEntries.LoadNewValues();
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug($"Command {Command} delayed because it is currently running...");
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug($"Command {Command} scheduled...");
// Register the cancel callback
var cancellationTokenSource = executeContext.CancellationTokenSource;
cancellationTokenSource.Token.Register(x => ((Command)x).Cancel(), Command);
Command.CancellationToken = cancellationTokenSource.Token;
try
{
status = await StartCommand(executeContext, commandResultEntries, builderContext);
}
finally
{
// Restore cancellation token (to avoid memory leak due to previous CancellationToken.Register
Command.CancellationToken = CancellationToken.None;
}
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
}
finally
{
if (!monitorExited)
{
Monitor.Exit(executeContext);
}
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// Re-output command log messages
foreach (var message in matchingResult.LogMessages)
{
executeContext.Logger.Log(message);
}
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return(status);
}
public static BuildResultCode BuildSlave(BuilderOptions options)
{
// Mount build path
((FileSystemProvider)VirtualFileSystem.ApplicationData).ChangeBasePath(options.BuildDirectory);
PrepareDatabases(options);
try
{
VirtualFileSystem.CreateDirectory("/data/");
VirtualFileSystem.CreateDirectory("/data/db/");
}
catch (Exception)
{
throw new OptionException("Invalid Build database path", "database");
}
// Open WCF channel with master builder
var namedPipeBinding = new NetNamedPipeBinding(NetNamedPipeSecurityMode.None)
{
SendTimeout = TimeSpan.FromSeconds(300.0)
};
var processBuilderRemote = ChannelFactory <IProcessBuilderRemote> .CreateChannel(namedPipeBinding, new EndpointAddress(options.SlavePipe));
try
{
RegisterRemoteLogger(processBuilderRemote);
// Create scheduler
var scheduler = new Scheduler();
var status = ResultStatus.NotProcessed;
// Schedule command
string buildPath = options.BuildDirectory;
Logger logger = options.Logger;
MicroThread microthread = scheduler.Add(async() =>
{
// Deserialize command and parameters
Command command = processBuilderRemote.GetCommandToExecute();
BuildParameterCollection parameters = processBuilderRemote.GetBuildParameters();
// Run command
var inputHashes = new DictionaryStore <InputVersionKey, ObjectId>(VirtualFileSystem.OpenStream("/data/db/InputHashes", VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite));
var builderContext = new BuilderContext(buildPath, inputHashes, parameters, 0, null);
var commandContext = new RemoteCommandContext(processBuilderRemote, command, builderContext, logger);
command.PreCommand(commandContext);
status = await command.DoCommand(commandContext);
command.PostCommand(commandContext, status);
// Returns result to master builder
processBuilderRemote.RegisterResult(commandContext.ResultEntry);
});
while (true)
{
scheduler.Run();
// Exit loop if no more micro threads
lock (scheduler.MicroThreads)
{
if (!scheduler.MicroThreads.Any())
{
break;
}
}
Thread.Sleep(0);
}
// Rethrow any exception that happened in microthread
if (microthread.Exception != null)
{
options.Logger.Fatal(microthread.Exception.ToString());
return(BuildResultCode.BuildError);
}
if (status == ResultStatus.Successful || status == ResultStatus.NotTriggeredWasSuccessful)
{
return(BuildResultCode.Successful);
}
return(BuildResultCode.BuildError);
}
finally
{
// Close WCF channel
// ReSharper disable SuspiciousTypeConversion.Global
((IClientChannel)processBuilderRemote).Close();
// ReSharper restore SuspiciousTypeConversion.Global
}
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List <BuildStep>();
// Process prerequisites build steps first
if (PrerequisiteSteps.Count > 0)
{
await CompleteCommands(executeContext, PrerequisiteSteps.ToList());
}
foreach (var child in Steps)
{
executeContext.ScheduleBuildStep(child);
buildStepsToWait.Add(child);
executedSteps.Add(child);
}
await CompleteCommands(executeContext, buildStepsToWait);
return(ComputeResultStatusFromExecutedSteps());
}
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary <string, string> variables)
{
if (buildStep.ExecutionId == 0)
{
if (buildStep.Parent != null && buildStep.Parent != instigator)
{
throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent");
}
if (buildStep.Parent == null)
{
buildStep.Parent = instigator;
}
// Compute content dependencies before scheduling the build
GenerateDependencies(buildStep);
// TODO: Big review of the log infrastructure of CompilerApp & BuildEngine!
// Create a logger that redirects to various places (BuildStep.Logger, timestampped log, global log, etc...)
var buildStepLogger = new BuildStepLogger(buildStep, Logger, startTime);
var logger = (Logger)buildStepLogger;
// Apply user-registered callbacks to the logger
buildStep.TransformExecuteContextLogger?.Invoke(ref logger);
// Create execute context
var executeContext = new ExecuteContext(this, builderContext, buildStep, logger)
{
Variables = new Dictionary <string, string>(variables)
};
//buildStep.ExpandStrings(executeContext);
if (runMode == Mode.Build)
{
MicroThread microThread = scheduler.Create();
// Set priority from this build step, if we have one.
if (buildStep.Priority.HasValue)
{
microThread.Priority = buildStep.Priority.Value;
}
buildStep.ExecutionId = microThread.Id;
microThread.Name = buildStep.ToString();
// Default:
// Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished.
// Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep.
// This should also reduce memory usage, and might improve cache coherency as well.
microThread.ScheduleMode = ScheduleMode.First;
microThread.Start(async() =>
{
// Wait for prerequisites
await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray());
// Check for failed prerequisites
var status = ResultStatus.NotProcessed;
if (buildStep.ArePrerequisitesSuccessful)
{
try
{
var outputObjectsGroups = executeContext.GetOutputObjectsGroups();
MicrothreadLocalDatabases.MountDatabase(outputObjectsGroups);
// Execute
status = await buildStep.Execute(executeContext, builderContext);
}
catch (TaskCanceledException e)
{
// Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again
executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully.");
executeContext.Logger.Warning(e.Message);
status = ResultStatus.Cancelled;
}
catch (Exception e)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + e);
status = ResultStatus.Failed;
}
finally
{
MicrothreadLocalDatabases.UnmountDatabase();
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
{
throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion.");
}
}
if (microThread.Exception != null)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception);
status = ResultStatus.Failed;
}
}
else
{
status = ResultStatus.NotTriggeredPrerequisiteFailed;
}
//if (completedTask.IsCanceled)
//{
// completedStep.Status = ResultStatus.Cancelled;
//}
var logType = LogMessageType.Info;
string logText = null;
switch (status)
{
case ResultStatus.Successful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Failed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredPrerequisiteFailed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed of previous failed prerequisites.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Cancelled:
logType = LogMessageType.Warning;
logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredWasSuccessful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotProcessed:
throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled");
}
if (logText != null)
{
var logMessage = new LogMessage(null, logType, logText);
executeContext.Logger.Log(logMessage);
}
buildStep.RegisterResult(executeContext, status);
stepCounter.AddStepResult(status);
});
}
else
{
buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete);
}
}
}
/// <summary>
/// Runs this instance.
/// </summary>
public BuildResultCode Run(Mode mode, bool writeIndexFile = true)
{
// When we setup the database ourself we have to take responsibility to close it after
var shouldCloseDatabase = ObjectDatabase == null;
OpenObjectDatabase(buildPath, indexName);
PreRun();
runMode = mode;
if (IsRunning)
{
throw new InvalidOperationException("An instance of this Builder is already running.");
}
// reset build cache from previous build run
cancellationTokenSource = new CancellationTokenSource();
Cancelled = false;
IsRunning = true;
DisableCompressionIds.Clear();
// Reseting result map
var inputHashes = FileVersionTracker.GetDefault();
{
var builderContext = new BuilderContext(inputHashes, TryExecuteRemote);
resultMap = ObjectDatabase;
scheduler = new Scheduler();
// Schedule the build
ScheduleBuildStep(builderContext, null, Root, InitialVariables);
// Create threads
var threads = Enumerable.Range(0, ThreadCount).Select(x => new Thread(SafeAction.Wrap(RunUntilEnd))
{
IsBackground = true
}).ToArray();
// Start threads
int threadId = 0;
foreach (var thread in threads)
{
thread.Name = (BuilderName ?? "Builder") + " worker thread " + (++threadId);
thread.Start();
}
// Wait for all threads to finish
foreach (var thread in threads)
{
thread.Join();
}
}
BuildResultCode result;
if (runMode == Mode.Build)
{
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error("Build cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure.");
Logger.Error("Build failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Info($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure.");
Logger.Info("Build is successful.");
result = BuildResultCode.Successful;
}
}
else
{
string modeName;
switch (runMode)
{
case Mode.Clean:
modeName = "Clean";
break;
case Mode.CleanAndDelete:
modeName = "Clean-and-delete";
break;
default:
throw new InvalidOperationException("Builder executed in unknown mode.");
}
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error(modeName + " has been cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error(modeName + " has failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Error(modeName + " has been successfully completed.");
result = BuildResultCode.Successful;
}
}
scheduler = null;
resultMap = null;
IsRunning = false;
if (shouldCloseDatabase)
{
CloseObjectDatabase();
}
return(result);
}
