protected CommandContextBase(Command command, BuilderContext builderContext)
{
CurrentCommand = command;
BuildParameters = builderContext.Parameters;
ResultEntry = new CommandResultEntry();
MetadataProvider = builderContext.MetadataProvider;
}
public override async Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List<BuildStep>();
// Process prerequisites build steps first
if (PrerequisiteSteps.Count > 0)
await CompleteCommands(executeContext, PrerequisiteSteps.ToList());
foreach (var child in Steps)
{
// Wait for all the tasks before the WaitBuildStep to be finished
if (child is WaitBuildStep)
{
await CompleteCommands(executeContext, buildStepsToWait);
}
else
{
executeContext.ScheduleBuildStep(child);
buildStepsToWait.Add(child);
}
executedSteps.Add(child);
}
await CompleteCommands(executeContext, buildStepsToWait);
return ComputeResultStatusFromExecutedSteps();
}
public override async Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
Steps = new List<BuildStep>();
var files = Enumerable.Empty<string>();
foreach (string pattern in SearchPattern)
{
string path = Path.GetDirectoryName(pattern);
string filePattern = Path.GetFileName(pattern);
if (!string.IsNullOrEmpty(path) && !string.IsNullOrEmpty(filePattern))
{
files = files.Concat(Directory.EnumerateFiles(path, filePattern));
}
else
{
files = files.Concat(Directory.EnumerateFiles(pattern));
}
}
var excludes = Enumerable.Empty<string>();
foreach (string pattern in ExcludePattern)
{
string path = Path.GetDirectoryName(pattern);
string filePattern = Path.GetFileName(pattern);
if (!string.IsNullOrEmpty(path) && !string.IsNullOrEmpty(filePattern))
{
excludes = excludes.Concat(Directory.EnumerateFiles(path, filePattern));
}
else
{
excludes = excludes.Concat(Directory.EnumerateFiles(pattern));
}
}
var buildStepToWait = new List<BuildStep>();
lock (Files)
{
Files = files.Where(x => !excludes.Contains(x));
foreach (string file in Files)
{
executeContext.Variables["FILE"] = file;
var fileBuildStep = Template.Clone();
((List<BuildStep>)Steps).Add(fileBuildStep);
buildStepToWait.Add(fileBuildStep);
executeContext.ScheduleBuildStep(fileBuildStep);
}
}
await CompleteCommands(executeContext, buildStepToWait);
return ResultStatus.Successful;
}
public override void Clean(IExecuteContext executeContext, BuilderContext builderContext, bool deleteOutput)
{
// try to retrieve result from one of the object store
var commandHash = Command.ComputeCommandHash(executeContext);
// If there was an error computing the hash, early exit
if (commandHash == ObjectId.Empty)
{
return;
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
var commandResultEntries = new ListStore<CommandResultEntry>(commandResultsFileStream) { AutoLoadNewValues = false };
commandResultEntries.LoadNewValues();
commandResultsFileStream.Close();
CommandResultEntry matchingResult = FindMatchingResult(executeContext, commandResultEntries.GetValues());
if (matchingResult != null)
{
if (deleteOutput)
{
foreach (KeyValuePair<ObjectUrl, ObjectId> outputObject in matchingResult.OutputObjects)
{
switch (outputObject.Key.Type)
{
case UrlType.File:
try
{
if (File.Exists(outputObject.Key.Path))
File.Delete(outputObject.Key.Path);
}
catch (Exception)
{
executeContext.Logger.Error("Unable to delete file: " + outputObject.Key.Path);
}
break;
case UrlType.ContentLink:
case UrlType.Content:
executeContext.ResultMap.Delete(outputObject.Value);
break;
}
}
}
foreach (CommandBuildStep spawnedStep in matchingResult.SpawnedCommands.Select(spawnedCommand => new CommandBuildStep(spawnedCommand)))
{
spawnedStep.Clean(executeContext, builderContext, deleteOutput);
}
}
executeContext.ResultMap.Delete(commandHash);
}
public async override Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List<BuildStep>();
while (true)
{
// interrupt the build if cancellation is required.
if (executeContext.CancellationTokenSource.Token.IsCancellationRequested)
return ResultStatus.Cancelled;
// Clean completed build steps
for (int index = buildStepsToWait.Count - 1; index >= 0; index--)
{
var buildStepToWait = buildStepsToWait[index];
if (buildStepToWait.Processed)
buildStepsToWait.RemoveAt(index);
}
// wait for a task to complete
if (buildStepsToWait.Count >= MaxParallelSteps)
await CompleteOneBuildStep(executeContext, buildStepsToWait);
// Should we check for all tasks or only high priority tasks? (priority < 0)
bool checkOnlyForHighPriorityTasks = buildStepsToWait.Count >= MaxParallelSteps;
// Transform item into build step
var buildStep = buildStepProvider.GetNextBuildStep(checkOnlyForHighPriorityTasks ? -1 : int.MaxValue);
// No job => passively wait
if (buildStep == null)
{
newWorkAvailable.WaitOne();
continue;
}
// Safeguard if the provided build step is already processed
if(buildStep.Processed)
continue;
if (buildStep is WaitBuildStep)
{
throw new InvalidOperationException("WaitBuildStep are not supported as direct child of DynamicBuildStep");
}
// Schedule build step
executeContext.ScheduleBuildStep(buildStep);
buildStepsToWait.Add(buildStep);
}
}
/// <inheritdoc/>
public override Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var steps = new List<BuildStep>();
var urlRoot = originalSourcePath.GetParent();
var stream = new MemoryStream(Encoding.UTF8.GetBytes(((EffectLogAsset)assetItem.Asset).Text));
using (var recordedEffectCompile = new EffectLogStore(stream))
{
recordedEffectCompile.LoadNewValues();
foreach (var entry in recordedEffectCompile.GetValues())
{
steps.Add(EffectCompileCommand.FromRequest(context, assetItem.Package, urlRoot, entry.Key));
}
}
Steps = steps;
return base.Execute(executeContext, builderContext);
}
public override async Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
Steps = new List<BuildStep>();
var urls = Enumerable.Empty<string>();
BuildStep parentList = Parent;
while (!(parentList is ListBuildStep))
{
parentList = parentList.Parent;
}
var parentListBuildStep = (ListBuildStep)parentList;
foreach (string tag in SearchTags)
{
urls = urls.Concat(parentListBuildStep.OutputObjects.Where(x => x.Value.Tags.Contains(tag)).Select(x => x.Key.ToString()));
}
var buildStepToWait = new List<BuildStep>();
lock (Urls)
{
Urls = urls;
foreach (string url in Urls)
{
executeContext.Variables["URL"] = url;
var fileBuildStep = Template.Clone();
((List<BuildStep>)Steps).Add(fileBuildStep);
buildStepToWait.Add(fileBuildStep);
executeContext.ScheduleBuildStep(fileBuildStep);
}
}
await CompleteCommands(executeContext, buildStepToWait);
return ResultStatus.Successful;
}
private async Task <ResultStatus> StartCommand(IExecuteContext executeContext, ListStore <CommandResultEntry> commandResultEntries, BuilderContext builderContext)
{
var logger = executeContext.Logger;
//await Scheduler.Yield();
ResultStatus status;
using (commandResultEntries)
{
logger.Debug("Starting command {0}...", Command.ToString());
// Creating the CommandResult object
var commandContext = new LocalCommandContext(executeContext, this, builderContext);
// Actually processing the command
if (Command.ShouldSpawnNewProcess() && builderContext.MaxParallelProcesses > 0)
{
while (!builderContext.CanSpawnParallelProcess())
{
await Task.Delay(1, Command.CancellationToken);
}
var address = "net.pipe://localhost/" + Guid.NewGuid();
var arguments = string.Format("--slave=\"{0}\" --build-path=\"{1}\" --profile=\"{2}\"", address, builderContext.BuildPath, builderContext.BuildProfile);
var startInfo = new ProcessStartInfo
{
FileName = builderContext.SlaveBuilderPath,
Arguments = arguments,
WorkingDirectory = Environment.CurrentDirectory,
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
// Start WCF pipe for communication with process
var processBuilderRemote = new ProcessBuilderRemote(commandContext, Command, builderContext.Parameters);
var host = new ServiceHost(processBuilderRemote);
host.AddServiceEndpoint(typeof(IProcessBuilderRemote), new NetNamedPipeBinding(NetNamedPipeSecurityMode.None)
{
MaxReceivedMessageSize = int.MaxValue
}, address);
host.Open();
var output = new List <string>();
var process = new Process {
StartInfo = startInfo
};
process.Start();
process.OutputDataReceived += (_, args) => LockProcessAndAddDataToList(process, output, args);
process.ErrorDataReceived += (_, args) => LockProcessAndAddDataToList(process, output, args);
process.BeginOutputReadLine();
process.BeginErrorReadLine();
// Attach debugger to newly created process
// Add a reference to EnvDTE80 in the csproj and uncomment this (and also the Thread.Sleep in BuildEngineCmmands), then start the master process without debugger to attach to a slave.
//var dte = (EnvDTE80.DTE2)System.Runtime.InteropServices.Marshal.GetActiveObject("VisualStudio.DTE.11.0");
//foreach (EnvDTE.Process dteProcess in dte.Debugger.LocalProcesses)
//{
// if (dteProcess.ProcessID == process.Id)
// {
// dteProcess.Attach();
// dte.Debugger.CurrentProcess = dteProcess;
// }
//}
Task[] tasksToWait = null;
while (!process.HasExited)
{
Thread.Sleep(1);
lock (spawnedCommandsToWait)
{
if (spawnedCommandsToWait.Count > 0)
{
tasksToWait = spawnedCommandsToWait.ToArray();
spawnedCommandsToWait.Clear();
}
}
if (tasksToWait != null)
{
await Task.WhenAll(tasksToWait);
tasksToWait = null;
}
}
host.Close();
builderContext.NotifyParallelProcessEnded();
if (process.ExitCode != 0)
{
logger.Debug("Remote command crashed with output:\n{0}", string.Join(Environment.NewLine, output));
}
if (processBuilderRemote.Result != null)
{
// Register results back locally
foreach (var outputObject in processBuilderRemote.Result.OutputObjects)
{
commandContext.RegisterOutput(outputObject.Key, outputObject.Value);
}
// Register log messages
foreach (var logMessage in processBuilderRemote.Result.LogMessages)
{
commandContext.Logger.Log(logMessage);
}
// Register tags
foreach (var tag in processBuilderRemote.Result.TagSymbols)
{
TagSymbol tagSymbol;
// Resolve tag locally
if (!Command.TagSymbols.TryGetValue(tag.Value, out tagSymbol))
{
// Should we ignore silently? (with warning)
throw new InvalidOperationException("Could not find tag symbol.");
}
commandContext.AddTag(tag.Key, tagSymbol);
}
}
status = Command.CancellationToken.IsCancellationRequested ? ResultStatus.Cancelled : (process.ExitCode == 0 ? ResultStatus.Successful : ResultStatus.Failed);
}
else
{
Command.PreCommand(commandContext);
if (!Command.BasePreCommandCalled)
{
throw new InvalidOperationException("base.PreCommand not called in command " + Command);
}
try
{
// Merge results from prerequisites
// TODO: This will prevent us from overwriting this asset with different content as it will result in a write conflict
// At some point we _might_ want to get rid of WaitBuildStep/ListBuildStep system and write a fully stateless input/output-based system; probably need further discussions
var fileProvider = ContentManager.FileProvider;
if (fileProvider != null)
{
var assetIndexMap = fileProvider.AssetIndexMap;
foreach (var prerequisiteStep in PrerequisiteSteps)
{
foreach (var output in prerequisiteStep.OutputObjectIds)
{
assetIndexMap[output.Key.Path] = output.Value;
}
}
}
status = await Command.DoCommand(commandContext);
}
catch (Exception ex)
{
executeContext.Logger.Error("Exception in command " + this + ": " + ex);
status = ResultStatus.Failed;
}
Command.PostCommand(commandContext, status);
if (!Command.BasePostCommandCalled)
{
throw new InvalidOperationException("base.PostCommand not called in command " + Command);
}
}
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
{
throw new InvalidDataException("The command " + Command + " returned ResultStatus.NotProcessed after completion.");
}
// Registering the result to the build cache
RegisterCommandResult(commandResultEntries, commandContext.ResultEntry, status);
}
return(status);
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore <CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
bool monitorExited = false;
var status = ResultStatus.NotProcessed;
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult = null;
try
{
ObjectId commandHash;
{
// try to retrieve result from one of the object store
commandHash = Command.ComputeCommandHash(executeContext);
// Early exit if the hash of the command failed
if (commandHash == ObjectId.Empty)
{
return(ResultStatus.Failed);
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore <CommandResultEntry>(commandResultsFileStream)
{
AutoLoadNewValues = false
};
commandResultEntries.LoadNewValues();
}
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} delayed because it is currently running...", Command.ToString());
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} scheduled...", Command.ToString());
// Register the cancel callback
var cancellationTokenSource = executeContext.CancellationTokenSource;
cancellationTokenSource.Token.Register(x => ((Command)x).Cancel(), Command);
Command.CancellationToken = cancellationTokenSource.Token;
try
{
status = await StartCommand(executeContext, commandResultEntries, builderContext);
}
finally
{
// Restore cancellation token (to avoid memory leak due to previous CancellationToken.Register
Command.CancellationToken = CancellationToken.None;
}
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
}
finally
{
if (!monitorExited)
{
Monitor.Exit(executeContext);
}
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// Replicate triggered builds
Debug.Assert(SpawnedStepsList.Count == 0);
foreach (Command spawnedCommand in matchingResult.SpawnedCommands)
{
var spawnedStep = new CommandBuildStep(spawnedCommand);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
}
// Re-output command log messages
foreach (var message in matchingResult.LogMessages)
{
executeContext.Logger.Log(message);
}
// Wait for all build steps to complete.
// TODO: Ideally, we should store and replicate the behavior of the command that spawned it
// (wait if it used ScheduleAndExecute, don't wait if it used RegisterSpawnedCommandWithoutScheduling)
await Task.WhenAll(SpawnedSteps.Select(x => x.ExecutedAsync()));
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return(status);
}
public static BuildResultCode BuildSlave(BuilderOptions options)
{
// Mount build path
((FileSystemProvider)VirtualFileSystem.ApplicationData).ChangeBasePath(options.BuildDirectory);
PrepareDatabases(options);
try
{
VirtualFileSystem.CreateDirectory("/data/");
VirtualFileSystem.CreateDirectory("/data/db/");
}
catch (Exception)
{
throw new OptionException("Invalid Build database path", "database");
}
// Open WCF channel with master builder
var namedPipeBinding = new NetNamedPipeBinding(NetNamedPipeSecurityMode.None)
{
SendTimeout = TimeSpan.FromSeconds(300.0)
};
var processBuilderRemote = ChannelFactory <IProcessBuilderRemote> .CreateChannel(namedPipeBinding, new EndpointAddress(options.SlavePipe));
try
{
RegisterRemoteLogger(processBuilderRemote);
// Create scheduler
var scheduler = new Scheduler();
var status = ResultStatus.NotProcessed;
// Schedule command
string buildPath = options.BuildDirectory;
Logger logger = options.Logger;
MicroThread microthread = scheduler.Add(async() =>
{
// Deserialize command and parameters
Command command = processBuilderRemote.GetCommandToExecute();
BuildParameterCollection parameters = processBuilderRemote.GetBuildParameters();
// Run command
var inputHashes = new DictionaryStore <InputVersionKey, ObjectId>(VirtualFileSystem.OpenStream("/data/db/InputHashes", VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite));
var builderContext = new BuilderContext(buildPath, inputHashes, parameters, 0, null);
var commandContext = new RemoteCommandContext(processBuilderRemote, command, builderContext, logger);
command.PreCommand(commandContext);
status = await command.DoCommand(commandContext);
command.PostCommand(commandContext, status);
// Returns result to master builder
processBuilderRemote.RegisterResult(commandContext.ResultEntry);
});
while (true)
{
scheduler.Run();
// Exit loop if no more micro threads
lock (scheduler.MicroThreads)
{
if (!scheduler.MicroThreads.Any())
{
break;
}
}
Thread.Sleep(0);
}
// Rethrow any exception that happened in microthread
if (microthread.Exception != null)
{
options.Logger.Fatal(microthread.Exception.ToString());
return(BuildResultCode.BuildError);
}
if (status == ResultStatus.Successful || status == ResultStatus.NotTriggeredWasSuccessful)
{
return(BuildResultCode.Successful);
}
return(BuildResultCode.BuildError);
}
finally
{
// Close WCF channel
// ReSharper disable SuspiciousTypeConversion.Global
((IClientChannel)processBuilderRemote).Close();
// ReSharper restore SuspiciousTypeConversion.Global
}
}
/// <summary> /// Execute the BuildStep, usually resulting in scheduling tasks in the scheduler /// </summary> /// <param name="executeContext">The execute context</param> /// <param name="builderContext">The builder context</param> /// <returns>A task returning <see cref="ResultStatus"/> indicating weither the execution has successed or failed.</returns> public abstract Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext);
public ExecuteContext(Builder builder, BuilderContext builderContext, BuildStep buildStep, Logger logger)
{
Logger = logger;
this.builderContext = builderContext;
this.builder = builder;
this.buildStep = buildStep;
buildTransaction = new BuildTransaction(null, buildStep.GetOutputObjectsGroups());
}
/// <summary>
/// Runs this instance.
/// </summary>
public BuildResultCode Run(Mode mode, bool writeIndexFile = true, bool enableMonitor = true)
{
// When we setup the database ourself we have to take responsibility to close it after
var shouldCloseDatabase = ObjectDatabase == null;
OpenObjectDatabase(buildPath, indexName);
PreRun();
runMode = mode;
if (IsRunning)
{
throw new InvalidOperationException("An instance of this Builder is already running.");
}
// reset build cache from previous build run
var parameters = new BuildParameterCollection();
cancellationTokenSource = new CancellationTokenSource();
Cancelled = false;
IsRunning = true;
DisableCompressionIds.Clear();
// Reseting result map
var inputHashes = FileVersionTracker.GetDefault();
{
var builderContext = new BuilderContext(buildPath, buildProfile, inputHashes, parameters, MaxParallelProcesses, SlaveBuilderPath);
resultMap = ObjectDatabase;
scheduler = new Scheduler();
if (enableMonitor)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId));
foreach (var monitorPipeName in MonitorPipeNames)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId, monitorPipeName));
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Start();
}
}
// Schedule the build
ScheduleBuildStep(builderContext, null, Root, InitialVariables);
// Create threads
var threads = Enumerable.Range(0, ThreadCount).Select(x => new Thread(SafeAction.Wrap(RunUntilEnd))
{
IsBackground = true
}).ToArray();
// Start threads
int threadId = 0;
foreach (var thread in threads)
{
thread.Name = (BuilderName ?? "Builder") + " worker thread " + (++threadId);
thread.Start();
}
// Wait for all threads to finish
foreach (var thread in threads)
{
thread.Join();
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Finish();
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Join();
}
}
threadMonitors.Clear();
BuildResultCode result;
if (runMode == Mode.Build)
{
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error("Build cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure.");
Logger.Error("Build failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Info($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure.");
Logger.Info("Build is successful.");
result = BuildResultCode.Successful;
}
}
else
{
string modeName;
switch (runMode)
{
case Mode.Clean:
modeName = "Clean";
break;
case Mode.CleanAndDelete:
modeName = "Clean-and-delete";
break;
default:
throw new InvalidOperationException("Builder executed in unknown mode.");
}
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error(modeName + " has been cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error(modeName + " has failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Error(modeName + " has been successfully completed.");
result = BuildResultCode.Successful;
}
}
scheduler = null;
resultMap = null;
IsRunning = false;
if (shouldCloseDatabase)
{
CloseObjectDatabase();
}
return(result);
}
public override Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
// Dependencies are done in ListBuildStep, Execute() should never be called directly.
return Task.FromResult(ResultStatus.Failed);
}
public async override Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List <BuildStep>();
while (true)
{
// interrupt the build if cancellation is required.
if (executeContext.CancellationTokenSource.Token.IsCancellationRequested)
{
return(ResultStatus.Cancelled);
}
// Clean completed build steps
for (int index = buildStepsToWait.Count - 1; index >= 0; index--)
{
var buildStepToWait = buildStepsToWait[index];
if (buildStepToWait.Processed)
{
buildStepsToWait.RemoveAt(index);
}
}
// wait for a task to complete
if (buildStepsToWait.Count >= MaxParallelSteps)
{
await CompleteOneBuildStep(executeContext, buildStepsToWait);
}
// Should we check for all tasks or only high priority tasks? (priority < 0)
bool checkOnlyForHighPriorityTasks = buildStepsToWait.Count >= MaxParallelSteps;
// Transform item into build step
var buildStep = buildStepProvider.GetNextBuildStep(checkOnlyForHighPriorityTasks ? -1 : int.MaxValue);
// No job => passively wait
if (buildStep == null)
{
newWorkAvailable.WaitOne();
continue;
}
// Safeguard if the provided build step is already processed
if (buildStep.Processed)
{
continue;
}
if (buildStep is WaitBuildStep)
{
throw new InvalidOperationException("WaitBuildStep are not supported as direct child of DynamicBuildStep");
}
// Schedule build step
executeContext.ScheduleBuildStep(buildStep);
buildStepsToWait.Add(buildStep);
}
}
public RemoteCommandContext(IProcessBuilderRemote processBuilderRemote, Command command, BuilderContext builderContext, Logger logger) : base(command, builderContext)
{
this.processBuilderRemote = processBuilderRemote;
this.logger = logger;
}
/// <summary>
/// Clean the build, deleting the command cache which is used to determine wheither a command has already been executed, and deleting the output objects if asked.
/// </summary>
/// <param name="executeContext">The execute context</param>
/// <param name="builderContext">The builder context</param>
/// <param name="deleteOutput">if true, every output object is also deleted, in addition of the command cache.</param>
public virtual void Clean(IExecuteContext executeContext, BuilderContext builderContext, bool deleteOutput)
{
// By default, do the same as Execute. This will apply for flow control steps (lists, enumerations...)
// Specific implementation exists for CommandBuildStep
Execute(executeContext, builderContext);
}
/// <summary> /// Execute the BuildStep, usually resulting in scheduling tasks in the scheduler /// </summary> /// <param name="executeContext">The execute context</param> /// <param name="builderContext">The builder context</param> /// <returns>A task returning <see cref="ResultStatus"/> indicating weither the execution has successed or failed.</returns> public abstract Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext);
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
Steps = new List <BuildStep>();
var urls = Enumerable.Empty <string>();
BuildStep parentList = Parent;
while (!(parentList is ListBuildStep))
{
parentList = parentList.Parent;
}
var parentListBuildStep = (ListBuildStep)parentList;
foreach (string tag in SearchTags)
{
urls = urls.Concat(parentListBuildStep.OutputObjects.Where(x => x.Value.Tags.Contains(tag)).Select(x => x.Key.ToString()));
}
var buildStepToWait = new List <BuildStep>();
lock (Urls)
{
Urls = urls;
foreach (string url in Urls)
{
executeContext.Variables["URL"] = url;
var fileBuildStep = Template.Clone();
((List <BuildStep>)Steps).Add(fileBuildStep);
buildStepToWait.Add(fileBuildStep);
executeContext.ScheduleBuildStep(fileBuildStep);
}
}
await CompleteCommands(executeContext, buildStepToWait);
return(ResultStatus.Successful);
}
public override Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
// Dependencies are done in ListBuildStep, Execute() should never be called directly.
return(Task.FromResult(ResultStatus.Failed));
}
protected CommandContextBase(Command command, BuilderContext builderContext)
{
CurrentCommand = command;
ResultEntry = new CommandResultEntry();
}
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary <string, string> variables)
{
if (buildStep.ExecutionId == 0)
{
if (buildStep.Parent != null && buildStep.Parent != instigator)
{
throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent");
}
if (buildStep.Parent == null)
{
buildStep.Parent = instigator;
}
// Compute content dependencies before scheduling the build
GenerateDependencies(buildStep);
// TODO: Big review of the log infrastructure of CompilerApp & BuildEngine!
// Create a logger that redirects to various places (BuildStep.Logger, timestampped log, global log, etc...)
var buildStepLogger = new BuildStepLogger(buildStep, Logger, startTime);
var logger = (Logger)buildStepLogger;
// Apply user-registered callbacks to the logger
buildStep.TransformExecuteContextLogger?.Invoke(ref logger);
// Create execute context
var executeContext = new ExecuteContext(this, builderContext, buildStep, logger)
{
Variables = new Dictionary <string, string>(variables)
};
//buildStep.ExpandStrings(executeContext);
if (runMode == Mode.Build)
{
MicroThread microThread = scheduler.Create();
// Find priority from this build step, or one of its parent.
var buildStepPriority = buildStep;
while (buildStepPriority != null)
{
if (buildStepPriority.Priority.HasValue)
{
microThread.Priority = buildStepPriority.Priority.Value;
break;
}
buildStepPriority = buildStepPriority.Parent;
}
buildStep.ExecutionId = microThread.Id;
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.RegisterBuildStep(buildStep, buildStepLogger.StepLogger);
}
microThread.Name = buildStep.ToString();
// Default:
// Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished.
// Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep.
// This should also reduce memory usage, and might improve cache coherency as well.
microThread.ScheduleMode = ScheduleMode.First;
microThread.Start(async() =>
{
// Wait for prerequisites
await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray());
// Check for failed prerequisites
var status = ResultStatus.NotProcessed;
if (buildStep.ArePrerequisitesSuccessful)
{
try
{
IEnumerable <IDictionary <ObjectUrl, OutputObject> > outputObjectsGroups = executeContext.GetOutputObjectsGroups();
MicrothreadLocalDatabases.MountDatabase(outputObjectsGroups);
// Execute
status = await buildStep.Execute(executeContext, builderContext);
}
catch (TaskCanceledException e)
{
// Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again
executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully.");
executeContext.Logger.Warning(e.Message);
status = ResultStatus.Cancelled;
}
catch (Exception e)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + e);
status = ResultStatus.Failed;
}
finally
{
MicrothreadLocalDatabases.UnmountDatabase();
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
{
throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion.");
}
}
if (microThread.Exception != null)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception);
status = ResultStatus.Failed;
}
}
else
{
status = ResultStatus.NotTriggeredPrerequisiteFailed;
}
//if (completedTask.IsCanceled)
//{
// completedStep.Status = ResultStatus.Cancelled;
//}
var logType = LogMessageType.Info;
string logText = null;
switch (status)
{
case ResultStatus.Successful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Failed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredPrerequisiteFailed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed of previous failed prerequisites.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Cancelled:
logType = LogMessageType.Warning;
logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredWasSuccessful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotProcessed:
throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled");
}
if (logText != null)
{
var logMessage = new LogMessage(null, logType, logText);
executeContext.Logger.Log(logMessage);
}
buildStep.RegisterResult(executeContext, status);
stepCounter.AddStepResult(status);
});
}
else
{
buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete);
}
}
}
private BuildResultCode BuildSlave()
{
// Mount build path
((FileSystemProvider)VirtualFileSystem.ApplicationData).ChangeBasePath(builderOptions.BuildDirectory);
PrepareDatabases();
VirtualFileSystem.CreateDirectory("/data/");
VirtualFileSystem.CreateDirectory("/data/db/");
// Open WCF channel with master builder
var namedPipeBinding = new NetNamedPipeBinding(NetNamedPipeSecurityMode.None) { SendTimeout = TimeSpan.FromSeconds(300.0) };
var processBuilderRemote = ChannelFactory<IProcessBuilderRemote>.CreateChannel(namedPipeBinding, new EndpointAddress(builderOptions.SlavePipe));
try
{
RegisterRemoteLogger(processBuilderRemote);
// Create scheduler
var scheduler = new Scheduler();
var status = ResultStatus.NotProcessed;
// Schedule command
string buildPath = builderOptions.BuildDirectory;
string buildProfile = builderOptions.BuildProfile;
Builder.SetupBuildPath(buildPath);
var logger = builderOptions.Logger;
MicroThread microthread = scheduler.Add(async () =>
{
// Deserialize command and parameters
Command command = processBuilderRemote.GetCommandToExecute();
BuildParameterCollection parameters = processBuilderRemote.GetBuildParameters();
// Run command
var inputHashes = FileVersionTracker.GetDefault();
var builderContext = new BuilderContext(buildPath, buildProfile, inputHashes, parameters, 0, null);
var commandContext = new RemoteCommandContext(processBuilderRemote, command, builderContext, logger);
IndexFileCommand.MountDatabase(commandContext.GetOutputObjectsGroups());
command.PreCommand(commandContext);
status = await command.DoCommand(commandContext);
command.PostCommand(commandContext, status);
// Returns result to master builder
processBuilderRemote.RegisterResult(commandContext.ResultEntry);
});
while (true)
{
scheduler.Run();
// Exit loop if no more micro threads
lock (scheduler.MicroThreads)
{
if (!scheduler.MicroThreads.Any())
break;
}
Thread.Sleep(0);
}
// Rethrow any exception that happened in microthread
if (microthread.Exception != null)
{
builderOptions.Logger.Fatal(microthread.Exception.ToString());
return BuildResultCode.BuildError;
}
if (status == ResultStatus.Successful || status == ResultStatus.NotTriggeredWasSuccessful)
return BuildResultCode.Successful;
return BuildResultCode.BuildError;
}
finally
{
// Close WCF channel
// ReSharper disable SuspiciousTypeConversion.Global
((IClientChannel)processBuilderRemote).Close();
// ReSharper restore SuspiciousTypeConversion.Global
}
}
public RemoteCommandContext(IProcessBuilderRemote processBuilderRemote, Command command, BuilderContext builderContext, LoggerResult logger)
: base(command, builderContext)
{
this.processBuilderRemote = processBuilderRemote;
this.logger = logger;
}
/// <summary>
/// Runs this instance.
/// </summary>
public BuildResultCode Run(Mode mode, bool writeIndexFile = true, bool enableMonitor = true)
{
runMode = mode;
if (IsRunning)
{
throw new InvalidOperationException("An instance of this Builder is already running.");
}
// reset build cache from previous build run
var parameters = new BuildParameterCollection();
cancellationTokenSource = new CancellationTokenSource();
Cancelled = false;
IsRunning = true;
DisableCompressionIds.Clear();
// Reseting result map
var inputHashes = FileVersionTracker.GetDefault();
{
var builderContext = new BuilderContext(buildPath, buildProfile, inputHashes, parameters, MaxParallelProcesses, SlaveBuilderPath);
if (!string.IsNullOrWhiteSpace(MetadataDatabaseDirectory))
{
var metadataProvider = new QueryMetadataProvider();
if (metadataProvider.Open(Path.Combine(MetadataDatabaseDirectory, QueryMetadataProvider.DefaultDatabaseFilename), false))
{
builderContext.MetadataProvider = metadataProvider;
}
}
resultMap = IndexFileCommand.ObjectDatabase;
scheduler = new Scheduler();
if (enableMonitor)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId));
foreach (var monitorPipeName in MonitorPipeNames)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId, monitorPipeName));
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Start();
}
}
ScheduleBuildStep(builderContext, null, Root, InitialVariables);
// Create threads
var threads = Enumerable.Range(0, ThreadCount).Select(x => new Thread(SafeAction.Wrap(RunUntilEnd))
{
IsBackground = true
}).ToArray();
// Start threads
int threadId = 0;
foreach (var thread in threads)
{
thread.Name = (BuilderName ?? "Builder") + " worker thread " + (++threadId);
thread.Start();
}
// Wait for all threads to finish
foreach (var thread in threads)
{
thread.Join();
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Finish();
}
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.Join();
}
}
threadMonitors.Clear();
BuildResultCode result;
if (runMode == Mode.Build)
{
Logger.Info("");
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error("Build cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error("Build finished in {0} steps. Command results: {1} succeeded, {2} up-to-date, {3} failed, {4} not triggered due to previous failure.",
stepCounter.Total, stepCounter.Get(ResultStatus.Successful), stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful),
stepCounter.Get(ResultStatus.Failed), stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed));
Logger.Error("Build failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Info("Build finished in {0} steps. Command results: {1} succeeded, {2} up-to-date, {3} failed, {4} not triggered due to previous failure.",
stepCounter.Total, stepCounter.Get(ResultStatus.Successful), stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful),
stepCounter.Get(ResultStatus.Failed), stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed));
Logger.Info("Build is successful.");
result = BuildResultCode.Successful;
}
}
else
{
// Clean input hashes file
if (VirtualFileSystem.FileExists(InputHashesFileFullPath))
{
try
{
VirtualFileSystem.FileDelete(InputHashesFileFullPath);
}
catch (IOException)
{
return(BuildResultCode.BuildError);
}
}
string modeName;
switch (runMode)
{
case Mode.Clean:
modeName = "Clean";
break;
case Mode.CleanAndDelete:
modeName = "Clean-and-delete";
break;
default:
throw new InvalidOperationException("Builder executed in unknown mode.");
}
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error(modeName + " has been cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error(modeName + " has failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Error(modeName + " has been successfully completed.");
result = BuildResultCode.Successful;
}
}
scheduler = null;
resultMap = null;
IsRunning = false;
return(result);
}
public ExecuteContext(Builder builder, BuilderContext builderContext, BuildStep buildStep)
{
Logger = new BuildStepLogger(buildStep, builder.Logger, builder.startTime);
this.builderContext = builderContext;
this.builder = builder;
this.buildStep = buildStep;
buildTransaction = new BuildTransaction(null, buildStep.GetOutputObjectsGroups());
}
public override async Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore<CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
bool monitorExited = false;
var status = ResultStatus.NotProcessed;
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult = null;
try
{
ObjectId commandHash;
{
// try to retrieve result from one of the object store
commandHash = Command.ComputeCommandHash(executeContext);
// Early exit if the hash of the command failed
if (commandHash == ObjectId.Empty)
{
return ResultStatus.Failed;
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore<CommandResultEntry>(commandResultsFileStream) { AutoLoadNewValues = false };
commandResultEntries.LoadNewValues();
}
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} delayed because it is currently running...", Command.ToString());
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} scheduled...", Command.ToString());
status = await StartCommand(executeContext, commandResultEntries, builderContext);
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
}
finally
{
if (!monitorExited)
{
Monitor.Exit(executeContext);
}
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// Replicate triggered builds
Debug.Assert(SpawnedStepsList.Count == 0);
foreach (Command spawnedCommand in matchingResult.SpawnedCommands)
{
var spawnedStep = new CommandBuildStep(spawnedCommand);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
}
// Re-output command log messages
foreach (var message in matchingResult.LogMessages)
{
executeContext.Logger.Log(message);
}
// Wait for all build steps to complete.
// TODO: Ideally, we should store and replicate the behavior of the command that spawned it
// (wait if it used ScheduleAndExecute, don't wait if it used RegisterSpawnedCommandWithoutScheduling)
await Task.WhenAll(SpawnedSteps.Select(x => x.ExecutedAsync()));
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return status;
}
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary<string, string> variables)
{
if (buildStep.ExecutionId == 0)
{
if (buildStep.Parent != null && buildStep.Parent != instigator)
throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent");
if (buildStep.Parent == null)
{
buildStep.Parent = instigator;
}
// Compute content dependencies before scheduling the build
GenerateDependencies(buildStep);
var executeContext = new ExecuteContext(this, builderContext, buildStep) { Variables = new Dictionary<string, string>(variables) };
//buildStep.ExpandStrings(executeContext);
if (runMode == Mode.Build)
{
MicroThread microThread = scheduler.Create();
// Find priority from this build step, or one of its parent.
var buildStepPriority = buildStep;
while (buildStepPriority != null)
{
if (buildStepPriority.Priority.HasValue)
{
microThread.Priority = buildStepPriority.Priority.Value;
break;
}
buildStepPriority = buildStepPriority.Parent;
}
buildStep.ExecutionId = microThread.Id;
foreach (var threadMonitor in threadMonitors)
{
threadMonitor.RegisterBuildStep(buildStep, ((BuildStepLogger)executeContext.Logger).StepLogger);
}
microThread.Name = buildStep.ToString();
// Default:
// Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished.
// Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep.
// This should also reduce memory usage, and might improve cache coherency as well.
microThread.ScheduleMode = ScheduleMode.First;
microThread.Start(async () =>
{
// Wait for prerequisites
await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray());
// Check for failed prerequisites
var status = ResultStatus.NotProcessed;
if (buildStep.ArePrerequisitesSuccessful)
{
try
{
IndexFileCommand.MountDatabase(executeContext.GetOutputObjectsGroups());
// Execute
status = await buildStep.Execute(executeContext, builderContext);
}
catch (TaskCanceledException e)
{
// Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again
executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully.");
executeContext.Logger.Warning(e.Message);
status = ResultStatus.Cancelled;
}
catch (Exception e)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + e);
status = ResultStatus.Failed;
}
finally
{
IndexFileCommand.UnmountDatabase();
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion.");
}
if (microThread.Exception != null)
{
executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception);
status = ResultStatus.Failed;
}
}
else
{
status = ResultStatus.NotTriggeredPrerequisiteFailed;
}
//if (completedTask.IsCanceled)
//{
// completedStep.Status = ResultStatus.Cancelled;
//}
var logType = LogMessageType.Info;
string logText = null;
switch (status)
{
case ResultStatus.Successful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Failed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredPrerequisiteFailed:
logType = LogMessageType.Error;
logText = "BuildStep {0} failed of previous failed prerequisites.".ToFormat(buildStep.ToString());
break;
case ResultStatus.Cancelled:
logType = LogMessageType.Warning;
logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotTriggeredWasSuccessful:
logType = LogMessageType.Verbose;
logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString());
break;
case ResultStatus.NotProcessed:
throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled");
}
if (logText != null)
{
var logMessage = new LogMessage(buildStep.Module, logType, logText);
Logger.Log(logMessage);
}
buildStep.RegisterResult(executeContext, status);
stepCounter.AddStepResult(status);
});
}
else
{
buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete);
}
}
}
/// <summary>
/// Runs this instance.
/// </summary>
public BuildResultCode Run(Mode mode, bool writeIndexFile = true, bool enableMonitor = true)
{
// When we setup the database ourself we have to take responsibility to close it after
var shouldCloseDatabase = IndexFileCommand.ObjectDatabase == null;
SetupBuildPath(buildPath, indexName);
PreRun();
runMode = mode;
if (IsRunning)
throw new InvalidOperationException("An instance of this Builder is already running.");
// reset build cache from previous build run
var parameters = new BuildParameterCollection();
cancellationTokenSource = new CancellationTokenSource();
Cancelled = false;
IsRunning = true;
DisableCompressionIds.Clear();
// Reseting result map
var inputHashes = FileVersionTracker.GetDefault();
{
var builderContext = new BuilderContext(buildPath, buildProfile, inputHashes, parameters, MaxParallelProcesses, SlaveBuilderPath);
resultMap = IndexFileCommand.ObjectDatabase;
scheduler = new Scheduler();
if (enableMonitor)
{
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId));
foreach (var monitorPipeName in MonitorPipeNames)
threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId, monitorPipeName));
foreach (var threadMonitor in threadMonitors)
threadMonitor.Start();
}
// Schedule the build
ScheduleBuildStep(builderContext, null, Root, InitialVariables);
// Create threads
var threads = Enumerable.Range(0, ThreadCount).Select(x => new Thread(SafeAction.Wrap(RunUntilEnd)) { IsBackground = true }).ToArray();
// Start threads
int threadId = 0;
foreach (var thread in threads)
{
thread.Name = (BuilderName ?? "Builder") + " worker thread " + (++threadId);
thread.Start();
}
// Wait for all threads to finish
foreach (var thread in threads)
{
thread.Join();
}
foreach (var threadMonitor in threadMonitors)
threadMonitor.Finish();
foreach (var threadMonitor in threadMonitors)
threadMonitor.Join();
}
threadMonitors.Clear();
BuildResultCode result;
if (runMode == Mode.Build)
{
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error("Build cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error("Build finished in {0} steps. Command results: {1} succeeded, {2} up-to-date, {3} failed, {4} not triggered due to previous failure.",
stepCounter.Total, stepCounter.Get(ResultStatus.Successful), stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful),
stepCounter.Get(ResultStatus.Failed), stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed));
Logger.Error("Build failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Info("Build finished in {0} steps. Command results: {1} succeeded, {2} up-to-date, {3} failed, {4} not triggered due to previous failure.",
stepCounter.Total, stepCounter.Get(ResultStatus.Successful), stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful),
stepCounter.Get(ResultStatus.Failed), stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed));
Logger.Info("Build is successful.");
result = BuildResultCode.Successful;
}
}
else
{
string modeName;
switch (runMode)
{
case Mode.Clean:
modeName = "Clean";
break;
case Mode.CleanAndDelete:
modeName = "Clean-and-delete";
break;
default:
throw new InvalidOperationException("Builder executed in unknown mode.");
}
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Error(modeName + " has been cancelled.");
result = BuildResultCode.Cancelled;
}
else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0)
{
Logger.Error(modeName + " has failed.");
result = BuildResultCode.BuildError;
}
else
{
Logger.Error(modeName + " has been successfully completed.");
result = BuildResultCode.Successful;
}
}
scheduler = null;
resultMap = null;
IsRunning = false;
if (shouldCloseDatabase && IndexFileCommand.ObjectDatabase != null)
{
IndexFileCommand.ObjectDatabase.Dispose();
IndexFileCommand.ObjectDatabase = null;
}
return result;
}
private async Task<ResultStatus> StartCommand(IExecuteContext executeContext, ListStore<CommandResultEntry> commandResultEntries, BuilderContext builderContext)
{
var logger = executeContext.Logger;
// Register the cancel callback
var cancellationTokenSource = executeContext.CancellationTokenSource;
cancellationTokenSource.Token.Register(x => ((Command)x).Cancel(), Command);
Command.CancellationToken = cancellationTokenSource.Token;
//await Scheduler.Yield();
ResultStatus status;
using (commandResultEntries)
{
logger.Debug("Starting command {0}...", Command.ToString());
// Creating the CommandResult object
var commandContext = new LocalCommandContext(executeContext, this, builderContext);
// Actually processing the command
if (Command.ShouldSpawnNewProcess() && builderContext.MaxParallelProcesses > 0)
{
while (!builderContext.CanSpawnParallelProcess())
{
await Task.Delay(1, Command.CancellationToken);
}
var address = "net.pipe://localhost/" + Guid.NewGuid();
var arguments = string.Format("--slave=\"{0}\" --build-path=\"{1}\" --profile=\"{2}\"", address, builderContext.BuildPath, builderContext.BuildProfile);
var startInfo = new ProcessStartInfo
{
FileName = builderContext.SlaveBuilderPath,
Arguments = arguments,
WorkingDirectory = Environment.CurrentDirectory,
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
// Start WCF pipe for communication with process
var processBuilderRemote = new ProcessBuilderRemote(commandContext, Command, builderContext.Parameters);
var host = new ServiceHost(processBuilderRemote);
host.AddServiceEndpoint(typeof(IProcessBuilderRemote), new NetNamedPipeBinding(NetNamedPipeSecurityMode.None) { MaxReceivedMessageSize = int.MaxValue }, address);
host.Open();
var output = new List<string>();
var process = new Process { StartInfo = startInfo };
process.Start();
process.OutputDataReceived += (_, args) => LockProcessAndAddDataToList(process, output, args);
process.ErrorDataReceived += (_, args) => LockProcessAndAddDataToList(process, output, args);
process.BeginOutputReadLine();
process.BeginErrorReadLine();
// Attach debugger to newly created process
// Add a reference to EnvDTE80 in the csproj and uncomment this (and also the Thread.Sleep in BuildEngineCmmands), then start the master process without debugger to attach to a slave.
//var dte = (EnvDTE80.DTE2)System.Runtime.InteropServices.Marshal.GetActiveObject("VisualStudio.DTE.11.0");
//foreach (EnvDTE.Process dteProcess in dte.Debugger.LocalProcesses)
//{
// if (dteProcess.ProcessID == process.Id)
// {
// dteProcess.Attach();
// dte.Debugger.CurrentProcess = dteProcess;
// }
//}
Task[] tasksToWait = null;
while (!process.HasExited)
{
Thread.Sleep(1);
lock (spawnedCommandsToWait)
{
if (spawnedCommandsToWait.Count > 0)
{
tasksToWait = spawnedCommandsToWait.ToArray();
spawnedCommandsToWait.Clear();
}
}
if (tasksToWait != null)
{
await Task.WhenAll(tasksToWait);
tasksToWait = null;
}
}
host.Close();
builderContext.NotifyParallelProcessEnded();
if (process.ExitCode != 0)
{
logger.Debug("Remote command crashed with output:\n{0}", string.Join(Environment.NewLine, output));
}
if (processBuilderRemote.Result != null)
{
// Register results back locally
foreach (var outputObject in processBuilderRemote.Result.OutputObjects)
{
commandContext.RegisterOutput(outputObject.Key, outputObject.Value);
}
// Register log messages
foreach (var logMessage in processBuilderRemote.Result.LogMessages)
{
commandContext.Logger.Log(logMessage);
}
// Register tags
foreach (var tag in processBuilderRemote.Result.TagSymbols)
{
TagSymbol tagSymbol;
// Resolve tag locally
if (!Command.TagSymbols.TryGetValue(tag.Value, out tagSymbol))
{
// Should we ignore silently? (with warning)
throw new InvalidOperationException("Could not find tag symbol.");
}
commandContext.AddTag(tag.Key, tagSymbol);
}
}
status = Command.CancellationToken.IsCancellationRequested ? ResultStatus.Cancelled : (process.ExitCode == 0 ? ResultStatus.Successful : ResultStatus.Failed);
}
else
{
Command.PreCommand(commandContext);
if (!Command.BasePreCommandCalled)
throw new InvalidOperationException("base.PreCommand not called in command " + Command);
try
{
status = await Command.DoCommand(commandContext);
}
catch (Exception ex)
{
executeContext.Logger.Error("Exception in command " + this + ": " + ex);
status = ResultStatus.Failed;
}
Command.PostCommand(commandContext, status);
if (!Command.BasePostCommandCalled)
throw new InvalidOperationException("base.PostCommand not called in command " + Command);
}
// Ensure the command set at least the result status
if (status == ResultStatus.NotProcessed)
throw new InvalidDataException("The command " + Command + " returned ResultStatus.NotProcessed after completion.");
// Registering the result to the build cache
RegisterCommandResult(commandResultEntries, commandContext.ResultEntry, status);
}
return status;
}
public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = new ForwardingLoggerResult(executeContext.Logger);
Step = step;
}
public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = new ForwardingLoggerResult(executeContext.Logger);
Step = step;
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
var buildStepsToWait = new List <BuildStep>();
// Process prerequisites build steps first
if (PrerequisiteSteps.Count > 0)
{
await CompleteCommands(executeContext, PrerequisiteSteps.ToList());
}
foreach (var child in Steps)
{
// Wait for all the tasks before the WaitBuildStep to be finished
if (child is WaitBuildStep)
{
await CompleteCommands(executeContext, buildStepsToWait);
}
else
{
executeContext.ScheduleBuildStep(child);
buildStepsToWait.Add(child);
}
executedSteps.Add(child);
}
await CompleteCommands(executeContext, buildStepsToWait);
return(ComputeResultStatusFromExecutedSteps());
}
/// <summary>
/// Clean the build, deleting the command cache which is used to determine wheither a command has already been executed, and deleting the output objects if asked.
/// </summary>
/// <param name="executeContext">The execute context</param>
/// <param name="builderContext">The builder context</param>
/// <param name="deleteOutput">if true, every output object is also deleted, in addition of the command cache.</param>
public virtual void Clean(IExecuteContext executeContext, BuilderContext builderContext, bool deleteOutput)
{
// By default, do the same as Execute. This will apply for flow control steps (lists, enumerations...)
// Specific implementation exists for CommandBuildStep
Execute(executeContext, builderContext);
}
protected CommandContextBase(Command command, BuilderContext builderContext)
{
CurrentCommand = command;
BuildParameters = builderContext.Parameters;
ResultEntry = new CommandResultEntry();
}