public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext) { var buildStepsToWait = new List <BuildStep>(); // Process prerequisites build steps first if (PrerequisiteSteps.Count > 0) { await CompleteCommands(executeContext, PrerequisiteSteps.ToList()); } foreach (var child in Steps) { executeContext.ScheduleBuildStep(child); buildStepsToWait.Add(child); executedSteps.Add(child); } await CompleteCommands(executeContext, buildStepsToWait); return(ComputeResultStatusFromExecutedSteps()); }
public RemoteCommandContext(IProcessBuilderRemote processBuilderRemote, Command command, BuilderContext builderContext, Logger logger) : base(command, builderContext) { this.processBuilderRemote = processBuilderRemote; this.logger = logger; }
/// <summary> /// Runs this instance. /// </summary> public BuildResultCode Run(Mode mode, bool writeIndexFile = true, bool enableMonitor = false) { // When we setup the database ourself we have to take responsibility to close it after var shouldCloseDatabase = ObjectDatabase == null; OpenObjectDatabase(buildPath, indexName); PreRun(); runMode = mode; if (IsRunning) { throw new InvalidOperationException("An instance of this Builder is already running."); } // reset build cache from previous build run cancellationTokenSource = new CancellationTokenSource(); Cancelled = false; IsRunning = true; DisableCompressionIds.Clear(); // Reseting result map var inputHashes = FileVersionTracker.GetDefault(); { var builderContext = new BuilderContext(inputHashes, TryExecuteRemote); resultMap = ObjectDatabase; scheduler = new Scheduler(); if (enableMonitor) { threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId)); foreach (var monitorPipeName in MonitorPipeNames) { threadMonitors.Add(new BuildThreadMonitor(scheduler, BuilderId, monitorPipeName)); } foreach (var threadMonitor in threadMonitors) { threadMonitor.Start(); } } // Schedule the build ScheduleBuildStep(builderContext, null, Root, InitialVariables); // Create threads var threads = Enumerable.Range(0, ThreadCount).Select(x => new Thread(SafeAction.Wrap(RunUntilEnd)) { IsBackground = true }).ToArray(); // Start threads int threadId = 0; foreach (var thread in threads) { thread.Name = (BuilderName ?? "Builder") + " worker thread " + (++threadId); thread.Start(); } // Wait for all threads to finish foreach (var thread in threads) { thread.Join(); } foreach (var threadMonitor in threadMonitors) { threadMonitor.Finish(); } foreach (var threadMonitor in threadMonitors) { threadMonitor.Join(); } } threadMonitors.Clear(); BuildResultCode result; if (runMode == Mode.Build) { if (cancellationTokenSource.IsCancellationRequested) { Logger.Error("Build cancelled."); result = BuildResultCode.Cancelled; } else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0) { Logger.Error($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure."); Logger.Error("Build failed."); result = BuildResultCode.BuildError; } else { Logger.Info($"Build finished in {stepCounter.Total} steps. Command results: {stepCounter.Get(ResultStatus.Successful)} succeeded, {stepCounter.Get(ResultStatus.NotTriggeredWasSuccessful)} up-to-date, {stepCounter.Get(ResultStatus.Failed)} failed, {stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed)} not triggered due to previous failure."); Logger.Info("Build is successful."); result = BuildResultCode.Successful; } } else { string modeName; switch (runMode) { case Mode.Clean: modeName = "Clean"; break; case Mode.CleanAndDelete: modeName = "Clean-and-delete"; break; default: throw new InvalidOperationException("Builder executed in unknown mode."); } if (cancellationTokenSource.IsCancellationRequested) { Logger.Error(modeName + " has been cancelled."); result = BuildResultCode.Cancelled; } else if (stepCounter.Get(ResultStatus.Failed) > 0 || stepCounter.Get(ResultStatus.NotTriggeredPrerequisiteFailed) > 0) { Logger.Error(modeName + " has failed."); result = BuildResultCode.BuildError; } else { Logger.Error(modeName + " has been successfully completed."); result = BuildResultCode.Successful; } } scheduler = null; resultMap = null; IsRunning = false; if (shouldCloseDatabase) { CloseObjectDatabase(); } return(result); }
private void ScheduleBuildStep(BuilderContext builderContext, BuildStep instigator, BuildStep buildStep, IDictionary <string, string> variables) { if (buildStep.ExecutionId == 0) { if (buildStep.Parent != null && buildStep.Parent != instigator) { throw new InvalidOperationException("Scheduling a BuildStep with a different instigator that its parent"); } if (buildStep.Parent == null) { buildStep.Parent = instigator; } // Compute content dependencies before scheduling the build GenerateDependencies(buildStep); // TODO: Big review of the log infrastructure of CompilerApp & BuildEngine! // Create a logger that redirects to various places (BuildStep.Logger, timestampped log, global log, etc...) var buildStepLogger = new BuildStepLogger(buildStep, Logger, startTime); var logger = (Logger)buildStepLogger; // Apply user-registered callbacks to the logger buildStep.TransformExecuteContextLogger?.Invoke(ref logger); // Create execute context var executeContext = new ExecuteContext(this, builderContext, buildStep, logger) { Variables = new Dictionary <string, string>(variables) }; //buildStep.ExpandStrings(executeContext); if (runMode == Mode.Build) { MicroThread microThread = scheduler.Create(); // Set priority from this build step, if we have one. if (buildStep.Priority.HasValue) { microThread.Priority = buildStep.Priority.Value; } buildStep.ExecutionId = microThread.Id; foreach (var threadMonitor in threadMonitors) { threadMonitor.RegisterBuildStep(buildStep, buildStepLogger.StepLogger); } microThread.Name = buildStep.ToString(); // Default: // Schedule continuations as early as possible to help EnumerableBuildStep finish when all its task are finished. // Otherwise, it would wait for all leaf to finish first before finishing parent EnumerableBuildStep. // This should also reduce memory usage, and might improve cache coherency as well. microThread.ScheduleMode = ScheduleMode.First; microThread.Start(async() => { // Wait for prerequisites await Task.WhenAll(buildStep.PrerequisiteSteps.Select(x => x.ExecutedAsync()).ToArray()); // Check for failed prerequisites var status = ResultStatus.NotProcessed; if (buildStep.ArePrerequisitesSuccessful) { try { var outputObjectsGroups = executeContext.GetOutputObjectsGroups(); MicrothreadLocalDatabases.MountDatabase(outputObjectsGroups); // Execute status = await buildStep.Execute(executeContext, builderContext); } catch (TaskCanceledException e) { // Benlitz: I'm NOT SURE this is the correct explanation, it might be a more subtle race condition, but I can't manage to reproduce it again executeContext.Logger.Warning("A child task of build step " + buildStep + " triggered a TaskCanceledException that was not caught by the parent task. The command has not handled cancellation gracefully."); executeContext.Logger.Warning(e.Message); status = ResultStatus.Cancelled; } catch (Exception e) { executeContext.Logger.Error("Exception in command " + buildStep + ": " + e); status = ResultStatus.Failed; } finally { MicrothreadLocalDatabases.UnmountDatabase(); // Ensure the command set at least the result status if (status == ResultStatus.NotProcessed) { throw new InvalidDataException("The build step " + buildStep + " returned ResultStatus.NotProcessed after completion."); } } if (microThread.Exception != null) { executeContext.Logger.Error("Exception in command " + buildStep + ": " + microThread.Exception); status = ResultStatus.Failed; } } else { status = ResultStatus.NotTriggeredPrerequisiteFailed; } //if (completedTask.IsCanceled) //{ // completedStep.Status = ResultStatus.Cancelled; //} var logType = LogMessageType.Info; string logText = null; switch (status) { case ResultStatus.Successful: logType = LogMessageType.Verbose; logText = "BuildStep {0} was successful.".ToFormat(buildStep.ToString()); break; case ResultStatus.Failed: logType = LogMessageType.Error; logText = "BuildStep {0} failed.".ToFormat(buildStep.ToString()); break; case ResultStatus.NotTriggeredPrerequisiteFailed: logType = LogMessageType.Error; logText = "BuildStep {0} failed of previous failed prerequisites.".ToFormat(buildStep.ToString()); break; case ResultStatus.Cancelled: logType = LogMessageType.Warning; logText = "BuildStep {0} cancelled.".ToFormat(buildStep.ToString()); break; case ResultStatus.NotTriggeredWasSuccessful: logType = LogMessageType.Verbose; logText = "BuildStep {0} is up-to-date and has been skipped".ToFormat(buildStep.ToString()); break; case ResultStatus.NotProcessed: throw new InvalidDataException("BuildStep has neither succeeded, failed, nor been cancelled"); } if (logText != null) { var logMessage = new LogMessage(null, logType, logText); executeContext.Logger.Log(logMessage); } buildStep.RegisterResult(executeContext, status); stepCounter.AddStepResult(status); }); } else { buildStep.Clean(executeContext, builderContext, runMode == Mode.CleanAndDelete); } } }
protected CommandContextBase(Command command, BuilderContext builderContext) { CurrentCommand = command; ResultEntry = new CommandResultEntry(); }
public static BuildResultCode BuildSlave(BuilderOptions options) { // Mount build path ((FileSystemProvider)VirtualFileSystem.ApplicationData).ChangeBasePath(options.BuildDirectory); PrepareDatabases(options); try { VirtualFileSystem.CreateDirectory("/data/"); VirtualFileSystem.CreateDirectory("/data/db/"); } catch (Exception) { throw new OptionException("Invalid Build database path", "database"); } // Open WCF channel with master builder var namedPipeBinding = new NetNamedPipeBinding(NetNamedPipeSecurityMode.None) { SendTimeout = TimeSpan.FromSeconds(300.0) }; var processBuilderRemote = ChannelFactory <IProcessBuilderRemote> .CreateChannel(namedPipeBinding, new EndpointAddress(options.SlavePipe)); try { RegisterRemoteLogger(processBuilderRemote); // Create scheduler var scheduler = new Scheduler(); var status = ResultStatus.NotProcessed; // Schedule command string buildPath = options.BuildDirectory; Logger logger = options.Logger; MicroThread microthread = scheduler.Add(async() => { // Deserialize command and parameters Command command = processBuilderRemote.GetCommandToExecute(); BuildParameterCollection parameters = processBuilderRemote.GetBuildParameters(); // Run command var inputHashes = new DictionaryStore <InputVersionKey, ObjectId>(VirtualFileSystem.OpenStream("/data/db/InputHashes", VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite)); var builderContext = new BuilderContext(buildPath, inputHashes, parameters, 0, null); var commandContext = new RemoteCommandContext(processBuilderRemote, command, builderContext, logger); command.PreCommand(commandContext); status = await command.DoCommand(commandContext); command.PostCommand(commandContext, status); // Returns result to master builder processBuilderRemote.RegisterResult(commandContext.ResultEntry); }); while (true) { scheduler.Run(); // Exit loop if no more micro threads lock (scheduler.MicroThreads) { if (!scheduler.MicroThreads.Any()) { break; } } Thread.Sleep(0); } // Rethrow any exception that happened in microthread if (microthread.Exception != null) { options.Logger.Fatal(microthread.Exception.ToString()); return(BuildResultCode.BuildError); } if (status == ResultStatus.Successful || status == ResultStatus.NotTriggeredWasSuccessful) { return(BuildResultCode.Successful); } return(BuildResultCode.BuildError); } finally { // Close WCF channel // ReSharper disable SuspiciousTypeConversion.Global ((IClientChannel)processBuilderRemote).Close(); // ReSharper restore SuspiciousTypeConversion.Global } }