public CommandBuildStep Add(Command command)
{
var commandBuildStep = new CommandBuildStep(command);
Add(commandBuildStep);
return(commandBuildStep);
}
public void CommandStarted(CommandBuildStep command)
{
lock (lockObject)
{
long startTime = stopWatch.ElapsedTicks;
commandExecutionIntervals.Add(command, new TimeInterval(startTime));
// Get a list of unique input files
var inputFiles = command.Command.GetInputFiles().Distinct().ToList();
// Store it aside, so that we're sure to remove the same entries during CommandEnded
commandInputFiles.Add(command, inputFiles);
// Setup start read time for each file entry
var inputHash = new HashSet <ObjectUrl>();
foreach (ObjectUrl inputUrl in inputFiles)
{
if (inputHash.Contains(inputUrl))
{
logger.Error($"The command '{command.Title}' has several times the file '{inputUrl.Path}' as input. Input Files must not be duplicated");
}
inputHash.Add(inputUrl);
ObjectAccesses inputAccesses;
if (!objectsAccesses.TryGetValue(inputUrl, out inputAccesses))
{
objectsAccesses.Add(inputUrl, inputAccesses = new ObjectAccesses());
}
inputAccesses.Reads.Add(new TimeInterval <BuildStep>(command, startTime));
}
}
}
public void NotifyCommandBuildStepFinished(CommandBuildStep commandBuildStep, ObjectId commandHash)
{
lock (builderContext.CommandsInProgress)
{
builderContext.CommandsInProgress.Remove(commandHash);
builder.ioMonitor.CommandEnded(commandBuildStep);
}
}
public void NotifyCommandBuildStepStarted(CommandBuildStep commandBuildStep, ObjectId commandHash)
{
lock (builderContext.CommandsInProgress)
{
if (!builderContext.CommandsInProgress.ContainsKey(commandHash))
{
builderContext.CommandsInProgress.Add(commandHash, commandBuildStep);
}
builder.ioMonitor.CommandStarted(commandBuildStep);
}
}
/// <summary>
/// Processes the results from a <see cref="CommandBuildStep"/>.
/// </summary>
/// <param name="executeContext">The execute context.</param>
/// <param name="buildStep">The build step.</param>
private void ProcessCommandBuildStepResult(IExecuteContext executeContext, CommandBuildStep buildStep)
{
foreach (var resultInputObject in buildStep.Command.GetInputFiles())
{
AddInputObject(resultInputObject, buildStep.Command);
}
if (buildStep.Result != null)
{
// Step1: Check inputs/outputs conflicts
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
CheckInputObject(executeContext, resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
CheckOutputObject(executeContext, resultOutputObject.Key, buildStep.Command);
}
// Step2: Add inputs/outputs
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
AddInputObject(resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
AddOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
}
}
// Forward logs
buildStep.Logger.CopyTo(Logger);
if (buildStep.Result != null)
{
// Resolve tags from TagSymbol
// TODO: Handle removed tags
foreach (var tag in buildStep.Result.TagSymbols)
{
var url = tag.Key;
// TODO: Improve search complexity?
if (outputObjects.TryGetValue(url, out var outputObject))
{
outputObject.Tags.Add(tag.Value);
}
}
}
}
public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = new ForwardingLoggerResult(executeContext.Logger);
Step = step;
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore <CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
bool monitorExited = false;
var status = ResultStatus.NotProcessed;
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult;
try
{
// try to retrieve result from one of the object store
var commandHash = Command.ComputeCommandHash(executeContext);
// Early exit if the hash of the command failed
if (commandHash == ObjectId.Empty)
{
return(ResultStatus.Failed);
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore <CommandResultEntry>(commandResultsFileStream)
{
AutoLoadNewValues = false
};
commandResultEntries.LoadNewValues();
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug($"Command {Command} delayed because it is currently running...");
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug($"Command {Command} scheduled...");
// Register the cancel callback
var cancellationTokenSource = executeContext.CancellationTokenSource;
cancellationTokenSource.Token.Register(x => ((Command)x).Cancel(), Command);
Command.CancellationToken = cancellationTokenSource.Token;
try
{
status = await StartCommand(executeContext, commandResultEntries, builderContext);
}
finally
{
// Restore cancellation token (to avoid memory leak due to previous CancellationToken.Register
Command.CancellationToken = CancellationToken.None;
}
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
}
finally
{
if (!monitorExited)
{
Monitor.Exit(executeContext);
}
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// Re-output command log messages
foreach (var message in matchingResult.LogMessages)
{
executeContext.Logger.Log(message);
}
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return(status);
}
public void CommandEnded(CommandBuildStep command)
{
lock (lockObject)
{
TimeInterval commandInterval = commandExecutionIntervals[command];
long startTime = commandInterval.StartTime;
long endTime = stopWatch.ElapsedTicks;
commandInterval.End(endTime);
commandExecutionIntervals.Remove(command);
foreach (var outputObject in command.Result.OutputObjects)
{
var outputUrl = outputObject.Key;
ObjectAccesses inputAccess;
if (objectsAccesses.TryGetValue(outputUrl, out inputAccess))
{
foreach (TimeInterval <BuildStep> input in inputAccess.Reads.Where(input => input.Object != command && input.Overlap(startTime, endTime)))
{
logger.Error($"Command {command} is writing {outputUrl} while command {input.Object} is reading it");
}
}
ObjectAccesses outputAccess;
if (!objectsAccesses.TryGetValue(outputUrl, out outputAccess))
{
objectsAccesses.Add(outputUrl, outputAccess = new ObjectAccesses());
}
foreach (var output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
if (outputObject.Value != output.Object.Value)
{
logger.Error($"Commands {command} and {output.Object} are both writing {outputUrl} at the same time, but they are different objects");
}
}
outputAccess.Writes.Add(new TimeInterval <KeyValuePair <BuildStep, ObjectId> >(new KeyValuePair <BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime));
}
foreach (ObjectUrl inputUrl in command.Result.InputDependencyVersions.Keys)
{
ObjectAccesses outputAccess;
if (objectsAccesses.TryGetValue(inputUrl, out outputAccess))
{
foreach (TimeInterval <KeyValuePair <BuildStep, ObjectId> > output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
logger.Error($"Command {output.Object} is writing {inputUrl} while command {command} is reading it");
}
}
}
// Notify that we're done reading input files
List <ObjectUrl> inputFiles;
if (commandInputFiles.TryGetValue(command, out inputFiles))
{
commandInputFiles.Remove(command);
foreach (ObjectUrl input in inputFiles)
{
objectsAccesses[input].Reads.Single(x => x.Object == command).End(endTime);
}
}
// "Garbage collection" of accesses
var newEarliestCommandAliveStartTime = commandExecutionIntervals.Count > 0 ? commandExecutionIntervals.Min(x => x.Value.StartTime) : endTime;
if (newEarliestCommandAliveStartTime > earliestCommandAliveStartTime)
{
earliestCommandAliveStartTime = newEarliestCommandAliveStartTime;
// We can remove objects whose all R/W accesses are "completed" (EndTime is set)
// and happened before all the current running commands started, since they won't affect us
foreach (var objectAccesses in objectsAccesses.ToList())
{
if (objectAccesses.Value.Reads.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime) &&
objectAccesses.Value.Writes.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime))
{
objectsAccesses.Remove(objectAccesses.Key);
}
}
}
}
}