public CommandBuildStep Add(Command command)
{
var commandBuildStep = new CommandBuildStep(command);
Add(commandBuildStep);
return(commandBuildStep);
}
public void CommandStarted(CommandBuildStep command)
{
lock (lockObject)
{
long startTime = stopWatch.ElapsedTicks;
commandExecutionIntervals.Add(command, new TimeInterval(startTime));
var inputHash = new HashSet <ObjectUrl>();
foreach (ObjectUrl inputUrl in command.Command.GetInputFiles())
{
if (inputHash.Contains(inputUrl))
{
logger.Error("The command '{0}' has several times the file '{1}' as input. Input Files must not be duplicated", command.Title, inputUrl.Path);
}
inputHash.Add(inputUrl);
List <TimeInterval <BuildStep> > inputReadAccess;
if (!readAccesses.TryGetValue(inputUrl, out inputReadAccess))
{
inputReadAccess = new List <TimeInterval <BuildStep> > {
new TimeInterval <BuildStep>(command, startTime)
};
readAccesses.Add(inputUrl, inputReadAccess);
}
else
{
inputReadAccess.Add(new TimeInterval <BuildStep>(command, startTime));
}
}
}
}
public void CommandStarted(CommandBuildStep command)
{
lock (lockObject)
{
long startTime = stopWatch.ElapsedTicks;
commandExecutionIntervals.Add(command, new TimeInterval(startTime));
// Get a list of unique input files
var inputFiles = command.Command.GetInputFiles().Distinct().ToList();
// Store it aside, so that we're sure to remove the same entries during CommandEnded
commandInputFiles.Add(command, inputFiles);
// Setup start read time for each file entry
var inputHash = new HashSet <ObjectUrl>();
foreach (ObjectUrl inputUrl in inputFiles)
{
if (inputHash.Contains(inputUrl))
{
logger.Error($"The command '{command.Title}' has several times the file '{inputUrl.Path}' as input. Input Files must not be duplicated");
}
inputHash.Add(inputUrl);
ObjectAccesses inputAccesses;
if (!objectsAccesses.TryGetValue(inputUrl, out inputAccesses))
{
objectsAccesses.Add(inputUrl, inputAccesses = new ObjectAccesses());
}
inputAccesses.Reads.Add(new TimeInterval <BuildStep>(command, startTime));
}
}
}
public void CommandStarted(CommandBuildStep command)
{
lock (lockObject)
{
long startTime = stopWatch.ElapsedTicks;
commandExecutionIntervals.Add(command, new TimeInterval(startTime));
// Get a list of unique input files
var inputFiles = command.Command.GetInputFiles().Distinct().ToList();
// Store it aside, so that we're sure to remove the same entries during CommandEnded
commandInputFiles.Add(command, inputFiles);
// Setup start read time for each file entry
var inputHash = new HashSet<ObjectUrl>();
foreach (ObjectUrl inputUrl in inputFiles)
{
if (inputHash.Contains(inputUrl))
logger.Error("The command '{0}' has several times the file '{1}' as input. Input Files must not be duplicated", command.Title, inputUrl.Path);
inputHash.Add(inputUrl);
List<TimeInterval<BuildStep>> inputReadAccess;
if (!readAccesses.TryGetValue(inputUrl, out inputReadAccess))
{
inputReadAccess = new List<TimeInterval<BuildStep>> { new TimeInterval<BuildStep>(command, startTime) };
readAccesses.Add(inputUrl, inputReadAccess);
}
else
{
inputReadAccess.Add(new TimeInterval<BuildStep>(command, startTime));
}
}
}
}
public void CommandEnded(CommandBuildStep command)
{
lock (lockObject)
{
TimeInterval commandInterval = commandExecutionIntervals[command];
long startTime = commandInterval.StartTime;
long endTime = stopWatch.ElapsedTicks;
commandInterval.End(endTime);
foreach (var outputObject in command.Result.OutputObjects)
{
var outputUrl = outputObject.Key;
List <TimeInterval <BuildStep> > inputReadAccess;
if (readAccesses.TryGetValue(outputUrl, out inputReadAccess))
{
foreach (TimeInterval <BuildStep> input in inputReadAccess.Where(input => input.Object != command && input.Overlap(startTime, endTime)))
{
logger.Error("Command {0} is writing {1} while command {2} is reading it", command, outputUrl, input.Object);
}
}
List <TimeInterval <KeyValuePair <BuildStep, ObjectId> > > outputWriteAccess;
if (!writeAccesses.TryGetValue(outputUrl, out outputWriteAccess))
{
outputWriteAccess = new List <TimeInterval <KeyValuePair <BuildStep, ObjectId> > > {
new TimeInterval <KeyValuePair <BuildStep, ObjectId> >(new KeyValuePair <BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime)
};
writeAccesses.Add(outputUrl, outputWriteAccess);
}
else
{
foreach (var output in outputWriteAccess.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
if (outputObject.Value != output.Object.Value)
{
logger.Error("Commands {0} and {1} are both writing {2} at the same time, but they are different objects", command, output.Object, outputUrl);
}
}
outputWriteAccess.Add(new TimeInterval <KeyValuePair <BuildStep, ObjectId> >(new KeyValuePair <BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime));
}
}
foreach (ObjectUrl inputUrl in command.Result.InputDependencyVersions.Keys)
{
List <TimeInterval <KeyValuePair <BuildStep, ObjectId> > > outputWriteAccess;
if (writeAccesses.TryGetValue(inputUrl, out outputWriteAccess))
{
foreach (TimeInterval <KeyValuePair <BuildStep, ObjectId> > output in outputWriteAccess.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
logger.Error("Command {0} is writing {1} while command {2} is reading it", output.Object, inputUrl, command);
}
}
}
foreach (ObjectUrl input in command.Command.GetInputFiles())
{
readAccesses[input].Single(x => x.Object == command).End(endTime);
}
}
}
/// <summary>
/// Processes the results from a <see cref="CommandBuildStep"/>.
/// </summary>
/// <param name="executeContext">The execute context.</param>
/// <param name="buildStep">The build step.</param>
private void ProcessCommandBuildStepResult(IExecuteContext executeContext, CommandBuildStep buildStep)
{
foreach (var resultInputObject in buildStep.Command.GetInputFiles())
{
AddInputObject(executeContext, resultInputObject, buildStep.Command);
}
if (buildStep.Result != null)
{
// Step1: Check inputs/outputs conflicts
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
CheckInputObject(executeContext, resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
CheckOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
}
// Step2: Add inputs/outputs
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
AddInputObject(executeContext, resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
AddOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
}
}
// Process recursively
// TODO: Wait for completion of spawned step in case Task didn't wait for them
foreach (var spawnedStep in buildStep.SpawnedSteps)
{
ProcessCommandBuildStepResult(executeContext, spawnedStep);
}
if (buildStep.Result != null)
{
// Resolve tags from TagSymbol
// TODO: Handle removed tags
foreach (var tagGroup in buildStep.Result
.TagSymbols
.Where(x => buildStep.Command.TagSymbols.ContainsKey(x.Value))
.GroupBy(x => x.Key, x => buildStep.Command.TagSymbols[x.Value].RealName))
{
var url = tagGroup.Key;
// TODO: Improve search complexity?
OutputObject outputObject;
if (outputObjects.TryGetValue(url, out outputObject))
{
outputObject.Tags.UnionWith(tagGroup);
}
}
}
}
public void NotifyCommandBuildStepFinished(CommandBuildStep commandBuildStep, ObjectId commandHash)
{
lock (builderContext.CommandsInProgress)
{
builderContext.CommandsInProgress.Remove(commandHash);
builder.ioMonitor.CommandEnded(commandBuildStep);
}
}
internal async Task <ResultStatus> SpawnCommand(Command command, IExecuteContext executeContext)
{
var spawnedStep = new CommandBuildStep(command);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
var resultStatus = (await spawnedStep.ExecutedAsync()).Status;
return(resultStatus);
}
public void NotifyCommandBuildStepStarted(CommandBuildStep commandBuildStep, ObjectId commandHash)
{
lock (builderContext.CommandsInProgress)
{
if (!builderContext.CommandsInProgress.ContainsKey(commandHash))
{
builderContext.CommandsInProgress.Add(commandHash, commandBuildStep);
}
builder.ioMonitor.CommandStarted(commandBuildStep);
}
}
/// <summary>
/// Processes the results from a <see cref="CommandBuildStep"/>.
/// </summary>
/// <param name="executeContext">The execute context.</param>
/// <param name="buildStep">The build step.</param>
private void ProcessCommandBuildStepResult(IExecuteContext executeContext, CommandBuildStep buildStep)
{
foreach (var resultInputObject in buildStep.Command.GetInputFiles())
{
AddInputObject(resultInputObject, buildStep.Command);
}
if (buildStep.Result != null)
{
// Step1: Check inputs/outputs conflicts
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
CheckInputObject(executeContext, resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
CheckOutputObject(executeContext, resultOutputObject.Key, buildStep.Command);
}
// Step2: Add inputs/outputs
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
AddInputObject(resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
AddOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
}
}
// Forward logs
buildStep.Logger.CopyTo(Logger);
if (buildStep.Result != null)
{
// Resolve tags from TagSymbol
// TODO: Handle removed tags
foreach (var tag in buildStep.Result.TagSymbols)
{
var url = tag.Key;
// TODO: Improve search complexity?
if (outputObjects.TryGetValue(url, out var outputObject))
{
outputObject.Tags.Add(tag.Value);
}
}
}
}
public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = new ForwardingLoggerResult(executeContext.Logger);
Step = step;
}
public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = executeContext.Logger;
Step = step;
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore <CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
bool monitorExited = false;
var status = ResultStatus.NotProcessed;
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult = null;
try
{
ObjectId commandHash;
{
// try to retrieve result from one of the object store
commandHash = Command.ComputeCommandHash(executeContext);
// Early exit if the hash of the command failed
if (commandHash == ObjectId.Empty)
{
return(ResultStatus.Failed);
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore <CommandResultEntry>(commandResultsFileStream)
{
AutoLoadNewValues = false
};
commandResultEntries.LoadNewValues();
}
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} delayed because it is currently running...", Command.ToString());
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} scheduled...", Command.ToString());
// Register the cancel callback
var cancellationTokenSource = executeContext.CancellationTokenSource;
cancellationTokenSource.Token.Register(x => ((Command)x).Cancel(), Command);
Command.CancellationToken = cancellationTokenSource.Token;
try
{
status = await StartCommand(executeContext, commandResultEntries, builderContext);
}
finally
{
// Restore cancellation token (to avoid memory leak due to previous CancellationToken.Register
Command.CancellationToken = CancellationToken.None;
}
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
}
finally
{
if (!monitorExited)
{
Monitor.Exit(executeContext);
}
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// Replicate triggered builds
Debug.Assert(SpawnedStepsList.Count == 0);
foreach (Command spawnedCommand in matchingResult.SpawnedCommands)
{
var spawnedStep = new CommandBuildStep(spawnedCommand);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
}
// Re-output command log messages
foreach (var message in matchingResult.LogMessages)
{
executeContext.Logger.Log(message);
}
// Wait for all build steps to complete.
// TODO: Ideally, we should store and replicate the behavior of the command that spawned it
// (wait if it used ScheduleAndExecute, don't wait if it used RegisterSpawnedCommandWithoutScheduling)
await Task.WhenAll(SpawnedSteps.Select(x => x.ExecutedAsync()));
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return(status);
}
public void NotifyCommandBuildStepStarted(CommandBuildStep commandBuildStep, ObjectId commandHash)
{
lock (builderContext.CommandsInProgress)
{
if (!builderContext.CommandsInProgress.ContainsKey(commandHash))
builderContext.CommandsInProgress.Add(commandHash, commandBuildStep);
builder.ioMonitor.CommandStarted(commandBuildStep);
}
}
internal async Task<ResultStatus> SpawnCommand(Command command, IExecuteContext executeContext)
{
var spawnedStep = new CommandBuildStep(command);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
var resultStatus = (await spawnedStep.ExecutedAsync()).Status;
return resultStatus;
}
public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
{
this.executeContext = executeContext;
logger = new ForwardingLoggerResult(executeContext.Logger);
Step = step;
}
/// <summary>
/// Processes the results from a <see cref="CommandBuildStep"/>.
/// </summary>
/// <param name="executeContext">The execute context.</param>
/// <param name="buildStep">The build step.</param>
private void ProcessCommandBuildStepResult(IExecuteContext executeContext, CommandBuildStep buildStep)
{
foreach (var resultInputObject in buildStep.Command.GetInputFiles())
{
AddInputObject(executeContext, resultInputObject, buildStep.Command);
}
if (buildStep.Result != null)
{
// Step1: Check inputs/outputs conflicts
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
CheckInputObject(executeContext, resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
CheckOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
}
// Step2: Add inputs/outputs
foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
{
AddInputObject(executeContext, resultInputObject.Key, buildStep.Command);
}
foreach (var resultOutputObject in buildStep.Result.OutputObjects)
{
AddOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
}
}
// Forward logs
buildStep.Logger.CopyTo(Logger);
// Process recursively
// TODO: Wait for completion of spawned step in case Task didn't wait for them
foreach (var spawnedStep in buildStep.SpawnedSteps)
{
ProcessCommandBuildStepResult(executeContext, spawnedStep);
}
if (buildStep.Result != null)
{
// Resolve tags from TagSymbol
// TODO: Handle removed tags
foreach (var tagGroup in buildStep.Result
.TagSymbols
.Where(x => buildStep.Command.TagSymbols.ContainsKey(x.Value))
.GroupBy(x => x.Key, x => buildStep.Command.TagSymbols[x.Value].RealName))
{
var url = tagGroup.Key;
// TODO: Improve search complexity?
OutputObject outputObject;
if (outputObjects.TryGetValue(url, out outputObject))
{
outputObject.Tags.UnionWith(tagGroup);
}
}
}
}
public override async Task<ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore<CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
bool monitorExited = false;
var status = ResultStatus.NotProcessed;
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult = null;
try
{
ObjectId commandHash;
{
// try to retrieve result from one of the object store
commandHash = Command.ComputeCommandHash(executeContext);
// Early exit if the hash of the command failed
if (commandHash == ObjectId.Empty)
{
return ResultStatus.Failed;
}
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore<CommandResultEntry>(commandResultsFileStream) { AutoLoadNewValues = false };
commandResultEntries.LoadNewValues();
}
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} delayed because it is currently running...", Command.ToString());
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
monitorExited = true;
executeContext.Logger.Debug("Command {0} scheduled...", Command.ToString());
status = await StartCommand(executeContext, commandResultEntries, builderContext);
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
}
finally
{
if (!monitorExited)
{
Monitor.Exit(executeContext);
}
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// Replicate triggered builds
Debug.Assert(SpawnedStepsList.Count == 0);
foreach (Command spawnedCommand in matchingResult.SpawnedCommands)
{
var spawnedStep = new CommandBuildStep(spawnedCommand);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
}
// Re-output command log messages
foreach (var message in matchingResult.LogMessages)
{
executeContext.Logger.Log(message);
}
// Wait for all build steps to complete.
// TODO: Ideally, we should store and replicate the behavior of the command that spawned it
// (wait if it used ScheduleAndExecute, don't wait if it used RegisterSpawnedCommandWithoutScheduling)
await Task.WhenAll(SpawnedSteps.Select(x => x.ExecutedAsync()));
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return status;
}
public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
{
ListStore <CommandResultEntry> commandResultEntries;
// Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
// NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
Monitor.Enter(executeContext);
ObjectId commandHash;
//await Task.Factory.StartNew(() =>
{
// try to retrieve result from one of the object store
commandHash = Command.ComputeCommandHash(executeContext);
var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
commandResultEntries = new ListStore <CommandResultEntry>(commandResultsFileStream)
{
AutoLoadNewValues = false
};
commandResultEntries.LoadNewValues();
}
//);
// if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
CommandResultEntry matchingResult;
var status = ResultStatus.NotProcessed;
if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
{
CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
if (stepInProgress != null)
{
Monitor.Exit(executeContext);
executeContext.Logger.Debug("Command {0} delayed because it is currently running...", Command.ToString());
status = (await stepInProgress.ExecutedAsync()).Status;
matchingResult = stepInProgress.Result;
}
else
{
executeContext.NotifyCommandBuildStepStarted(this, commandHash);
Monitor.Exit(executeContext);
executeContext.Logger.Debug("Command {0} scheduled...", Command.ToString());
status = await StartCommand(executeContext, commandResultEntries, builderContext);
executeContext.NotifyCommandBuildStepFinished(this, commandHash);
}
}
else
{
Monitor.Exit(executeContext);
}
// The command has not been executed
if (matchingResult != null)
{
using (commandResultEntries)
{
// the command was not started because it is already up-to-date (retrieved from cache and no change in external files since last execution)
executeContext.Logger.Verbose("Command {0} is up-to-date, skipping...", Command.ToString());
// Replicate triggered builds
Debug.Assert(SpawnedStepsList.Count == 0);
foreach (Command spawnedCommand in matchingResult.SpawnedCommands)
{
var spawnedStep = new CommandBuildStep(spawnedCommand);
SpawnedStepsList.Add(spawnedStep);
executeContext.ScheduleBuildStep(spawnedStep);
}
// Wait for all build steps to complete.
// TODO: Ideally, we should store and replicate the behavior of the command that spawned it
// (wait if it used ScheduleAndExecute, don't wait if it used RegisterSpawnedCommandWithoutScheduling)
await Task.WhenAll(SpawnedSteps.Select(x => x.ExecutedAsync()));
status = ResultStatus.NotTriggeredWasSuccessful;
RegisterCommandResult(commandResultEntries, matchingResult, status);
}
}
return(status);
}
public void CommandEnded(CommandBuildStep command)
{
lock (lockObject)
{
TimeInterval commandInterval = commandExecutionIntervals[command];
long startTime = commandInterval.StartTime;
long endTime = stopWatch.ElapsedTicks;
commandInterval.End(endTime);
foreach (var outputObject in command.Result.OutputObjects)
{
var outputUrl = outputObject.Key;
List<TimeInterval<BuildStep>> inputReadAccess;
if (readAccesses.TryGetValue(outputUrl, out inputReadAccess))
{
foreach (TimeInterval<BuildStep> input in inputReadAccess.Where(input => input.Object != command && input.Overlap(startTime, endTime)))
{
logger.Error("Command {0} is writing {1} while command {2} is reading it", command, outputUrl, input.Object);
}
}
List<TimeInterval<KeyValuePair<BuildStep, ObjectId>>> outputWriteAccess;
if (!writeAccesses.TryGetValue(outputUrl, out outputWriteAccess))
{
outputWriteAccess = new List<TimeInterval<KeyValuePair<BuildStep, ObjectId>>> { new TimeInterval<KeyValuePair<BuildStep, ObjectId>>(new KeyValuePair<BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime) };
writeAccesses.Add(outputUrl, outputWriteAccess);
}
else
{
foreach (var output in outputWriteAccess.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
if (outputObject.Value != output.Object.Value)
logger.Error("Commands {0} and {1} are both writing {2} at the same time, but they are different objects", command, output.Object, outputUrl);
}
outputWriteAccess.Add(new TimeInterval<KeyValuePair<BuildStep, ObjectId>>(new KeyValuePair<BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime));
}
}
foreach (ObjectUrl inputUrl in command.Result.InputDependencyVersions.Keys)
{
List<TimeInterval<KeyValuePair<BuildStep, ObjectId>>> outputWriteAccess;
if (writeAccesses.TryGetValue(inputUrl, out outputWriteAccess))
{
foreach (TimeInterval<KeyValuePair<BuildStep, ObjectId>> output in outputWriteAccess.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
logger.Error("Command {0} is writing {1} while command {2} is reading it", output.Object, inputUrl, command);
}
}
}
// Notify that we're done reading input files
List<ObjectUrl> inputFiles;
if (commandInputFiles.TryGetValue(command, out inputFiles))
{
commandInputFiles.Remove(command);
foreach (ObjectUrl input in inputFiles)
{
readAccesses[input].Single(x => x.Object == command).End(endTime);
}
}
}
}
public void NotifyCommandBuildStepFinished(CommandBuildStep commandBuildStep, ObjectId commandHash)
{
lock (builderContext.CommandsInProgress)
{
builderContext.CommandsInProgress.Remove(commandHash);
builder.ioMonitor.CommandEnded(commandBuildStep);
}
}
public void CommandEnded(CommandBuildStep command)
{
lock (lockObject)
{
TimeInterval commandInterval = commandExecutionIntervals[command];
long startTime = commandInterval.StartTime;
long endTime = stopWatch.ElapsedTicks;
commandInterval.End(endTime);
commandExecutionIntervals.Remove(command);
foreach (var outputObject in command.Result.OutputObjects)
{
var outputUrl = outputObject.Key;
ObjectAccesses inputAccess;
if (objectsAccesses.TryGetValue(outputUrl, out inputAccess))
{
foreach (TimeInterval <BuildStep> input in inputAccess.Reads.Where(input => input.Object != command && input.Overlap(startTime, endTime)))
{
logger.Error($"Command {command} is writing {outputUrl} while command {input.Object} is reading it");
}
}
ObjectAccesses outputAccess;
if (!objectsAccesses.TryGetValue(outputUrl, out outputAccess))
{
objectsAccesses.Add(outputUrl, outputAccess = new ObjectAccesses());
}
foreach (var output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
if (outputObject.Value != output.Object.Value)
{
logger.Error($"Commands {command} and {output.Object} are both writing {outputUrl} at the same time, but they are different objects");
}
}
outputAccess.Writes.Add(new TimeInterval <KeyValuePair <BuildStep, ObjectId> >(new KeyValuePair <BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime));
}
foreach (ObjectUrl inputUrl in command.Result.InputDependencyVersions.Keys)
{
ObjectAccesses outputAccess;
if (objectsAccesses.TryGetValue(inputUrl, out outputAccess))
{
foreach (TimeInterval <KeyValuePair <BuildStep, ObjectId> > output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
logger.Error($"Command {output.Object} is writing {inputUrl} while command {command} is reading it");
}
}
}
// Notify that we're done reading input files
List <ObjectUrl> inputFiles;
if (commandInputFiles.TryGetValue(command, out inputFiles))
{
commandInputFiles.Remove(command);
foreach (ObjectUrl input in inputFiles)
{
objectsAccesses[input].Reads.Single(x => x.Object == command).End(endTime);
}
}
// "Garbage collection" of accesses
var newEarliestCommandAliveStartTime = commandExecutionIntervals.Count > 0 ? commandExecutionIntervals.Min(x => x.Value.StartTime) : endTime;
if (newEarliestCommandAliveStartTime > earliestCommandAliveStartTime)
{
earliestCommandAliveStartTime = newEarliestCommandAliveStartTime;
// We can remove objects whose all R/W accesses are "completed" (EndTime is set)
// and happened before all the current running commands started, since they won't affect us
foreach (var objectAccesses in objectsAccesses.ToList())
{
if (objectAccesses.Value.Reads.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime) &&
objectAccesses.Value.Writes.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime))
{
objectsAccesses.Remove(objectAccesses.Key);
}
}
}
}
}
public void CommandEnded(CommandBuildStep command)
{
lock (lockObject)
{
TimeInterval commandInterval = commandExecutionIntervals[command];
long startTime = commandInterval.StartTime;
long endTime = stopWatch.ElapsedTicks;
commandInterval.End(endTime);
commandExecutionIntervals.Remove(command);
foreach (var outputObject in command.Result.OutputObjects)
{
var outputUrl = outputObject.Key;
ObjectAccesses inputAccess;
if (objectsAccesses.TryGetValue(outputUrl, out inputAccess))
{
foreach (TimeInterval<BuildStep> input in inputAccess.Reads.Where(input => input.Object != command && input.Overlap(startTime, endTime)))
{
logger.Error("Command {0} is writing {1} while command {2} is reading it", command, outputUrl, input.Object);
}
}
ObjectAccesses outputAccess;
if (!objectsAccesses.TryGetValue(outputUrl, out outputAccess))
{
objectsAccesses.Add(outputUrl, outputAccess = new ObjectAccesses());
}
foreach (var output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
if (outputObject.Value != output.Object.Value)
logger.Error("Commands {0} and {1} are both writing {2} at the same time, but they are different objects", command, output.Object, outputUrl);
}
outputAccess.Writes.Add(new TimeInterval<KeyValuePair<BuildStep, ObjectId>>(new KeyValuePair<BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime));
}
foreach (ObjectUrl inputUrl in command.Result.InputDependencyVersions.Keys)
{
ObjectAccesses outputAccess;
if (objectsAccesses.TryGetValue(inputUrl, out outputAccess))
{
foreach (TimeInterval<KeyValuePair<BuildStep, ObjectId>> output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
{
logger.Error("Command {0} is writing {1} while command {2} is reading it", output.Object, inputUrl, command);
}
}
}
// Notify that we're done reading input files
List<ObjectUrl> inputFiles;
if (commandInputFiles.TryGetValue(command, out inputFiles))
{
commandInputFiles.Remove(command);
foreach (ObjectUrl input in inputFiles)
{
objectsAccesses[input].Reads.Single(x => x.Object == command).End(endTime);
}
}
// "Garbage collection" of accesses
var newEarliestCommandAliveStartTime = commandExecutionIntervals.Count > 0 ? commandExecutionIntervals.Min(x => x.Value.StartTime) : endTime;
if (newEarliestCommandAliveStartTime > earliestCommandAliveStartTime)
{
earliestCommandAliveStartTime = newEarliestCommandAliveStartTime;
// We can remove objects whose all R/W accesses are "completed" (EndTime is set)
// and happened before all the current running commands started, since they won't affect us
foreach (var objectAccesses in objectsAccesses.ToList())
{
if (objectAccesses.Value.Reads.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime)
&& objectAccesses.Value.Writes.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime))
objectsAccesses.Remove(objectAccesses.Key);
}
}
}
}
