Exemplo n.º 1
0
        public CommandBuildStep Add(Command command)
        {
            var commandBuildStep = new CommandBuildStep(command);

            Add(commandBuildStep);
            return(commandBuildStep);
        }
Exemplo n.º 2
0
        public void CommandStarted(CommandBuildStep command)
        {
            lock (lockObject)
            {
                long startTime = stopWatch.ElapsedTicks;
                commandExecutionIntervals.Add(command, new TimeInterval(startTime));

                // Get a list of unique input files
                var inputFiles = command.Command.GetInputFiles().Distinct().ToList();
                // Store it aside, so that we're sure to remove the same entries during CommandEnded
                commandInputFiles.Add(command, inputFiles);

                // Setup start read time for each file entry
                var inputHash = new HashSet <ObjectUrl>();
                foreach (ObjectUrl inputUrl in inputFiles)
                {
                    if (inputHash.Contains(inputUrl))
                    {
                        logger.Error($"The command '{command.Title}' has several times the file '{inputUrl.Path}' as input. Input Files must not be duplicated");
                    }
                    inputHash.Add(inputUrl);

                    ObjectAccesses inputAccesses;
                    if (!objectsAccesses.TryGetValue(inputUrl, out inputAccesses))
                    {
                        objectsAccesses.Add(inputUrl, inputAccesses = new ObjectAccesses());
                    }
                    inputAccesses.Reads.Add(new TimeInterval <BuildStep>(command, startTime));
                }
            }
        }
Exemplo n.º 3
0
 public void NotifyCommandBuildStepFinished(CommandBuildStep commandBuildStep, ObjectId commandHash)
 {
     lock (builderContext.CommandsInProgress)
     {
         builderContext.CommandsInProgress.Remove(commandHash);
         builder.ioMonitor.CommandEnded(commandBuildStep);
     }
 }
Exemplo n.º 4
0
            public void NotifyCommandBuildStepStarted(CommandBuildStep commandBuildStep, ObjectId commandHash)
            {
                lock (builderContext.CommandsInProgress)
                {
                    if (!builderContext.CommandsInProgress.ContainsKey(commandHash))
                    {
                        builderContext.CommandsInProgress.Add(commandHash, commandBuildStep);
                    }

                    builder.ioMonitor.CommandStarted(commandBuildStep);
                }
            }
Exemplo n.º 5
0
        /// <summary>
        /// Processes the results from a <see cref="CommandBuildStep"/>.
        /// </summary>
        /// <param name="executeContext">The execute context.</param>
        /// <param name="buildStep">The build step.</param>
        private void ProcessCommandBuildStepResult(IExecuteContext executeContext, CommandBuildStep buildStep)
        {
            foreach (var resultInputObject in buildStep.Command.GetInputFiles())
            {
                AddInputObject(resultInputObject, buildStep.Command);
            }

            if (buildStep.Result != null)
            {
                // Step1: Check inputs/outputs conflicts
                foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
                {
                    CheckInputObject(executeContext, resultInputObject.Key, buildStep.Command);
                }

                foreach (var resultOutputObject in buildStep.Result.OutputObjects)
                {
                    CheckOutputObject(executeContext, resultOutputObject.Key, buildStep.Command);
                }

                // Step2: Add inputs/outputs
                foreach (var resultInputObject in buildStep.Result.InputDependencyVersions)
                {
                    AddInputObject(resultInputObject.Key, buildStep.Command);
                }

                foreach (var resultOutputObject in buildStep.Result.OutputObjects)
                {
                    AddOutputObject(executeContext, resultOutputObject.Key, resultOutputObject.Value, buildStep.Command);
                }
            }

            // Forward logs
            buildStep.Logger.CopyTo(Logger);

            if (buildStep.Result != null)
            {
                // Resolve tags from TagSymbol
                // TODO: Handle removed tags
                foreach (var tag in buildStep.Result.TagSymbols)
                {
                    var url = tag.Key;

                    // TODO: Improve search complexity?
                    if (outputObjects.TryGetValue(url, out var outputObject))
                    {
                        outputObject.Tags.Add(tag.Value);
                    }
                }
            }
        }
Exemplo n.º 6
0
        public void CommandEnded(CommandBuildStep command)
        {
            lock (lockObject)
            {
                TimeInterval commandInterval = commandExecutionIntervals[command];

                long startTime = commandInterval.StartTime;
                long endTime   = stopWatch.ElapsedTicks;
                commandInterval.End(endTime);

                commandExecutionIntervals.Remove(command);

                foreach (var outputObject in command.Result.OutputObjects)
                {
                    var            outputUrl = outputObject.Key;
                    ObjectAccesses inputAccess;
                    if (objectsAccesses.TryGetValue(outputUrl, out inputAccess))
                    {
                        foreach (TimeInterval <BuildStep> input in inputAccess.Reads.Where(input => input.Object != command && input.Overlap(startTime, endTime)))
                        {
                            logger.Error($"Command {command} is writing {outputUrl} while command {input.Object} is reading it");
                        }
                    }

                    ObjectAccesses outputAccess;
                    if (!objectsAccesses.TryGetValue(outputUrl, out outputAccess))
                    {
                        objectsAccesses.Add(outputUrl, outputAccess = new ObjectAccesses());
                    }

                    foreach (var output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
                    {
                        if (outputObject.Value != output.Object.Value)
                        {
                            logger.Error($"Commands {command} and {output.Object} are both writing {outputUrl} at the same time, but they are different objects");
                        }
                    }

                    outputAccess.Writes.Add(new TimeInterval <KeyValuePair <BuildStep, ObjectId> >(new KeyValuePair <BuildStep, ObjectId>(command, outputObject.Value), startTime, endTime));
                }

                foreach (ObjectUrl inputUrl in command.Result.InputDependencyVersions.Keys)
                {
                    ObjectAccesses outputAccess;
                    if (objectsAccesses.TryGetValue(inputUrl, out outputAccess))
                    {
                        foreach (TimeInterval <KeyValuePair <BuildStep, ObjectId> > output in outputAccess.Writes.Where(output => output.Object.Key != command && output.Overlap(startTime, endTime)))
                        {
                            logger.Error($"Command {output.Object} is writing {inputUrl} while command {command} is reading it");
                        }
                    }
                }

                // Notify that we're done reading input files
                List <ObjectUrl> inputFiles;
                if (commandInputFiles.TryGetValue(command, out inputFiles))
                {
                    commandInputFiles.Remove(command);
                    foreach (ObjectUrl input in inputFiles)
                    {
                        objectsAccesses[input].Reads.Single(x => x.Object == command).End(endTime);
                    }
                }

                // "Garbage collection" of accesses
                var newEarliestCommandAliveStartTime = commandExecutionIntervals.Count > 0 ? commandExecutionIntervals.Min(x => x.Value.StartTime) : endTime;
                if (newEarliestCommandAliveStartTime > earliestCommandAliveStartTime)
                {
                    earliestCommandAliveStartTime = newEarliestCommandAliveStartTime;

                    // We can remove objects whose all R/W accesses are "completed" (EndTime is set)
                    // and happened before all the current running commands started, since they won't affect us
                    foreach (var objectAccesses in objectsAccesses.ToList())
                    {
                        if (objectAccesses.Value.Reads.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime) &&
                            objectAccesses.Value.Writes.All(x => x.EndTime != 0 && x.EndTime < earliestCommandAliveStartTime))
                        {
                            objectsAccesses.Remove(objectAccesses.Key);
                        }
                    }
                }
            }
        }
Exemplo n.º 7
0
 public LocalCommandContext(IExecuteContext executeContext, CommandBuildStep step, BuilderContext builderContext) : base(step.Command, builderContext)
 {
     this.executeContext = executeContext;
     logger = new ForwardingLoggerResult(executeContext.Logger);
     Step   = step;
 }
Exemplo n.º 8
0
        public override async Task <ResultStatus> Execute(IExecuteContext executeContext, BuilderContext builderContext)
        {
            ListStore <CommandResultEntry> commandResultEntries;

            // Prevent several command build step to evaluate wheither they should start at the same time. This increase the efficiency of the builder by avoiding the same command to be executed several time in parallel
            // NOTE: Careful here, there's no try/finally block around the monitor Enter/Exit, so no non-fatal exception is allowed!
            Monitor.Enter(executeContext);
            bool monitorExited = false;
            var  status        = ResultStatus.NotProcessed;
            // if any external input has changed since the last execution (or if we don't have a successful execution in cache, trigger the command
            CommandResultEntry matchingResult;

            try
            {
                // try to retrieve result from one of the object store
                var commandHash = Command.ComputeCommandHash(executeContext);
                // Early exit if the hash of the command failed
                if (commandHash == ObjectId.Empty)
                {
                    return(ResultStatus.Failed);
                }

                var commandResultsFileStream = executeContext.ResultMap.OpenStream(commandHash, VirtualFileMode.OpenOrCreate, VirtualFileAccess.ReadWrite, VirtualFileShare.ReadWrite);
                commandResultEntries = new ListStore <CommandResultEntry>(commandResultsFileStream)
                {
                    AutoLoadNewValues = false
                };
                commandResultEntries.LoadNewValues();

                if (ShouldExecute(executeContext, commandResultEntries.GetValues(), commandHash, out matchingResult))
                {
                    CommandBuildStep stepInProgress = executeContext.IsCommandCurrentlyRunning(commandHash);
                    if (stepInProgress != null)
                    {
                        Monitor.Exit(executeContext);
                        monitorExited = true;
                        executeContext.Logger.Debug($"Command {Command} delayed because it is currently running...");
                        status         = (await stepInProgress.ExecutedAsync()).Status;
                        matchingResult = stepInProgress.Result;
                    }
                    else
                    {
                        executeContext.NotifyCommandBuildStepStarted(this, commandHash);
                        Monitor.Exit(executeContext);
                        monitorExited = true;

                        executeContext.Logger.Debug($"Command {Command} scheduled...");

                        // Register the cancel callback
                        var cancellationTokenSource = executeContext.CancellationTokenSource;
                        cancellationTokenSource.Token.Register(x => ((Command)x).Cancel(), Command);

                        Command.CancellationToken = cancellationTokenSource.Token;

                        try
                        {
                            status = await StartCommand(executeContext, commandResultEntries, builderContext);
                        }
                        finally
                        {
                            // Restore cancellation token (to avoid memory leak due to previous CancellationToken.Register
                            Command.CancellationToken = CancellationToken.None;
                        }

                        executeContext.NotifyCommandBuildStepFinished(this, commandHash);
                    }
                }
            }
            finally
            {
                if (!monitorExited)
                {
                    Monitor.Exit(executeContext);
                }
            }

            // The command has not been executed
            if (matchingResult != null)
            {
                using (commandResultEntries)
                {
                    // Re-output command log messages
                    foreach (var message in matchingResult.LogMessages)
                    {
                        executeContext.Logger.Log(message);
                    }

                    status = ResultStatus.NotTriggeredWasSuccessful;
                    RegisterCommandResult(commandResultEntries, matchingResult, status);
                }
            }

            return(status);
        }