/// <summary> /// Called after worker finishes materializing inputs for a pip /// </summary> public void OnInputMaterializationCompletion(Pip pip, IPipExecutionEnvironment environment) { Contract.Assert(pip.PipType == PipType.Process || pip.PipType == PipType.Ipc); var fileContentManager = environment.State.FileContentManager; // Update the pip state Transition(pip.PipId, WorkerPipState.Prepped); if (!IsContentTrackingEnabled) { return; } fileContentManager.CollectPipInputsToMaterialize( environment.PipTable, pip, files: null, filter: artifact => { if (artifact.IsFile && artifact.FileArtifact.IsSourceFile) { // Do not register the source files as the available content. return(false); } bool added = TryAddAvailableContent(artifact); if (artifact.IsFile) { // Don't attempt to add anything. Just need to register the available content return(false); } else { // Process directories to visit files unless they were already added return(!added); } }, serviceFilter: servicePipId => { bool added = TryAddAvailableServiceInputContent(servicePipId); // Don't attempt to add anything. Just need to register the available content return(false); }); }
/// <summary> /// Indicates if the pip execution step is tracked for the pip running time displayed in critical path printout /// </summary> public static bool IncludeInRunningTime(this PipExecutionStep step, IPipExecutionEnvironment environment) { switch (step) { // These steps pertain to distribution and thus should not be considered for running time // which is expected to be comparable whether the pip runs remotely or not case PipExecutionStep.ChooseWorkerCpu: case PipExecutionStep.ChooseWorkerCacheLookup: return(false); case PipExecutionStep.MaterializeOutputs: // If we materialize outputs in background, then do not include the duration in running time. return(!environment.MaterializeOutputsInBackground); default: return(true); } }
internal void Start(EngineSchedule schedule) { Contract.Requires(schedule != null); Contract.Requires(schedule.Scheduler != null); Contract.Requires(schedule.SchedulingQueue != null); Contract.Assert(AttachCompletion.IsCompleted && AttachCompletion.GetAwaiter().GetResult(), "ProcessBuildRequests called before finishing attach on worker"); m_workerPipStateManager = new WorkerServicePipStateManager(this); m_pipTable = schedule.PipTable; m_pipQueue = schedule.SchedulingQueue; m_scheduler = schedule.Scheduler; m_operationTracker = m_scheduler.OperationTracker; m_environment = m_scheduler; m_environment.ContentFingerprinter.FingerprintSalt = BuildStartData.FingerprintSalt; m_environment.State.PipEnvironment.MasterEnvironmentVariables = BuildStartData.EnvironmentVariables; m_resultSerializer = new ExecutionResultSerializer(maxSerializableAbsolutePathIndex: schedule.MaxSerializedAbsolutePath, executionContext: m_scheduler.Context); m_forwardingEventListener = new ForwardingEventListener(this); }
internal ProcessRunnablePip( LoggingContext phaseLoggingContext, PipId pipId, int priority, Func <RunnablePip, Task> executionFunc, IPipExecutionEnvironment environment, ushort cpuUsageInPercents = 0, Pip pip = null) : base(phaseLoggingContext, pipId, PipType.Process, priority, executionFunc, environment, pip) { if (cpuUsageInPercents > 100) { m_weightBasedOnHistoricCpuUsage = (int)Math.Ceiling(cpuUsageInPercents / 100.0); } else { // If cpu usage is less than 100%, just use the lowest possible weight. m_weightBasedOnHistoricCpuUsage = Process.MinWeight; } }
public PerProcessPipPerformanceInformationStoreTests(ITestOutputHelper output) : base(output) { m_context = BuildXLContext.CreateInstanceForTesting(); m_loggingContext = CreateLoggingContextForTest(); m_configuration = ConfigurationHelpers.GetDefaultForTesting(m_context.PathTable, AbsolutePath.Create(m_context.PathTable, Path.Combine(TemporaryDirectory, "config.ds"))); m_perPipPerformanceInformationStore = new PerProcessPipPerformanceInformationStore(MaxNumPipTelemetryBatches, m_configuration.Logging.AriaIndividualMessageSizeLimitBytes); m_runnablePips = new Hashtable(); var pipTable = new PipTable( m_context.PathTable, m_context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3, debug: false); m_executionEnvironment = new DummyPipExecutionEnvironment(m_loggingContext, m_context, m_configuration, pipTable: pipTable); m_individualPipInfoStringLength = PerProcessPipPerformanceInformationStore.SerializePipPerfInfo(CreateSamplePip(0)).Length; }
/// <summary> /// Checks if the <paramref name="servicePipId"/> has already been started for the requested workerHost/workerId. /// If it hasn't, executes the requested service pip in a given <paramref name="environment"/>. /// </summary> /// <returns> /// <code>false</code> if the requested service has already failed and <code>true</code> otherwise. /// </returns> private Task <bool> EnsureServiceRunningAsync(PipId servicePipId, IPipExecutionEnvironment environment) { Contract.Requires(environment != null); ServiceTracking result = m_startedServices.GetOrAdd( key: servicePipId, data: this, addValueFactory: (key, scheduler) => { var serviceStartupCompletion = TaskSourceSlim.Create <bool>(); Task <PipResult> serviceCompletion = StartServiceOrShutdownServiceAsync( environment, servicePipId, serviceStartupCompletion, isStartup: true); return(new ServiceTracking(environment, serviceStartupCompletion, serviceCompletion)); }).Item.Value; return(result.WaitForProcessStartAsync()); }
/// <inheritdoc /> public override async Task <bool> TryRunServiceDependenciesAsync( IPipExecutionEnvironment environment, PipId pipId, IEnumerable <PipId> servicePips, LoggingContext loggingContext) { if (m_finalizationPipToServicePipMap.TryGetValue(pipId, out var serviceForFinalizationPip)) { servicePips = servicePips.Concat(new[] { serviceForFinalizationPip }); } if (!servicePips.Any()) { return(true); } var results = await Task.WhenAll( servicePips.Select(servicePipId => EnsureServiceRunningAsync(servicePipId, environment))); return(results.All(succeeded => succeeded)); }
internal RunnablePip( LoggingContext phaseLoggingContext, PipId pipId, PipType type, int priority, Func <RunnablePip, Task> executionFunc, IPipExecutionEnvironment environment, Pip pip = null) { Contract.Requires(phaseLoggingContext != null); Contract.Requires(environment != null); PipId = pipId; PipType = type; Priority = priority; OperationContext = OperationContext.CreateUntracked(phaseLoggingContext); m_executionFunc = executionFunc; Environment = environment; Transition(PipExecutionStep.Start); ScheduleTime = DateTime.UtcNow; Performance = new RunnablePipPerformanceInfo(ScheduleTime); m_pip = pip; }
/// <summary> /// Creates a default renderer that can be used for <see cref="IPipExecutionEnvironment.PipFragmentRenderer"/>. /// </summary> public static PipFragmentRenderer CreatePipFragmentRenderer(this IPipExecutionEnvironment env) { return(new PipFragmentRenderer(env.Context.PathTable, mId => env.IpcProvider.LoadAndRenderMoniker(mId), env.ContentFingerprinter.ContentHashLookupFunction)); }
/// <nodoc /> public ServiceTracking(IPipExecutionEnvironment environment, TaskSourceSlim <bool> startupCompletion, Task <PipResult> serviceCompletion) { Environment = environment; StartupCompletion = startupCompletion; ServiceExecutionCompletion = serviceCompletion; }
private async Task <PipResult> StartServiceOrShutdownServiceAsync( IPipExecutionEnvironment environment, PipId pipId, TaskSourceSlim <bool> serviceLaunchCompletion, bool isStartup) { // Ensure task does not block current thread await Task.Yield(); var loggingContext = m_executePhaseLoggingContext; try { var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId); if (isStartup) { Logger.Log.ScheduleServicePipStarting( loggingContext, serviceProcess.GetDescription(m_context)); Interlocked.Increment(ref m_runningServicesCount); } using (var operationContext = m_operationTracker.StartOperation( isStartup ? PipExecutorCounter.ExecuteServiceDuration : PipExecutorCounter.ExecuteServiceShutdownDuration, pipId, PipType.Process, loggingContext)) { var serviceStartTask = PipExecutor.ExecuteServiceStartOrShutdownAsync( #pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call // Bug #1155822: There is a race condition where service start/shutdown can // cause crash in the operation tracker because the parent operation is already completed // this is not fully understood, but the tracking of details of services operations is not // important so this disables it OperationContext.CreateUntracked(loggingContext), #pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call environment, serviceProcess, processId => { serviceLaunchCompletion.TrySetResult(isStartup); }); using ( operationContext.StartAsyncOperation( isStartup ? PipExecutorCounter.ExecuteServiceStartupLaunchDuration : PipExecutorCounter.ExecuteServiceShutdownLaunchDuration)) { Analysis.IgnoreResult( await Task.WhenAny(serviceLaunchCompletion.Task, serviceStartTask), justification: "Task<Task> doesn't contain any data." ); } var result = await serviceStartTask; return(PipResult.CreateWithPointPerformanceInfo(result.Result)); } } finally { // Attempt to set the result to false to indicate service startup failure. // This will not succeed if the service is already started // and the result is set to true. if (serviceLaunchCompletion.TrySetResult(false)) { var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId); Logger.Log.ScheduleServiceTerminatedBeforeStartupWasSignaled( loggingContext, serviceProcess.GetDescription(m_context)); } } }
/// <nodoc/> public DirectoryArtifactContext(IPipExecutionEnvironment pipExecutionEnvironment) { Contract.Requires(pipExecutionEnvironment != null); m_pipExecutionEnvironment = pipExecutionEnvironment; }
/// <summary> /// Gets the cacheable pip abstraction for the process. This data is cached so that subsequent /// calls do not create a new cacheable pip. /// </summary> public CacheableProcess GetCacheableProcess(Process process, IPipExecutionEnvironment environment) { m_cacheablePip = m_cacheablePip ?? new CacheableProcess(process, environment); return(m_cacheablePip); }
public override Task <bool> TryRunServiceDependenciesAsync(IPipExecutionEnvironment environment, IEnumerable <PipId> servicePipDependencies, LoggingContext loggingContext) { return(BoolTask.True); }
/// <nodoc/> public WorkerNotificationManager(WorkerService workerService, EngineSchedule schedule, IPipExecutionEnvironment environment, DistributionServices services) { WorkerService = workerService; DistributionServices = services; m_scheduler = schedule.Scheduler; m_environment = environment; m_forwardingEventListener = new ForwardingEventListener(this); m_sendCancellationSource = new CancellationTokenSource(); m_pipResultListener = new PipResultListener(this, schedule, environment); m_sendThread = new Thread(() => SendNotifications(m_sendCancellationSource.Token)); }
public PipResultListener(WorkerNotificationManager notificationManager, EngineSchedule schedule, IPipExecutionEnvironment environment) { m_notificationManager = notificationManager; m_resultSerializer = new ExecutionResultSerializer(maxSerializableAbsolutePathIndex: schedule.MaxSerializedAbsolutePath, executionContext: schedule.Scheduler.Context); m_pipTable = schedule.PipTable; m_environment = environment; }