public void InitializeWorkerSetupCost(Pip pip) { for (int i = 0; i < m_context.Workers.Count; i++) { var worker = m_context.Workers[i]; WorkerSetupCosts[i] = new WorkerSetupCost() { Worker = worker }; } }
private void InitializeWorkerSetupCost(Pip pip) { for (int i = 0; i < m_context.Workers.Count; i++) { var worker = m_context.Workers[i]; WorkerSetupCosts[i] = new WorkerSetupCost() { Worker = worker, AcquiredSlots = pip.PipType == PipType.Ipc ? worker.AcquiredIpcSlots : worker.AcquiredProcessSlots }; } }
public void InitializeWorkerSetupCost(RunnablePip runnable) { for (int i = 0; i < m_context.Workers.Count; i++) { var worker = m_context.Workers[i]; int acquiredSlots = worker.AcquiredProcessSlots; if (runnable.IsLight) { // For light process and IPC pips, use Light slots to order the workers. acquiredSlots = worker.AcquiredLightSlots; } WorkerSetupCosts[i] = new WorkerSetupCost() { Worker = worker, AcquiredSlots = acquiredSlots }; } }
/// <summary> /// The result contains estimated amount of work for each worker /// </summary> public void EstimateAndSortSetupCostPerWorker(RunnablePip runnablePip) { if (m_context.MustRunOnMaster(runnablePip)) { // Only estimate setup costs for pips which can execute remotely return; } var pip = runnablePip.Pip; for (int i = 0; i < m_context.Workers.Count; i++) { WorkerSetupCosts[i] = new WorkerSetupCost() { Worker = m_context.Workers[i], }; } // The block below collects process input file artifacts and hashes // Currently there is no logic to keep from sending the same hashes twice // Consider a model where hashes for files are requested by worker using (var pooledFileSet = Pools.GetFileArtifactSet()) { var pipInputs = pooledFileSet.Instance; m_context.m_fileContentManager.CollectPipInputsToMaterialize( m_context.m_pipTable, pip, pipInputs, // Service pip cost is not considered as this is shared among many clients and is a one-time cost per worker serviceFilter: servicePipId => false); m_visitedHashes.Clear(); foreach (var fileInput in pipInputs) { if (!fileInput.IsOutputFile) { continue; } if (pip.PipType == PipType.Ipc && !m_context.m_executedProcessOutputs.Contains(fileInput)) { // Only executed process outputs are considered for IPC pip affinity continue; } FileContentInfo fileContentInfo = m_context.m_fileContentManager.GetInputContent(fileInput).FileContentInfo; if (!m_visitedHashes.Add(fileContentInfo.Hash)) { continue; } // How many bytes we have to copy. long fileSize = fileContentInfo.HasKnownLength ? fileContentInfo.Length : TypicalFileLength; for (int idx = 0; idx < m_context.Workers.Count; ++idx) { if (!WorkerSetupCosts[idx].Worker.HasContent(fileInput)) { WorkerSetupCosts[idx].SetupBytes += fileSize; } } } if (pip.PipType == PipType.Ipc) { for (int idx = 0; idx < m_context.Workers.Count; ++idx) { WorkerSetupCosts[idx].AcquiredIpcSlots = m_context.Workers[idx].AcquiredIpcSlots; } } } Array.Sort(WorkerSetupCosts); }