public void InitializeWorkerSetupCost(Pip pip)
{
for (int i = 0; i < m_context.Workers.Count; i++)
{
var worker = m_context.Workers[i];
WorkerSetupCosts[i] = new WorkerSetupCost()
{
Worker = worker
};
}
}
private void InitializeWorkerSetupCost(Pip pip)
{
for (int i = 0; i < m_context.Workers.Count; i++)
{
var worker = m_context.Workers[i];
WorkerSetupCosts[i] = new WorkerSetupCost()
{
Worker = worker,
AcquiredSlots = pip.PipType == PipType.Ipc ? worker.AcquiredIpcSlots : worker.AcquiredProcessSlots
};
}
}
public void InitializeWorkerSetupCost(RunnablePip runnable)
{
for (int i = 0; i < m_context.Workers.Count; i++)
{
var worker = m_context.Workers[i];
int acquiredSlots = worker.AcquiredProcessSlots;
if (runnable.IsLight)
{
// For light process and IPC pips, use Light slots to order the workers.
acquiredSlots = worker.AcquiredLightSlots;
}
WorkerSetupCosts[i] = new WorkerSetupCost()
{
Worker = worker,
AcquiredSlots = acquiredSlots
};
}
}
/// <summary>
/// The result contains estimated amount of work for each worker
/// </summary>
public void EstimateAndSortSetupCostPerWorker(RunnablePip runnablePip)
{
if (m_context.MustRunOnMaster(runnablePip))
{
// Only estimate setup costs for pips which can execute remotely
return;
}
var pip = runnablePip.Pip;
for (int i = 0; i < m_context.Workers.Count; i++)
{
WorkerSetupCosts[i] = new WorkerSetupCost()
{
Worker = m_context.Workers[i],
};
}
// The block below collects process input file artifacts and hashes
// Currently there is no logic to keep from sending the same hashes twice
// Consider a model where hashes for files are requested by worker
using (var pooledFileSet = Pools.GetFileArtifactSet())
{
var pipInputs = pooledFileSet.Instance;
m_context.m_fileContentManager.CollectPipInputsToMaterialize(
m_context.m_pipTable,
pip,
pipInputs,
// Service pip cost is not considered as this is shared among many clients and is a one-time cost per worker
serviceFilter: servicePipId => false);
m_visitedHashes.Clear();
foreach (var fileInput in pipInputs)
{
if (!fileInput.IsOutputFile)
{
continue;
}
if (pip.PipType == PipType.Ipc && !m_context.m_executedProcessOutputs.Contains(fileInput))
{
// Only executed process outputs are considered for IPC pip affinity
continue;
}
FileContentInfo fileContentInfo = m_context.m_fileContentManager.GetInputContent(fileInput).FileContentInfo;
if (!m_visitedHashes.Add(fileContentInfo.Hash))
{
continue;
}
// How many bytes we have to copy.
long fileSize = fileContentInfo.HasKnownLength ? fileContentInfo.Length : TypicalFileLength;
for (int idx = 0; idx < m_context.Workers.Count; ++idx)
{
if (!WorkerSetupCosts[idx].Worker.HasContent(fileInput))
{
WorkerSetupCosts[idx].SetupBytes += fileSize;
}
}
}
if (pip.PipType == PipType.Ipc)
{
for (int idx = 0; idx < m_context.Workers.Count; ++idx)
{
WorkerSetupCosts[idx].AcquiredIpcSlots = m_context.Workers[idx].AcquiredIpcSlots;
}
}
}
Array.Sort(WorkerSetupCosts);
}
