/// <summary>
/// Constructor
/// </summary>
public DispatcherQueue(PipQueue pipQueue, int maxParallelDegree)
{
m_pipQueue = pipQueue;
MaxParallelDegree = maxParallelDegree;
m_numRunning = 0;
m_processesQueued = 0;
m_stack = new ConcurrentStack <int>();
for (int i = MaxParallelDegree - 1; i >= 0; i--)
{
m_stack.Push(i);
}
}
/// <summary>
/// Constructor
/// </summary>
public DispatcherQueue(PipQueue pipQueue, int maxParallelDegree, bool useWeight = false)
{
m_pipQueue = pipQueue;
MaxParallelDegree = maxParallelDegree;
m_numAcquiredSlots = 0;
m_numQueuedProcessPips = 0;
UseWeight = useWeight;
m_stack = new ConcurrentStack <int>();
for (int i = MaxParallelDegree - 1; i >= 0; i--)
{
m_stack.Push(i);
}
}
/// <summary>
/// Choose a worker based on setup cost
/// </summary>
private Worker ChooseWorker(RunnablePip runnablePip, WorkerSetupCost[] workerSetupCosts, out WorkerResource?limitingResource)
{
if (MustRunOnMaster(runnablePip))
{
// This is shortcut for the single-machine builds and distributed workers.
return(LocalWorker.TryAcquire(runnablePip, out limitingResource, loadFactor: MaxLoadFactor) ? LocalWorker : null);
}
ResetStatus();
var pendingWorkerSelectionPipCount = PipQueue.GetNumQueuedByKind(DispatcherKind.ChooseWorkerCpu);
bool loadBalanceWorkers = false;
if (runnablePip.PipType == PipType.Process)
{
if (pendingWorkerSelectionPipCount + m_totalAcquiredProcessSlots < (m_totalProcessSlots / 2))
{
// When there is a limited amount of work (less than half the total capacity of
// the all the workers). We load balance so that each worker gets
// its share of the work and the work can complete faster
loadBalanceWorkers = true;
}
}
long setupCostForBestWorker = workerSetupCosts[0].SetupBytes;
limitingResource = null;
foreach (var loadFactor in m_workerBalancedLoadFactors)
{
if (!loadBalanceWorkers && loadFactor < 1)
{
// Not load balancing so allow worker to be filled to capacity at least
continue;
}
for (int i = 0; i < workerSetupCosts.Length; i++)
{
var worker = workerSetupCosts[i].Worker;
if (worker.TryAcquire(runnablePip, out limitingResource, loadFactor: loadFactor))
{
runnablePip.Performance.SetInputMaterializationCost(ByteSizeFormatter.ToMegabytes((ulong)setupCostForBestWorker), ByteSizeFormatter.ToMegabytes((ulong)workerSetupCosts[i].SetupBytes));
return(worker);
}
}
}
return(null);
}
internal void Start(EngineSchedule schedule)
{
Contract.Requires(schedule != null);
Contract.Requires(schedule.Scheduler != null);
Contract.Requires(schedule.SchedulingQueue != null);
Contract.Assert(AttachCompletion.IsCompleted && AttachCompletion.GetAwaiter().GetResult(), "ProcessBuildRequests called before finishing attach on worker");
m_workerPipStateManager = new WorkerServicePipStateManager(this);
m_pipTable = schedule.PipTable;
m_pipQueue = schedule.SchedulingQueue;
m_scheduler = schedule.Scheduler;
m_operationTracker = m_scheduler.OperationTracker;
m_environment = m_scheduler;
m_environment.ContentFingerprinter.FingerprintSalt = BuildStartData.FingerprintSalt;
m_environment.State.PipEnvironment.MasterEnvironmentVariables = BuildStartData.EnvironmentVariables;
m_resultSerializer = new ExecutionResultSerializer(maxSerializableAbsolutePathIndex: schedule.MaxSerializedAbsolutePath, executionContext: m_scheduler.Context);
m_forwardingEventListener = new ForwardingEventListener(this);
}
public void Empty()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
using (var pipQueue = new PipQueue(new ScheduleConfiguration()))
{
pipQueue.SetAsFinalized();
pipQueue.DrainQueues();
XAssert.IsTrue(pipQueue.IsFinished);
}
}
}
public async Task Stress()
{
const int N = 5;
const int M = N * N;
var context = BuildXLContext.CreateInstanceForTesting();
var loggingContext = CreateLoggingContextForTest();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
var executionEnvironment = new PipQueueTestExecutionEnvironment(
context,
config,
pipTable,
Path.Combine(TestOutputDirectory, "temp"),
TryGetSubstSourceAndTarget(out string substSource, out string substTarget) ? (substSource, substTarget) : default((string, string)?),
GetSandboxConnection());
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
}
executionEnvironment.MarkExecuted(pip);
return(result);
};
string executable = CmdHelper.OsShellExe;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
// This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);
using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 1: create some files
var baseFileArtifacts = new List <FileArtifact>();
for (int i = 0; i < N; i++)
{
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
baseFileArtifacts.Add(destinationArtifact);
PipData contents = PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
i.ToString(CultureInfo.InvariantCulture));
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
var pipId = pipTable.Add((uint)(i + 1), writeFile);
var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(DispatcherKind.IO);
phase1PipQueue.Enqueue(runnable);
}
phase1PipQueue.SetAsFinalized();
phase1PipQueue.DrainQueues();
await Task.WhenAll(
Enumerable.Range(0, 2).Select(
async range =>
{
using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 2: do some more with those files
var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >();
var checkerTasks = new ConcurrentQueue <Task>();
Action <PipId, Task <PipResult> > callback =
(id, task) =>
{
XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
XAssert.IsFalse(task.Result.Status.IndicatesFailure());
Tuple <string, int> t;
if (!pips.TryRemove(id, out t))
{
XAssert.Fail();
}
checkerTasks.Enqueue(
Task.Run(
() =>
{
string actual = File.ReadAllText(t.Item1).Trim();
// TODO: Make this async
XAssert.AreEqual(actual, t.Item2.ToString());
}));
};
var r = new Random(0);
for (int i = 0; i < M; i++)
{
int sourceIndex = r.Next(baseFileArtifacts.Count);
FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
Pip pip;
DispatcherKind queueKind;
switch (r.Next(2))
{
case 0:
pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
queueKind = DispatcherKind.IO;
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
case 1:
string workingDirectory =
OperatingSystemHelper.IsUnixOS ? "/tmp" :
Environment.GetFolderPath(Environment.SpecialFolder.Windows);
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
var pipData = OperatingSystemHelper.IsUnixOS ?
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);
queueKind = DispatcherKind.CPU;
pip = new Process(
executableArtifact,
workingDirectoryAbsolutePath,
pipData,
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
default:
Contract.Assert(false);
continue;
}
var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);
Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
{
var task = taskFactory(runnablePip);
var pipResult = await task;
callback(pipId, task);
};
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(queueKind);
phase2PipQueue.Enqueue(runnable);
if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
{
Contract.Assert(false);
}
}
phase2PipQueue.SetAsFinalized();
phase2PipQueue.DrainQueues();
XAssert.AreEqual(0, pips.Count);
await Task.WhenAll(checkerTasks);
}
}));
}
}
}
}
/// <summary>
/// Choose a worker based on setup cost
/// </summary>
private Worker ChooseWorker(RunnablePip runnablePip, WorkerSetupCost[] workerSetupCosts, out WorkerResource?limitingResource)
{
if (MustRunOnMaster(runnablePip))
{
// This is shortcut for the single-machine builds and distributed workers.
return(LocalWorker.TryAcquire(runnablePip, out limitingResource, loadFactor: MaxLoadFactor) ? LocalWorker : null);
}
ResetStatus();
var pendingWorkerSelectionPipCount = PipQueue.GetNumQueuedByKind(DispatcherKind.ChooseWorkerCpu) + PipQueue.GetNumRunningByKind(DispatcherKind.ChooseWorkerCpu);
bool loadBalanceWorkers = false;
if (runnablePip.PipType == PipType.Process)
{
if (pendingWorkerSelectionPipCount + m_totalAcquiredProcessSlots < (m_totalProcessSlots / 2))
{
// When there is a limited amount of work (less than half the total capacity of
// the all the workers). We load balance so that each worker gets
// its share of the work and the work can complete faster
loadBalanceWorkers = true;
}
}
double?disableLoadBalanceMultiplier = EngineEnvironmentSettings.DisableLoadBalanceMultiplier;
// Disable load-balance if there is a multiplier specified including 0.
loadBalanceWorkers &= !disableLoadBalanceMultiplier.HasValue;
long setupCostForBestWorker = workerSetupCosts[0].SetupBytes;
limitingResource = null;
foreach (var loadFactor in m_workerBalancedLoadFactors)
{
if (!loadBalanceWorkers && loadFactor < 1)
{
// Not load balancing so allow worker to be filled to capacity at least
continue;
}
for (int i = 0; i < workerSetupCosts.Length; i++)
{
var worker = workerSetupCosts[i].Worker;
if (worker.TryAcquire(runnablePip, out limitingResource, loadFactor: loadFactor))
{
runnablePip.Performance.SetInputMaterializationCost(ByteSizeFormatter.ToMegabytes((ulong)setupCostForBestWorker), ByteSizeFormatter.ToMegabytes((ulong)workerSetupCosts[i].SetupBytes));
return(worker);
}
// If the worker is not chosen due to the lack of process slots,
// do not try the next worker immediately if 'BuildXLDisableLoadBalanceMultiplier' is specified.
// We first check whether the number of pips waiting for a worker is less than the total slots times with the multiplier.
// For example, if the multiplier is 1 and totalWorkerSlots is 100, then we do not try the next worker
// if there are less than 100 pips waiting for a worker.
// For Cosine builds, executing pips on a new worker is expensive due to the input materialization.
// It is usually faster to wait for the busy worker to be available compared to trying on another worker.
if (limitingResource == WorkerResource.AvailableProcessSlots &&
disableLoadBalanceMultiplier.HasValue &&
pendingWorkerSelectionPipCount < (worker.TotalProcessSlots * disableLoadBalanceMultiplier.Value))
{
limitingResource = WorkerResource.DisableLoadBalance;
return(null);
}
}
}
return(null);
}
/// <summary>
/// Constructor
/// </summary>
public DispatcherQueue(PipQueue pipQueue, int maxParallelDegree)
{
m_pipQueue = pipQueue;
MaxParallelDegree = maxParallelDegree;
m_numRunning = 0;
}
/// <summary>
/// Runs the scheduler allowing various options to be specifically set
/// </summary>
public ScheduleRunResult RunSchedulerSpecific(
PipGraph graph,
SchedulerTestHooks testHooks = null,
SchedulerState schedulerState = null,
RootFilter filter = null,
TempCleaner tempCleaner = null)
{
var config = new CommandLineConfiguration(Configuration);
// Populating the configuration may modify the configuration, so it should occur first.
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext);
FileAccessWhitelist whitelist = new FileAccessWhitelist(Context);
whitelist.Initialize(config);
IReadOnlyList <string> junctionRoots = Configuration.Engine.DirectoriesToTranslate?.Select(a => a.ToPath.ToString(Context.PathTable)).ToList();
var map = VolumeMap.TryCreateMapOfAllLocalVolumes(LoggingContext, junctionRoots);
var optionalAccessor = TryGetJournalAccessor(map);
// Although scan change journal is enabled, but if we cannot create an enabled journal accessor, then create a disabled one.
m_journalState = map == null || !optionalAccessor.IsValid
? JournalState.DisabledJournal
: JournalState.CreateEnabledJournal(map, optionalAccessor.Value);
if (config.Schedule.IncrementalScheduling)
{
// Ensure that we can scan the journal when incremental scheduling is enabled.
XAssert.IsTrue(m_journalState.IsEnabled, "Incremental scheduling requires that journal is enabled");
}
// Seal the translator if not sealed
DirectoryTranslator.Seal();
// .....................................................................................
// some dummy setup in order to get a PreserveOutputsSalt.txt file and an actual salt
// .....................................................................................
string dummyCacheDir = Path.Combine(TemporaryDirectory, "Out", "Cache");
Directory.CreateDirectory(dummyCacheDir); // EngineSchedule tries to put the PreserveOutputsSalt.txt here
ContentHash?previousOutputsSalt =
EngineSchedule.PreparePreviousOutputsSalt(LoggingContext, Context.PathTable, config);
Contract.Assert(previousOutputsSalt.HasValue);
// .....................................................................................
testHooks = testHooks ?? new SchedulerTestHooks();
Contract.Assert(!(config.Engine.CleanTempDirectories && tempCleaner == null));
using (var queue = new PipQueue(config.Schedule))
using (var testQueue = new TestPipQueue(queue, LoggingContext, initiallyPaused: false))
using (var testScheduler = new TestScheduler(
graph: graph,
pipQueue: testQueue,
context: Context,
fileContentTable: FileContentTable,
loggingContext: LoggingContext,
cache: Cache,
configuration: config,
journalState: m_journalState,
fileAccessWhitelist: whitelist,
fingerprintSalt: Configuration.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable),
tempCleaner: tempCleaner,
previousInputsSalt: previousOutputsSalt.Value,
successfulPips: null,
failedPips: null,
ipcProvider: null,
directoryTranslator: DirectoryTranslator,
testHooks: testHooks))
{
if (filter == null)
{
EngineSchedule.TryGetPipFilter(LoggingContext, Context, config, config, Expander.TryGetRootByMountName, out filter);
}
XAssert.IsTrue(testScheduler.InitForMaster(LoggingContext, filter, schedulerState), "Failed to initialized test scheduler");
testScheduler.Start(LoggingContext);
bool success = testScheduler.WhenDone().GetAwaiter().GetResult();
testScheduler.SaveFileChangeTrackerAsync(LoggingContext).Wait();
return(new ScheduleRunResult
{
Graph = graph,
Config = config,
Success = success,
PipResults = testScheduler.PipResults,
PipExecutorCounters = testScheduler.PipExecutionCounters,
PathSets = testScheduler.PathSets,
ProcessPipCountersByFilter = testScheduler.ProcessPipCountersByFilter,
ProcessPipCountersByTelemetryTag = testScheduler.ProcessPipCountersByTelemetryTag,
SchedulerState = new SchedulerState(testScheduler)
});
}
}
