private static Scheduler CreateInternal(
PipExecutionContext context,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew();
var fileAccessWhiteList = new FileAccessWhitelist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: Events.StaticContext,
configuration: configuration,
fileAccessWhitelist: fileAccessWhiteList,
testHooks: testHooks,
buildEngineFingerprint: null,
tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp"))));
}
public ChooseWorkerCacheLookup(
LoggingContext loggingContext,
IConfiguration config,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue) : base(loggingContext, config, workers, pipQueue, DispatcherKind.ChooseWorkerCacheLookup, config.Schedule.MaxChooseWorkerCacheLookup)
{
}
public static Tuple <Scheduler, EngineCache> CreateWithCaching(
PipExecutionContext context,
LoggingContext loggingContext,
IConfiguration configuration,
PipGraph.Builder graphBuilder,
IPipQueue queue)
{
Contract.Requires(graphBuilder != null);
Contract.Requires(context != null);
Contract.Requires(queue != null);
var cacheLayer = new EngineCache(
new InMemoryArtifactContentCache(),
new InMemoryTwoPhaseFingerprintStore());
Scheduler scheduler = CreateInternal(
context,
loggingContext,
graphBuilder.Build(),
queue,
cacheLayer,
configuration);
return(Tuple.Create(scheduler, cacheLayer));
}
private static Scheduler CreateInternal(
PipExecutionContext context,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew();
var fileAccessWhiteList = new FileAccessWhitelist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: Events.StaticContext,
configuration: configuration,
fileAccessWhitelist: fileAccessWhiteList,
testHooks: testHooks,
buildEngineFingerprint: null));
}
public ChooseWorkerCacheLookup(
LoggingContext loggingContext,
IScheduleConfiguration scheduleConfig,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue) : base(loggingContext, workers, pipQueue, DispatcherKind.ChooseWorkerCacheLookup, scheduleConfig.MaxChooseWorkerCacheLookup, scheduleConfig.ModuleAffinityEnabled())
{
m_workerBalancedLoadFactors = ReadOnlyArray <double> .FromWithoutCopy(0.5, 1, 2, 3);
}
public ChooseWorkerCacheLookup(
LoggingContext loggingContext,
IScheduleConfiguration scheduleConfig,
bool distributeCacheLookups,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue) : base(loggingContext, workers, pipQueue, DispatcherKind.ChooseWorkerCacheLookup, scheduleConfig.MaxChooseWorkerCacheLookup, scheduleConfig.ModuleAffinityEnabled())
{
m_distributeCacheLookups = distributeCacheLookups;
}
public ChooseWorkerCacheLookup(
LoggingContext loggingContext,
int maxParallelDegree,
bool distributeCacheLookups,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue) : base(loggingContext, workers, pipQueue, DispatcherKind.ChooseWorkerCacheLookup, maxParallelDegree)
{
m_distributeCacheLookups = distributeCacheLookups;
}
public TestPipQueue(IPipQueue innerQueue, LoggingContext loggingContext, bool initiallyPaused = false)
{
m_innerQueue = innerQueue;
m_loggingContext = loggingContext;
m_paused = initiallyPaused;
if (m_paused)
{
m_pausedQueueActions = new Queue <Action>();
}
}
/// <summary>
/// Constructor
/// </summary>
public LocalWorker(IScheduleConfiguration scheduleConfig, IPipQueue pipQueue, IDetoursEventListener detoursListener, PipExecutionContext context)
: base(workerId: 0, name: "#0 (Local)", context: context)
{
TotalProcessSlots = scheduleConfig.EffectiveMaxProcesses;
TotalCacheLookupSlots = scheduleConfig.MaxCacheLookup;
TotalLightSlots = scheduleConfig.MaxLightProcesses;
TotalMaterializeInputSlots = scheduleConfig.MaxMaterialize;
m_detoursListener = detoursListener;
m_pipQueue = pipQueue;
Start();
}
public ChooseWorkerCpu(
LoggingContext loggingContext,
int maxParallelDegree,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue,
PipGraph pipGraph,
FileContentManager fileContentManager) : base(loggingContext, workers, pipQueue, DispatcherKind.ChooseWorkerCpu, maxParallelDegree)
{
m_pipTable = pipGraph.PipTable;
m_executedProcessOutputs = new ContentTrackingSet(pipGraph);
m_fileContentManager = fileContentManager;
m_pipSetupCostPool = new ObjectPool <PipSetupCosts>(() => new PipSetupCosts(this), costs => costs, size: maxParallelDegree);
}
public ChooseWorkerCpu(
LoggingContext loggingContext,
IConfiguration config,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue,
PipGraph pipGraph,
FileContentManager fileContentManager) : base(loggingContext, config, workers, pipQueue, DispatcherKind.ChooseWorkerCpu, config.Schedule.MaxChooseWorkerCpu)
{
m_pipTable = pipGraph.PipTable;
m_executedProcessOutputs = new ContentTrackingSet(pipGraph);
m_fileContentManager = fileContentManager;
m_pipSetupCostPool = new ObjectPool <PipSetupCosts>(() => new PipSetupCosts(this), costs => costs, size: config.Schedule.MaxChooseWorkerCpu);
}
/// <summary>
/// Constructor.
/// </summary>
public LocalWorkerWithRemoting(
IScheduleConfiguration scheduleConfig,
ISandboxConfiguration sandboxConfig,
IPipQueue pipQueue,
IDetoursEventListener detoursListener,
PipExecutionContext pipExecutionContext)
: base(scheduleConfig, pipQueue, detoursListener, pipExecutionContext)
{
m_localExecutionSemaphore = new SemaphoreSlim(scheduleConfig.MaxProcesses, scheduleConfig.MaxProcesses);
m_remotingThreshold = (int)(scheduleConfig.MaxProcesses * scheduleConfig.RemotingThresholdMultiplier);
m_processCanRunRemoteTags = scheduleConfig.ProcessCanRunRemoteTags.Select(t => StringTable.AddString(t)).ToHashSet();
m_processMustRunLocalTags = scheduleConfig.ProcessMustRunLocalTags.Select(t => StringTable.AddString(t)).ToHashSet();
m_sandboxConfig = sandboxConfig;
}
protected ChooseWorkerContext(
LoggingContext loggingContext,
IReadOnlyList <Worker> workers,
IPipQueue pipQueue,
DispatcherKind kind,
int maxParallelDegree)
{
Workers = workers;
PipQueue = pipQueue;
LocalWorker = (LocalWorker)workers[0];
LoggingContext = loggingContext;
Kind = kind;
MaxParallelDegree = maxParallelDegree;
foreach (var worker in Workers)
{
worker.ResourcesChanged += OnWorkerResourcesChanged;
}
}
