public void It_should_set_IsOperationRunning_to_true()
{
var operationTracker = new OperationTracker();
operationTracker.BeginOperation();
Assert.That(operationTracker.IsOperationRunning, Is.True);
}
public void It_should_not_decrement_OperationsRunning_below_zero()
{
var operationTracker = new OperationTracker();
operationTracker.EndOperation();
Assert.That(operationTracker.OperationsRunning, Is.EqualTo(0));
}
public void TestOperationsHelper(bool parallel)
{
LoggingContext log = new LoggingContext("op");
var operationTracker = new OperationTracker(new LoggingContext("test"));
int length = 100000;
using (var globalContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, new PipId(15234), PipType.Process, log))
using (globalContext.StartOperation(PipExecutorCounter.ExecutePipStepDuration))
{
For(length, i =>
{
using (var context = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, log))
using (context.StartOperation(PipExecutorCounter.ExecutePipStepDuration))
using (var outerContext = context.StartAsyncOperation(PipExecutorCounter.FileContentManagerTryMaterializeOuterDuration))
using (outerContext.StartOperation(PipExecutorCounter.FileContentManagerTryMaterializeDuration))
{
}
using (var outerContext = globalContext.StartAsyncOperation(PipExecutorCounter.FileContentManagerTryMaterializeOuterDuration))
using (outerContext.StartOperation(PipExecutorCounter.FileContentManagerTryMaterializeDuration))
{
}
}, parallel);
}
}
/// <summary>
/// Sets logging context and start time of the pip
/// </summary>
public void Start(OperationTracker tracker, LoggingContext loggingContext)
{
Contract.Assert(Step == PipExecutionStep.Start || IsDistributedWorker);
OperationContext = tracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, PipId, PipType, loggingContext, OperationCompleted);
StartTime = DateTime.UtcNow;
}
public void It_should_increment_OperationsRunning()
{
var operationTracker = new OperationTracker();
operationTracker.BeginOperation();
Assert.That(operationTracker.OperationsRunning, Is.EqualTo(1));
}
public void It_should_invoke_setTargetPropertyAction_with_a_value_of_true()
{
var isOperationRunning = false;
var operationTracker = new OperationTracker(value => { isOperationRunning = value; });
operationTracker.BeginOperation();
Assert.That(isOperationRunning, Is.True);
}
internal void Start(LoggingContext loggingContext, OperationTracker operationTracker)
{
Contract.Requires(!IsStarted);
Contract.Ensures(IsStarted);
m_executePhaseLoggingContext = loggingContext;
m_operationTracker = operationTracker;
IsStarted = true;
}
public void Setup()
{
_operationTracker = new OperationTracker();
_fileSystem = new InMemoryFilesystem();
_node = new LocalNode(_operationTracker, _fileSystem);
_fileSystem.AddRoot(Path.GetPathRoot(Path.GetTempPath()));
_fileSystem.CreateDirectory(Path.GetTempPath());
}
protected virtual IAnchorRunspaceProxy GetRunspace()
{
IAnchorRunspaceProxy result;
using (OperationTracker.Create(this.anchorContext.Logger, "Creating a new runspace.", new object[0]))
{
result = AnchorRunspaceProxy.CreateRunspaceForDatacenterAdmin(this.anchorContext, "Mailbox Load Balance");
}
return(result);
}
OperationTracker CreateTrackerForNewOperation(string operationName, CancellationToken operationCancellation)
{
if (currentOperationTracker != null && !currentOperationTracker.cancellation.IsCancellationRequested)
{
throw new InvalidOperationException(
string.Format("Impossible to start new operation '{0}' while previous one '{1}' is not finished or cancelled",
operationName, currentOperationTracker.name));
}
currentOperationTracker = new OperationTracker(this, operationName, operationCancellation);
return(currentOperationTracker);
}
public void OperationTracker_Basics1()
{
var o = new OperationTracker();
//
// Enter a few times.
//
var g1 = o.Enter();
var g2 = o.Enter();
//
// Double-dispose is fine.
//
g1.Dispose();
g1.Dispose();
//
// Initiate dispose.
//
var d = o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
.AsTask()
#endif
;
Assert.IsFalse(d.IsCompleted);
//
// Cannot enter again.
//
Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
//
// Exit the last operation.
//
g2.Dispose();
//
// Tracker is disposed now.
//
d.Wait();
//
// Cannot enter again.
//
Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
}
protected override void ProcessRequest()
{
using (OperationTracker.Create(this.logger, "Rebalancing {0} from {1} to {2}.", new object[]
{
this.rebalanceData.RebalanceInformation,
this.rebalanceData.SourceDatabase,
this.rebalanceData.TargetDatabase
}))
{
this.logger.Log(MigrationEventType.Information, "MOVE: Moving {0} from {1} to {2}.", new object[]
{
this.rebalanceData.RebalanceInformation,
this.rebalanceData.SourceDatabase,
this.rebalanceData.TargetDatabase
});
Band[] bands = this.rebalanceData.RebalanceInformation.Metrics.OfType <Band>().ToArray <Band>();
TopologyExtractorFactory entitySelectorFactory = this.serviceContext.TopologyExtractorFactoryContextPool.GetContext(this.clientFactory, bands, this.nonMovableOrgs, this.logger).GetEntitySelectorFactory();
DirectoryDatabase database = this.directoryProvider.GetDatabase(this.rebalanceData.SourceDatabase.Guid);
LoadContainer container = entitySelectorFactory.GetExtractor(database).ExtractTopology();
IOperationRetryManager operationRetryManager = LoadBalanceOperationRetryManager.Create(this.logger);
foreach (LoadMetric loadMetric in this.rebalanceData.RebalanceInformation.Metrics)
{
EntitySelector selector = loadMetric.GetSelector(container, this.rebalanceData.ConstraintSetIdentity, this.rebalanceData.RebalanceInformation[loadMetric]);
if (selector.IsEmpty)
{
this.logger.Log(MigrationEventType.Information, "Could not find any mailbox for metric {0} in database {1}.", new object[]
{
loadMetric,
this.rebalanceData.SourceDatabase
});
}
else
{
this.logger.Log(MigrationEventType.Information, "Found mailboxes matching the metric {0} in database {1}. Requesting the injections.", new object[]
{
loadMetric,
this.rebalanceData.SourceDatabase
});
operationRetryManager.TryRun(delegate
{
DirectoryDatabase database2 = (DirectoryDatabase)this.directoryProvider.GetDirectoryObject(this.rebalanceData.TargetDatabase.DirectoryObjectIdentity);
using (IInjectorService injectorClientForDatabase = this.clientFactory.GetInjectorClientForDatabase(database2))
{
injectorClientForDatabase.InjectMoves(this.rebalanceData.TargetDatabase.Guid, this.rebalanceData.RebalanceBatchName, selector.GetEntities(this.rebalanceData.TargetDatabase));
}
});
}
}
}
}
protected override void ProcessRequest()
{
using (OperationTracker.Create(this.logger, "Applying processor {0} to mailbox {1}.", new object[]
{
this.Processor.GetType().FullName,
this.Mailbox.Identity
}))
{
using (RunspaceReservation runspaceReservation = this.cmdletPool.AcquireRunspace())
{
this.Processor.ProcessMailbox(this.Mailbox, runspaceReservation.Runspace);
}
}
}
private void RefreshCachedValue()
{
using (OperationTracker.Create(this.logger, "Refreshing topology for {0}.", new object[]
{
this.directoryObject
}))
{
this.cachedValue = this.topologyExtractor.ExtractTopology();
ExAssert.RetailAssert(this.cachedValue != null, "ExtractTopology for directoryObject: {0} should never return null. TopologyExtractor: {1}", new object[]
{
this.directoryObject,
this.topologyExtractor
});
}
}
private static async Task <PipResult> Execute(BuildXLContext context, FileContentTable fileContentTable, IConfiguration config, Pip pip)
{
Contract.Requires(context != null);
Contract.Requires(fileContentTable != null);
Contract.Requires(config != null);
Contract.Requires(pip != null);
var loggingContext = BuildXLTestBase.CreateLoggingContextForTest();
var operationTracker = new OperationTracker(loggingContext);
using (var env = new Test.BuildXL.Scheduler.Utils.DummyPipExecutionEnvironment(loggingContext, context, config, fileContentTable))
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, env.LoggingContext))
{
return(await Test.BuildXL.Scheduler.TestPipExecutor.ExecuteAsync(operationContext, env, pip));
}
}
private IPhysicalMailbox LoadMailboxData()
{
IPhysicalMailbox result;
using (OperationTracker.Create(this.logger, "Retrieving single mailbox {0} data from database {1}", new object[]
{
this.Guid,
this.database.Identity
}))
{
using (IPhysicalDatabase physicalDatabaseConnection = this.clientFactory.GetPhysicalDatabaseConnection(this.database))
{
result = (physicalDatabaseConnection.GetMailbox(this.Guid) ?? EmptyPhysicalMailbox.Instance);
}
}
return(result);
}
private void BeginProcessingMailbox(DirectoryMailbox mailbox)
{
IRequestQueue requestQueue = (mailbox.Parent != null) ? this.context.QueueManager.GetProcessingQueue(mailbox.Parent) : this.context.QueueManager.MainProcessingQueue;
string[] source = LoadBalanceADSettings.Instance.Value.ExcludedMailboxProcessors.Split(new char[]
{
','
});
IList <MailboxProcessor> list = this.processors(this.context.QueueManager) ?? ((IList <MailboxProcessor>)Array <MailboxProcessor> .Empty);
foreach (MailboxProcessor mailboxProcessor in list)
{
if (source.Contains(mailboxProcessor.Name))
{
this.context.Logger.LogVerbose("Skipping processor {0} because it is disabled.", new object[]
{
mailboxProcessor.Name
});
}
else if (!mailboxProcessor.ShouldProcess(mailbox))
{
this.context.Logger.LogVerbose("Processor {0} doesn't want to process the mailbox, ignored.", new object[]
{
mailboxProcessor.Name
});
}
else if (mailboxProcessor.RequiresRunspace)
{
requestQueue.EnqueueRequest(new ProcessMailboxRequest(mailbox, mailboxProcessor, this.context.Logger, this.context.CmdletPool));
}
else
{
using (OperationTracker.Create(this.context.Logger, "Applying processor {0} to mailbox {1}.", new object[]
{
mailboxProcessor.GetType().FullName,
mailbox.Identity
}))
{
mailboxProcessor.ProcessMailbox(mailbox, null);
}
}
}
}
internal void Start(LoggingContext loggingContext, OperationTracker operationTracker)
{
Contract.Requires(!IsStarted);
Contract.Ensures(IsStarted);
m_executePhaseLoggingContext = loggingContext;
m_operationTracker = operationTracker;
foreach (var servicePipId in m_pipGraph.GetServicePipIds())
{
var serviceMutable = (ProcessMutablePipState)m_pipGraph.PipTable.GetMutable(servicePipId);
foreach (var finalizationPipId in serviceMutable.ServiceInfo.FinalizationPipIds)
{
m_finalizationPipToServicePipMap[finalizationPipId] = servicePipId;
}
}
IsStarted = true;
}
protected override void ProcessRequest()
{
this.logger.LogInformation("Starting to drain database {0} with batch name {1}.", new object[]
{
this.directoryDatabase.Identity,
this.batchName
});
IOperationRetryManager operationRetryManager = LoadBalanceOperationRetryManager.Create(this.logger);
using (OperationTracker.Create(this.logger, "Moving mailboxes out of {0}.", new object[]
{
this.directoryDatabase.Identity
}))
{
foreach (DirectoryMailbox mailboxToMove2 in this.directoryDatabase.GetMailboxes())
{
DirectoryMailbox mailboxToMove = mailboxToMove2;
operationRetryManager.TryRun(delegate
{
this.moveInjector.InjectMoveForMailbox(mailboxToMove, this.batchName);
});
}
}
this.logger.LogInformation("Draining database {0}: Cleaning soft deleted mailboxes.", new object[]
{
this.directoryDatabase.Identity
});
using (OperationTracker.Create(this.logger, "Starting soft deleted cleanup for {0}.", new object[]
{
this.directoryDatabase.Identity
}))
{
this.serviceContext.CleanupSoftDeletedMailboxesOnDatabase(this.directoryDatabase.Identity, ByteQuantifiedSize.Zero);
}
this.logger.LogInformation("Finished processing the draining of database {0}.", new object[]
{
this.directoryDatabase.Identity
});
if (this.OnDrainFinished != null)
{
this.OnDrainFinished(this.directoryDatabase);
}
}
private PerProcessPipPerformanceInformation CreateSamplePip(int index)
{
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, m_executionEnvironment, pip);
}
return(result);
};
var pathTable = m_context.PathTable;
var executable = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, X("/x/pkgs/tool.exe")));
var dependencies = new HashSet <FileArtifact> {
executable
};
var processBuilder = new ProcessBuilder()
.WithExecutable(executable)
.WithWorkingDirectory(AbsolutePath.Create(pathTable, X("/x/obj/working")))
.WithArguments(PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-loadargs"))
.WithStandardDirectory(AbsolutePath.Create(pathTable, X("/x/obj/working.std")))
.WithDependencies(dependencies)
.WithContext(m_context);
var dataBuilder = new PipDataBuilder(m_context.PathTable.StringTable);
var pipData = dataBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping);
var pip = processBuilder.WithArguments(pipData).Build();
var pipId = m_executionEnvironment.PipTable.Add((uint)(index + 1), pip);
var runnableProcessPip = (ProcessRunnablePip)(RunnablePip.Create(m_loggingContext, m_executionEnvironment, pipId, PipType.Process, 0, taskFactory, 0));
m_runnablePips.Add(index, runnableProcessPip); // For verification
return(GeneratePipInfoWithRunnablePipAndIndex(ref runnableProcessPip, index));
}
private void PopulateDirectoryObjectFromIdentity(LoadEntity entity)
{
using (OperationTracker.Create(this.logger, "Re-hydrating directory object of type {0} on load entity.", new object[]
{
entity.DirectoryObjectIdentity.ObjectType
}))
{
if (entity.DirectoryObjectIdentity != null)
{
try
{
entity.DirectoryObject = this.directory.GetDirectoryObject(entity.DirectoryObjectIdentity);
}
catch (LocalizedException exception)
{
this.logger.LogError(exception, "Failed to rehydrate object with identity '{0}'.", new object[]
{
entity.DirectoryObjectIdentity
});
}
}
}
}
public void TestSanitizeForJSON(string oldValue, string expectValue)
{
var sanitizedDescription = OperationTracker.SanitizeForJSON(oldValue);
XAssert.AreEqual(expectValue, sanitizedDescription);
}
public void OperationTracker_Concurrent()
{
//
// Create enough competing threads so we have concurrency the moment we dispose the tracker.
//
int n = Environment.ProcessorCount * 4;
using var c = new CountdownEvent(n);
using var e = new ManualResetEvent(initialState: false);
var tasks = new Thread[n];
var t = new OperationTracker();
//
// Each thread will enter, then signal having arrived within the tracked scope, and then wait for a signal to continue.
//
for (int i = 0; i < n; i++)
{
var u = new Thread(() =>
{
using var _ = t.Enter();
c.Signal();
e.WaitOne();
Thread.Yield();
});
tasks[i] = u;
u.Start();
}
//
// Wait for all threads to arrive, then let them loose.
//
c.Wait();
e.Set();
//
// Initiate and await completion of closing the tracker.
//
t.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
.AsTask()
#endif
.Wait();
//
// Join all threads.
//
foreach (var u in tasks)
{
u.Join();
}
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
public MailboxProvisioningResult GetDatabase(MailboxProvisioningData provisioningData)
{
MailboxProvisioningResult result;
using (OperationTracker.Create(this.logger, "Selecting database for '{0}' using {1}.", new object[]
{
provisioningData,
base.GetType().Name
}))
{
MailboxProvisioningResult mailboxProvisioningResult = new MailboxProvisioningResult
{
Status = MailboxProvisioningResultStatus.ConstraintCouldNotBeSatisfied,
MailboxProvisioningConstraints = provisioningData.MailboxProvisioningConstraints
};
IMailboxProvisioningConstraints provisioningConstraint = provisioningData.MailboxProvisioningConstraints ?? new MailboxProvisioningConstraints();
IEnumerable <LoadContainer> databasesMatchingConstraint = this.GetDatabasesMatchingConstraint(provisioningConstraint);
List <LoadContainer> list = new List <LoadContainer>(300);
foreach (LoadContainer loadContainer in databasesMatchingConstraint)
{
mailboxProvisioningResult.Status = MailboxProvisioningResultStatus.InsufficientCapacity;
LoadMetric loadMetric;
long num;
long num2;
if (!loadContainer.CanAcceptRegularLoad)
{
this.logger.Log(MigrationEventType.Instrumentation, "Database '{0}' cannot accept regular load, skipping.", new object[]
{
loadContainer.DirectoryObjectIdentity
});
}
else if (!loadContainer.AvailableCapacity.SupportsAdditional(provisioningData.ConsumedLoad, out loadMetric, out num, out num2))
{
this.logger.Log(MigrationEventType.Instrumentation, "Database '{0}' does not have sufficient capacity for the provisioning request. The {1} requested units of {2} would exceed the {3} available. Skipped.", new object[]
{
loadContainer.DirectoryObjectIdentity,
num,
loadMetric,
num2
});
}
else
{
list.Add(loadContainer);
}
}
if (list.Any <LoadContainer>())
{
LoadContainer loadContainer2 = list[new Random().Next(0, list.Count)];
mailboxProvisioningResult.Status = MailboxProvisioningResultStatus.Valid;
mailboxProvisioningResult.Database = loadContainer2.DirectoryObjectIdentity;
loadContainer2.CommittedLoad += provisioningData.ConsumedLoad;
this.logger.Log(MigrationEventType.Instrumentation, "Selected database {0} with {1} max, {2} consumed and {3} available.", new object[]
{
loadContainer2.DirectoryObjectIdentity,
loadContainer2.MaximumLoad,
loadContainer2.ConsumedLoad,
loadContainer2.AvailableCapacity
});
}
result = mailboxProvisioningResult;
}
return(result);
}
public void OperationTracker_Basics2()
{
var o = new OperationTracker();
//
// Enter a few times.
//
var g1 = o.Enter();
var g2 = o.Enter();
//
// Initiate dispose a first time.
//
var d1 = o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
.AsTask()
#endif
;
Assert.IsFalse(d1.IsCompleted);
//
// Cannot enter again.
//
Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
//
// Initiate dispose a second time.
//
var d2 = o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
.AsTask()
#endif
;
Assert.IsFalse(d2.IsCompleted);
//
// Exit one of the operations.
//
g2.Dispose();
Assert.IsFalse(d1.IsCompleted);
Assert.IsFalse(d2.IsCompleted);
//
// Cannot enter again.
//
Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
//
// Exit the other operation.
//
g1.Dispose();
//
// Tracker is disposed now.
//
d1.Wait();
d2.Wait();
//
// Cannot enter again.
//
Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
//
// Dispose is a no-op now.
//
o.DisposeAsync()
#if NET5_0 || NETCOREAPP3_1
.AsTask()
#endif
.Wait();
//
// Cannot enter again.
//
Assert.ThrowsException <ObjectDisposedException>(() => o.Enter());
}
public async Task Stress()
{
const int N = 5;
const int M = N * N;
var context = BuildXLContext.CreateInstanceForTesting();
var loggingContext = CreateLoggingContextForTest();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
var executionEnvironment = new PipQueueTestExecutionEnvironment(
context,
config,
pipTable,
Path.Combine(TestOutputDirectory, "temp"),
TryGetSubstSourceAndTarget(out string substSource, out string substTarget) ? (substSource, substTarget) : default((string, string)?),
GetSandboxConnection());
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
}
executionEnvironment.MarkExecuted(pip);
return(result);
};
string executable = CmdHelper.OsShellExe;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
// This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);
using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 1: create some files
var baseFileArtifacts = new List <FileArtifact>();
for (int i = 0; i < N; i++)
{
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
baseFileArtifacts.Add(destinationArtifact);
PipData contents = PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
i.ToString(CultureInfo.InvariantCulture));
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
var pipId = pipTable.Add((uint)(i + 1), writeFile);
var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(DispatcherKind.IO);
phase1PipQueue.Enqueue(runnable);
}
phase1PipQueue.SetAsFinalized();
phase1PipQueue.DrainQueues();
await Task.WhenAll(
Enumerable.Range(0, 2).Select(
async range =>
{
using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 2: do some more with those files
var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >();
var checkerTasks = new ConcurrentQueue <Task>();
Action <PipId, Task <PipResult> > callback =
(id, task) =>
{
XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
XAssert.IsFalse(task.Result.Status.IndicatesFailure());
Tuple <string, int> t;
if (!pips.TryRemove(id, out t))
{
XAssert.Fail();
}
checkerTasks.Enqueue(
Task.Run(
() =>
{
string actual = File.ReadAllText(t.Item1).Trim();
// TODO: Make this async
XAssert.AreEqual(actual, t.Item2.ToString());
}));
};
var r = new Random(0);
for (int i = 0; i < M; i++)
{
int sourceIndex = r.Next(baseFileArtifacts.Count);
FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
Pip pip;
DispatcherKind queueKind;
switch (r.Next(2))
{
case 0:
pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
queueKind = DispatcherKind.IO;
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
case 1:
string workingDirectory =
OperatingSystemHelper.IsUnixOS ? "/tmp" :
Environment.GetFolderPath(Environment.SpecialFolder.Windows);
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
var pipData = OperatingSystemHelper.IsUnixOS ?
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);
queueKind = DispatcherKind.CPU;
pip = new Process(
executableArtifact,
workingDirectoryAbsolutePath,
pipData,
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
default:
Contract.Assert(false);
continue;
}
var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);
Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
{
var task = taskFactory(runnablePip);
var pipResult = await task;
callback(pipId, task);
};
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(queueKind);
phase2PipQueue.Enqueue(runnable);
if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
{
Contract.Assert(false);
}
}
phase2PipQueue.SetAsFinalized();
phase2PipQueue.DrainQueues();
XAssert.AreEqual(0, pips.Count);
await Task.WhenAll(checkerTasks);
}
}));
}
}
}
}
internal Tracker(OperationTracker parent)
{
_parent = parent;
}