private ProcessSemaphoreInfo[] GetAdditionalResourceInfo(ProcessRunnablePip runnableProcess) { if (TotalRamMb == null || TotalCommitMb == null || runnableProcess.Environment.Configuration.Schedule.UseHistoricalRamUsageInfo != true) { // Not tracking working set or commit memory return(null); } if (!m_ramSemaphoreNameId.IsValid) { m_ramSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(RamSemaphoreName); m_ramSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_ramSemaphoreNameId, ProcessExtensions.PercentageResourceLimit); } if (!m_commitSemaphoreNameId.IsValid) { m_commitSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(CommitSemaphoreName); m_commitSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_commitSemaphoreNameId, ProcessExtensions.PercentageResourceLimit); } var expectedMemoryCounters = GetExpectedMemoryCounters(runnableProcess); return(new ProcessSemaphoreInfo[] { ProcessExtensions.GetNormalizedPercentageResource( m_ramSemaphoreNameId, usage: expectedMemoryCounters.PeakWorkingSetMb, total: TotalRamMb.Value), ProcessExtensions.GetNormalizedPercentageResource( m_commitSemaphoreNameId, usage: expectedMemoryCounters.PeakCommitUsageMb, total: TotalCommitMb.Value), }); }
private ProcessSemaphoreInfo[] GetAdditionalResourceInfo(ProcessRunnablePip runnableProcess, ProcessMemoryCounters expectedMemoryCounters) { using (var semaphoreInfoListWrapper = s_semaphoreInfoListPool.GetInstance()) { var semaphores = semaphoreInfoListWrapper.Instance; if (runnableProcess.Process.RequiresAdmin && runnableProcess.Environment.Configuration.Sandbox.AdminRequiredProcessExecutionMode.ExecuteExternalVm() && runnableProcess.Environment.Configuration.Sandbox.VmConcurrencyLimit > 0) { semaphores.Add(new ProcessSemaphoreInfo( runnableProcess.Environment.Context.StringTable.AddString(PipInVmSemaphoreName), value: 1, limit: runnableProcess.Environment.Configuration.Sandbox.VmConcurrencyLimit)); } if (TotalRamMb == null || runnableProcess.Environment.Configuration.Schedule.UseHistoricalRamUsageInfo != true) { // Not tracking working set return(semaphores.ToArray()); } if (!m_ramSemaphoreNameId.IsValid) { m_ramSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(RamSemaphoreName); m_ramSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_ramSemaphoreNameId, ProcessExtensions.PercentageResourceLimit); } bool enableLessAggresiveMemoryProjection = runnableProcess.Environment.Configuration.Schedule.EnableLessAggresiveMemoryProjection; var ramSemaphoreInfo = ProcessExtensions.GetNormalizedPercentageResource( m_ramSemaphoreNameId, usage: enableLessAggresiveMemoryProjection ? expectedMemoryCounters.AverageWorkingSetMb : expectedMemoryCounters.PeakWorkingSetMb, total: TotalRamMb.Value); semaphores.Add(ramSemaphoreInfo); if (runnableProcess.Environment.Configuration.Schedule.EnableHistoricCommitMemoryProjection) { if (!m_commitSemaphoreNameId.IsValid) { m_commitSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(CommitSemaphoreName); m_commitSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_commitSemaphoreNameId, ProcessExtensions.PercentageResourceLimit); } var commitSemaphoreInfo = ProcessExtensions.GetNormalizedPercentageResource( m_commitSemaphoreNameId, usage: enableLessAggresiveMemoryProjection ? expectedMemoryCounters.AverageCommitSizeMb : expectedMemoryCounters.PeakCommitSizeMb, total: TotalCommitMb.Value); semaphores.Add(commitSemaphoreInfo); } return(semaphores.ToArray()); } }
public void TestSemaphoreSet_MultipleSemaphoreAcquisition() { SemaphoreSet <int> semaphores = new SemaphoreSet <int>(); int[] limits = new int[] { 1, 3, 7, 10 }; for (int i = 0; i < limits.Length; i++) { int semaphoreIndex = semaphores.CreateSemaphore(i, limits[i]); XAssert.AreEqual(i, semaphoreIndex); } // Try acquiring more than on semaphore at once for (int i = 0; i < 4; i++) { bool acquired = semaphores.TryAcquireResources(ItemResources.Create(new int[] { 0, 1, 0, 1 })); bool expectAcquired = i != 3; } // Create a new copy with independent usages semaphores = semaphores.CreateSharingCopy(); // Try acquiring more than on semaphore at once with count > 1 for (int i = 0; i < 4; i++) { bool acquired = semaphores.TryAcquireResources(ItemResources.Create(new int[] { 0, 0, 2, 2 })); bool expectAcquired = i != 3; } }
/// <summary> /// Gets the ItemResources for the given process's semaphores using the given semaphore set /// </summary> public static ItemResources GetSemaphoreResources <TList>( SemaphoreSet <StringId> semaphoreSet, TList semaphores, Func <ProcessSemaphoreInfo, int> getLimit = null) where TList : IReadOnlyList <ProcessSemaphoreInfo> { if (semaphores.Count == 0) { return(ItemResources.Empty); } int max = -1; for (int i = 0; i < semaphores.Count; i++) { var semaphore = semaphores[i]; var limit = getLimit?.Invoke(semaphore) ?? semaphore.Limit; max = Math.Max(max, semaphoreSet.CreateSemaphore(semaphore.Name, limit)); } if (max < 0) { return(ItemResources.Empty); } int[] semaphoreIncrements = new int[max + 1]; for (int i = 0; i < semaphores.Count; i++) { var semaphore = semaphores[i]; var limit = getLimit?.Invoke(semaphore) ?? semaphore.Limit; int semaphoreIndex = semaphoreSet.CreateSemaphore(semaphore.Name, limit); semaphoreIncrements[semaphoreIndex] = semaphore.Value; } return(ItemResources.Create(semaphoreIncrements)); }
private ProcessSemaphoreInfo[] GetAdditionalResourceInfo(ProcessRunnablePip runnableProcess) { if (TotalMemoryMb == null || runnableProcess.Environment.Configuration.Schedule.UseHistoricalRamUsageInfo != true) { // Not tracking total memory return(null); } int expectedUsage = GetExpectedRamUsageMb(runnableProcess); if (!m_ramSemaphoreNameId.IsValid) { m_ramSemaphoreNameId = runnableProcess.Environment.Context.StringTable.AddString(RamSemaphoreName); m_ramSemaphoreIndex = m_workerSemaphores.CreateSemaphore(m_ramSemaphoreNameId, ProcessExtensions.PercentageResourceLimit); } return(new ProcessSemaphoreInfo[] { ProcessExtensions.GetNormalizedPercentageResource( m_ramSemaphoreNameId, usage: expectedUsage, total: TotalMemoryMb.Value), }); }
public async Task Semaphores() { var semaphoreLimits = new int[] { 1, 2, 3, 4 }; SemaphoreSet <int> semaphores = new SemaphoreSet <int>(); Func <int, ItemResources> itemResourceGetter = index => { int[] semaphoreIncrements; switch (index) { case 2: case 4: case 6: semaphoreIncrements = new[] { 0, 1 }; break; default: semaphoreIncrements = null; break; } return(ItemResources.Create(semaphoreIncrements)); }; var created = new TaskCompletionSource <int> [10]; var completed = new TaskCompletionSource <int> [10]; for (int i = 0; i < completed.Length; i++) { completed[i] = new TaskCompletionSource <int>(); created[i] = new TaskCompletionSource <int>(); } Func <int, Task> taskCreator = index => { created[index].SetResult(index); return(completed[index].Task); }; using ( var q = new ConcurrentDrainingPriorityQueue <int, Task>( taskCreator, maxDegreeOfParallelism: 0, itemResourceGetter: itemResourceGetter, semaphores: semaphores)) using (var item6Queued = new ManualResetEvent(false)) using (var item6Dequeued = new ManualResetEvent(false)) { q.ItemSemaphoreQueued += (sender, e) => { XAssert.AreEqual(e.Item, 6); item6Queued.Set(); }; q.ItemSemaphoreDequeued += (sender, e) => { XAssert.AreEqual(e.Item, 6); item6Dequeued.Set(); }; for (int i = 0; i < 10; i++) { q.Enqueue(10 - i, i); } for (int i = 0; i < semaphoreLimits.Length; i++) { int index = semaphores.CreateSemaphore(i, semaphoreLimits[i]); XAssert.AreEqual(index, i); } q.MaxDegreeOfParallelism = 3; completed[0].SetResult(0); completed[1].SetResult(1); // skip 2, building up resource use... completed[3].SetResult(3); // skip 4, building up resource use... completed[5].SetResult(5); for (int i = 0; i < 5; i++) { await created[i].Task; } // item 6 cannot run, as 2 and 4 are using up the two semaphore slots XAssert.IsFalse(created[6].Task.IsCompleted); // Don't block, use await --- otherwise deadlock might arise as TPL may schedule a task continuation on the current thread. await item6Queued.ToTask(); completed[2].SetResult(2); // now that item 2 has finished, item 6 can dequeue and run... // Don't block, use await --- otherwise deadlock might arise as TPL may schedule a task continuation on the current thread. await item6Dequeued.ToTask(); completed[4].SetResult(4); completed[6].SetResult(6); for (int i = 7; i < 10; i++) { completed[i].SetResult(i); } await q.WhenDone(); for (int i = 0; i < 10; i++) { XAssert.IsTrue(created[i].Task.IsCompleted); } XAssert.AreEqual(0, q.PriorityQueued); XAssert.AreEqual(0, q.SemaphoreQueued); XAssert.AreEqual(0, q.Running); } }
public void TestSemaphoreSet() { SemaphoreSet <int> semaphores = new SemaphoreSet <int>(); int[] limits = new int[] { 1, 2, 3, 10 }; int originalLimitsLength = limits.Length; for (int i = 0; i < limits.Length; i++) { int semaphoreIndex = semaphores.CreateSemaphore(i, limits[i]); XAssert.AreEqual(i, semaphoreIndex); } for (int i = 0; i < limits.Length; i++) { var limit = limits[i]; for (int usage = 1; usage <= limit + 1; usage++) { int semaphoreIndex = semaphores.CreateSemaphore(i, limits[i]); XAssert.AreEqual(i, semaphoreIndex); var semaphoreIncrements = new int[i + 1]; semaphoreIncrements[i] = 1; bool acquired = semaphores.TryAcquireResources(ItemResources.Create(semaphoreIncrements)); bool expectAcquired = usage <= limit; XAssert.AreEqual(expectAcquired, acquired); } } // Now verify semaphores can be acquired in the copy var copiedSemaphores = semaphores.CreateSharingCopy(); for (int i = 0; i < limits.Length; i++) { var limit = limits[i]; for (int usage = 1; usage <= limit + 1; usage++) { var semaphoreIncrements = new int[i + 1]; semaphoreIncrements[i] = 1; bool acquired = copiedSemaphores.TryAcquireResources(ItemResources.Create(semaphoreIncrements)); bool expectAcquired = usage <= limit; XAssert.AreEqual(expectAcquired, acquired); } } limits = limits.Concat(new int[] { 6, 8 }).ToArray(); // Create new semaphores in the copy and verify that semaphores can be used in original for (int i = 0; i < limits.Length; i++) { int semaphoreIndex = copiedSemaphores.CreateSemaphore(i, limits[i]); XAssert.AreEqual(i, semaphoreIndex); } for (int i = originalLimitsLength; i < limits.Length; i++) { var limit = limits[i]; for (int usage = 1; usage <= limit + 1; usage++) { var semaphoreIncrements = new int[i + 1]; semaphoreIncrements[i] = 1; bool acquired = semaphores.TryAcquireResources(ItemResources.Create(semaphoreIncrements)); bool expectAcquired = usage <= limit; XAssert.AreEqual(expectAcquired, acquired); } } // Now verify new semaphores can be acquired in the copy for (int i = originalLimitsLength; i < limits.Length; i++) { var limit = limits[i]; for (int usage = 1; usage <= limit + 1; usage++) { var semaphoreIncrements = new int[i + 1]; semaphoreIncrements[i] = 1; bool acquired = copiedSemaphores.TryAcquireResources(ItemResources.Create(semaphoreIncrements)); bool expectAcquired = usage <= limit; XAssert.AreEqual(expectAcquired, acquired); } } // Now release the resources in the copy for (int i = 0; i < limits.Length; i++) { var limit = limits[i]; for (int usage = 1; usage <= limit; usage++) { var semaphoreIncrements = new int[i + 1]; semaphoreIncrements[i] = 1; copiedSemaphores.ReleaseResources(ItemResources.Create(semaphoreIncrements)); } } // Verify that resources still can't be acquired on original for (int i = 0; i < limits.Length; i++) { var semaphoreIncrements = new int[i + 1]; semaphoreIncrements[i] = 1; bool acquired = semaphores.TryAcquireResources(ItemResources.Create(semaphoreIncrements)); XAssert.IsFalse(acquired); } }