private List <List <Pip> > TopSort(List <Pip> pips)
{
var sortedPipGroups = new List <List <Pip> >();
var modules = new List <Pip>();
var specs = new List <Pip>();
var values = new List <Pip>();
// Service related are service-shutdown process pip, service finalization (IPC) pip, service-start process pip.
var serviceRelatedPips = new List <Pip>();
var otherPips = new List <Pip>();
foreach (var pip in pips)
{
if (pip is ModulePip)
{
modules.Add(pip);
}
else if (pip is SpecFilePip)
{
specs.Add(pip);
}
else if (pip is ValuePip)
{
values.Add(pip);
}
else if (ServicePipKindUtil.IsServiceStartShutdownOrFinalizationPip(pip))
{
serviceRelatedPips.Add(pip);
}
else
{
otherPips.Add(pip);
}
}
sortedPipGroups.Add(modules);
sortedPipGroups.Add(specs);
sortedPipGroups.Add(values);
// Special service related pips must go in sequential order.
sortedPipGroups.AddRange(serviceRelatedPips.Select(pip => new List <Pip>()
{
pip
}));
TopSortInternal(otherPips, sortedPipGroups);
sortedPipGroups = StableSortPips(pips, sortedPipGroups);
return(sortedPipGroups);
}
public void TestTopSort()
{
// Create fragment :
//
// z -> C --------+----> x -> A -> w
// | |
// | |
// + -> y -> B -> v
var fragment = CreatePipGraphFragment(nameof(TestTopSort), useTopSort: true);
var processBuilderA = fragment.GetProcessBuilder();
var argumentsBuilderA = new ArgumentsBuilder(processBuilderA);
var x = fragment.CreateOutputFile("x");
argumentsBuilderA
.AddInputFileOption("/input:", fragment.CreateSourceFile("w"))
.AddOutputFileOption("/output:", x.Path)
.Finish();
(Process processA, ProcessOutputs outputsA) = fragment.ScheduleProcessBuilder(processBuilderA);
var processBuilderB = fragment.GetProcessBuilder();
var argumentsBuilderB = new ArgumentsBuilder(processBuilderB);
var y = fragment.CreateOutputFile("y");
argumentsBuilderB
.AddInputFileOption("/input:", fragment.CreateSourceFile("v"))
.AddInputFileOption("/input:", outputsA.TryGetOutputFile(x.Path, out var xAsOutput) ? xAsOutput : FileArtifact.Invalid)
.AddOutputFileOption("/output:", y.Path)
.Finish();
(Process processB, ProcessOutputs outputsB) = fragment.ScheduleProcessBuilder(processBuilderB);
var processBuilderC = fragment.GetProcessBuilder();
var argumentsBuilderC = new ArgumentsBuilder(processBuilderC);
var z = fragment.CreateOutputFile("z");
argumentsBuilderC
.AddInputFileOption("/input:", outputsB.TryGetOutputFile(y.Path, out var yAsOutput) ? yAsOutput : FileArtifact.Invalid)
.AddInputFileOption("/input:", outputsA.TryGetOutputFile(x.Path, out var xAsOutput2) ? xAsOutput2 : FileArtifact.Invalid)
.AddOutputFileOption("/output:", z.Path)
.Finish();
(Process processC, ProcessOutputs outputsC) = fragment.ScheduleProcessBuilder(processBuilderC);
// Drop z and y.
var serviceRelatedPips = new TestPipGraphFragmentUtils.ServiceRelatedPips();
(IIpcMoniker moniker, PipId servicePipId) = TestPipGraphFragmentUtils.CreateService(fragment, serviceRelatedPips);
var addZBuilder = fragment.GetIpcProcessBuilder();
new ArgumentsBuilder(addZBuilder)
.AddStringOption("--command ", "addFile")
.AddIpcMonikerOption("--ipcMoniker ", moniker)
.AddInputFileOption("--file ", z)
.Finish();
IpcPip ipcPipZ = fragment.ScheduleIpcPip(
moniker,
servicePipId,
addZBuilder,
fragment.CreateOutputFile("addZ"),
false);
var addYBuilder = fragment.GetIpcProcessBuilder();
new ArgumentsBuilder(addYBuilder)
.AddStringOption("--command ", "addFile")
.AddIpcMonikerOption("--ipcMoniker ", moniker)
.AddInputFileOption("--file ", y)
.Finish();
IpcPip ipcPipY = fragment.ScheduleIpcPip(
moniker,
servicePipId,
addYBuilder,
fragment.CreateOutputFile("addY"),
false);
var sortedPips = new PipGraphFragmentTopSort(fragment.PipGraph).Sort();
XAssert.AreEqual(10, sortedPips.Count); // There are ten layers.
XAssert.IsTrue(sortedPips[0].All(p => p is ModulePip)); // 0th layer is module pips.
XAssert.IsTrue(sortedPips[1].All(p => p is SpecFilePip)); // 1st layer is spec pips.
XAssert.IsTrue(sortedPips[2].All(p => p is ValuePip)); // 2nd layer is value pips.
for (int i = 3; i <= 5; ++i)
{
// 3rd, 4th, and 5th layer are service related pips.
XAssert.AreEqual(1, sortedPips[i].Count);
XAssert.IsTrue(ServicePipKindUtil.IsServiceStartShutdownOrFinalizationPip(sortedPips[i][0]));
}
// 6th layer contains pip A and create drop pip.
XAssert.AreEqual(2, sortedPips[6].Count);
XAssert.Contains(sortedPips[6], serviceRelatedPips.Create, processA);
// 7th layer contains pip B.
XAssert.AreEqual(1, sortedPips[7].Count);
XAssert.Contains(sortedPips[7], processB);
// 8th layer contains pip C and create drop pip y.
XAssert.AreEqual(2, sortedPips[8].Count);
XAssert.Contains(sortedPips[8], ipcPipY, processC);
// 9th layer contains drop pip z.
XAssert.AreEqual(1, sortedPips[9].Count);
XAssert.Contains(sortedPips[9], ipcPipZ);
}
/// <inheritdoc />
public GraphPatchingStatistics PartiallyReloadGraph(HashSet <AbsolutePath> affectedSpecs)
{
Contract.Requires(affectedSpecs != null);
var startTime = DateTime.UtcNow;
var mustSkipDueToTransitivity = new VisitationTracker(m_oldPipGraph);
var mustPostponeDueToServicePips = new VisitationTracker(m_oldPipGraph);
var toPostpone = new SortedList <uint, Pip>();
MultiValueDictionary <int, NodeId> nodesByHeight = m_oldPipGraph.TopSort();
int numReloadedPips = 0;
int numNotReloadablePips = 0;
m_builder.SealDirectoryTable.StartPatching();
// go one level at a time:
// - process nodes at the same height in parallel
// - postpone service-related pips because they must be added in the order of creation
// (because, for example, pip builder adds forward edges from service client to service finalization pips)
for (int height = 0; height < nodesByHeight.Count; height++)
{
Parallel.ForEach(
nodesByHeight[height],
new ParallelOptions {
MaxDegreeOfParallelism = m_maxDegreeOfParallelism
},
body: (node) =>
{
var pipId = node.ToPipId();
var pipType = m_oldPipTable.GetPipType(pipId);
// skip non-reloadable pips
if (!s_reloadablePipTypes.Contains(pipType))
{
Interlocked.Increment(ref numNotReloadablePips);
return;
}
var pip = HydratePip(pipId);
AbsolutePath?producingSpec = GetProducerSpecFile(pip);
// check if this node must be skipped due to its spec file being affected
if (producingSpec == null || affectedSpecs.Contains(producingSpec.Value))
{
mustSkipDueToTransitivity.MarkVisited(node);
MarkAllDependentsVisited(mustSkipDueToTransitivity, node);
return;
}
// check if this pip is a service-related pip which must be postponed
if (ServicePipKindUtil.IsServiceRelatedPip(pip))
{
SynchronizedAddToSortedList(toPostpone, pip);
MarkAllDependentsVisited(mustPostponeDueToServicePips, node);
return;
}
// check if this node must be postponed because it depends on a node that was already postponed
if (mustPostponeDueToServicePips.WasVisited(node))
{
SynchronizedAddToSortedList(toPostpone, pip);
MarkAllDependentsVisited(mustPostponeDueToServicePips, node);
return;
}
// check if this node must be skipped due to transitivity
ThrowIfVisited(mustSkipDueToTransitivity, pip);
// everything passed: reload this node
ReloadPip(pip, ref numReloadedPips);
});
}
// add postponed nodes sequentially in the order of creation
foreach (var pip in toPostpone.Values)
{
var serviceKind = ServicePipKindUtil.ServiceKind(pip);
if (serviceKind != ServicePipKind.ServiceShutdown && serviceKind != ServicePipKind.ServiceFinalization)
{
// 'shutdown' and 'finalization' are exception because of forward edges that are added
// during construction from service client pips to service shutdown/finalization pips.
ThrowIfVisited(mustSkipDueToTransitivity, pip);
}
ReloadPip(pip, ref numReloadedPips);
}
m_builder.SealDirectoryTable.FinishPatching();
return(new GraphPatchingStatistics
{
ElapsedMilliseconds = (int)DateTime.UtcNow.Subtract(startTime).TotalMilliseconds,
NumPipsReloaded = numReloadedPips,
NumPipsAutomaticallyAdded = m_builder.PipCount - numReloadedPips,
NumPipsNotReloadable = numNotReloadablePips,
NumPipsSkipped = mustSkipDueToTransitivity.VisitedCount,
AffectedSpecs = affectedSpecs,
});
}
