Example #1
0
        private List <List <Pip> > TopSort(List <Pip> pips)
        {
            var sortedPipGroups = new List <List <Pip> >();
            var modules         = new List <Pip>();
            var specs           = new List <Pip>();
            var values          = new List <Pip>();

            // Service related are service-shutdown process pip, service finalization (IPC) pip, service-start process pip.
            var serviceRelatedPips = new List <Pip>();
            var otherPips          = new List <Pip>();

            foreach (var pip in pips)
            {
                if (pip is ModulePip)
                {
                    modules.Add(pip);
                }
                else if (pip is SpecFilePip)
                {
                    specs.Add(pip);
                }
                else if (pip is ValuePip)
                {
                    values.Add(pip);
                }
                else if (ServicePipKindUtil.IsServiceStartShutdownOrFinalizationPip(pip))
                {
                    serviceRelatedPips.Add(pip);
                }
                else
                {
                    otherPips.Add(pip);
                }
            }

            sortedPipGroups.Add(modules);
            sortedPipGroups.Add(specs);
            sortedPipGroups.Add(values);

            // Special service related pips must go in sequential order.
            sortedPipGroups.AddRange(serviceRelatedPips.Select(pip => new List <Pip>()
            {
                pip
            }));

            TopSortInternal(otherPips, sortedPipGroups);
            sortedPipGroups = StableSortPips(pips, sortedPipGroups);

            return(sortedPipGroups);
        }
Example #2
0
        public void TestTopSort()
        {
            // Create fragment :
            //
            // z -> C --------+----> x -> A -> w
            //      |         |
            //      |         |
            //      + -> y -> B -> v

            var fragment          = CreatePipGraphFragment(nameof(TestTopSort), useTopSort: true);
            var processBuilderA   = fragment.GetProcessBuilder();
            var argumentsBuilderA = new ArgumentsBuilder(processBuilderA);
            var x = fragment.CreateOutputFile("x");

            argumentsBuilderA
            .AddInputFileOption("/input:", fragment.CreateSourceFile("w"))
            .AddOutputFileOption("/output:", x.Path)
            .Finish();
            (Process processA, ProcessOutputs outputsA) = fragment.ScheduleProcessBuilder(processBuilderA);

            var processBuilderB   = fragment.GetProcessBuilder();
            var argumentsBuilderB = new ArgumentsBuilder(processBuilderB);
            var y = fragment.CreateOutputFile("y");

            argumentsBuilderB
            .AddInputFileOption("/input:", fragment.CreateSourceFile("v"))
            .AddInputFileOption("/input:", outputsA.TryGetOutputFile(x.Path, out var xAsOutput) ? xAsOutput : FileArtifact.Invalid)
            .AddOutputFileOption("/output:", y.Path)
            .Finish();
            (Process processB, ProcessOutputs outputsB) = fragment.ScheduleProcessBuilder(processBuilderB);

            var processBuilderC   = fragment.GetProcessBuilder();
            var argumentsBuilderC = new ArgumentsBuilder(processBuilderC);
            var z = fragment.CreateOutputFile("z");

            argumentsBuilderC
            .AddInputFileOption("/input:", outputsB.TryGetOutputFile(y.Path, out var yAsOutput) ? yAsOutput : FileArtifact.Invalid)
            .AddInputFileOption("/input:", outputsA.TryGetOutputFile(x.Path, out var xAsOutput2) ? xAsOutput2 : FileArtifact.Invalid)
            .AddOutputFileOption("/output:", z.Path)
            .Finish();
            (Process processC, ProcessOutputs outputsC) = fragment.ScheduleProcessBuilder(processBuilderC);

            // Drop z and y.
            var serviceRelatedPips = new TestPipGraphFragmentUtils.ServiceRelatedPips();

            (IIpcMoniker moniker, PipId servicePipId) = TestPipGraphFragmentUtils.CreateService(fragment, serviceRelatedPips);

            var addZBuilder = fragment.GetIpcProcessBuilder();

            new ArgumentsBuilder(addZBuilder)
            .AddStringOption("--command ", "addFile")
            .AddIpcMonikerOption("--ipcMoniker ", moniker)
            .AddInputFileOption("--file ", z)
            .Finish();

            IpcPip ipcPipZ = fragment.ScheduleIpcPip(
                moniker,
                servicePipId,
                addZBuilder,
                fragment.CreateOutputFile("addZ"),
                false);

            var addYBuilder = fragment.GetIpcProcessBuilder();

            new ArgumentsBuilder(addYBuilder)
            .AddStringOption("--command ", "addFile")
            .AddIpcMonikerOption("--ipcMoniker ", moniker)
            .AddInputFileOption("--file ", y)
            .Finish();

            IpcPip ipcPipY = fragment.ScheduleIpcPip(
                moniker,
                servicePipId,
                addYBuilder,
                fragment.CreateOutputFile("addY"),
                false);

            var sortedPips = new PipGraphFragmentTopSort(fragment.PipGraph).Sort();

            XAssert.AreEqual(10, sortedPips.Count);                   // There are ten layers.
            XAssert.IsTrue(sortedPips[0].All(p => p is ModulePip));   // 0th layer is module pips.
            XAssert.IsTrue(sortedPips[1].All(p => p is SpecFilePip)); // 1st layer is spec pips.
            XAssert.IsTrue(sortedPips[2].All(p => p is ValuePip));    // 2nd layer is value pips.

            for (int i = 3; i <= 5; ++i)
            {
                // 3rd, 4th, and 5th layer are service related pips.
                XAssert.AreEqual(1, sortedPips[i].Count);
                XAssert.IsTrue(ServicePipKindUtil.IsServiceStartShutdownOrFinalizationPip(sortedPips[i][0]));
            }

            // 6th layer contains pip A and create drop pip.
            XAssert.AreEqual(2, sortedPips[6].Count);
            XAssert.Contains(sortedPips[6], serviceRelatedPips.Create, processA);

            // 7th layer contains pip B.
            XAssert.AreEqual(1, sortedPips[7].Count);
            XAssert.Contains(sortedPips[7], processB);

            // 8th layer contains pip C and create drop pip y.
            XAssert.AreEqual(2, sortedPips[8].Count);
            XAssert.Contains(sortedPips[8], ipcPipY, processC);

            // 9th layer contains drop pip z.
            XAssert.AreEqual(1, sortedPips[9].Count);
            XAssert.Contains(sortedPips[9], ipcPipZ);
        }
Example #3
0
        /// <inheritdoc />
        public GraphPatchingStatistics PartiallyReloadGraph(HashSet <AbsolutePath> affectedSpecs)
        {
            Contract.Requires(affectedSpecs != null);

            var startTime = DateTime.UtcNow;
            var mustSkipDueToTransitivity    = new VisitationTracker(m_oldPipGraph);
            var mustPostponeDueToServicePips = new VisitationTracker(m_oldPipGraph);
            var toPostpone = new SortedList <uint, Pip>();

            MultiValueDictionary <int, NodeId> nodesByHeight = m_oldPipGraph.TopSort();
            int numReloadedPips      = 0;
            int numNotReloadablePips = 0;

            m_builder.SealDirectoryTable.StartPatching();

            // go one level at a time:
            //   - process nodes at the same height in parallel
            //   - postpone service-related pips because they must be added in the order of creation
            //     (because, for example, pip builder adds forward edges from service client to service finalization pips)
            for (int height = 0; height < nodesByHeight.Count; height++)
            {
                Parallel.ForEach(
                    nodesByHeight[height],
                    new ParallelOptions {
                    MaxDegreeOfParallelism = m_maxDegreeOfParallelism
                },
                    body: (node) =>
                {
                    var pipId   = node.ToPipId();
                    var pipType = m_oldPipTable.GetPipType(pipId);

                    // skip non-reloadable pips
                    if (!s_reloadablePipTypes.Contains(pipType))
                    {
                        Interlocked.Increment(ref numNotReloadablePips);
                        return;
                    }

                    var pip = HydratePip(pipId);
                    AbsolutePath?producingSpec = GetProducerSpecFile(pip);

                    // check if this node must be skipped due to its spec file being affected
                    if (producingSpec == null || affectedSpecs.Contains(producingSpec.Value))
                    {
                        mustSkipDueToTransitivity.MarkVisited(node);
                        MarkAllDependentsVisited(mustSkipDueToTransitivity, node);
                        return;
                    }

                    // check if this pip is a service-related pip which must be postponed
                    if (ServicePipKindUtil.IsServiceRelatedPip(pip))
                    {
                        SynchronizedAddToSortedList(toPostpone, pip);
                        MarkAllDependentsVisited(mustPostponeDueToServicePips, node);
                        return;
                    }

                    // check if this node must be postponed because it depends on a node that was already postponed
                    if (mustPostponeDueToServicePips.WasVisited(node))
                    {
                        SynchronizedAddToSortedList(toPostpone, pip);
                        MarkAllDependentsVisited(mustPostponeDueToServicePips, node);
                        return;
                    }

                    // check if this node must be skipped due to transitivity
                    ThrowIfVisited(mustSkipDueToTransitivity, pip);

                    // everything passed: reload this node
                    ReloadPip(pip, ref numReloadedPips);
                });
            }

            // add postponed nodes sequentially in the order of creation
            foreach (var pip in toPostpone.Values)
            {
                var serviceKind = ServicePipKindUtil.ServiceKind(pip);
                if (serviceKind != ServicePipKind.ServiceShutdown && serviceKind != ServicePipKind.ServiceFinalization)
                {
                    // 'shutdown' and 'finalization' are exception because of forward edges that are added
                    // during construction from service client pips to service shutdown/finalization pips.
                    ThrowIfVisited(mustSkipDueToTransitivity, pip);
                }

                ReloadPip(pip, ref numReloadedPips);
            }

            m_builder.SealDirectoryTable.FinishPatching();

            return(new GraphPatchingStatistics
            {
                ElapsedMilliseconds = (int)DateTime.UtcNow.Subtract(startTime).TotalMilliseconds,
                NumPipsReloaded = numReloadedPips,
                NumPipsAutomaticallyAdded = m_builder.PipCount - numReloadedPips,
                NumPipsNotReloadable = numNotReloadablePips,
                NumPipsSkipped = mustSkipDueToTransitivity.VisitedCount,
                AffectedSpecs = affectedSpecs,
            });
        }