Example #1
0
        private static void SimulateActual(PipExecutionData data, int actualConcurrency, MultiWriter writers)
        {
            // simulate with actual concurrency
            Console.WriteLine("Simulating actual build");
            SimulationResult actualSimulation = new SimulationResult(data, data.AggregateCosts);

            actualSimulation.Simulate(actualConcurrency);
            Console.WriteLine("Done");
            Console.WriteLine();

            WriteActualAndSimulationResults(data, actualSimulation);


            writers.WriteLine("Edge Count: {0}", data.DataflowGraph.EdgeCount);
            writers.WriteLine("Pip Type Counts:");
            foreach (var pipType in EnumTraits <PipType> .EnumerateValues())
            {
                writers.WriteLine("{0}: {1}", data.PipTypeCounts[(int)pipType].ToString().PadLeft(10), pipType);
            }

            writers.WriteLine("Processes with timing information:{0} ", data.DataflowGraph.Nodes.Where(node => data.GetPipType(node) == PipType.Process && data.StartTimes[node] != 0).Count());

            writers.WriteLine("Actual Total Build Time: {0} min", data.TotalDuration.ToMinutes());
            writers.WriteLine("Actual Concurrency: {0}", data.ActualConcurrency);
            writers.WriteLine("Simulated total build time (using actual concurrency): {0} min", actualSimulation.TotalTime.ToMinutes());

            // write down info for each critical path
            ulong criticalPathCost = WriteCriticalPathToResult(writers, data, actualSimulation);
        }
Example #2
0
        public Dictionary <TEnum, (long total, double ratePerSecond)> GetRates()
        {
            var result = new Dictionary <TEnum, (long total, double ratePerSecond)>();

            lock (_activityLock)
            {
                ShiftWindow(_clock.UtcNow);

                Snapshot?lastSnapshot  = _activityWindow.LastOrDefault();
                Snapshot?firstSnapshot = _activityWindow.Count > 1 ? _activityWindow.First() : null;

                foreach (var v in EnumTraits <TEnum> .EnumerateValues())
                {
                    long   total         = lastSnapshot?.Counters[v].Value ?? 0;
                    double ratePerSecond = 0;

                    if (lastSnapshot != null && firstSnapshot != null)
                    {
                        var duration = lastSnapshot.SnapshotTime.Subtract(firstSnapshot.SnapshotTime);
                        var diff     = lastSnapshot.Counters[v].Value - firstSnapshot.Counters[v].Value;
                        ratePerSecond = ((double)diff) / duration.TotalSeconds;
                    }

                    result[v] = (total, ratePerSecond);
                }
            }

            return(result);
        }
Example #3
0
        public override Task <BoolResult> GarbageCollectAsync(OperationContext context)
        {
            return(context.PerformOperationAsync(Tracer,
                                                 () =>
            {
                // Get exclusive lock to prevent concurrent access while deleting column families
                // without this, we might try to read from a column family which does not exist
                return _keyValueStore.UseExclusive(
                    (store, db) =>
                {
                    var otherColumnGroup = db.GetFormerColumnGroup();

                    store.Put(nameof(GlobalKeys.ActiveColummGroup), otherColumnGroup.ToString());

                    foreach (var column in EnumTraits <Columns> .EnumerateValues())
                    {
                        var columnName = db.NameOf(column, otherColumnGroup);

                        // Clear the column family by dropping and recreating
                        store.DropColumnFamily(columnName);
                        store.CreateColumnFamily(columnName);
                    }

                    _activeColumnsGroup = otherColumnGroup;
                    return BoolResult.SuccessTask;
                },
                    this).ThrowOnError();
            },
                                                 counter: Counters[ContentLocationDatabaseCounters.GarbageCollectContent],
                                                 isCritical: true));
        }
Example #4
0
 /// <inheritdoc />
 public override IEnumerable <(Counter, string name)> GetCounters()
 {
     foreach (var counterEnum in EnumTraits <TEnum> .EnumerateValues())
     {
         yield return(this[counterEnum], counterEnum.ToString());
     }
 }
Example #5
0
        private TestEnum FromInteger(ulong integerValue)
        {
            Contract.Assume(Enum.IsDefined(typeof(TestEnum), (int)integerValue));
            TestEnum value;
            bool     found = EnumTraits <TestEnum> .TryConvert(integerValue, out value);

            XAssert.IsTrue(found, "Failed to find a value");
            return(value);
        }
Example #6
0
            public void PrintCounts(MultiWriter logWriter)
            {
                foreach (var value in EnumTraits <TEnum> .EnumerateValues())
                {
                    logWriter.WriteLine("{0}: {1}", value, m_counts[EnumTraits <TEnum> .ToInteger(value)]);
                }

                logWriter.WriteLine();
            }
Example #7
0
        /// <summary>
        /// Gets a counter index given an enum name.
        /// </summary>
        public ushort GetCounterIndex(TEnum counterId)
        {
            ulong counterIdValue = EnumTraits <TEnum> .ToInteger(counterId);

            Contract.Assume(counterIdValue >= CounterIdOffset);
            ulong relativeCounterIdValue = counterIdValue - CounterIdOffset;

            return(checked ((ushort)relativeCounterIdValue));
        }
Example #8
0
        /// <summary>
        /// Appends <paramref name="other"/> into a current collection instance.
        /// </summary>
        public void Append(CounterCollection <TEnum> other)
        {
            foreach (var value in EnumTraits <TEnum> .EnumerateValues())
            {
                if (IsStopwatch(value))
                {
                    AddToCounter(value, other.GetElapsedTime(value));
                }

                AddToCounter(value, other.GetCounterValue(value));
            }
        }
Example #9
0
        private string ToDebuggerDisplay()
        {
            var sb = new StringBuilder();

            foreach (var enumValue in EnumTraits <TEnum> .EnumerateValues())
            {
                var counter = this[enumValue];
                sb.AppendLine($"[{enumValue,-50}]: {counter}");
            }

            return(sb.ToString());
        }
Example #10
0
        public void CanConvertFromInteger()
        {
            XAssert.AreEqual(TestEnum.A, FromInteger((ulong)TestEnum.A));
            XAssert.AreEqual(TestEnum.B, FromInteger((ulong)TestEnum.B));
            XAssert.AreEqual(TestEnum.C, FromInteger((ulong)TestEnum.C));

            TestEnum notFound;

            EnumTraits <TestEnum> .TryConvert(123, out notFound);

            XAssert.AreEqual(default(TestEnum), notFound);
        }
Example #11
0
        public void Set(TEnum column, object value)
        {
            if (value == null)
            {
                return;
            }

            var stringValue = value.ToString();
            var columnIndex = EnumTraits <TEnum> .ToInteger(column);

            m_maxColumnLengths[columnIndex] = Math.Max(m_maxColumnLengths[columnIndex], stringValue.Length);
            m_currentRow[columnIndex]       = stringValue;
        }
Example #12
0
            static Inner()
            {
                PassThroughOperationKind = CreateOperationKind("[Pass-Through]", false);

                var prefix = "PipExecutorCounter.";

                foreach (var counter in EnumTraits <PipExecutorCounter> .EnumerateValues())
                {
                    int trackedCacheCounterId = IsTrackedCacheCounter(counter) ? s_trackedCacheCounterKinds.Count : -1;

                    if (!CounterCollection <PipExecutorCounter> .IsStopwatch(counter))
                    {
                        s_pipExecutorCounterKinds.Add(Invalid);
                        continue;
                    }

                    var counterName = counter.ToString();
                    if (counterName.EndsWith(DurationSuffix, StringComparison.OrdinalIgnoreCase))
                    {
                        counterName = counterName.Substring(0, counterName.Length - DurationSuffix.Length);
                    }

                    counterName = prefix + counterName;

                    var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ IsPipTypeSpecificCounter(counter), trackedCacheCounterId: trackedCacheCounterId);
                    s_pipExecutorCounterKinds.Add(operationKind);

                    if (trackedCacheCounterId != -1)
                    {
                        s_trackedCacheCounterKinds.Add(operationKind);
                    }
                }

                prefix = "PipExecutionStep.";
                foreach (var counter in EnumTraits <PipExecutionStep> .EnumerateValues())
                {
                    var counterName = counter.ToString();
                    counterName = prefix + counterName;
                    var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ true);
                    s_pipExecutionStepKinds.Add(operationKind);
                }

                prefix = "Operation.";
                foreach (var counter in EnumTraits <OperationCounter> .EnumerateValues())
                {
                    var counterName = counter.ToString();
                    counterName = prefix + counterName;
                    var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ false);
                    s_operationCounterKinds.Add(operationKind);
                }
            }
        public static ReadUsnJournalStatus ReadUsnJournalReadStatus(this ChangeJournalServiceProtocolReader reader)
        {
            Contract.Requires(reader != null);

            var readStatusValue = reader.ReadByte();
            ReadUsnJournalStatus readStatus;

            if (!EnumTraits <ReadUsnJournalStatus> .TryConvert(readStatusValue, out readStatus))
            {
                throw new BuildXLException("Invalid ReadUsnJournalStatus");
            }

            return(readStatus);
        }
Example #14
0
        public DisplayTable(string columnDelimeter, bool defaultHeader = true)
        {
            m_maxColumnLengths = new int[EnumTraits <TEnum> .ValueCount];
            m_columnDelimeter  = columnDelimeter;

            if (defaultHeader)
            {
                NextRow();
                foreach (var value in EnumTraits <TEnum> .EnumerateValues())
                {
                    Set(value, value.ToString());
                }
            }
        }
Example #15
0
        public static CounterCollection <TEnum> operator +(CounterCollection <TEnum> x, CounterCollection <TEnum> y)
        {
            var result = new CounterCollection <TEnum>();

            foreach (var value in EnumTraits <TEnum> .EnumerateValues())
            {
                if (IsStopwatch(value))
                {
                    result.AddToCounter(value, x.GetElapsedTime(value) + y.GetElapsedTime(value));
                }

                result.AddToCounter(value, x.GetCounterValue(value) + y.GetCounterValue(value));
            }

            return(result);
        }
        public static QueryUsnJournalResult ReadQueryUsnJournalResult(this ChangeJournalServiceProtocolReader reader)
        {
            Contract.Requires(reader != null);

            var queryStatusValue = reader.ReadInt32();
            QueryUsnJournalStatus queryStatus;

            if (!EnumTraits <QueryUsnJournalStatus> .TryConvert(queryStatusValue, out queryStatus))
            {
                throw new BuildXLException("Invalid QueryUsnJournalStatus");
            }

            if (queryStatus == QueryUsnJournalStatus.Success)
            {
                QueryUsnJournalData data = reader.ReadUsnJournalData();
                return(new QueryUsnJournalResult(queryStatus, data));
            }

            return(new QueryUsnJournalResult(queryStatus, data: null));
        }
Example #17
0
        /// <nodoc />
        public static CounterSet ToCounterSet <TEnum>(this CounterCollection <TEnum> counters)
            where TEnum : struct
        {
            CounterSet counterSet = new CounterSet();

            foreach (var counterEnum in EnumTraits <TEnum> .EnumerateValues())
            {
                var counter     = counters[counterEnum];
                var counterName = counterEnum.ToString();

                counterSet.Add($"{counterName}.Count", (long)counter.Value, counter.Name);

                if (counter.IsStopwatch)
                {
                    counterSet.Add($"{counterName}.AverageMs", counter.Value != 0 ? (long)counter.Duration.TotalMilliseconds / counter.Value : 0);
                    counterSet.Add($"{counterName}.DurationMs", (long)counter.Duration.TotalMilliseconds);
                }
            }

            return(counterSet);
        }
Example #18
0
        /// <summary>
        /// Converts <see cref="CounterCollection"/> as statistics in the form of dictionary from statistic names to values.
        /// </summary>
        public Dictionary <string, long> AsStatistics(string namePrefix = null)
        {
            Dictionary <string, long> counters = new Dictionary <string, long>();

            foreach (TEnum counterName in EnumTraits <TEnum> .EnumerateValues())
            {
                bool isStopwatch = IsStopwatch(counterName);

                string nameString = string.Format(
                    CultureInfo.InvariantCulture,
                    "{0}{1}{2}",
                    !string.IsNullOrWhiteSpace(namePrefix) ? namePrefix + "." : string.Empty,
                    counterName,
                    isStopwatch ? "Ms" : string.Empty);
                long counterValue = isStopwatch
                    ? (long)GetElapsedTime(counterName).TotalMilliseconds
                    : GetCounterValue(counterName);
                counters.Add(nameString, counterValue);
            }

            return(counters);
        }
Example #19
0
 public void TestFindByMemberValue()
 {
     Assert.Equal(TestEnum.Value1, EnumTraits <TestEnum> .FindByMemberValue("Value 1"));
     Assert.Equal(TestEnum.Value2, EnumTraits <TestEnum> .FindByMemberValue("Value 2"));
     Assert.Null(EnumTraits <TestEnum> .FindByMemberValue("other"));
 }
Example #20
0
 public void Increment(TEnum value)
 {
     m_counts[EnumTraits <TEnum> .ToInteger(value)]++;
 }
Example #21
0
 public EnumCounter()
 {
     m_counts = new int[EnumTraits <TEnum> .EnumerateValues().Count()];
 }
Example #22
0
        static void Main(string[] args)
        {
            PipExecutionData data = new PipExecutionData();

            if (args.Length != 1)
            {
                throw new ArgumentException("Specify directory path as only argument on command line");
            }

            string directory = args[0];
            string graphPath = Path.Combine(args[0], "graph.json");

            if (!File.Exists(graphPath))
            {
                throw new ArgumentException(string.Format("Graph does not exist at location: {0}", graphPath));
            }


            string resultsFormat = string.Format(@"{0}\sim\{{0}}", directory);

            Console.WriteLine("Loading graph...");

            Directory.CreateDirectory(string.Format(resultsFormat, ""));
            data.ReadAndCacheJsonGraph(Path.Combine(directory, "graph.json"));

            Console.WriteLine("Done");
            Console.WriteLine();

            Console.WriteLine("Saving runtime time table:");

            PipRuntimeTimeTable runtimeTable = new PipRuntimeTimeTable();

            foreach (var node in data.DataflowGraph.Nodes)
            {
                if (data.PipTypes[node] == PipType.Process)
                {
                    var  stableId     = (long)data.SemiStableHashes[node];
                    var  duration     = TimeSpan.FromTicks((long)data.Durations[node]);
                    uint milliseconds = (uint)Math.Min(duration.TotalMilliseconds, uint.MaxValue);
                    runtimeTable[stableId] = new PipHistoricPerfData(PipHistoricPerfData.DefaultTimeToLive, milliseconds);
                }
            }

            runtimeTable.SaveAsync(Path.Combine(directory, "RunningTimeTable")).Wait();

            Console.WriteLine("Done");
            Console.WriteLine();


            using (StreamWriter sw = new StreamWriter(resultsFormat.FormatWith("result.txt")))
            {
                var writers = new MultiWriter(sw, Console.Out);
                writers.WriteLine("Edge Count: {0}", data.DataflowGraph.EdgeCount);
                writers.WriteLine("Pip Type Counts:");
                foreach (var pipType in EnumTraits <PipType> .EnumerateValues())
                {
                    writers.WriteLine("{0}: {1}", data.PipTypeCounts[(int)pipType].ToString().PadLeft(10), pipType);
                }

                writers.WriteLine("Processes with timing information:{0} ", data.DataflowGraph.Nodes.Where(node => data.PipTypes[node] == PipType.Process && data.StartTimes[node] != 0).Count());

                File.WriteAllLines(resultsFormat.FormatWith("actual.durations.csv"), data.Spans.Where(ps => data.PipTypes[ps.Id] == PipType.Process).Select(ps => string.Join(",", data.GetName(ps.Id), ps.Duration.ToMinutes().ToString())));
                File.WriteAllLines(resultsFormat.FormatWith("actual.durations.txt"), data.Spans.Select(ps => ps.Duration.ToMinutes().ToString()));
                File.WriteAllLines(resultsFormat.FormatWith("actual.starts.txt"), data.Spans.Select(ps => ps.StartTime.ToMinutes().ToString()));
                File.WriteAllLines(resultsFormat.FormatWith("actual.ends.txt"), data.Spans.Select(ps => ps.EndTime.ToMinutes().ToString()));

                CreateSpanImage(resultsFormat.FormatWith("actual.png"), data.Spans, data.TotalDuration, Math.Max(data.ActualConcurrency, 1));

                SimulationResult actualSimulation = null;
                if (true)
                {
                    Console.WriteLine("Simulating actual build");
                    actualSimulation = new SimulationResult(data, data.AggregateCosts);
                    actualSimulation.Simulate((uint)data.ActualConcurrency);

                    File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.durations.txt"), actualSimulation.GetSpans().Select(ps => ps.Duration.ToMinutes().ToString()));
                    File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.starts.txt"), actualSimulation.GetSpans().Select(ps => ps.StartTime.ToMinutes().ToString()));
                    File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.ends.txt"), actualSimulation.GetSpans().Select(ps => ps.EndTime.ToMinutes().ToString()));

                    string csvFormat = "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}";


                    File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.txt"), new string[] { csvFormat.FormatWith("Id", "Thread", "Minimum Start Time", "Start Time", "End Time", "Duration", "Incoming", "Outgoing") }.Concat(actualSimulation.GetSpans().Select(ps =>
                                                                                                                                                                                                                                                                             csvFormat.FormatWith(
                                                                                                                                                                                                                                                                                 ps.Id.Value,
                                                                                                                                                                                                                                                                                 ps.Thread,
                                                                                                                                                                                                                                                                                 actualSimulation.MinimumStartTimes[ps.Id].ToMinutes(),
                                                                                                                                                                                                                                                                                 ps.StartTime.ToMinutes(),
                                                                                                                                                                                                                                                                                 ps.EndTime.ToMinutes(),
                                                                                                                                                                                                                                                                                 ps.Duration.ToMinutes(),
                                                                                                                                                                                                                                                                                 data.DataflowGraph.GetIncomingEdgesCount(ps.Id),
                                                                                                                                                                                                                                                                                 data.DataflowGraph.GetIncomingEdgesCount(ps.Id)))));

                    CreateSimulationImage(actualSimulation, resultsFormat.FormatWith("actualSimulation.png"));

                    Console.WriteLine("Done");
                }

                Console.WriteLine();


                writers.WriteLine("Actual Total Build Time: {0} min", data.TotalDuration.ToMinutes());
                writers.WriteLine("Actual Concurrency: {0}", data.ActualConcurrency);
                if (actualSimulation != null)
                {
                    writers.WriteLine("Simulated total build time (using actual concurrency): {0} min", actualSimulation.TotalTime.ToMinutes());
                }

                int nameWidth;

                int   count            = 0;
                ulong criticalPathCost = 0;
                foreach (var p in data.GetSortedPips(20, true, n => data.PipTypes[n] == PipType.Process, n => data.AggregateCosts[n]))
                {
                    List <NodeId> criticalPath      = new List <NodeId>();
                    NodeId        criticalChainNode = p.Node;
                    while (criticalChainNode.IsValid())
                    {
                        criticalPath.Add(criticalChainNode);
                        criticalChainNode = data.CriticalChain[criticalChainNode];
                    }

                    p.Priority.Max(ref criticalPathCost);

                    writers.WriteLine("Critical Path {0}:", count++);
                    writers.WriteLine("Critical Path Cost: {0} min", p.Priority.ToMinutes());
                    writers.WriteLine("Critical Path Length: {0}", criticalPath.Count);
                    nameWidth = criticalPath.Select(n => data.GetName(n).Length).Max();
                    writers.WriteLine("Critical Path:\n    {0}", string.Join(Environment.NewLine + " -> ", criticalPath
                                                                             //.Where(n => data.Durations[n] > 0)
                                                                             .Select(n => string.Format("{0} ({1} min) [{2}] <{3}>", data.GetName(n).PadLeft(nameWidth), data.Durations[n].ToMinutes(), data.PipIds[n], data.PipTypes[n]))));
                }


                nameWidth = data.LongestRunningPips.Select(n => data.GetName(n.Node).Length).Max();
                writers.WriteLine();
                writers.WriteLine("Top 20 Long Running Pips:");
                foreach (var p in data.LongestRunningPips)
                {
                    writers.WriteLine("{0} [{2}]: {1} min", data.GetName(p.Node).PadLeft(nameWidth), p.Priority.ToMinutes(), data.PipIds[p.Node]);
                }

                writers.WriteLine();
                writers.WriteLine("Bottom 20 Shortest Running Processes:");
                foreach (var p in data.ShortestRunningProcesses)
                {
                    writers.WriteLine("{0} [{2}]: {1} min", data.GetName(p.Node).PadLeft(nameWidth), p.Priority.ToMinutes(), data.PipIds[p.Node]);
                }

                int    simulationCount = 1;
                int    increment       = 70;
                int?[] threadCounts    = new int?[simulationCount];
                threadCounts[0] = 60;
                //threadCounts[0] = 60;
                //threadCounts[4] = 330;
                //threadCounts[5] = 400;
                //threadCounts[6] = 470;
                SimulationResult[] results = new SimulationResult[simulationCount];
                Parallel.For(0, simulationCount, i =>
                {
                    var threadCount = threadCounts[i] ?? ((i + 1) * increment);
                    Console.WriteLine("Simulating {0}...", threadCount);
                    SimulationResult result = new SimulationResult(data, data.AggregateCosts);
                    result.Simulate((uint)threadCount);
                    results[i] = result;
                    Console.WriteLine("Done {0}", threadCount);

                    //Console.WriteLine("Simulating adjusted{0}...", threadCount);
                    //SimulationResult adjusted = new SimulationResult(data, data.ComputeAggregateCosts(result.GetAdjustedDurations(criticalPathCost)));
                    //result.Simulate((uint)threadCount);
                    //results[i + simulationCount] = result;
                    //Console.WriteLine("Done adjusted {0}", threadCount);

                    //i = i + (results.Length / 2);

                    //Console.WriteLine("Simulating {0}...", i);
                    //SimulationResult adjusted = new SimulationResult(data, result.EndTimes);
                    //adjusted.Simulate((uint)result.Threads.Count);
                    //results[i] = adjusted;
                    //Console.WriteLine("Done {0}", i);
                });

                results = results.OrderBy(s => s.Threads.Count).ToArray();

                string format = "{0}, {1}, {5}, {6}";
                writers.WriteLine();
                writers.WriteLine("Results");
                writers.WriteLine(format,
                                  "Thread Count",                  // 0
                                  "Total Execution Time (min)",    // 1
                                  "Critical Path Length",          // 2
                                  "Critical Path Tail Pip",        // 3
                                  "Critical Path Idle Time (min)", // 4
                                  "Average Utilization",           // 5
                                  "Max Path Length",               // 6
                                  "Executed Pips"                  // 7
                                  );

                for (int i = 0; i < results.Length; i++)
                {
                    var result = results[i];
                    writers.WriteLine(format,
                                      result.Threads.Count,
                                      result.TotalTime.ToMinutes(),
                                      result.CriticalPath.Executions.Count,
                                      result.CriticalPath.Executions.Last().Id.Value,
                                      result.CriticalPath.IdleTime.ToMinutes(),
                                      Math.Round((double)result.TotalActiveTime / (double)((ulong)result.Threads.Count * result.TotalTime), 3),
                                      result.Threads.Select(t => t.Executions.Count).Max(),
                                      result.Threads.Select(t => t.Executions.Count).Sum());

                    CreateSimulationImage(result, resultsFormat.FormatWith("simulation.{0}.png".FormatWith(result.Threads.Count)));
                }
            }
        }
Example #23
0
 public void CanConvertToInteger()
 {
     XAssert.AreEqual((ulong)TestEnum.A, EnumTraits <TestEnum> .ToInteger(TestEnum.A));
     XAssert.AreEqual((ulong)TestEnum.B, EnumTraits <TestEnum> .ToInteger(TestEnum.B));
     XAssert.AreEqual((ulong)TestEnum.C, EnumTraits <TestEnum> .ToInteger(TestEnum.C));
 }