Exemplo n.º 1
0
        private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
                                                                        out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
        {
            var stratCol                = new List <uint>();
            var stratVal                = new List <ReadOnlyMemory <char> >();
            var auc                     = new List <Double>();
            var drAtK                   = new List <Double>();
            var drAtP                   = new List <Double>();
            var drAtNumAnomalies        = new List <Double>();
            var thresholdAtK            = new List <Single>();
            var thresholdAtP            = new List <Single>();
            var thresholdAtNumAnomalies = new List <Single>();
            var numAnoms                = new List <long>();

            var scores       = new List <Single>();
            var labels       = new List <Single>();
            var names        = new List <ReadOnlyMemory <char> >();
            var topKStratCol = new List <uint>();
            var topKStratVal = new List <ReadOnlyMemory <char> >();

            bool hasStrats = Utils.Size(dictionaries) > 0;

            addAgg =
                (stratColKey, stratColVal, agg) =>
            {
                agg.Finish();
                stratCol.Add(stratColKey);
                stratVal.Add(stratColVal);
                auc.Add(agg.Auc);
                drAtK.Add(agg.DrAtK);
                drAtP.Add(agg.DrAtP);
                drAtNumAnomalies.Add(agg.DrAtNumAnomalies);
                thresholdAtK.Add(agg.ThresholdAtK);
                thresholdAtP.Add(agg.ThresholdAtP);
                thresholdAtNumAnomalies.Add(agg.ThresholdAtNumAnomalies);
                numAnoms.Add(agg.AggCounters.NumAnomalies);

                names.AddRange(agg.Names.Take(agg.NumTopExamples));
                scores.AddRange(agg.Scores.Take(agg.NumTopExamples));
                labels.AddRange(agg.Labels.Take(agg.NumTopExamples));

                if (hasStrats)
                {
                    topKStratCol.AddRange(agg.Scores.Select(x => stratColKey));
                    topKStratVal.AddRange(agg.Scores.Select(x => stratColVal));
                }
            };

            consolidate =
                () =>
            {
                var overallDvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
                }
                overallDvBldr.AddColumn(BinaryClassifierEvaluator.Auc, NumberType.R8, auc.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.DrAtK, NumberType.R8, drAtK.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.DrAtPFpr, NumberType.R8, drAtP.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.DrAtNumPos, NumberType.R8, drAtNumAnomalies.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.ThreshAtK, NumberType.R4, thresholdAtK.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.ThreshAtP, NumberType.R4, thresholdAtP.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.ThreshAtNumPos, NumberType.R4, thresholdAtNumAnomalies.ToArray());
                overallDvBldr.AddColumn(OverallMetrics.NumAnomalies, NumberType.I8, numAnoms.ToArray());

                var topKdvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    topKdvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, topKStratCol.ToArray());
                    topKdvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, topKStratVal.ToArray());
                }
                topKdvBldr.AddColumn(TopKResultsColumns.Instance, TextType.Instance, names.ToArray());
                topKdvBldr.AddColumn(TopKResultsColumns.AnomalyScore, NumberType.R4, scores.ToArray());
                topKdvBldr.AddColumn(TopKResultsColumns.Label, NumberType.R4, labels.ToArray());

                var result = new Dictionary <string, IDataView>();
                result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
                result.Add(TopKResults, topKdvBldr.GetDataView());

                return(result);
            };
        }
Exemplo n.º 2
0
        private protected override void GetAggregatorConsolidationFuncs(TAgg aggregator, AggregatorDictionaryBase[] dictionaries,
                                                                        out Action <uint, ReadOnlyMemory <char>, TAgg> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
        {
            var stratCol   = new List <uint>();
            var stratVal   = new List <ReadOnlyMemory <char> >();
            var isWeighted = new List <bool>();
            var l1         = new List <TMetrics>();
            var l2         = new List <TMetrics>();
            var rms        = new List <TMetrics>();
            var loss       = new List <TMetrics>();
            var rSquared   = new List <TMetrics>();

            bool hasStrats = Utils.Size(dictionaries) > 0;
            bool hasWeight = aggregator.Weighted;

            addAgg =
                (stratColKey, stratColVal, agg) =>
            {
                Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");

                stratCol.Add(stratColKey);
                stratVal.Add(stratColVal);
                isWeighted.Add(false);
                l1.Add(agg.UnweightedCounters.L1);
                l2.Add(agg.UnweightedCounters.L2);
                rms.Add(agg.UnweightedCounters.Rms);
                loss.Add(agg.UnweightedCounters.Loss);
                rSquared.Add(agg.UnweightedCounters.RSquared);
                if (agg.Weighted)
                {
                    stratCol.Add(stratColKey);
                    stratVal.Add(stratColVal);
                    isWeighted.Add(true);
                    l1.Add(agg.WeightedCounters.L1);
                    l2.Add(agg.WeightedCounters.L2);
                    rms.Add(agg.WeightedCounters.Rms);
                    loss.Add(agg.WeightedCounters.Loss);
                    rSquared.Add(agg.WeightedCounters.RSquared);
                }
            };

            consolidate =
                () =>
            {
                var overallDvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), 0, dictionaries.Length, stratCol.ToArray());
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
                }
                if (hasWeight)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BoolType.Instance, isWeighted.ToArray());
                }
                aggregator.AddColumn(overallDvBldr, L1, l1.ToArray());
                aggregator.AddColumn(overallDvBldr, L2, l2.ToArray());
                aggregator.AddColumn(overallDvBldr, Rms, rms.ToArray());
                aggregator.AddColumn(overallDvBldr, Loss, loss.ToArray());
                aggregator.AddColumn(overallDvBldr, RSquared, rSquared.ToArray());

                var result = new Dictionary <string, IDataView>();
                result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
                return(result);
            };
        }
Exemplo n.º 3
0
        private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
                                                                        out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
        {
            var stratCol     = new List <uint>();
            var stratVal     = new List <ReadOnlyMemory <char> >();
            var isWeighted   = new List <bool>();
            var nmi          = new List <Double>();
            var avgMinScores = new List <Double>();
            var dbi          = new List <Double>();

            bool hasStrats = Utils.Size(dictionaries) > 0;
            bool hasWeight = aggregator.Weighted;

            addAgg =
                (stratColKey, stratColVal, agg) =>
            {
                Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");
                Host.Check(agg.UnweightedCounters.CalculateDbi == aggregator.UnweightedCounters.CalculateDbi,
                           "All aggregators must either compute DBI or not compute DBI");

                stratCol.Add(stratColKey);
                stratVal.Add(stratColVal);
                isWeighted.Add(false);
                nmi.Add(agg.UnweightedCounters.Nmi);
                avgMinScores.Add(agg.UnweightedCounters.AvgMinScores);
                if (agg.UnweightedCounters.CalculateDbi)
                {
                    dbi.Add(agg.UnweightedCounters.Dbi);
                }
                if (agg.Weighted)
                {
                    stratCol.Add(stratColKey);
                    stratVal.Add(stratColVal);
                    isWeighted.Add(true);
                    nmi.Add(agg.WeightedCounters.Nmi);
                    avgMinScores.Add(agg.WeightedCounters.AvgMinScores);
                    if (agg.WeightedCounters.CalculateDbi)
                    {
                        dbi.Add(agg.WeightedCounters.Dbi);
                    }
                }
            };

            consolidate =
                () =>
            {
                var overallDvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextDataViewType.Instance, stratVal.ToArray());
                }
                if (hasWeight)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BooleanDataViewType.Instance, isWeighted.ToArray());
                }
                overallDvBldr.AddColumn(Nmi, NumberDataViewType.Double, nmi.ToArray());
                overallDvBldr.AddColumn(AvgMinScore, NumberDataViewType.Double, avgMinScores.ToArray());
                if (aggregator.UnweightedCounters.CalculateDbi)
                {
                    overallDvBldr.AddColumn(Dbi, NumberDataViewType.Double, dbi.ToArray());
                }

                var result = new Dictionary <string, IDataView>
                {
                    { MetricKinds.OverallMetrics, overallDvBldr.GetDataView() }
                };

                return(result);
            };
        }
        private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
                                                                        out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
        {
            var stratCol   = new List <uint>();
            var stratVal   = new List <ReadOnlyMemory <char> >();
            var isWeighted = new List <bool>();
            var ndcg       = new List <Double[]>();
            var dcg        = new List <Double[]>();

            var groupName     = new List <ReadOnlyMemory <char> >();
            var groupNdcg     = new List <Double[]>();
            var groupDcg      = new List <Double[]>();
            var groupMaxDcg   = new List <Double[]>();
            var groupStratCol = new List <uint>();
            var groupStratVal = new List <ReadOnlyMemory <char> >();

            bool hasStrats    = Utils.Size(dictionaries) > 0;
            bool hasWeight    = aggregator.Weighted;
            bool groupSummary = aggregator.UnweightedCounters.GroupSummary;

            addAgg =
                (stratColKey, stratColVal, agg) =>
            {
                Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");
                Host.Check(agg.UnweightedCounters.GroupSummary == aggregator.UnweightedCounters.GroupSummary,
                           "All aggregators must either compute group summary or not compute group summary");

                stratCol.Add(stratColKey);
                stratVal.Add(stratColVal);
                isWeighted.Add(false);
                ndcg.Add(agg.UnweightedCounters.Ndcg);
                dcg.Add(agg.UnweightedCounters.Dcg);
                if (agg.UnweightedCounters.GroupSummary)
                {
                    groupStratCol.AddRange(agg.UnweightedCounters.GroupDcg.Select(x => stratColKey));
                    groupStratVal.AddRange(agg.UnweightedCounters.GroupDcg.Select(x => stratColVal));
                    groupName.AddRange(agg.GroupId.Select(sb => sb.ToString().AsMemory()));
                    groupNdcg.AddRange(agg.UnweightedCounters.GroupNdcg);
                    groupDcg.AddRange(agg.UnweightedCounters.GroupDcg);
                    groupMaxDcg.AddRange(agg.UnweightedCounters.GroupMaxDcg);
                }

                if (agg.Weighted)
                {
                    stratCol.Add(stratColKey);
                    stratVal.Add(stratColVal);
                    isWeighted.Add(true);
                    ndcg.Add(agg.WeightedCounters.Ndcg);
                    dcg.Add(agg.WeightedCounters.Dcg);
                }
            };

            consolidate =
                () =>
            {
                var overallDvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
                }
                if (hasWeight)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BoolType.Instance, isWeighted.ToArray());
                }
                overallDvBldr.AddColumn(Ndcg, aggregator.GetSlotNames, NumberType.R8, ndcg.ToArray());
                overallDvBldr.AddColumn(Dcg, aggregator.GetSlotNames, NumberType.R8, dcg.ToArray());

                var groupDvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    groupDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, groupStratCol.ToArray());
                    groupDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, groupStratVal.ToArray());
                }
                if (groupSummary)
                {
                    groupDvBldr.AddColumn(GroupId, TextType.Instance, groupName.ToArray());
                    groupDvBldr.AddColumn(Ndcg, aggregator.GetGroupSummarySlotNames("NDCG"), NumberType.R8, groupNdcg.ToArray());
                    groupDvBldr.AddColumn(Dcg, aggregator.GetGroupSummarySlotNames("DCG"), NumberType.R8, groupDcg.ToArray());
                    groupDvBldr.AddColumn(MaxDcg, aggregator.GetGroupSummarySlotNames("MaxDCG"), NumberType.R8, groupMaxDcg.ToArray());
                }

                var result = new Dictionary <string, IDataView>();
                result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
                if (groupSummary)
                {
                    result.Add(GroupSummary, groupDvBldr.GetDataView());
                }
                return(result);
            };
        }
Exemplo n.º 5
0
        private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
                                                                        out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
        {
            var stratCol     = new List <uint>();
            var stratVal     = new List <ReadOnlyMemory <char> >();
            var isWeighted   = new List <bool>();
            var l1           = new List <Double>();
            var l2           = new List <Double>();
            var dist         = new List <Double>();
            var perLabelL1   = new List <Double[]>();
            var perLabelL2   = new List <Double[]>();
            var perLabelRms  = new List <Double[]>();
            var perLabelLoss = new List <Double[]>();

            bool hasStrats = Utils.Size(dictionaries) > 0;
            bool hasWeight = aggregator.Weighted;

            addAgg =
                (stratColKey, stratColVal, agg) =>
            {
                Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");

                stratCol.Add(stratColKey);
                stratVal.Add(stratColVal);
                isWeighted.Add(false);
                l1.Add(agg.UnweightedCounters.L1);
                l2.Add(agg.UnweightedCounters.L2);
                dist.Add(agg.UnweightedCounters.Dist);
                perLabelL1.Add(agg.UnweightedCounters.PerLabelL1);
                perLabelL2.Add(agg.UnweightedCounters.PerLabelL2);
                perLabelRms.Add(agg.UnweightedCounters.PerLabelRms);
                perLabelLoss.Add(agg.UnweightedCounters.PerLabelLoss);
                if (agg.Weighted)
                {
                    stratCol.Add(stratColKey);
                    stratVal.Add(stratColVal);
                    isWeighted.Add(true);
                    l1.Add(agg.WeightedCounters.L1);
                    l2.Add(agg.WeightedCounters.L2);
                    dist.Add(agg.WeightedCounters.Dist);
                    perLabelL1.Add(agg.WeightedCounters.PerLabelL1);
                    perLabelL2.Add(agg.WeightedCounters.PerLabelL2);
                    perLabelRms.Add(agg.WeightedCounters.PerLabelRms);
                    perLabelLoss.Add(agg.WeightedCounters.PerLabelLoss);
                }
            };

            consolidate =
                () =>
            {
                var overallDvBldr = new ArrayDataViewBuilder(Host);
                if (hasStrats)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
                }
                if (hasWeight)
                {
                    overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BoolType.Instance, isWeighted.ToArray());
                }
                overallDvBldr.AddColumn(PerLabelL1, aggregator.GetSlotNames, NumberType.R8, perLabelL1.ToArray());
                overallDvBldr.AddColumn(PerLabelL2, aggregator.GetSlotNames, NumberType.R8, perLabelL2.ToArray());
                overallDvBldr.AddColumn(PerLabelRms, aggregator.GetSlotNames, NumberType.R8, perLabelRms.ToArray());
                overallDvBldr.AddColumn(PerLabelLoss, aggregator.GetSlotNames, NumberType.R8, perLabelLoss.ToArray());
                overallDvBldr.AddColumn(L1, NumberType.R8, l1.ToArray());
                overallDvBldr.AddColumn(L2, NumberType.R8, l2.ToArray());
                overallDvBldr.AddColumn(Dist, NumberType.R8, dist.ToArray());
                var result = new Dictionary <string, IDataView>();
                result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
                return(result);
            };
        }