private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
{
var stratCol = new List <uint>();
var stratVal = new List <ReadOnlyMemory <char> >();
var auc = new List <Double>();
var drAtK = new List <Double>();
var drAtP = new List <Double>();
var drAtNumAnomalies = new List <Double>();
var thresholdAtK = new List <Single>();
var thresholdAtP = new List <Single>();
var thresholdAtNumAnomalies = new List <Single>();
var numAnoms = new List <long>();
var scores = new List <Single>();
var labels = new List <Single>();
var names = new List <ReadOnlyMemory <char> >();
var topKStratCol = new List <uint>();
var topKStratVal = new List <ReadOnlyMemory <char> >();
bool hasStrats = Utils.Size(dictionaries) > 0;
addAgg =
(stratColKey, stratColVal, agg) =>
{
agg.Finish();
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
auc.Add(agg.Auc);
drAtK.Add(agg.DrAtK);
drAtP.Add(agg.DrAtP);
drAtNumAnomalies.Add(agg.DrAtNumAnomalies);
thresholdAtK.Add(agg.ThresholdAtK);
thresholdAtP.Add(agg.ThresholdAtP);
thresholdAtNumAnomalies.Add(agg.ThresholdAtNumAnomalies);
numAnoms.Add(agg.AggCounters.NumAnomalies);
names.AddRange(agg.Names.Take(agg.NumTopExamples));
scores.AddRange(agg.Scores.Take(agg.NumTopExamples));
labels.AddRange(agg.Labels.Take(agg.NumTopExamples));
if (hasStrats)
{
topKStratCol.AddRange(agg.Scores.Select(x => stratColKey));
topKStratVal.AddRange(agg.Scores.Select(x => stratColVal));
}
};
consolidate =
() =>
{
var overallDvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
}
overallDvBldr.AddColumn(BinaryClassifierEvaluator.Auc, NumberType.R8, auc.ToArray());
overallDvBldr.AddColumn(OverallMetrics.DrAtK, NumberType.R8, drAtK.ToArray());
overallDvBldr.AddColumn(OverallMetrics.DrAtPFpr, NumberType.R8, drAtP.ToArray());
overallDvBldr.AddColumn(OverallMetrics.DrAtNumPos, NumberType.R8, drAtNumAnomalies.ToArray());
overallDvBldr.AddColumn(OverallMetrics.ThreshAtK, NumberType.R4, thresholdAtK.ToArray());
overallDvBldr.AddColumn(OverallMetrics.ThreshAtP, NumberType.R4, thresholdAtP.ToArray());
overallDvBldr.AddColumn(OverallMetrics.ThreshAtNumPos, NumberType.R4, thresholdAtNumAnomalies.ToArray());
overallDvBldr.AddColumn(OverallMetrics.NumAnomalies, NumberType.I8, numAnoms.ToArray());
var topKdvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
topKdvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, topKStratCol.ToArray());
topKdvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, topKStratVal.ToArray());
}
topKdvBldr.AddColumn(TopKResultsColumns.Instance, TextType.Instance, names.ToArray());
topKdvBldr.AddColumn(TopKResultsColumns.AnomalyScore, NumberType.R4, scores.ToArray());
topKdvBldr.AddColumn(TopKResultsColumns.Label, NumberType.R4, labels.ToArray());
var result = new Dictionary <string, IDataView>();
result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
result.Add(TopKResults, topKdvBldr.GetDataView());
return(result);
};
}
private protected override void GetAggregatorConsolidationFuncs(TAgg aggregator, AggregatorDictionaryBase[] dictionaries,
out Action <uint, ReadOnlyMemory <char>, TAgg> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
{
var stratCol = new List <uint>();
var stratVal = new List <ReadOnlyMemory <char> >();
var isWeighted = new List <bool>();
var l1 = new List <TMetrics>();
var l2 = new List <TMetrics>();
var rms = new List <TMetrics>();
var loss = new List <TMetrics>();
var rSquared = new List <TMetrics>();
bool hasStrats = Utils.Size(dictionaries) > 0;
bool hasWeight = aggregator.Weighted;
addAgg =
(stratColKey, stratColVal, agg) =>
{
Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(false);
l1.Add(agg.UnweightedCounters.L1);
l2.Add(agg.UnweightedCounters.L2);
rms.Add(agg.UnweightedCounters.Rms);
loss.Add(agg.UnweightedCounters.Loss);
rSquared.Add(agg.UnweightedCounters.RSquared);
if (agg.Weighted)
{
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(true);
l1.Add(agg.WeightedCounters.L1);
l2.Add(agg.WeightedCounters.L2);
rms.Add(agg.WeightedCounters.Rms);
loss.Add(agg.WeightedCounters.Loss);
rSquared.Add(agg.WeightedCounters.RSquared);
}
};
consolidate =
() =>
{
var overallDvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), 0, dictionaries.Length, stratCol.ToArray());
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
}
if (hasWeight)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BoolType.Instance, isWeighted.ToArray());
}
aggregator.AddColumn(overallDvBldr, L1, l1.ToArray());
aggregator.AddColumn(overallDvBldr, L2, l2.ToArray());
aggregator.AddColumn(overallDvBldr, Rms, rms.ToArray());
aggregator.AddColumn(overallDvBldr, Loss, loss.ToArray());
aggregator.AddColumn(overallDvBldr, RSquared, rSquared.ToArray());
var result = new Dictionary <string, IDataView>();
result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
return(result);
};
}
private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
{
var stratCol = new List <uint>();
var stratVal = new List <ReadOnlyMemory <char> >();
var isWeighted = new List <bool>();
var nmi = new List <Double>();
var avgMinScores = new List <Double>();
var dbi = new List <Double>();
bool hasStrats = Utils.Size(dictionaries) > 0;
bool hasWeight = aggregator.Weighted;
addAgg =
(stratColKey, stratColVal, agg) =>
{
Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");
Host.Check(agg.UnweightedCounters.CalculateDbi == aggregator.UnweightedCounters.CalculateDbi,
"All aggregators must either compute DBI or not compute DBI");
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(false);
nmi.Add(agg.UnweightedCounters.Nmi);
avgMinScores.Add(agg.UnweightedCounters.AvgMinScores);
if (agg.UnweightedCounters.CalculateDbi)
{
dbi.Add(agg.UnweightedCounters.Dbi);
}
if (agg.Weighted)
{
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(true);
nmi.Add(agg.WeightedCounters.Nmi);
avgMinScores.Add(agg.WeightedCounters.AvgMinScores);
if (agg.WeightedCounters.CalculateDbi)
{
dbi.Add(agg.WeightedCounters.Dbi);
}
}
};
consolidate =
() =>
{
var overallDvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextDataViewType.Instance, stratVal.ToArray());
}
if (hasWeight)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BooleanDataViewType.Instance, isWeighted.ToArray());
}
overallDvBldr.AddColumn(Nmi, NumberDataViewType.Double, nmi.ToArray());
overallDvBldr.AddColumn(AvgMinScore, NumberDataViewType.Double, avgMinScores.ToArray());
if (aggregator.UnweightedCounters.CalculateDbi)
{
overallDvBldr.AddColumn(Dbi, NumberDataViewType.Double, dbi.ToArray());
}
var result = new Dictionary <string, IDataView>
{
{ MetricKinds.OverallMetrics, overallDvBldr.GetDataView() }
};
return(result);
};
}
private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
{
var stratCol = new List <uint>();
var stratVal = new List <ReadOnlyMemory <char> >();
var isWeighted = new List <bool>();
var ndcg = new List <Double[]>();
var dcg = new List <Double[]>();
var groupName = new List <ReadOnlyMemory <char> >();
var groupNdcg = new List <Double[]>();
var groupDcg = new List <Double[]>();
var groupMaxDcg = new List <Double[]>();
var groupStratCol = new List <uint>();
var groupStratVal = new List <ReadOnlyMemory <char> >();
bool hasStrats = Utils.Size(dictionaries) > 0;
bool hasWeight = aggregator.Weighted;
bool groupSummary = aggregator.UnweightedCounters.GroupSummary;
addAgg =
(stratColKey, stratColVal, agg) =>
{
Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");
Host.Check(agg.UnweightedCounters.GroupSummary == aggregator.UnweightedCounters.GroupSummary,
"All aggregators must either compute group summary or not compute group summary");
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(false);
ndcg.Add(agg.UnweightedCounters.Ndcg);
dcg.Add(agg.UnweightedCounters.Dcg);
if (agg.UnweightedCounters.GroupSummary)
{
groupStratCol.AddRange(agg.UnweightedCounters.GroupDcg.Select(x => stratColKey));
groupStratVal.AddRange(agg.UnweightedCounters.GroupDcg.Select(x => stratColVal));
groupName.AddRange(agg.GroupId.Select(sb => sb.ToString().AsMemory()));
groupNdcg.AddRange(agg.UnweightedCounters.GroupNdcg);
groupDcg.AddRange(agg.UnweightedCounters.GroupDcg);
groupMaxDcg.AddRange(agg.UnweightedCounters.GroupMaxDcg);
}
if (agg.Weighted)
{
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(true);
ndcg.Add(agg.WeightedCounters.Ndcg);
dcg.Add(agg.WeightedCounters.Dcg);
}
};
consolidate =
() =>
{
var overallDvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
}
if (hasWeight)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BoolType.Instance, isWeighted.ToArray());
}
overallDvBldr.AddColumn(Ndcg, aggregator.GetSlotNames, NumberType.R8, ndcg.ToArray());
overallDvBldr.AddColumn(Dcg, aggregator.GetSlotNames, NumberType.R8, dcg.ToArray());
var groupDvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
groupDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, groupStratCol.ToArray());
groupDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, groupStratVal.ToArray());
}
if (groupSummary)
{
groupDvBldr.AddColumn(GroupId, TextType.Instance, groupName.ToArray());
groupDvBldr.AddColumn(Ndcg, aggregator.GetGroupSummarySlotNames("NDCG"), NumberType.R8, groupNdcg.ToArray());
groupDvBldr.AddColumn(Dcg, aggregator.GetGroupSummarySlotNames("DCG"), NumberType.R8, groupDcg.ToArray());
groupDvBldr.AddColumn(MaxDcg, aggregator.GetGroupSummarySlotNames("MaxDCG"), NumberType.R8, groupMaxDcg.ToArray());
}
var result = new Dictionary <string, IDataView>();
result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
if (groupSummary)
{
result.Add(GroupSummary, groupDvBldr.GetDataView());
}
return(result);
};
}
private protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, AggregatorDictionaryBase[] dictionaries,
out Action <uint, ReadOnlyMemory <char>, Aggregator> addAgg, out Func <Dictionary <string, IDataView> > consolidate)
{
var stratCol = new List <uint>();
var stratVal = new List <ReadOnlyMemory <char> >();
var isWeighted = new List <bool>();
var l1 = new List <Double>();
var l2 = new List <Double>();
var dist = new List <Double>();
var perLabelL1 = new List <Double[]>();
var perLabelL2 = new List <Double[]>();
var perLabelRms = new List <Double[]>();
var perLabelLoss = new List <Double[]>();
bool hasStrats = Utils.Size(dictionaries) > 0;
bool hasWeight = aggregator.Weighted;
addAgg =
(stratColKey, stratColVal, agg) =>
{
Host.Check(agg.Weighted == hasWeight, "All aggregators must either be weighted or unweighted");
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(false);
l1.Add(agg.UnweightedCounters.L1);
l2.Add(agg.UnweightedCounters.L2);
dist.Add(agg.UnweightedCounters.Dist);
perLabelL1.Add(agg.UnweightedCounters.PerLabelL1);
perLabelL2.Add(agg.UnweightedCounters.PerLabelL2);
perLabelRms.Add(agg.UnweightedCounters.PerLabelRms);
perLabelLoss.Add(agg.UnweightedCounters.PerLabelLoss);
if (agg.Weighted)
{
stratCol.Add(stratColKey);
stratVal.Add(stratColVal);
isWeighted.Add(true);
l1.Add(agg.WeightedCounters.L1);
l2.Add(agg.WeightedCounters.L2);
dist.Add(agg.WeightedCounters.Dist);
perLabelL1.Add(agg.WeightedCounters.PerLabelL1);
perLabelL2.Add(agg.WeightedCounters.PerLabelL2);
perLabelRms.Add(agg.WeightedCounters.PerLabelRms);
perLabelLoss.Add(agg.WeightedCounters.PerLabelLoss);
}
};
consolidate =
() =>
{
var overallDvBldr = new ArrayDataViewBuilder(Host);
if (hasStrats)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratCol, GetKeyValueGetter(dictionaries), (ulong)dictionaries.Length, stratCol.ToArray());
overallDvBldr.AddColumn(MetricKinds.ColumnNames.StratVal, TextType.Instance, stratVal.ToArray());
}
if (hasWeight)
{
overallDvBldr.AddColumn(MetricKinds.ColumnNames.IsWeighted, BoolType.Instance, isWeighted.ToArray());
}
overallDvBldr.AddColumn(PerLabelL1, aggregator.GetSlotNames, NumberType.R8, perLabelL1.ToArray());
overallDvBldr.AddColumn(PerLabelL2, aggregator.GetSlotNames, NumberType.R8, perLabelL2.ToArray());
overallDvBldr.AddColumn(PerLabelRms, aggregator.GetSlotNames, NumberType.R8, perLabelRms.ToArray());
overallDvBldr.AddColumn(PerLabelLoss, aggregator.GetSlotNames, NumberType.R8, perLabelLoss.ToArray());
overallDvBldr.AddColumn(L1, NumberType.R8, l1.ToArray());
overallDvBldr.AddColumn(L2, NumberType.R8, l2.ToArray());
overallDvBldr.AddColumn(Dist, NumberType.R8, dist.ToArray());
var result = new Dictionary <string, IDataView>();
result.Add(MetricKinds.OverallMetrics, overallDvBldr.GetDataView());
return(result);
};
}
