public void Verify(List <BooleanStatistic> statistics, CorrelationMatrix correlationMatrix, double meanConfidence = 0.1, double correlationConfidence = 0.1, bool onlyWarn = false, Dictionary <string, double> diffie = null)
{
var samples = GetSamples(1000, statistics);
var taken = new HashSet <string>();
foreach (var outerStat in statistics)
{
if (!NameToColumnIndex.ContainsKey(outerStat.Name))
{
continue;
}
var samplesForOuterStat = GetValue(NameToColumnIndex[outerStat.Name], samples);
var meanForOuterStat = TrueRatio(samplesForOuterStat);
ThrowIfExceedsConfidence(meanForOuterStat, outerStat.TrueRatio, meanConfidence, "Mean for " + outerStat.Name, onlyWarn);
foreach (var innerStat in statistics)
{
var combo = innerStat.Name + outerStat.Name;
if (!NameToColumnIndex.ContainsKey(innerStat.Name))
{
continue;
}
if (taken.Contains(combo))
{
continue;
}
if (innerStat.Name == outerStat.Name)
{
continue;
}
if (outerStat.TrueRatio == 100 || outerStat.TrueRatio == 0 || innerStat.TrueRatio == 100 || innerStat.TrueRatio == 0)
{
continue;
}
var samplesForInnerStat = GetValue(NameToColumnIndex[innerStat.Name], samples);
var cor = StatisticHelper.CalculateCorrelation(samplesForOuterStat, samplesForInnerStat);
var orgCor = correlationMatrix.GetValue(outerStat.Name, innerStat.Name).Value;
var realDiff = ThrowIfExceedsConfidence(cor, orgCor, correlationConfidence, "Correlation between " + innerStat.Name + " and " + outerStat.Name, onlyWarn);
if (diffie != null)
{
if (!diffie.ContainsKey(combo))
{
diffie.Add(combo, 0);
}
diffie[combo] += realDiff;
}
taken.Add(outerStat.Name + innerStat.Name);
}
}
}
public static CorrelationMatrix Create(Dictionary <string, Statistic> statisticsDic, double correlationThreshold = 0.3)
{
var matrix = new CorrelationMatrix(statisticsDic.Count, statisticsDic.Keys.ToArray());
var taken = new HashSet <string>();
foreach (var outerStat in statisticsDic.Keys) //vertical
{
foreach (var innerStat in statisticsDic.Keys) //horizont
{
if (taken.Contains(outerStat + innerStat) || taken.Contains(innerStat + outerStat))
{
continue;
}
var valuesOuter = statisticsDic[outerStat].GetSamples();
var valuesInner = statisticsDic[innerStat].GetSamples();
if (valuesInner == null || valuesOuter == null || valuesOuter.Length < 2 || valuesInner.Length < 2)
{
matrix.SetValue(outerStat, innerStat, null, null);
continue;
}
int?numberOfSamples = null;
if (valuesOuter.Length != valuesInner.Length)
{
numberOfSamples = Math.Min(valuesInner.Length, valuesOuter.Length);
valuesInner = valuesInner.Take(numberOfSamples.Value).ToArray();
valuesOuter = valuesOuter.Take(numberOfSamples.Value).ToArray();
}
var value = matrix.SetValue(outerStat, innerStat, StatisticHelper.CalculateCorrelation(valuesOuter, valuesInner), numberOfSamples);
taken.Add(outerStat + innerStat);
}
}
return(matrix);
}
public void BuildCoverianceMatrix(List <BooleanStatistic> statistics, CorrelationMatrix correlationMatrix, Func <double[, ], double[, ]> getSigma)
{
var taken = new HashSet <string>();
NameToColumnIndex = new Dictionary <string, int>();
ColumnIndexToName = new Dictionary <int, string>();
if (statistics.Any(s => string.IsNullOrEmpty(s.Name)))
{
throw new ArgumentException("All statistics.name must be non-null");
}
var commonprob = new double[statistics.Count, statistics.Count];
int outerIndex = -1;
int innerIndex = -1;
foreach (var outerStat in statistics)
{
outerIndex++;
NameToColumnIndex.Add(outerStat.Name, outerIndex);
ColumnIndexToName.Add(outerIndex, outerStat.Name);
foreach (var innerStat in statistics)
{
innerIndex++;
var key = GetKey(outerIndex, innerIndex);
if (taken.Contains(key) || taken.Contains(GetKey(innerIndex, outerIndex)))
{
continue;
}
taken.Add(key);
if (outerIndex == innerIndex)
{
if (outerStat.Name != innerStat.Name)
{
throw new ArgumentException("Bug..");
}
commonprob[outerIndex, innerIndex] = outerStat.TrueRatio;
continue;
}
var correlation = correlationMatrix.GetValue(outerStat.Name, innerStat.Name);
if (Double.IsNaN(correlation.Value))
{
throw new ArgumentException("Correlation cannot be 0");
}
if (correlation.Value > 1.0)
{
correlation.Value = 1.0;
}
var value = GetPaGivenB(outerStat, innerStat, correlation.Value);
commonprob[outerIndex, innerIndex] = value;
commonprob[innerIndex, outerIndex] = value;
}
innerIndex = -1;
}
CovarianceMatrix = getSigma(commonprob);
}
