/// <summary>
/// Gets the calibration score for the specified expert <c>e</c>.
/// </summary>
/// <returns>The calibration score.</returns>
/// <param name="e">The expert.</param>
public double GetCalibrationScore(Expert e)
{
var p = GetInterquantileRanges(Quantiles);
var score = 0d;
var s = GetEmpiricalDistributions(e, Quantiles);
for (int i = 0; i < p.Length; i++)
{
double lscore = 0;
var epsilon = Double.Parse("10e-5");
if (Math.Abs(p[i]) < epsilon && Math.Abs(s[i]) > epsilon)
{
throw new ArgumentException("Specified quantiles are incompatible with datas. There" +
"is no absolute continuity between interpolated probabilities and empirical " +
"distribution. See http://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence for " +
"more details.");
}
if (s[i] > 0 & p[i] > 0)
{
lscore = s[i] * Math.Log(s[i] / p[i]);
}
score += lscore;
}
var nvar = Variables.Count();
return(1 - ChiSquared.CDF(p.Length - 1, 2 * nvar * score));
}
public void AddExpert(Expert e)
{
this.Experts.Add(e);
foreach (var variable in e.Estimates.Keys)
{
this.Variables.Add(variable);
}
}
/// <summary>
/// Gets the information score for the specified expert <c>e</c>.
/// </summary>
/// <returns>The information score.</returns>
/// <param name="e">The expert.</param>
public double GetInformationScore(Expert e)
{
var score = 0d;
foreach (var v in Variables.OfType <CalibrationVariable>())
{
var lscore = GetInformationScore(v, e);
score += lscore;
}
return(score / Variables.Count());
}
public static void ExampleBook(string[] args)
{
Console.WriteLine("Hello World!");
var item1 = new CalibrationVariable("Item 1", 27);
var item2 = new CalibrationVariable("Item 2", 45);
var item3 = new CalibrationVariable("Item 3", 60);
var item4 = new CalibrationVariable("Item 4", 28);
var expert1 = new Expert("Expert 1");
expert1.AddEstimate(item1, new PERTEstimate(25, 32, 37, .05));
expert1.AddEstimate(item2, new PERTEstimate(35, 42, 47, .05));
expert1.AddEstimate(item3, new PERTEstimate(50, 57, 62, .05));
expert1.AddEstimate(item4, new PERTEstimate(25, 32, 37, .05));
var expert2 = new Expert("Expert 2");
expert2.AddEstimate(item1, new PERTEstimate(0, 45, 78, .05));
expert2.AddEstimate(item2, new PERTEstimate(0, 45, 78, .05));
expert2.AddEstimate(item3, new PERTEstimate(0, 45, 78, .05));
expert2.AddEstimate(item4, new PERTEstimate(0, 45, 78, .05));
var expert3 = new Expert("Expert 3");
expert3.AddEstimate(item1, new PERTEstimate(40, 45, 52, .05));
expert3.AddEstimate(item2, new PERTEstimate(35, 40, 47, .05));
expert3.AddEstimate(item3, new PERTEstimate(39, 44, 51, .05));
expert3.AddEstimate(item4, new PERTEstimate(41, 46, 53, .05));
var manager = new ExpertManager();
manager.AddExpert(expert1);
manager.AddExpert(expert2);
manager.AddExpert(expert3);
manager.SetQuantiles(new [] { 0.05, .5, .95 });
Console.WriteLine("-- Calibration");
Console.WriteLine(string.Join("\n", manager.GetCalibrationScores().Select(x => "I(" + x.Item1.Name + ") = " + x.Item2)));
Console.WriteLine();
Console.WriteLine("-- Information");
Console.WriteLine(string.Join("\n", manager.GetInformationScores().Select(x => "C(" + x.Item1.Name + ") = " + x.Item2)));
Console.WriteLine();
Console.WriteLine("-- Weight");
Console.WriteLine(string.Join("\n", manager.GetWeights().Select(x => "W(" + x.Item1.Name + ") = " + x.Item2)));
Console.WriteLine();
}
/// <summary>
/// Gets the information score for the specified variable <c>v</c> and specified expert <c>e</c>.
/// </summary>
/// <returns>The information score.</returns>
/// <param name="v">The variable.</param>
/// <param name="e">The expert.</param>
public double GetInformationScore(CalibrationVariable v, Expert e)
{
var expertOpinion = e.Estimates[v];
var p = GetInterquantileRanges(Quantiles);
var r = GetInterpolatedDistribution(v, expertOpinion, Quantiles);
var score = 0d;
for (int i = 0; i < p.Length; i++)
{
var lscore = (p[i] * Math.Log(p[i] / r[i]));
score += lscore;
}
return(score);
}
/// <summary>
/// Gets the empirical distributions for the specified expert <c>e</c>.
/// </summary>
/// <returns>The empirical distributions.</returns>
/// <param name="e">E.</param>
double[] GetEmpiricalDistributions(Expert e, double[] quantiles)
{
var res = new double[quantiles.Length + 1];
foreach (var v in Variables.OfType <CalibrationVariable> ())
{
var trueValue = v.TrueValue;
var estimates = e.Estimates[v];
var i = GetInterval(trueValue, estimates, quantiles);
res[i]++;
}
var nvar = Variables.Count();
return(res.Select(x => x / nvar).ToArray());
}
